Weigh evals you can watch
2026-10-20 22:34 UTC

V-10

exchange-triage · run 2026-10-19-prompts

Input

What every variant was given, as the suite chose to show it.

Broker: Variant desk

Hello,

Reviewing options for Timberline Walk Apartments LLC — 44 units at 7800 Timberline Dr, Portland, OR 97219 (Multnomah County). Wood frame (unprotected), 4 stories, built 1996, no sprinklers.

TIV $6,100,000. Target premium $58,000. One large water loss 2021, $86,000 paid.

Regards.

Outputs

One column per variant. Checks are the suite's binary checks on this output; a judge that gave no verdict is unanswered and not counted.

v1

result
ok
prompt tokens
4,320
output tokens
952
cost
$0.0248
latency
17,104 ms · 3 calls
at
2026-10-19 22:28
stages
intake extraction grounding appetite
decision
DECLINED
fitScore
28
retries
0
fieldsWrong
1
checkverdict
extraction_schemapass
fields_correctfail
construction_canonicalfail
nulls_stay_nullnot applicable
decision_correctpass
attribution_jevpass
attribution_claudefail
{
 "decision": "DECLINED",
 "expectedDecision": "DECLINED",
 "decisionCorrect": true,
 "fitScore": 28,
 "confidence": 0.9671428571428571,
 "reasons": [
  "\"Wood frame (unprotected)\" matches excluded construction: unprotected wood frame over 3 stories"
 ],
 "extraction": {
  "insuredName": {
   "value": "Timberline Walk Apartments LLC",
   "confidence": 0.97,
   "source": "email"
  },
  "propertyAddress": {
   "value": "7800 Timberline Dr, Portland, OR 97219",
   "confidence": 0.97,
   "source": "email"
  },
  "state": {
   "value": "OR",
   "confidence": 0.99,
   "source": "email"
  },
  "county": {
   "value": "Multnomah",
   "confidence": 0.98,
   "source": "email"
  },
  "units": {
   "value": 44,
   "confidence": 0.98,
   "source": "email"
  },
  "tiv": {
   "value": 6100000,
   "confidence": 0.98,
   "source": "email"
  },
  "constructionType": {
   "value": "Wood frame (unprotected)",
   "confidence": 0.9,
   "source": "email"
  },
  "yearBuilt": {
   "value": 1996,
   "confidence": 0.98,
   "source": "email"
  },
  "targetPremium": {
   "value": 58000,
   "confidence": 0.98,
   "source": "email"
  },
  "subclass": {
   "value": "Mid-rise apartments",
   "confidence": 0.5,
   "source": "email"
  },
  "priorLossSummary": {
   "value": "One large water loss in 2021, $86,000 paid.",
   "confidence": 0.95,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Timberline Walk Apartments LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "7800 Timberline Dr, Portland, OR 97219",
   "source": "email"
  },
  {
   "field": "state",
   "value": "OR",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Multnomah",
   "source": "email"
  },
  {
   "field": "units",
   "value": 44,
   "source": "email"
  },
  {
   "field": "tiv",
   "value": 6100000,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Wood frame (unprotected)",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 1996,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 58000,
   "source": "email"
  },
  {
   "field": "subclass",
   "value": "Mid-rise apartments",
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "One large water loss in 2021, $86,000 paid.",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Timberline Walk Apartments LLC",
   "got": "Timberline Walk Apartments LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "OR",
   "got": "OR",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Multnomah",
   "got": "Multnomah",
   "correct": true
  },
  {
   "field": "units",
   "expected": 44,
   "got": 44,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": 6100000,
   "got": 6100000,
   "correct": true
  },
  {
   "field": "constructionType",
   "expected": "Frame (unprotected)",
   "got": "Wood frame (unprotected)",
   "correct": false
  },
  {
   "field": "yearBuilt",
   "expected": 1996,
   "got": 1996,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 58000,
   "got": 58000,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "subclass",
   "issue": "ambiguous",
   "detail": "not clearly supported by source: 'Mid-rise apartments' does not appear in the email; only 4 stories is stated, so the subclass is inferred."
  }
 ]
}

v2

result
ok
prompt tokens
4,742
output tokens
949
cost
$0.0231
latency
13,962 ms · 3 calls
at
2026-10-19 22:27
stages
intake extraction grounding appetite
decision
DECLINED
fitScore
28
retries
0
fieldsWrong
0
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalpass
nulls_stay_nullnot applicable
decision_correctpass
attribution_jevfail
attribution_claudefail
{
 "decision": "DECLINED",
 "expectedDecision": "DECLINED",
 "decisionCorrect": true,
 "fitScore": 28,
 "confidence": 0.9742857142857143,
 "reasons": [
  "\"Frame (unprotected)\" matches excluded construction: unprotected wood frame over 3 stories"
 ],
 "extraction": {
  "insuredName": {
   "value": "Timberline Walk Apartments LLC",
   "confidence": 0.97,
   "source": "email"
  },
  "propertyAddress": {
   "value": "7800 Timberline Dr, Portland, OR 97219",
   "confidence": 0.97,
   "source": "email"
  },
  "state": {
   "value": "OR",
   "confidence": 0.99,
   "source": "email"
  },
  "county": {
   "value": "Multnomah",
   "confidence": 0.97,
   "source": "email"
  },
  "units": {
   "value": 44,
   "confidence": 0.98,
   "source": "email"
  },
  "tiv": {
   "value": 6100000,
   "confidence": 0.98,
   "source": "email"
  },
  "constructionType": {
   "value": "Frame (unprotected)",
   "confidence": 0.95,
   "source": "email"
  },
  "yearBuilt": {
   "value": 1996,
   "confidence": 0.98,
   "source": "email"
  },
  "targetPremium": {
   "value": 58000,
   "confidence": 0.98,
   "source": "email"
  },
  "subclass": {
   "value": "Garden apartments",
   "confidence": 0.4,
   "source": "email"
  },
  "priorLossSummary": {
   "value": "One large water loss in 2021, $86,000 paid.",
   "confidence": 0.95,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Timberline Walk Apartments LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "7800 Timberline Dr, Portland, OR 97219",
   "source": "email"
  },
  {
   "field": "state",
   "value": "OR",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Multnomah",
   "source": "email"
  },
  {
   "field": "units",
   "value": 44,
   "source": "email"
  },
  {
   "field": "tiv",
   "value": 6100000,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Frame (unprotected)",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 1996,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 58000,
   "source": "email"
  },
  {
   "field": "subclass",
   "value": "Garden apartments",
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "One large water loss in 2021, $86,000 paid.",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Timberline Walk Apartments LLC",
   "got": "Timberline Walk Apartments LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "OR",
   "got": "OR",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Multnomah",
   "got": "Multnomah",
   "correct": true
  },
  {
   "field": "units",
   "expected": 44,
   "got": 44,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": 6100000,
   "got": 6100000,
   "correct": true
  },
  {
   "field": "constructionType",
   "expected": "Frame (unprotected)",
   "got": "Frame (unprotected)",
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 1996,
   "got": 1996,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 58000,
   "got": 58000,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "subclass",
   "issue": "ambiguous",
   "detail": "not clearly supported by source: 'Garden apartments' is not stated; email says 4 stories, which is inferred and arguably inconsistent with garden-style."
  }
 ]
}

v2-repeat

result
ok
prompt tokens
4,742
output tokens
954
cost
$0.0219
latency
13,503 ms · 3 calls
at
2026-10-19 22:27
stages
intake extraction grounding appetite
decision
DECLINED
fitScore
28
retries
0
fieldsWrong
0
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalpass
nulls_stay_nullnot applicable
decision_correctpass
attribution_jevfail
attribution_claudefail
{
 "decision": "DECLINED",
 "expectedDecision": "DECLINED",
 "decisionCorrect": true,
 "fitScore": 28,
 "confidence": 0.9585714285714284,
 "reasons": [
  "\"Frame (unprotected)\" matches excluded construction: unprotected wood frame over 3 stories"
 ],
 "extraction": {
  "insuredName": {
   "value": "Timberline Walk Apartments LLC",
   "confidence": 0.95,
   "source": "email"
  },
  "propertyAddress": {
   "value": "7800 Timberline Dr, Portland, OR 97219",
   "confidence": 0.95,
   "source": "email"
  },
  "state": {
   "value": "OR",
   "confidence": 0.98,
   "source": "email"
  },
  "county": {
   "value": "Multnomah",
   "confidence": 0.97,
   "source": "email"
  },
  "units": {
   "value": 44,
   "confidence": 0.97,
   "source": "email"
  },
  "tiv": {
   "value": 6100000,
   "confidence": 0.97,
   "source": "email"
  },
  "constructionType": {
   "value": "Frame (unprotected)",
   "confidence": 0.92,
   "source": "email"
  },
  "yearBuilt": {
   "value": 1996,
   "confidence": 0.97,
   "source": "email"
  },
  "targetPremium": {
   "value": 58000,
   "confidence": 0.97,
   "source": "email"
  },
  "subclass": {
   "value": "Garden apartments",
   "confidence": 0.4,
   "source": "email"
  },
  "priorLossSummary": {
   "value": "One large water loss in 2021, $86,000 paid",
   "confidence": 0.95,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Timberline Walk Apartments LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "7800 Timberline Dr, Portland, OR 97219",
   "source": "email"
  },
  {
   "field": "state",
   "value": "OR",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Multnomah",
   "source": "email"
  },
  {
   "field": "units",
   "value": 44,
   "source": "email"
  },
  {
   "field": "tiv",
   "value": 6100000,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Frame (unprotected)",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 1996,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 58000,
   "source": "email"
  },
  {
   "field": "subclass",
   "value": "Garden apartments",
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "One large water loss in 2021, $86,000 paid",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Timberline Walk Apartments LLC",
   "got": "Timberline Walk Apartments LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "OR",
   "got": "OR",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Multnomah",
   "got": "Multnomah",
   "correct": true
  },
  {
   "field": "units",
   "expected": 44,
   "got": 44,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": 6100000,
   "got": 6100000,
   "correct": true
  },
  {
   "field": "constructionType",
   "expected": "Frame (unprotected)",
   "got": "Frame (unprotected)",
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 1996,
   "got": 1996,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 58000,
   "got": 58000,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "warn"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": [
  {
   "field": "subclass",
   "issue": "ambiguous",
   "detail": "not clearly supported by source: 'Garden apartments' is not stated; the email only says 4 stories, which is inferred and not clearly garden-style."
  }
 ]
}

v3

result
ok
prompt tokens
5,026
output tokens
888
cost
$0.0224
latency
12,733 ms · 3 calls
at
2026-10-19 22:27
stages
intake extraction grounding appetite
decision
DECLINED
fitScore
28
retries
0
fieldsWrong
0
checkverdict
extraction_schemapass
fields_correctpass
construction_canonicalpass
nulls_stay_nullnot applicable
decision_correctpass
attribution_jevpass
attribution_claudepass
{
 "decision": "DECLINED",
 "expectedDecision": "DECLINED",
 "decisionCorrect": true,
 "fitScore": 28,
 "confidence": 0.9714285714285713,
 "reasons": [
  "\"Frame (unprotected)\" matches excluded construction: unprotected wood frame over 3 stories"
 ],
 "extraction": {
  "insuredName": {
   "value": "Timberline Walk Apartments LLC",
   "confidence": 0.97,
   "source": "email"
  },
  "propertyAddress": {
   "value": "7800 Timberline Dr, Portland, OR 97219",
   "confidence": 0.97,
   "source": "email"
  },
  "state": {
   "value": "OR",
   "confidence": 0.99,
   "source": "email"
  },
  "county": {
   "value": "Multnomah",
   "confidence": 0.98,
   "source": "email"
  },
  "units": {
   "value": 44,
   "confidence": 0.98,
   "source": "email"
  },
  "tiv": {
   "value": 6100000,
   "confidence": 0.98,
   "source": "email"
  },
  "constructionType": {
   "value": "Frame (unprotected)",
   "confidence": 0.93,
   "source": "email"
  },
  "yearBuilt": {
   "value": 1996,
   "confidence": 0.98,
   "source": "email"
  },
  "targetPremium": {
   "value": 58000,
   "confidence": 0.98,
   "source": "email"
  },
  "subclass": {
   "value": null,
   "confidence": 0.9,
   "source": null
  },
  "priorLossSummary": {
   "value": "One large water loss in 2021, $86,000 paid.",
   "confidence": 0.95,
   "source": "email"
  }
 },
 "extracted": [
  {
   "field": "insuredName",
   "value": "Timberline Walk Apartments LLC",
   "source": "email"
  },
  {
   "field": "propertyAddress",
   "value": "7800 Timberline Dr, Portland, OR 97219",
   "source": "email"
  },
  {
   "field": "state",
   "value": "OR",
   "source": "email"
  },
  {
   "field": "county",
   "value": "Multnomah",
   "source": "email"
  },
  {
   "field": "units",
   "value": 44,
   "source": "email"
  },
  {
   "field": "tiv",
   "value": 6100000,
   "source": "email"
  },
  {
   "field": "constructionType",
   "value": "Frame (unprotected)",
   "source": "email"
  },
  {
   "field": "yearBuilt",
   "value": 1996,
   "source": "email"
  },
  {
   "field": "targetPremium",
   "value": 58000,
   "source": "email"
  },
  {
   "field": "priorLossSummary",
   "value": "One large water loss in 2021, $86,000 paid.",
   "source": "email"
  }
 ],
 "fields": [
  {
   "field": "insuredName",
   "expected": "Timberline Walk Apartments LLC",
   "got": "Timberline Walk Apartments LLC",
   "correct": true
  },
  {
   "field": "state",
   "expected": "OR",
   "got": "OR",
   "correct": true
  },
  {
   "field": "county",
   "expected": "Multnomah",
   "got": "Multnomah",
   "correct": true
  },
  {
   "field": "units",
   "expected": 44,
   "got": 44,
   "correct": true
  },
  {
   "field": "tiv",
   "expected": 6100000,
   "got": 6100000,
   "correct": true
  },
  {
   "field": "constructionType",
   "expected": "Frame (unprotected)",
   "got": "Frame (unprotected)",
   "correct": true
  },
  {
   "field": "yearBuilt",
   "expected": 1996,
   "got": 1996,
   "correct": true
  },
  {
   "field": "targetPremium",
   "expected": 58000,
   "got": 58000,
   "correct": true
  }
 ],
 "retries": 0,
 "guardrails": [
  {
   "node": "extraction",
   "guardrail": "schema",
   "status": "pass"
  },
  {
   "node": "grounding",
   "guardrail": "grounding",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "missing_fields",
   "status": "pass"
  },
  {
   "node": "appetite",
   "guardrail": "confidence",
   "status": "pass"
  }
 ],
 "flags": []
}