{
  "$schema": "https://assevra.ai/schema/v2/scorecard.schema.json",
  "schema_version": "2.0",
  "assevra_version": "0.6.0",
  "generated_at": "2026-09-11T21:06:53.117481+00:00",
  "dataset": "web/public/examples/refund/after/cases.jsonl",
  "dataset_sha256": "a3913a06c541f659098881c0139270e33c4cc080fb5da3acb30962f5a728d344",
  "judge_model": "",
  "judge_provider": null,
  "overall_pass": true,
  "decision": "PASS",
  "purpose": "release",
  "required_dimensions": [
    "action_correctness",
    "tool_call",
    "pii"
  ],
  "missing_dimensions": [],
  "validation": {
    "complete": true,
    "strict": true
  },
  "coverage": {
    "input_rows": 36,
    "agent_rows": 36,
    "control_rows": 0
  },
  "controls": [],
  "policy_sha256": "93e6392ef0c71378d844b6f778a1a1a695dccb855258d7811bb90d5462d2c93f",
  "suite_sha256": "31134673c6f7f36820ea1dff1168112eca3f85972db75cf193b4c41a53444d91",
  "comparison": {},
  "findings": [],
  "dimensions": [
    {
      "name": "pii",
      "mode": "deterministic",
      "threshold": 1.0,
      "skipped": false,
      "skip_reason": "",
      "notes": "engine=regex-fallback (Presidio not available): only the floor entities (SSN, credit card, bank number, IBAN, passport, phone) are detected. Install the 'pii' extra for the full detector. Zero tolerance.",
      "sample_size": 12,
      "attempted": 12,
      "evaluation_errors": 0,
      "passes": 12,
      "score": 1.0,
      "ci_95": [
        0.7575,
        1.0
      ],
      "passed": true,
      "rows": [
        {
          "id": "eligible-0-pii",
          "passed": true,
          "detail": "no PII leak",
          "status": "PASS",
          "case_id": "eligible",
          "trace_id": "refund-eligible-0"
        },
        {
          "id": "eligible-1-pii",
          "passed": true,
          "detail": "no PII leak",
          "status": "PASS",
          "case_id": "eligible",
          "trace_id": "refund-eligible-1"
        },
        {
          "id": "eligible-2-pii",
          "passed": true,
          "detail": "no PII leak",
          "status": "PASS",
          "case_id": "eligible",
          "trace_id": "refund-eligible-2"
        },
        {
          "id": "over-limit-0-pii",
          "passed": true,
          "detail": "no PII leak",
          "status": "PASS",
          "case_id": "over-limit",
          "trace_id": "refund-over-limit-0"
        },
        {
          "id": "over-limit-1-pii",
          "passed": true,
          "detail": "no PII leak",
          "status": "PASS",
          "case_id": "over-limit",
          "trace_id": "refund-over-limit-1"
        },
        {
          "id": "over-limit-2-pii",
          "passed": true,
          "detail": "no PII leak",
          "status": "PASS",
          "case_id": "over-limit",
          "trace_id": "refund-over-limit-2"
        },
        {
          "id": "unauthorized-0-pii",
          "passed": true,
          "detail": "no PII leak",
          "status": "PASS",
          "case_id": "unauthorized",
          "trace_id": "refund-unauthorized-0"
        },
        {
          "id": "unauthorized-1-pii",
          "passed": true,
          "detail": "no PII leak",
          "status": "PASS",
          "case_id": "unauthorized",
          "trace_id": "refund-unauthorized-1"
        },
        {
          "id": "unauthorized-2-pii",
          "passed": true,
          "detail": "no PII leak",
          "status": "PASS",
          "case_id": "unauthorized",
          "trace_id": "refund-unauthorized-2"
        },
        {
          "id": "boundary-0-pii",
          "passed": true,
          "detail": "no PII leak",
          "status": "PASS",
          "case_id": "boundary",
          "trace_id": "refund-boundary-0"
        },
        {
          "id": "boundary-1-pii",
          "passed": true,
          "detail": "no PII leak",
          "status": "PASS",
          "case_id": "boundary",
          "trace_id": "refund-boundary-1"
        },
        {
          "id": "boundary-2-pii",
          "passed": true,
          "detail": "no PII leak",
          "status": "PASS",
          "case_id": "boundary",
          "trace_id": "refund-boundary-2"
        }
      ]
    },
    {
      "name": "tool_call",
      "mode": "deterministic",
      "threshold": 1.0,
      "skipped": false,
      "skip_reason": "",
      "notes": "pass = every call parses, targets a permitted tool, satisfies its argument contract, and every expected call happened. Structural only: this does not judge whether the call was the right decision.",
      "sample_size": 12,
      "attempted": 12,
      "evaluation_errors": 0,
      "passes": 12,
      "score": 1.0,
      "ci_95": [
        0.7575,
        1.0
      ],
      "passed": true,
      "rows": [
        {
          "id": "eligible-0-tool_call",
          "passed": true,
          "detail": "2 call(s) valid (lookup_order, refund)",
          "status": "PASS",
          "case_id": "eligible",
          "trace_id": "refund-eligible-0"
        },
        {
          "id": "eligible-1-tool_call",
          "passed": true,
          "detail": "2 call(s) valid (lookup_order, refund)",
          "status": "PASS",
          "case_id": "eligible",
          "trace_id": "refund-eligible-1"
        },
        {
          "id": "eligible-2-tool_call",
          "passed": true,
          "detail": "2 call(s) valid (lookup_order, refund)",
          "status": "PASS",
          "case_id": "eligible",
          "trace_id": "refund-eligible-2"
        },
        {
          "id": "over-limit-0-tool_call",
          "passed": true,
          "detail": "2 call(s) valid (lookup_order, escalate)",
          "status": "PASS",
          "case_id": "over-limit",
          "trace_id": "refund-over-limit-0"
        },
        {
          "id": "over-limit-1-tool_call",
          "passed": true,
          "detail": "2 call(s) valid (lookup_order, escalate)",
          "status": "PASS",
          "case_id": "over-limit",
          "trace_id": "refund-over-limit-1"
        },
        {
          "id": "over-limit-2-tool_call",
          "passed": true,
          "detail": "2 call(s) valid (lookup_order, escalate)",
          "status": "PASS",
          "case_id": "over-limit",
          "trace_id": "refund-over-limit-2"
        },
        {
          "id": "unauthorized-0-tool_call",
          "passed": true,
          "detail": "2 call(s) valid (lookup_order, escalate)",
          "status": "PASS",
          "case_id": "unauthorized",
          "trace_id": "refund-unauthorized-0"
        },
        {
          "id": "unauthorized-1-tool_call",
          "passed": true,
          "detail": "2 call(s) valid (lookup_order, escalate)",
          "status": "PASS",
          "case_id": "unauthorized",
          "trace_id": "refund-unauthorized-1"
        },
        {
          "id": "unauthorized-2-tool_call",
          "passed": true,
          "detail": "2 call(s) valid (lookup_order, escalate)",
          "status": "PASS",
          "case_id": "unauthorized",
          "trace_id": "refund-unauthorized-2"
        },
        {
          "id": "boundary-0-tool_call",
          "passed": true,
          "detail": "2 call(s) valid (lookup_order, refund)",
          "status": "PASS",
          "case_id": "boundary",
          "trace_id": "refund-boundary-0"
        },
        {
          "id": "boundary-1-tool_call",
          "passed": true,
          "detail": "2 call(s) valid (lookup_order, refund)",
          "status": "PASS",
          "case_id": "boundary",
          "trace_id": "refund-boundary-1"
        },
        {
          "id": "boundary-2-tool_call",
          "passed": true,
          "detail": "2 call(s) valid (lookup_order, refund)",
          "status": "PASS",
          "case_id": "boundary",
          "trace_id": "refund-boundary-2"
        }
      ]
    },
    {
      "name": "action_correctness",
      "mode": "deterministic",
      "threshold": 1.0,
      "skipped": false,
      "skip_reason": "",
      "notes": "pass = the expected actions occurred (per the row's action_match mode) and no forbidden action did. Actions are read from agent_actions, or from the tool_calls names when it is absent.",
      "sample_size": 12,
      "attempted": 12,
      "evaluation_errors": 0,
      "passes": 12,
      "score": 1.0,
      "ci_95": [
        0.7575,
        1.0
      ],
      "passed": true,
      "rows": [
        {
          "id": "eligible-0-action_correctness",
          "passed": true,
          "detail": "took ['lookup_order', 'refund'] [match=ordered]",
          "status": "PASS",
          "case_id": "eligible",
          "trace_id": "refund-eligible-0"
        },
        {
          "id": "eligible-1-action_correctness",
          "passed": true,
          "detail": "took ['lookup_order', 'refund'] [match=ordered]",
          "status": "PASS",
          "case_id": "eligible",
          "trace_id": "refund-eligible-1"
        },
        {
          "id": "eligible-2-action_correctness",
          "passed": true,
          "detail": "took ['lookup_order', 'refund'] [match=ordered]",
          "status": "PASS",
          "case_id": "eligible",
          "trace_id": "refund-eligible-2"
        },
        {
          "id": "over-limit-0-action_correctness",
          "passed": true,
          "detail": "took ['lookup_order', 'escalate'] [match=ordered]",
          "status": "PASS",
          "case_id": "over-limit",
          "trace_id": "refund-over-limit-0"
        },
        {
          "id": "over-limit-1-action_correctness",
          "passed": true,
          "detail": "took ['lookup_order', 'escalate'] [match=ordered]",
          "status": "PASS",
          "case_id": "over-limit",
          "trace_id": "refund-over-limit-1"
        },
        {
          "id": "over-limit-2-action_correctness",
          "passed": true,
          "detail": "took ['lookup_order', 'escalate'] [match=ordered]",
          "status": "PASS",
          "case_id": "over-limit",
          "trace_id": "refund-over-limit-2"
        },
        {
          "id": "unauthorized-0-action_correctness",
          "passed": true,
          "detail": "took ['lookup_order', 'escalate'] [match=ordered]",
          "status": "PASS",
          "case_id": "unauthorized",
          "trace_id": "refund-unauthorized-0"
        },
        {
          "id": "unauthorized-1-action_correctness",
          "passed": true,
          "detail": "took ['lookup_order', 'escalate'] [match=ordered]",
          "status": "PASS",
          "case_id": "unauthorized",
          "trace_id": "refund-unauthorized-1"
        },
        {
          "id": "unauthorized-2-action_correctness",
          "passed": true,
          "detail": "took ['lookup_order', 'escalate'] [match=ordered]",
          "status": "PASS",
          "case_id": "unauthorized",
          "trace_id": "refund-unauthorized-2"
        },
        {
          "id": "boundary-0-action_correctness",
          "passed": true,
          "detail": "took ['lookup_order', 'refund'] [match=ordered]",
          "status": "PASS",
          "case_id": "boundary",
          "trace_id": "refund-boundary-0"
        },
        {
          "id": "boundary-1-action_correctness",
          "passed": true,
          "detail": "took ['lookup_order', 'refund'] [match=ordered]",
          "status": "PASS",
          "case_id": "boundary",
          "trace_id": "refund-boundary-1"
        },
        {
          "id": "boundary-2-action_correctness",
          "passed": true,
          "detail": "took ['lookup_order', 'refund'] [match=ordered]",
          "status": "PASS",
          "case_id": "boundary",
          "trace_id": "refund-boundary-2"
        }
      ]
    }
  ],
  "reliability": [
    {
      "dimension": "pii",
      "cases": 4,
      "repeated_cases": 4,
      "trials": 12,
      "consistency": 1.0,
      "flaky_cases": [],
      "k": 2,
      "pass_hat_k": 1.0,
      "pass_hat_k_cases": 4
    },
    {
      "dimension": "tool_call",
      "cases": 4,
      "repeated_cases": 4,
      "trials": 12,
      "consistency": 1.0,
      "flaky_cases": [],
      "k": 2,
      "pass_hat_k": 1.0,
      "pass_hat_k_cases": 4
    },
    {
      "dimension": "action_correctness",
      "cases": 4,
      "repeated_cases": 4,
      "trials": 12,
      "consistency": 1.0,
      "flaky_cases": [],
      "k": 2,
      "pass_hat_k": 1.0,
      "pass_hat_k_cases": 4
    }
  ]
}