{
  "cases": [
    {
      "errorsWithinExact": 1,
      "exact": 20,
      "id": "input.BanSubstrings",
      "nativeScopeRejected": 1,
      "rows": 21,
      "status": "unsupported"
    },
    {
      "errorsWithinExact": 3,
      "exact": 19,
      "id": "input.Regex",
      "nativeScopeRejected": 1,
      "rows": 20,
      "status": "unsupported"
    },
    {
      "errorsWithinExact": 1,
      "exact": 21,
      "id": "output.BanSubstrings",
      "nativeScopeRejected": 1,
      "rows": 22,
      "status": "unsupported"
    },
    {
      "errorsWithinExact": 0,
      "exact": 8,
      "id": "output.Deanonymize",
      "nativeScopeRejected": 0,
      "rows": 8,
      "status": "identical"
    },
    {
      "errorsWithinExact": 0,
      "exact": 20,
      "id": "output.JSON",
      "nativeScopeRejected": 3,
      "rows": 23,
      "status": "unsupported"
    },
    {
      "errorsWithinExact": 0,
      "exact": 9,
      "id": "output.NoRefusalLight",
      "nativeScopeRejected": 0,
      "rows": 9,
      "status": "identical"
    },
    {
      "errorsWithinExact": 2,
      "exact": 16,
      "id": "output.ReadingTime",
      "nativeScopeRejected": 0,
      "rows": 16,
      "status": "identical"
    },
    {
      "errorsWithinExact": 3,
      "exact": 20,
      "id": "output.Regex",
      "nativeScopeRejected": 1,
      "rows": 21,
      "status": "unsupported"
    }
  ],
  "comparator": "Exact cleaned-text digest, validity and risk, with matched error classes counted separately. Native scope refusals remain in the denominator.",
  "corpus": {
    "name": "Authored fixed text and recorded policy settings",
    "source": {
      "commit": "a7d96a424d079048028dc690c3853767a7ee009b",
      "file": "evals/suites/llm-guard-passive/summary.json",
      "sha256": "449c8ab56cc007d18cec7002b33c336d2731ec66c3829fe9bda52bf2ca53f33f"
    }
  },
  "engine": {
    "clean": true,
    "commit": "a7d96a424d079048028dc690c3853767a7ee009b",
    "name": "Keryn native passive text policy scanners"
  },
  "metrics": [
    {
      "denominator": 8,
      "id": "identical-cases",
      "label": "Scanner groups with exact agreement on every row",
      "numerator": 3,
      "of": "all scanner groups tested",
      "value": 0.375
    },
    {
      "denominator": 140,
      "id": "exact-outcomes",
      "label": "Rows with exact outcome agreement",
      "numerator": 133,
      "of": "all recorded rows including errors and native scope refusals",
      "value": 0.95
    },
    {
      "denominator": 140,
      "id": "matching-errors",
      "label": "Rows with matching error outcomes",
      "numerator": 10,
      "of": "all recorded rows",
      "value": 0.0714
    },
    {
      "denominator": 140,
      "id": "scope-refusals",
      "label": "Rows refused by native scope",
      "numerator": 7,
      "of": "all recorded rows",
      "value": 0.05
    }
  ],
  "question": "Which authored policy checks match the pinned passive scanners on the same inputs?",
  "reference": {
    "commit": "168c1034ffdb33837e7ae6fd6a16b80567c1be03",
    "name": "LLM Guard read-only scanner methods",
    "repo": "https://github.com/protectai/llm-guard",
    "unicode": "16.0.0"
  },
  "schema": "keryn-eval-receipt/1",
  "scope": [
    "Fixture agreement checks the measured scanner contracts and preserves explicit native scope refusals.",
    "Other scanner families and production accuracy remain outside this comparison. No live model or external target was called."
  ],
  "suite": "llm-guard-passive",
  "summary": {
    "cases": 8,
    "exact": 133,
    "matchingErrors": 10,
    "nativeScopeRejected": 7,
    "rows": 140,
    "statuses": {
      "differs": 0,
      "error": 0,
      "identical": 3,
      "incomplete": 0,
      "no-reference": 0,
      "timeout": 0,
      "unsupported": 5
    },
    "upstreamScannersInventoried": 39
  },
  "title": "LLM Guard passive scanner parity"
}
