{
  "cases": [
    {
      "cancelled": 0,
      "engineErrors": 1,
      "engineUnsupported": 0,
      "expectedScopeRefusals": 0,
      "id": "bleu",
      "matchingErrors": 1,
      "matchingGrades": 9,
      "matchingRows": 10,
      "referenceErrors": 1,
      "referenceUnsupported": 0,
      "rows": 10,
      "status": "identical"
    },
    {
      "cancelled": 0,
      "engineErrors": 0,
      "engineUnsupported": 0,
      "expectedScopeRefusals": 0,
      "id": "contains-html",
      "matchingErrors": 0,
      "matchingGrades": 5,
      "matchingRows": 5,
      "referenceErrors": 0,
      "referenceUnsupported": 0,
      "rows": 5,
      "status": "identical"
    },
    {
      "cancelled": 0,
      "engineErrors": 0,
      "engineUnsupported": 0,
      "expectedScopeRefusals": 0,
      "id": "contains-sql",
      "matchingErrors": 0,
      "matchingGrades": 7,
      "matchingRows": 7,
      "referenceErrors": 0,
      "referenceUnsupported": 0,
      "rows": 7,
      "status": "identical"
    },
    {
      "cancelled": 0,
      "engineErrors": 0,
      "engineUnsupported": 0,
      "expectedScopeRefusals": 0,
      "id": "contains-xml",
      "matchingErrors": 0,
      "matchingGrades": 6,
      "matchingRows": 6,
      "referenceErrors": 0,
      "referenceUnsupported": 0,
      "rows": 6,
      "status": "identical"
    },
    {
      "cancelled": 0,
      "engineErrors": 2,
      "engineUnsupported": 0,
      "expectedScopeRefusals": 0,
      "id": "gleu",
      "matchingErrors": 2,
      "matchingGrades": 6,
      "matchingRows": 8,
      "referenceErrors": 2,
      "referenceUnsupported": 0,
      "rows": 8,
      "status": "identical"
    },
    {
      "cancelled": 0,
      "engineErrors": 0,
      "engineUnsupported": 0,
      "expectedScopeRefusals": 0,
      "id": "guardrails",
      "matchingErrors": 0,
      "matchingGrades": 10,
      "matchingRows": 10,
      "referenceErrors": 0,
      "referenceUnsupported": 0,
      "rows": 10,
      "status": "identical"
    },
    {
      "cancelled": 0,
      "engineErrors": 0,
      "engineUnsupported": 0,
      "expectedScopeRefusals": 0,
      "id": "is-html",
      "matchingErrors": 0,
      "matchingGrades": 10,
      "matchingRows": 10,
      "referenceErrors": 0,
      "referenceUnsupported": 0,
      "rows": 10,
      "status": "identical"
    },
    {
      "cancelled": 0,
      "engineErrors": 1,
      "engineUnsupported": 3,
      "expectedScopeRefusals": 3,
      "id": "is-sql",
      "matchingErrors": 1,
      "matchingGrades": 12,
      "matchingRows": 13,
      "referenceErrors": 1,
      "referenceUnsupported": 0,
      "rows": 16,
      "status": "unsupported"
    },
    {
      "cancelled": 0,
      "engineErrors": 2,
      "engineUnsupported": 1,
      "expectedScopeRefusals": 1,
      "id": "is-xml",
      "matchingErrors": 2,
      "matchingGrades": 8,
      "matchingRows": 10,
      "referenceErrors": 2,
      "referenceUnsupported": 0,
      "rows": 11,
      "status": "unsupported"
    },
    {
      "cancelled": 0,
      "engineErrors": 0,
      "engineUnsupported": 3,
      "expectedScopeRefusals": 3,
      "id": "meteor",
      "matchingErrors": 0,
      "matchingGrades": 0,
      "matchingRows": 0,
      "referenceErrors": 3,
      "referenceUnsupported": 0,
      "rows": 3,
      "status": "unsupported"
    },
    {
      "cancelled": 0,
      "engineErrors": 0,
      "engineUnsupported": 1,
      "expectedScopeRefusals": 1,
      "id": "rouge-l",
      "matchingErrors": 0,
      "matchingGrades": 4,
      "matchingRows": 4,
      "referenceErrors": 0,
      "referenceUnsupported": 0,
      "rows": 5,
      "status": "unsupported"
    },
    {
      "cancelled": 0,
      "engineErrors": 1,
      "engineUnsupported": 1,
      "expectedScopeRefusals": 1,
      "id": "rouge-n",
      "matchingErrors": 1,
      "matchingGrades": 6,
      "matchingRows": 7,
      "referenceErrors": 1,
      "referenceUnsupported": 0,
      "rows": 8,
      "status": "unsupported"
    },
    {
      "cancelled": 0,
      "engineErrors": 1,
      "engineUnsupported": 1,
      "expectedScopeRefusals": 1,
      "id": "rouge-s",
      "matchingErrors": 1,
      "matchingGrades": 4,
      "matchingRows": 5,
      "referenceErrors": 1,
      "referenceUnsupported": 0,
      "rows": 6,
      "status": "unsupported"
    }
  ],
  "comparator": "Row status, pass and recorded reasons match exactly. Grade scores must differ by at most 1e-12. Matching errors and expected scope refusals remain in the denominator and are counted separately from matching grades.",
  "corpus": {
    "name": "Authored fixed outputs and deterministic assertions",
    "sha256": "c051a88c0b05de99dbd75678cbeb9c4e14afe5f6d166d1b30e24a67a4378a70c",
    "source": {
      "commit": "3034ba83fd4d70a42e2c18d4e5dce8dafa2e1150",
      "file": "docs/adversarial-eval/results/promptfoo-structured.json",
      "sha256": "c4af8fa9026c24289a9da9c54116ed3cca6a5b15e9823156ddbb1f289182966d"
    }
  },
  "engine": {
    "clean": true,
    "commit": "3034ba83fd4d70a42e2c18d4e5dce8dafa2e1150",
    "name": "Keryn Eval native config importer and replay engine"
  },
  "metrics": [
    {
      "denominator": 13,
      "id": "identical-cases",
      "label": "Assertion types with matching outcomes on every row",
      "numerator": 7,
      "of": "all offline assertion types compared",
      "value": 0.5385
    },
    {
      "denominator": 105,
      "id": "outcome-agreement",
      "label": "Rows with matching outcomes and scores",
      "numerator": 95,
      "of": "all fixture rows, including errors",
      "value": 0.9048
    },
    {
      "denominator": 105,
      "id": "matching-grades",
      "label": "Rows with matching pass/fail grades and scores",
      "numerator": 87,
      "of": "all fixture rows, including errors",
      "value": 0.8286
    },
    {
      "denominator": 105,
      "id": "matching-errors",
      "label": "Rows both graders reported as errors",
      "numerator": 8,
      "of": "all fixture rows, including errors",
      "value": 0.0762
    },
    {
      "denominator": 105,
      "id": "expected-scope-refusals",
      "label": "Rows with expected native scope refusals",
      "numerator": 10,
      "of": "all fixture rows",
      "value": 0.0952
    }
  ],
  "question": "Do the two graders return the same outcome and score on authored deterministic examples?",
  "reference": {
    "entrypointSha256": "da98ac904e8fad04142f719246dcaa9d751f14a7341e1cb91a321ba94d8edbc2",
    "lockSha256": "43ee984ea9c63f6ef9769018f48b8939fadc404b0d5a55e679ab92ed7e0a1b66",
    "name": "Promptfoo assertion API",
    "repo": "https://github.com/promptfoo/promptfoo",
    "version": "0.125.1"
  },
  "schema": "keryn-eval-receipt/1",
  "scope": [
    "Offline fixture comparison with the recorded Promptfoo package and the native Keryn config/replay path.",
    "The historical 433-config Promptfoo differential is measured separately. This comparison makes no claim about live providers or broader corpus coverage."
  ],
  "suite": "promptfoo-structured",
  "summary": {
    "cases": 13,
    "expectedScopeRefusals": 10,
    "matchingErrors": 8,
    "matchingGrades": 87,
    "matchingRows": 95,
    "rows": 105,
    "statuses": {
      "differs": 0,
      "error": 0,
      "identical": 7,
      "incomplete": 0,
      "no-reference": 0,
      "timeout": 0,
      "unsupported": 6
    }
  },
  "title": "Promptfoo offline assertion parity"
}
