{
  "cases": [
    {
      "cwe": "CWE-1164",
      "id": "juliet-cpp/104778-v1.1.0",
      "reason": "Forge ships no cpp rule for CWE-1164",
      "status": "unsupported"
    },
    {
      "badPathFlagged": true,
      "cwe": "CWE-476",
      "findingsDigest": "a70c57c7d0f86bdca2fbe977ca51b15365d336bfd4ef4e9b3641397516d129d3",
      "goodPathsFlagged": 0,
      "id": "juliet-cpp/240691-v2.0.0",
      "status": "identical"
    },
    {
      "badPathFlagged": true,
      "cwe": "CWE-121",
      "findingsDigest": "4fe826577e0743b38d3843911be02a5501f62b838e8b4015145d854532bc43d1",
      "goodPathsFlagged": 0,
      "id": "juliet-cpp/64472-v1.0.0",
      "status": "identical"
    },
    {
      "cwe": "CWE-134",
      "id": "juliet-cpp/79386-v1.0.0",
      "reason": "Forge ships no cpp rule for CWE-134",
      "status": "unsupported"
    },
    {
      "cwe": "CWE-476",
      "id": "juliet-java/139834-v1.0.0",
      "reason": "Forge ships no java rule for CWE-476",
      "status": "unsupported"
    },
    {
      "badPathFlagged": true,
      "cwe": "CWE-78",
      "findingsDigest": "bb408a2660f89fd0f83593a6da8d6bc9a0d490172f7af38fb6ecffdecae50525",
      "goodPathsFlagged": 0,
      "id": "juliet-java/144350-v1.0.0",
      "status": "identical"
    },
    {
      "badPathFlagged": true,
      "cwe": "CWE-89",
      "findingsDigest": "80f271bd3666c7647e22afc2b85d6d9036deb9c6092f419e2b51d1a7e891a39b",
      "goodPathsFlagged": 0,
      "id": "juliet-java/146140-v1.0.0",
      "status": "identical"
    }
  ],
  "comparator": "Per NIST Juliet case (mixed good/bad): `identical` when Forge reports a `flow` finding of the case's CWE (CWE-120 counts for CWE-121: both are buffer-copy overflows) whose sink lies in a `bad*` function or on a flaw line from the NIST manifest, and no such finding in a `good*` function. `unsupported` when Forge ships no rule for that CWE in that language. Otherwise `differs`. Raw finding counts are never used.",
  "corpus": {
    "cases": 7,
    "groundTruth": {
      "file": "evals/range/receipts/juliet-smoke-ground-truth.json",
      "sha256": "5e709b84dc4b254876217fa1d61f7d582aeff429eb9b8aca12274c2c586d3b08"
    },
    "license": "Public domain (NIST SARD)",
    "name": "NIST Juliet starter cases from the Keryn test range (evals/range): Java 1.3 (SARD suite 109) 3 cases, C/C++ 1.3.1 (SARD suite 116) 4 cases",
    "pins": "evals/range/targets.lock.json (NIST ZIP sha256 and case ids)",
    "repo": "https://samate.nist.gov/SARD"
  },
  "engine": {
    "clean": true,
    "commit": "98492c26edc478fa3db60997e409affb89728fad",
    "description": "Keryn Forge (crates/keryn-forge) via examples/forge_scan: JVM producer on the javac output of the range's Juliet build, LLVM producer on `clang -S -emit-llvm` of the range's C/C++ compile commands.",
    "name": "keryn-forge"
  },
  "metrics": [
    {
      "denominator": 7,
      "id": "identical-cases",
      "label": "Cases where the bad path is found and no good path is flagged",
      "numerator": 4,
      "of": "Juliet starter cases in the range",
      "value": 0.5714
    },
    {
      "denominator": 4,
      "id": "supported-identical",
      "label": "Of the cases whose CWE Forge has a rule for, those handled correctly",
      "numerator": 4,
      "of": "starter cases with a Forge rule for their CWE and language",
      "value": 1
    }
  ],
  "question": "On the test range's seven Juliet starter cases, does Forge find the bad path of each case's CWE without flagging its good paths?",
  "reference": {
    "name": "NIST Juliet per-case SARIF manifests (expected flaw locations)",
    "role": "Ground truth shipped with the corpus (evals/range/receipts/juliet-smoke-ground-truth.json)."
  },
  "schema": "keryn-eval-receipt/1",
  "scope": [
    "Seven starter cases: a build and integration check, not a representative accuracy benchmark. The full Juliet corpora are not measured yet.",
    "No Juliet executable is run."
  ],
  "suite": "forge-juliet-smoke",
  "summary": {
    "cases": 7,
    "statuses": {
      "differs": 0,
      "error": 0,
      "identical": 4,
      "no-reference": 0,
      "timeout": 0,
      "unsupported": 3
    }
  },
  "title": "NIST Juliet starter cases (Java, C/C++): Keryn Forge"
}
