{
  "cases": [
    {
      "ecosystem": "Maven",
      "engine": {
        "reasons": [
          "an analysed file imports org.apache.logging.log4j.LogManager, under the package's groupId (which artifact provides it is not resolved)"
        ],
        "verdict": "reachable"
      },
      "expected": "reachable",
      "id": "mvn-01-group-import",
      "input": {
        "sha256": "8d53ec28b98b90549f214b67d01aaee88d286260406d432e39ef8ce44a45c23f"
      },
      "status": "identical",
      "truth": "used"
    },
    {
      "ecosystem": "Maven",
      "engine": {
        "reasons": [
          "no analysed file imports under the groupId, but a Maven groupId is not reliably the Java package prefix"
        ],
        "verdict": "unknown"
      },
      "expected": "reachable",
      "id": "mvn-02-group-mismatch",
      "input": {
        "sha256": "2bb192423121ab6ada6f87430dfe1a27e02f1b549180da1f88684f96f06444db"
      },
      "status": "differs",
      "truth": "used"
    },
    {
      "ecosystem": "Maven",
      "engine": {
        "reasons": [
          "no analysed file imports under the groupId, but a Maven groupId is not reliably the Java package prefix"
        ],
        "verdict": "unknown"
      },
      "expected": "unknown",
      "id": "mvn-03-not-imported",
      "input": {
        "sha256": "5cdc907e687b90bf6092ea283b24509d60cc0f1d0b6b46a7bde2adbcd551fec7"
      },
      "status": "identical",
      "truth": "undecidable"
    },
    {
      "ecosystem": "npm",
      "engine": {
        "reasons": [
          "an analysed file imports express; the advisory names no vulnerable function, so any use counts"
        ],
        "verdict": "reachable"
      },
      "expected": "reachable",
      "id": "npm-01-default-import",
      "input": {
        "sha256": "d5a406dd67853f5cac253ba3a459614c15d2a799d2caf6c95e404c3d6b1b9162"
      },
      "status": "identical",
      "truth": "used"
    },
    {
      "ecosystem": "npm",
      "engine": {
        "reasons": [
          "a direct dependency that none of the 1 analysed file(s) imports"
        ],
        "verdict": "not-reachable-in-analysed-code"
      },
      "expected": "not-reachable-in-analysed-code",
      "id": "npm-02-unused-direct",
      "input": {
        "sha256": "de1a8aba3119601da65cec673b4f1ddd895116214b8f46df5d05da4ceeb55910"
      },
      "status": "identical",
      "truth": "unused"
    },
    {
      "ecosystem": "npm",
      "engine": {
        "reasons": [
          "an analysed file imports lodash; the advisory names no vulnerable function, so any use counts"
        ],
        "verdict": "reachable"
      },
      "expected": "reachable",
      "id": "npm-03-require-subpath",
      "input": {
        "sha256": "3346d5640b533e491fe28bd8ad9d84a348d89d0e948ed7cfaf1ea73c087dd872"
      },
      "status": "identical",
      "truth": "used"
    },
    {
      "ecosystem": "npm",
      "engine": {
        "reasons": [
          "an analysed file imports @aws-sdk/client-s3; the advisory names no vulnerable function, so any use counts"
        ],
        "verdict": "reachable"
      },
      "expected": "reachable",
      "id": "npm-04-scoped",
      "input": {
        "sha256": "d7b0adecefe9b2ff1febb4a64ece17a540c0d65fdc660eb88ded671437dd598e"
      },
      "status": "identical",
      "truth": "used"
    },
    {
      "ecosystem": "npm",
      "engine": {
        "reasons": [
          "a direct dependency that none of the 1 analysed file(s) imports"
        ],
        "verdict": "not-reachable-in-analysed-code"
      },
      "expected": "not-reachable-in-analysed-code",
      "id": "npm-05-test-only",
      "input": {
        "sha256": "c74cc488d8874b83535aa336e2ebd785fd558b39748e9774d47c490c8115e0cc"
      },
      "status": "identical",
      "truth": "unused"
    },
    {
      "ecosystem": "npm",
      "engine": {
        "reasons": [
          "a transitive dependency: other packages may call it, and calls through them are not followed"
        ],
        "verdict": "unknown"
      },
      "expected": "unknown",
      "id": "npm-06-transitive",
      "input": {
        "sha256": "de1a8aba3119601da65cec673b4f1ddd895116214b8f46df5d05da4ceeb55910"
      },
      "status": "identical",
      "truth": "undecidable"
    },
    {
      "ecosystem": "npm",
      "engine": {
        "reasons": [
          "src/load.js imports dynamically, so an import of the package could be hidden"
        ],
        "verdict": "unknown"
      },
      "expected": "unknown",
      "id": "npm-07-dynamic-require",
      "input": {
        "sha256": "af8c28dbbbe9cc23999d26a2d3ae72ad0b3ec2f5e5c6e1ad063dbfaa07861f88"
      },
      "status": "identical",
      "truth": "undecidable"
    },
    {
      "ecosystem": "npm",
      "engine": {
        "reasons": [
          "src/t.js imports lodash and names template, a function the advisory lists"
        ],
        "verdict": "reachable"
      },
      "expected": "reachable",
      "id": "npm-08-symbol-used",
      "input": {
        "sha256": "928cc03f1f16500c0155adda9af2cb3249c3be8916013c6abc8e556afe7bf1ba"
      },
      "status": "identical",
      "truth": "used"
    },
    {
      "ecosystem": "npm",
      "engine": {
        "reasons": [
          "1 import(s) of the package, but no analysed file names a function the advisory lists (template)"
        ],
        "verdict": "not-reachable-in-analysed-code"
      },
      "expected": "not-reachable-in-analysed-code",
      "id": "npm-09-symbol-unused",
      "input": {
        "sha256": "fa5cfe2c703dc0b688529df05d211085a3569c4c183e6af3d78d634fe6583a22"
      },
      "status": "identical",
      "truth": "unused"
    },
    {
      "ecosystem": "npm",
      "engine": {
        "reasons": [
          "an analysed file imports zod; the advisory names no vulnerable function, so any use counts"
        ],
        "verdict": "reachable"
      },
      "expected": "reachable",
      "id": "npm-10-reexport",
      "input": {
        "sha256": "807700cf903538e3f678f7802ea67c62e3195592083b4b3d832ad82aec3d5239"
      },
      "status": "identical",
      "truth": "used"
    },
    {
      "ecosystem": "npm",
      "engine": {
        "reasons": [
          "a direct dependency that none of the 1 analysed file(s) imports"
        ],
        "verdict": "not-reachable-in-analysed-code"
      },
      "expected": "not-reachable-in-analysed-code",
      "id": "npm-11-type-only",
      "input": {
        "sha256": "4772692ab1c04aafa37dac812212c6dba46730ebf48b229f7286afb54c8a12c4"
      },
      "status": "identical",
      "truth": "unused"
    },
    {
      "ecosystem": "npm",
      "engine": {
        "reasons": [
          "an analysed file imports axios; the advisory names no vulnerable function, so any use counts"
        ],
        "verdict": "reachable"
      },
      "expected": "reachable",
      "id": "npm-12-vue-sfc",
      "input": {
        "sha256": "0f2f3e9d80658f81c8c831b8bad115afd4ea514d76e653d28450fc6aad0f9341"
      },
      "status": "identical",
      "truth": "used"
    },
    {
      "ecosystem": "npm",
      "engine": {
        "reasons": [
          "a direct dependency that none of the 1 analysed file(s) imports"
        ],
        "verdict": "not-reachable-in-analysed-code"
      },
      "expected": "not-reachable-in-analysed-code",
      "id": "npm-13-similar-name",
      "input": {
        "sha256": "8f7cab889eb4ba535f38bee5ce0c70b279b2a23e27155a214669324a8ede71b2"
      },
      "status": "identical",
      "truth": "unused"
    },
    {
      "ecosystem": "npm",
      "engine": {
        "reasons": [
          "a direct dependency that none of the 1 analysed file(s) imports"
        ],
        "verdict": "not-reachable-in-analysed-code"
      },
      "expected": "not-reachable-in-analysed-code",
      "id": "npm-14-commented-import",
      "input": {
        "sha256": "ea9a7f5a05fa003cdaacce5221829cffa9986ab5af338bb7409bd5eadd6a4973"
      },
      "status": "identical",
      "truth": "unused"
    },
    {
      "ecosystem": "npm",
      "engine": {
        "reasons": [
          "an analysed file imports marked; the advisory names no vulnerable function, so any use counts"
        ],
        "verdict": "reachable"
      },
      "expected": "reachable",
      "id": "npm-15-dynamic-import-literal",
      "input": {
        "sha256": "493ac50e9936063e4500f29f9efc49bdd16ed18b838bb673322bf90e28afe00a"
      },
      "status": "identical",
      "truth": "used"
    },
    {
      "ecosystem": "PyPI",
      "engine": {
        "reasons": [
          "an analysed file imports requests; the advisory names no vulnerable function, so any use counts"
        ],
        "verdict": "reachable"
      },
      "expected": "reachable",
      "id": "py-01-import",
      "input": {
        "sha256": "15375b7b29daa33f835023a4c81450c165812e238b68ae4872004f56839fce2f"
      },
      "status": "identical",
      "truth": "used"
    },
    {
      "ecosystem": "PyPI",
      "engine": {
        "reasons": [
          "an analysed file imports yaml; the advisory names no vulnerable function, so any use counts"
        ],
        "verdict": "reachable"
      },
      "expected": "reachable",
      "id": "py-02-pyyaml",
      "input": {
        "sha256": "efbf61944e1262fc803c061c0bdf4f9f4e2181fda711f1015f3bd370f9ccda1a"
      },
      "status": "identical",
      "truth": "used"
    },
    {
      "ecosystem": "PyPI",
      "engine": {
        "reasons": [
          "an analysed file imports PIL; the advisory names no vulnerable function, so any use counts"
        ],
        "verdict": "reachable"
      },
      "expected": "reachable",
      "id": "py-03-pillow-from",
      "input": {
        "sha256": "73d40c8f568d8ef39dde2b369344ded79cac73e428302308fc3b57dfbc1ecb70"
      },
      "status": "identical",
      "truth": "used"
    },
    {
      "ecosystem": "PyPI",
      "engine": {
        "reasons": [
          "a direct dependency that none of the 1 analysed file(s) imports"
        ],
        "verdict": "not-reachable-in-analysed-code"
      },
      "expected": "not-reachable-in-analysed-code",
      "id": "py-04-unused-direct",
      "input": {
        "sha256": "199d99f150844f74c7d2a2aed478271df9cab105dcba6bea18d0279032adecad"
      },
      "status": "identical",
      "truth": "unused"
    },
    {
      "ecosystem": "PyPI",
      "engine": {
        "reasons": [
          "app/plug.py imports dynamically, so an import of the package could be hidden"
        ],
        "verdict": "unknown"
      },
      "expected": "unknown",
      "id": "py-05-importlib",
      "input": {
        "sha256": "b434b4e96ef18c2f25d70f62b2bdb6aafb422110573d0f7fa59204a5aa1ea80c"
      },
      "status": "identical",
      "truth": "undecidable"
    },
    {
      "ecosystem": "PyPI",
      "engine": {
        "reasons": [
          "the module names of some-obscure-dist are not known (no vendored top_level.txt), so a missing import is not evidence"
        ],
        "verdict": "unknown"
      },
      "expected": "unknown",
      "id": "py-06-unknown-mapping",
      "input": {
        "sha256": "48ea28039da7978bcebd8dbac72dc1970be70b5065fd6d8612f64a169bc075c7"
      },
      "status": "identical",
      "truth": "undecidable"
    },
    {
      "ecosystem": "PyPI",
      "engine": {
        "reasons": [
          "1 import(s) of the package, but no analysed file names a function the advisory lists (load)"
        ],
        "verdict": "not-reachable-in-analysed-code"
      },
      "expected": "not-reachable-in-analysed-code",
      "id": "py-07-symbol-unused",
      "input": {
        "sha256": "863bd117737899f96a408dbe744420f5baaa827f980bd3a78f4c3d9b5b7de51f"
      },
      "status": "identical",
      "truth": "unused"
    },
    {
      "ecosystem": "PyPI",
      "engine": {
        "reasons": [
          "app/conf.py imports yaml and names load, a function the advisory lists"
        ],
        "verdict": "reachable"
      },
      "expected": "reachable",
      "id": "py-08-symbol-used",
      "input": {
        "sha256": "0f6e2056aa6af53cc0f29f0a52e7b132a2d312dea525d34e6a252e979ad6f067"
      },
      "status": "identical",
      "truth": "used"
    },
    {
      "ecosystem": "PyPI",
      "engine": {
        "reasons": [
          "the module names of pytest-mock are not known (no vendored top_level.txt), so a missing import is not evidence"
        ],
        "verdict": "unknown"
      },
      "expected": "not-reachable-in-analysed-code",
      "id": "py-09-tests-only",
      "input": {
        "sha256": "d5025f976262bc89b37d39f26629257b372e36963b9a40452219e444d21b7c98"
      },
      "status": "differs",
      "truth": "unused"
    },
    {
      "ecosystem": "PyPI",
      "engine": {
        "reasons": [
          "an analysed file imports foo_impl; the advisory names no vulnerable function, so any use counts"
        ],
        "verdict": "reachable"
      },
      "expected": "reachable",
      "id": "py-10-top-level-txt",
      "input": {
        "sha256": "6efb752a3dae3cca07e55143eac830ffbeb82b5ea9fe67fc4c0ae48cced78cd7"
      },
      "status": "identical",
      "truth": "used"
    },
    {
      "ecosystem": "PyPI",
      "engine": {
        "reasons": [
          "an analysed file imports requests.adapters; the advisory names no vulnerable function, so any use counts"
        ],
        "verdict": "reachable"
      },
      "expected": "reachable",
      "id": "py-11-submodule",
      "input": {
        "sha256": "47a923cc12e5e3ab463e904fded6c41f6f9d39f6b6bba70a3fb8a2dea56c3cc5"
      },
      "status": "identical",
      "truth": "used"
    },
    {
      "ecosystem": "PyPI",
      "engine": {
        "reasons": [
          "a transitive dependency: other packages may call it, and calls through them are not followed"
        ],
        "verdict": "unknown"
      },
      "expected": "unknown",
      "id": "py-12-transitive",
      "input": {
        "sha256": "199d99f150844f74c7d2a2aed478271df9cab105dcba6bea18d0279032adecad"
      },
      "status": "identical",
      "truth": "undecidable"
    }
  ],
  "comparator": "sca-reachability/1: the engine's per-finding verdict against an authored truth. used -> reachable, unused -> not-reachable-in-analysed-code, undecidable -> unknown. A wrong not-reachable is counted apart from an abstention.",
  "corpus": {
    "license": "repository license; synthetic authored data",
    "manifest": "evals/suites/sca-reachability/cases.json",
    "name": "30 authored cases: 15 npm, 12 PyPI, 3 Maven",
    "sha256": "2bb2d91deab1a03cb6021c55de4102fc93a5392c4e8b6b4aa19cc684ef0737c9"
  },
  "engine": {
    "clean": true,
    "commit": "b7c1a6278e6317ce4de2f579901950c2a1df4523",
    "description": "The dependency stage's import join over first-party source; no network, no Deep pass.",
    "name": "keryn import reachability (ts/packages/sca/src/import-reachability.ts)"
  },
  "metrics": [
    {
      "denominator": 30,
      "id": "identical-cases",
      "label": "Cases where the verdict equals the authored truth",
      "numerator": 28,
      "of": "all authored cases",
      "value": 0.9333
    },
    {
      "denominator": 8,
      "id": "not-reachable-precision",
      "label": "Not-reachable verdicts that are right",
      "numerator": 8,
      "of": "cases the engine called not reachable in analysed code",
      "value": 1
    },
    {
      "denominator": 14,
      "id": "reachable-precision",
      "label": "Reachable verdicts that are right",
      "numerator": 14,
      "of": "cases the engine called reachable",
      "value": 1
    },
    {
      "denominator": 15,
      "id": "used-recall",
      "label": "Used packages found reachable",
      "numerator": 14,
      "of": "cases whose vulnerable code is used",
      "value": 0.9333
    },
    {
      "denominator": 9,
      "id": "unused-recall",
      "label": "Unused packages called not reachable",
      "numerator": 8,
      "of": "cases whose vulnerable code is not used by shipped code",
      "value": 0.8889
    },
    {
      "denominator": 6,
      "id": "unknown-on-undecidable",
      "label": "Undecidable cases left unknown",
      "numerator": 6,
      "of": "cases a static import read cannot decide",
      "value": 1
    }
  ],
  "question": "When Keryn says a vulnerable dependency is reachable, or not reachable in the analysed code, is that what the code does?",
  "reference": {
    "name": "independently authored truth labels",
    "role": "Bounded regression expectations, including traps (similar names, type-only and commented-out imports, test-only use, groupId/package mismatch), not a real-world reachability oracle.",
    "version": "authored-reach-1"
  },
  "schema": "keryn-eval-receipt/1",
  "scope": [
    "Static imports and, where an advisory names functions, a name match in importing files.",
    "No call graph, no data flow, no framework or reflection model; calls made by other packages are never followed.",
    "Synthetic cases: precision and recall here are not real-world rates."
  ],
  "suite": "sca-reachability",
  "summary": {
    "cases": 30,
    "statuses": {
      "differs": 2,
      "error": 0,
      "identical": 28,
      "incomplete": 0,
      "no-reference": 0,
      "timeout": 0,
      "unsupported": 0
    },
    "wrongNotReachable": 0,
    "wrongReachable": 0
  },
  "title": "Dependency reachability from imports (JavaScript/TypeScript, Python, Java)"
}
