feat(scanner): implement Payload extractors

README.md

@@ -128,6 +128,11 @@ interface Options {
 }
 ```
 
+Additional APIs are available at:
+
+- [from](./workspaces/scanner/docs/from.md)
+- [extractors](./workspaces/scanner/docs/extractors.md)
+
 ## Workspaces
 
 Click on one of the links to access the documentation of the workspace:

workspaces/scanner/docs/extractors.md

@@ -0,0 +1,103 @@
+# Extractors APIs
+
+A set of APIs to efficiently extract and aggregate data from a NodeSecure payload.
+
+## Usage example
+
+Here's how to extract a list of contacts (maintainers, authors) from the Fastify framework:
+
+```ts
+import { Extractors, from } from "@nodesecure/scanner";
+
+const payload = await from("fastify");
+
+const extractor = new Extractors.Payload(
+  payload,
+  [
+    new Extractors.Probes.ContactExtractor()
+  ]
+);
+
+const { contacts } = extractor.extractAndMerge();
+console.log(contacts);
+```
+
+## Probes
+
+A probe is a worker designed to collect a specific type of data from a NodeSecure payload.
+
+Available probes include:
+
+
+| name | level |
+| --- | --- |
+| ContactExtractor | manifest |
+| LicensesExtractor | manifest |
+| SizeExtractor | manifest |
+
+All probes follow the same `ProbeExtractor` interface, which acts as an iterator-like contract:
+
+```ts
+export interface ProbeExtractor<Defs> {
+  level: ProbeExtractorLevel;
+  next(...args: any[]): void;
+  done(): Defs;
+}
+```
+
+Depending on the scope of the data being processed, probes are implemented at two distinct levels:
+
+- Packument (operates at the package registry level, across multiple versions of a package)
+- Manifest (operates at the level of a specific dependency's package.json)
+
+Each probe level defines its own `next()` method signature:
+
+```ts
+export interface PackumentProbeExtractor<Defs> extends ProbeExtractor<Defs> {
+  level: "packument";
+  next(name: string, dependency: Scanner.Dependency): void;
+}
+
+export interface ManifestProbeExtractor<Defs> extends ProbeExtractor<Defs> {
+  level: "manifest";
+  next(
+    spec: string,
+    dependencyVersion: Scanner.DependencyVersion,
+    parent: ProbeExtractorManifestParent
+  ): void;
+}
+```
+
+## API
+
+> [!NOTE]
+> generic `T` is defined as extending from `ProbeExtractor<any>[]`
+
+### constructor(data: Scanner.Payload | Scanner.Payload[ "dependencies" ], probes: [ ...T ])
+
+Creates a new extractor instance using the provided payload and probes.
+
+### extract(): ExtractProbeResult< T >
+
+Executes each probe and returns their results as an array, for example:
+
+```js
+[
+  { "probe1", "xxx" },
+  { "probe2", "xxx" }
+]
+```
+
+> [!WARNING]
+> The method can only be used once because the result will be cached.
+
+### extractAndMerge(): MergedExtractProbeResult< T >
+
+Runs the probes and deeply merges their results into a single record, for example:
+
+```js
+{
+  "probe1": "xxx",
+  "probe2": "xxx"
+}
+```

workspaces/scanner/package.json

@@ -48,6 +48,7 @@
   },
   "homepage": "https://github.com/NodeSecure/tree/master/workspaces/scanner#readme",
   "dependencies": {
+    "@fastify/deepmerge": "^2.0.1",
     "@nodesecure/conformance": "^1.0.0",
     "@nodesecure/contact": "^1.0.0",
     "@nodesecure/flags": "^2.4.0",
@@ -62,6 +63,7 @@
     "@nodesecure/vulnera": "^2.0.1",
     "@openally/mutex": "^1.0.0",
     "pacote": "^18.0.6",
-    "semver": "^7.5.4"
+    "semver": "^7.5.4",
+    "type-fest": "^4.30.2"
   }
 }

workspaces/scanner/src/extractors/index.ts

@@ -0,0 +1,20 @@
+// Import Internal Dependencies
+import {
+  Payload,
+  type ProbeExtractor,
+  type PackumentProbeExtractor,
+  type ManifestProbeExtractor
+} from "./payload.js";
+
+import * as Probes from "./probes/index.js";
+
+export const Extractors = {
+  Payload,
+  Probes
+} as const;
+
+export type {
+  ProbeExtractor,
+  PackumentProbeExtractor,
+  ManifestProbeExtractor
+};

workspaces/scanner/src/extractors/payload.ts

@@ -0,0 +1,98 @@
+// Import Third-party Dependencies
+import type { Simplify } from "type-fest";
+// @ts-ignore
+import deepmerge from "@fastify/deepmerge";
+
+// Import Internal Dependencies
+import * as Scanner from "../types.js";
+import { isNodesecurePayload } from "../utils/index.js";
+
+// CONSTANTS
+const kFastMerge = deepmerge({ all: true });
+
+type MergeDeep<T extends unknown[]> =
+    T extends [a: infer A, ...rest: infer R] ? A & MergeDeep<R> : {};
+
+export type ExtractProbeResult<
+  T extends ProbeExtractor<any>[]
+> = {
+  [K in keyof T]: T[K] extends ProbeExtractor<any> ? ReturnType<T[K]["done"]> : never;
+};
+export type MergedExtractProbeResult<
+  T extends ProbeExtractor<any>[]
+> = Simplify<MergeDeep<ExtractProbeResult<T>>>;
+
+export type ProbeExtractorLevel = "packument" | "manifest";
+export type ProbeExtractorManifestParent = {
+  name: string;
+  dependency: Scanner.Dependency;
+};
+
+export interface ProbeExtractor<Defs> {
+  level: ProbeExtractorLevel;
+  next(...args: any[]): void;
+  done(): Defs;
+}
+
+export interface PackumentProbeExtractor<Defs> extends ProbeExtractor<Defs> {
+  level: "packument";
+  next(name: string, dependency: Scanner.Dependency): void;
+}
+
+export interface ManifestProbeExtractor<Defs> extends ProbeExtractor<Defs> {
+  level: "manifest";
+  next(
+    spec: string,
+    dependencyVersion: Scanner.DependencyVersion,
+    parent: ProbeExtractorManifestParent
+  ): void;
+}
+
+export class Payload<T extends ProbeExtractor<any>[]> {
+  private dependencies: Scanner.Payload["dependencies"];
+  private probes: Record<ProbeExtractorLevel, T>;
+  private cachedResult: ExtractProbeResult<T>;
+
+  constructor(
+    data: Scanner.Payload | Scanner.Payload["dependencies"],
+    probes: [...T]
+  ) {
+    this.dependencies = isNodesecurePayload(data) ?
+      data.dependencies :
+      data;
+
+    this.probes = probes.reduce((data, probe) => {
+      data[probe.level].push(probe);
+
+      return data;
+    }, { packument: [] as unknown as T, manifest: [] as unknown as T });
+  }
+
+  extract() {
+    if (this.cachedResult) {
+      return this.cachedResult;
+    }
+
+    for (const [name, dependency] of Object.entries(this.dependencies)) {
+      this.probes.packument.forEach((probe) => probe.next(name, dependency));
+      if (this.probes.manifest.length > 0) {
+        for (const [spec, depVersion] of Object.entries(dependency.versions)) {
+          this.probes.manifest.forEach((probe) => probe.next(spec, depVersion, { name, dependency }));
+        }
+      }
+    }
+
+    this.cachedResult = [
+      ...this.probes.packument.map((probe) => probe.done()),
+      ...this.probes.manifest.map((probe) => probe.done())
+    ] as ExtractProbeResult<T>;
+
+    return this.cachedResult;
+  }
+
+  extractAndMerge() {
+    return kFastMerge(
+      ...this.extract()
+    ) as unknown as MergedExtractProbeResult<T>;
+  }
+}

workspaces/scanner/src/extractors/probes/ContactExtractor.class.ts

@@ -0,0 +1,54 @@
+// Import Third-party Dependencies
+import type { Contact } from "@nodesecure/npm-types";
+
+// Import Internal Dependencies
+import type {
+  ManifestProbeExtractor,
+  ProbeExtractorManifestParent
+} from "../payload.js";
+import type { DependencyVersion } from "../../types.js";
+
+export type ContactExtractorResult = {
+  contacts: Record<string, number>;
+};
+
+export class ContactExtractor implements ManifestProbeExtractor<ContactExtractorResult> {
+  level = "manifest" as const;
+
+  #contacts: ContactExtractorResult["contacts"] = Object.create(null);
+  #packages: Set<string> = new Set();
+
+  #addContact(
+    user: Contact | null
+  ) {
+    if (!user || !user.email) {
+      return;
+    }
+
+    this.#contacts[user.email] = user.email in this.#contacts ?
+      ++this.#contacts[user.email] : 1;
+  }
+
+  next(
+    _: string,
+    version: DependencyVersion,
+    parent: ProbeExtractorManifestParent
+  ) {
+    const { author } = version;
+    const { name, dependency } = parent;
+
+    this.#addContact(author);
+    if (!this.#packages.has(name)) {
+      dependency.metadata.maintainers.forEach(
+        (maintainer) => this.#addContact(maintainer)
+      );
+      this.#packages.add(name);
+    }
+  }
+
+  done() {
+    return {
+      contacts: this.#contacts
+    };
+  }
+}

workspaces/scanner/src/extractors/probes/LicensesExtractor.class.ts

@@ -0,0 +1,33 @@
+// Import Internal Dependencies
+import type {
+  ManifestProbeExtractor
+} from "../payload.js";
+import type { DependencyVersion } from "../../types.js";
+
+export type LicensesExtractorResult = {
+  licenses: Record<string, number>;
+};
+
+export class LicensesExtractor implements ManifestProbeExtractor<LicensesExtractorResult> {
+  level = "manifest" as const;
+
+  #licenses: LicensesExtractorResult["licenses"] = Object.create(null);
+
+  next(
+    _: string,
+    version: DependencyVersion
+  ) {
+    const { uniqueLicenseIds } = version;
+
+    for (const licenseName of uniqueLicenseIds) {
+      this.#licenses[licenseName] = licenseName in this.#licenses ?
+        ++this.#licenses[licenseName] : 1;
+    }
+  }
+
+  done() {
+    return {
+      licenses: this.#licenses
+    };
+  }
+}