From 5b6dc926301cc1c3a3b73a25a218f3dfa7fc3b79 Mon Sep 17 00:00:00 2001 From: cgombauld Date: Tue, 20 Jan 2026 16:21:01 +0100 Subject: [PATCH 1/4] feat(scanner): integrate built-in stats in the response of depWalker --- .changeset/wet-bears-notice.md | 5 + .../scanner/src/class/DateProvider.class.ts | 16 +++ .../scanner/src/class/StatsCollector.class.ts | 49 +++++++++ workspaces/scanner/src/depWalker.ts | 43 ++++++-- .../src/registry/NpmRegistryProvider.ts | 3 +- .../src/registry/PackumentExtractor.ts | 14 +-- workspaces/scanner/src/types.ts | 45 ++++++-- .../scanner/test/StatsCollector.spec.ts | 100 ++++++++++++++++++ workspaces/scanner/test/depWalker.spec.ts | 25 ++++- 9 files changed, 261 insertions(+), 39 deletions(-) create mode 100644 .changeset/wet-bears-notice.md create mode 100644 workspaces/scanner/src/class/DateProvider.class.ts create mode 100644 workspaces/scanner/src/class/StatsCollector.class.ts create mode 100644 workspaces/scanner/test/StatsCollector.spec.ts diff --git a/.changeset/wet-bears-notice.md b/.changeset/wet-bears-notice.md new file mode 100644 index 00000000..80659191 --- /dev/null +++ b/.changeset/wet-bears-notice.md @@ -0,0 +1,5 @@ +--- +"@nodesecure/scanner": minor +--- + +feat(scanner): integrate built-in stats in the response of depWalker diff --git a/workspaces/scanner/src/class/DateProvider.class.ts b/workspaces/scanner/src/class/DateProvider.class.ts new file mode 100644 index 00000000..e225bd5a --- /dev/null +++ b/workspaces/scanner/src/class/DateProvider.class.ts @@ -0,0 +1,16 @@ +export interface DateProvider { + now(): number; + oneYearAgo(): Date; +} + +export class SystemDateProvider implements DateProvider { + now(): number { + return Date.now(); + } + oneYearAgo(): Date { + const date = new Date(); + date.setFullYear(date.getFullYear() - 1); + + return date; + } +} diff --git a/workspaces/scanner/src/class/StatsCollector.class.ts b/workspaces/scanner/src/class/StatsCollector.class.ts new file mode 100644 index 00000000..f50d268c --- /dev/null +++ b/workspaces/scanner/src/class/StatsCollector.class.ts @@ -0,0 +1,49 @@ +// Import Internal Dependencies +import { SystemDateProvider, type DateProvider } from "./DateProvider.class.ts"; +import type { ApiStats, Stats } from "../types.ts"; + +export class StatsCollector { + #apiCalls: ApiStats[] = []; + #dateProvider: DateProvider; + #startedAt: number; + constructor(dateProvider: DateProvider = new SystemDateProvider()) { + this.#dateProvider = dateProvider; + this.#startedAt = this.#dateProvider.now(); + } + + track any>(name: string, fn: T): ReturnType { + const startedAt = this.#dateProvider.now(); + try { + const result = fn(); + if (result instanceof Promise) { + return result.finally(() => this.#addApiStat(name, startedAt) + ) as ReturnType; + } + + this.#addApiStat(name, startedAt); + + return result; + } + catch (error) { + this.#addApiStat(name, startedAt); + throw error; + } + } + + #addApiStat(name: string, startedAt: number) { + this.#apiCalls.push({ + name, + startedAt, + executionTime: this.#dateProvider.now() - startedAt + }); + } + + getStats(): Stats { + return { + startedAt: this.#startedAt, + executionTime: this.#dateProvider.now() - this.#startedAt, + apiCalls: this.#apiCalls, + apiCallsCount: this.#apiCalls.length + }; + } +} diff --git a/workspaces/scanner/src/depWalker.ts b/workspaces/scanner/src/depWalker.ts index d32ace95..426e0be7 100644 --- a/workspaces/scanner/src/depWalker.ts +++ b/workspaces/scanner/src/depWalker.ts @@ -3,6 +3,8 @@ import path from "node:path"; import { readFileSync } from "node:fs"; // Import Third-party Dependencies +import pacote from "pacote"; +import * as npmRegistrySDK from "@nodesecure/npm-registry-sdk"; import { Mutex, MutexRelease } from "@openally/mutex"; import { extractAndResolve, @@ -26,7 +28,8 @@ import { getManifestLinks, NPM_TOKEN } from "./utils/index.ts"; -import { NpmRegistryProvider } from "./registry/NpmRegistryProvider.ts"; +import { NpmRegistryProvider, type NpmApiClient } from "./registry/NpmRegistryProvider.ts"; +import { StatsCollector } from "./class/StatsCollector.class.ts"; import { RegistryTokenStore } from "./registry/RegistryTokenStore.ts"; import { TempDirectory } from "./class/TempDirectory.class.ts"; import { Logger, ScannerLoggerEvents } from "./class/logger.class.ts"; @@ -94,7 +97,6 @@ type InitialPayload = Partial & { rootDependency: Payload["rootDependency"]; - metadata: Payload["metadata"]; }; export async function depWalker( @@ -113,7 +115,7 @@ export async function depWalker( npmRcConfig } = options; - const startedAt = Date.now(); + const statsCollector = new StatsCollector(); const isRemoteScanning = typeof location === "undefined"; const tokenStore = new RegistryTokenStore(npmRcConfig, NPM_TOKEN.token); @@ -130,18 +132,36 @@ export async function depWalker( }, scannerVersion: packageVersion, vulnerabilityStrategy, - warnings: [], - metadata: { - startedAt, - executionTime: 0 - } + warnings: [] }; const dependencies: Map = new Map(); const highlightedPackages: Set = new Set(); const npmTreeWalker = new npm.TreeWalker({ - registry + registry, + providers: { + pacote: { + manifest: (spec, opts) => statsCollector.track(`pacote.manifest ${spec}`, () => pacote.manifest(spec, opts)), + packument: (spec, opts) => statsCollector.track(`pacote.packument ${spec}`, () => pacote.packument(spec, opts)) + } + } }); + const npmApiClient: NpmApiClient = { + packument: (name, opts) => statsCollector.track( + `npmRegistrySDK.packument ${name}`, + () => npmRegistrySDK.packument(name, opts) + ), + + packumentVersion: (name, version, opts) => statsCollector.track( + `npmRegistrySDK.packumentVersion ${name}@${version}`, + () => npmRegistrySDK.packumentVersion(name, version, opts) + ), + + org: (namespace) => statsCollector.track( + `npmRegistrySDK.org ${namespace}`, + () => npmRegistrySDK.org(namespace) + ) + }; { logger .start(ScannerLoggerEvents.analysis.tree) @@ -181,7 +201,8 @@ export async function depWalker( operationsQueue.push( new NpmRegistryProvider(name, version, { registry, - tokenStore + tokenStore, + npmApiClient }).enrichDependencyVersion(dep, dependencyConfusionWarnings, org) ); @@ -350,7 +371,7 @@ export async function depWalker( packages: [...highlightedPackages] }; payload.dependencies = Object.fromEntries(dependencies); - payload.metadata.executionTime = Date.now() - startedAt; + payload.metadata = statsCollector.getStats(); return payload as Payload; } diff --git a/workspaces/scanner/src/registry/NpmRegistryProvider.ts b/workspaces/scanner/src/registry/NpmRegistryProvider.ts index c6d8b323..aa503164 100644 --- a/workspaces/scanner/src/registry/NpmRegistryProvider.ts +++ b/workspaces/scanner/src/registry/NpmRegistryProvider.ts @@ -11,7 +11,8 @@ import * as i18n from "@nodesecure/i18n"; import { isHTTPError } from "@openally/httpie"; // Import Internal Dependencies -import { PackumentExtractor, type DateProvider } from "./PackumentExtractor.ts"; +import { PackumentExtractor } from "./PackumentExtractor.ts"; +import type { DateProvider } from "../class/DateProvider.class.ts"; import { fetchNpmAvatars } from "./fetchNpmAvatars.ts"; import type { Dependency, diff --git a/workspaces/scanner/src/registry/PackumentExtractor.ts b/workspaces/scanner/src/registry/PackumentExtractor.ts index 29b7e463..8ae46636 100644 --- a/workspaces/scanner/src/registry/PackumentExtractor.ts +++ b/workspaces/scanner/src/registry/PackumentExtractor.ts @@ -6,10 +6,7 @@ import { packageJSONIntegrityHash } from "@nodesecure/mama"; import type { Dependency } from "../types.ts"; - -export interface DateProvider { - oneYearAgo(): Date; -} +import { SystemDateProvider, type DateProvider } from "../class/DateProvider.class.ts"; export interface PackumentExtractorOptions { dateProvider?: DateProvider; @@ -103,12 +100,3 @@ export class PackumentExtractor { return result; } } - -class SystemDateProvider implements DateProvider { - oneYearAgo(): Date { - const date = new Date(); - date.setFullYear(date.getFullYear() - 1); - - return date; - } -} diff --git a/workspaces/scanner/src/types.ts b/workspaces/scanner/src/types.ts index 86b1018e..178fcff0 100644 --- a/workspaces/scanner/src/types.ts +++ b/workspaces/scanner/src/types.ts @@ -184,6 +184,40 @@ export type GlobalWarning = { message: string; } & ( | DependencyConfusionWarning); +export type ApiStats = { + /** + * UNIX Timestamp just before the api call start + */ + startedAt: number; + /** + * Execution time in milliseconds + */ + executionTime: number; + + /** + * Name of the api call + */ + name: string; +}; + +export type Stats = { + /** + * UNIX Timestamp when the scan started + */ + startedAt: number; + /** + * Execution time in milliseconds + */ + executionTime: number; + + /** + * Number of external API calls + */ + apiCallsCount: number; + + apiCalls: ApiStats[]; +}; + export interface Payload { /** Payload unique id */ id: string; @@ -207,16 +241,7 @@ export interface Payload { /** Vulnerability strategy name (npm, snyk, node) */ vulnerabilityStrategy: Vulnera.Kind; - metadata: { - /** - * UNIX Timestamp when the scan started - */ - startedAt: number; - /** - * Execution time in milliseconds - */ - executionTime: number; - }; + metadata: Stats; } export type SemverRange = string | "*"; diff --git a/workspaces/scanner/test/StatsCollector.spec.ts b/workspaces/scanner/test/StatsCollector.spec.ts new file mode 100644 index 00000000..c8597c11 --- /dev/null +++ b/workspaces/scanner/test/StatsCollector.spec.ts @@ -0,0 +1,100 @@ +// Import Node.js Dependencies +import { describe, it } from "node:test"; +import assert from "node:assert"; + +// Import Internal Dependencies +import type { DateProvider } from "../src/class/DateProvider.class.ts"; +import { StatsCollector } from "../src/class/StatsCollector.class.ts"; + +describe("StatsCollectors", () => { + it("should get the expected global start and execution time", () => { + const dateProvider = new FakeDateProvider(); + dateProvider.setNow(1658512000000); + const statsCollector = new StatsCollector(dateProvider); + dateProvider.setNow(1658512001000); + const { startedAt, executionTime } = statsCollector.getStats(); + assert.strictEqual(startedAt, 1658512000000); + assert.strictEqual(executionTime, 1000); + }); + + it("should still record the exexution time if the function being tracked throws", () => { + const dateProvider = new FakeDateProvider(); + dateProvider.setNow(1658512000000); + const statsCollector = new StatsCollector(dateProvider); + assert.throws(() => { + statsCollector.track("api/test/1", () => { + dateProvider.setNow(1658512001000); + throw new Error("oh no!"); + }); + }); + + const { apiCalls, apiCallsCount } = statsCollector.getStats(); + assert.strictEqual(apiCallsCount, 1); + assert.deepEqual(apiCalls, [ + { + name: "api/test/1", + startedAt: 1658512000000, + executionTime: 1000 + } + + ]); + }); + + it("should be able to track the start and execution time of external api call", async() => { + let hasFnOneBeenCalled = false; + let hasFnTwoBeenCalled = false; + const dateProvider = new FakeDateProvider(); + dateProvider.setNow(1658512000000); + const statsCollector = new StatsCollector(dateProvider); + dateProvider.setNow(1658512001001); + const promise = statsCollector.track("api/test/1", () => { + hasFnOneBeenCalled = true; + + return Promise.resolve(1); + }); + + dateProvider.setNow(1658512002000); + const promiseResult = await promise; + + dateProvider.setNow(1658512003000); + const fnResult = statsCollector.track("api/test/2", () => { + hasFnTwoBeenCalled = true; + dateProvider.setNow(1658512004000); + + return null; + }); + dateProvider.setNow(1658512005000); + const { apiCalls, apiCallsCount } = statsCollector.getStats(); + assert.strictEqual(promiseResult, 1); + assert.strictEqual(fnResult, null); + assert.strictEqual(hasFnOneBeenCalled, true); + assert.strictEqual(hasFnTwoBeenCalled, true); + assert.strictEqual(apiCallsCount, 2); + assert.deepEqual(apiCalls, [ + { + name: "api/test/1", + startedAt: 1658512001001, + executionTime: 999 + }, + { + name: "api/test/2", + startedAt: 1658512003000, + executionTime: 1000 + } + ]); + }); +}); + +class FakeDateProvider implements DateProvider { + #now: number; + now(): number { + return this.#now; + } + oneYearAgo(): Date { + return new Date(Date.now() - (365 * 24 * 60 * 60 * 1000)); + } + + setNow(now: number) { + this.#now = now; + } +} diff --git a/workspaces/scanner/test/depWalker.spec.ts b/workspaces/scanner/test/depWalker.spec.ts index a7efc545..c913cd4f 100644 --- a/workspaces/scanner/test/depWalker.spec.ts +++ b/workspaces/scanner/test/depWalker.spec.ts @@ -294,9 +294,9 @@ test("fetch payload of pacote on the npm registry", async() => { "scannerVersion", "vulnerabilityStrategy", "warnings", - "metadata", "highlighted", - "dependencies" + "dependencies", + "metadata" ]); assert.strictEqual(typeof result.rootDependency.integrity, "string"); }); @@ -314,9 +314,9 @@ test("fetch payload of pacote on the gitlab registry", async() => { "scannerVersion", "vulnerabilityStrategy", "warnings", - "metadata", "highlighted", - "dependencies" + "dependencies", + "metadata" ]); assert.strictEqual(typeof result.rootDependency.integrity, "string"); }); @@ -340,6 +340,23 @@ test("highlight contacts from a remote package", async() => { ); }); +test("should collect stats", async() => { + Vulnera.setStrategy(Vulnera.strategies.GITHUB_ADVISORY); + const { logger } = errorLogger(); + test.after(() => logger.removeAllListeners()); + + const { metadata } = await depWalker( + pkgGitdeps, + structuredClone(kDefaultWalkerOptions), + logger + ); + + assert.strictEqual(typeof metadata.startedAt, "number"); + assert.strictEqual(typeof metadata.executionTime, "number"); + assert.strictEqual(Array.isArray(metadata.apiCalls), true); + assert.strictEqual(metadata.apiCallsCount, 25); +}); + describe("scanner.cwd()", () => { test("should parse author, homepage and links for a local package who doesn't exist on the remote registry", async() => { const result = await workingDir(path.join(kFixturePath, "non-npm-package")); From 00c300823926cd817bba841908721438ea4454e9 Mon Sep 17 00:00:00 2001 From: Clement Gombauld Date: Wed, 21 Jan 2026 10:53:01 +0100 Subject: [PATCH 2/4] Update workspaces/scanner/src/class/DateProvider.class.ts Co-authored-by: Thomas.G --- workspaces/scanner/src/class/DateProvider.class.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/workspaces/scanner/src/class/DateProvider.class.ts b/workspaces/scanner/src/class/DateProvider.class.ts index e225bd5a..27beba3b 100644 --- a/workspaces/scanner/src/class/DateProvider.class.ts +++ b/workspaces/scanner/src/class/DateProvider.class.ts @@ -7,6 +7,7 @@ export class SystemDateProvider implements DateProvider { now(): number { return Date.now(); } + oneYearAgo(): Date { const date = new Date(); date.setFullYear(date.getFullYear() - 1); From e869fa1c409b40b8383f53ef2e11adc82a8a3311 Mon Sep 17 00:00:00 2001 From: Clement Gombauld Date: Wed, 21 Jan 2026 10:53:12 +0100 Subject: [PATCH 3/4] Update workspaces/scanner/src/class/StatsCollector.class.ts Co-authored-by: Thomas.G --- workspaces/scanner/src/class/StatsCollector.class.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/workspaces/scanner/src/class/StatsCollector.class.ts b/workspaces/scanner/src/class/StatsCollector.class.ts index f50d268c..556ab022 100644 --- a/workspaces/scanner/src/class/StatsCollector.class.ts +++ b/workspaces/scanner/src/class/StatsCollector.class.ts @@ -20,13 +20,10 @@ export class StatsCollector { ) as ReturnType; } - this.#addApiStat(name, startedAt); - return result; } - catch (error) { + finally { this.#addApiStat(name, startedAt); - throw error; } } From 0d74e7996a333c7c88803b0aea93bde819a29e5b Mon Sep 17 00:00:00 2001 From: Clement Gombauld Date: Wed, 21 Jan 2026 10:53:36 +0100 Subject: [PATCH 4/4] Update workspaces/scanner/src/class/StatsCollector.class.ts Co-authored-by: Thomas.G --- .../scanner/src/class/DateProvider.class.ts | 2 +- .../scanner/src/class/StatsCollector.class.ts | 6 ++++- workspaces/scanner/test/depWalker.spec.ts | 22 +++++-------------- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/workspaces/scanner/src/class/DateProvider.class.ts b/workspaces/scanner/src/class/DateProvider.class.ts index 27beba3b..87a8761c 100644 --- a/workspaces/scanner/src/class/DateProvider.class.ts +++ b/workspaces/scanner/src/class/DateProvider.class.ts @@ -7,7 +7,7 @@ export class SystemDateProvider implements DateProvider { now(): number { return Date.now(); } - + oneYearAgo(): Date { const date = new Date(); date.setFullYear(date.getFullYear() - 1); diff --git a/workspaces/scanner/src/class/StatsCollector.class.ts b/workspaces/scanner/src/class/StatsCollector.class.ts index 556ab022..194ce60d 100644 --- a/workspaces/scanner/src/class/StatsCollector.class.ts +++ b/workspaces/scanner/src/class/StatsCollector.class.ts @@ -6,6 +6,7 @@ export class StatsCollector { #apiCalls: ApiStats[] = []; #dateProvider: DateProvider; #startedAt: number; + constructor(dateProvider: DateProvider = new SystemDateProvider()) { this.#dateProvider = dateProvider; this.#startedAt = this.#dateProvider.now(); @@ -20,10 +21,13 @@ export class StatsCollector { ) as ReturnType; } + this.#addApiStat(name, startedAt); + return result; } - finally { + catch (err) { this.#addApiStat(name, startedAt); + throw err; } } diff --git a/workspaces/scanner/test/depWalker.spec.ts b/workspaces/scanner/test/depWalker.spec.ts index c913cd4f..1be9a17f 100644 --- a/workspaces/scanner/test/depWalker.spec.ts +++ b/workspaces/scanner/test/depWalker.spec.ts @@ -167,6 +167,11 @@ test("execute depWalker on pkg.gitdeps", async(test) => { phase: "tarball-scan" } ]); + const { metadata } = result; + assert.strictEqual(typeof metadata.startedAt, "number"); + assert.strictEqual(typeof metadata.executionTime, "number"); + assert.strictEqual(Array.isArray(metadata.apiCalls), true); + assert.strictEqual(metadata.apiCallsCount, 25); }); test("execute depWalker on typo-squatting (with location)", async(test) => { @@ -340,23 +345,6 @@ test("highlight contacts from a remote package", async() => { ); }); -test("should collect stats", async() => { - Vulnera.setStrategy(Vulnera.strategies.GITHUB_ADVISORY); - const { logger } = errorLogger(); - test.after(() => logger.removeAllListeners()); - - const { metadata } = await depWalker( - pkgGitdeps, - structuredClone(kDefaultWalkerOptions), - logger - ); - - assert.strictEqual(typeof metadata.startedAt, "number"); - assert.strictEqual(typeof metadata.executionTime, "number"); - assert.strictEqual(Array.isArray(metadata.apiCalls), true); - assert.strictEqual(metadata.apiCallsCount, 25); -}); - describe("scanner.cwd()", () => { test("should parse author, homepage and links for a local package who doesn't exist on the remote registry", async() => { const result = await workingDir(path.join(kFixturePath, "non-npm-package"));