feat: properly implement streaming, worker chunking and proper worker thread scheduler

Author: ovflowd

bin/cli.mjs

@@ -7,11 +7,28 @@ import { Command, Option } from 'commander';
 import commands from './commands/index.mjs';
 import interactive from './commands/interactive.mjs';
 import { errorWrap } from './utils.mjs';
+import logger from '../src/logger/index.mjs';
 
 const program = new Command()
   .name('@nodejs/doc-kit')
   .description('CLI tool to generate the Node.js API documentation');
 
+// Add global log level option
+program.addOption(
+  new Option('--log-level <level>', 'Log level')
+    .choices(['debug', 'info', 'warn', 'error', 'fatal'])
+    .default('info')
+);
+
+// Set log level before any command runs
+program.hook('preAction', thisCommand => {
+  const logLevel = thisCommand.opts().logLevel;
+
+  if (logLevel) {
+    logger.setLogLevel(logLevel);
+  }
+});
+
 // Registering commands
 commands.forEach(({ name, description, options, action }) => {
   const cmd = program.command(name).description(description);

bin/commands/generate.mjs

@@ -3,10 +3,7 @@ import { resolve } from 'node:path';
 
 import { coerce } from 'semver';
 
-import {
-  DOC_NODE_CHANGELOG_URL,
-  DOC_NODE_VERSION,
-} from '../../src/constants.mjs';
+import { NODE_CHANGELOG_URL, NODE_VERSION } from '../../src/constants.mjs';
 import { publicGenerators } from '../../src/generators/index.mjs';
 import createGenerator from '../../src/generators.mjs';
 import { parseChangelog, parseIndex } from '../../src/parsers/markdown.mjs';
@@ -18,7 +15,10 @@ const availableGenerators = Object.keys(publicGenerators);
 
 // Half of available logical CPUs guarantees in general all physical CPUs are being used
 // which in most scenarios is the best way to maximize performance
-const optimalThreads = Math.floor(cpus().length / 2) + 1;
+// When spawning more than a said number of threads, the overhead of context switching
+// and CPU contention starts to degrade performance rather than improve it.
+// Therefore, we set the optimal threads to half the number of CPU cores, with a minimum of 6.
+const optimalThreads = Math.min(Math.floor(cpus().length / 2), 6);
 
 /**
  * @typedef {Object} Options
@@ -88,7 +88,7 @@ export default {
       prompt: {
         type: 'text',
         message: 'Enter Node.js version',
-        initialValue: DOC_NODE_VERSION,
+        initialValue: NODE_VERSION,
       },
     },
     changelog: {
@@ -97,7 +97,7 @@ export default {
       prompt: {
         type: 'text',
         message: 'Enter changelog URL',
-        initialValue: DOC_NODE_CHANGELOG_URL,
+        initialValue: NODE_CHANGELOG_URL,
       },
     },
     gitRef: {

src/__tests__/generators.test.mjs

@@ -0,0 +1,137 @@
+import { ok, strictEqual } from 'node:assert';
+import { describe, it } from 'node:test';
+
+import createGenerator from '../generators.mjs';
+import { isAsyncGenerator } from '../streaming.mjs';
+
+describe('createGenerator', () => {
+  // Simple mock input for testing
+  const mockInput = [
+    {
+      file: { stem: 'test', basename: 'test.md' },
+      tree: { type: 'root', children: [] },
+    },
+  ];
+
+  // Mock options with minimal required fields
+  const mockOptions = {
+    input: '/tmp/test',
+    output: '/tmp/output',
+    generators: ['metadata'],
+    version: { major: 22, minor: 0, patch: 0 },
+    releases: [],
+    index: [],
+    gitRef: 'https://github.com/nodejs/node/tree/HEAD',
+    threads: 1,
+    chunkSize: 20,
+    typeMap: {},
+  };
+
+  it('should create a generator orchestrator with runGenerators method', () => {
+    const { runGenerators } = createGenerator(mockInput);
+
+    ok(runGenerators);
+    strictEqual(typeof runGenerators, 'function');
+  });
+
+  it('should return the ast input directly when generators list is empty', async () => {
+    const { runGenerators } = createGenerator(mockInput);
+
+    const result = await runGenerators({
+      ...mockOptions,
+      generators: ['ast'],
+    });
+
+    // The 'ast' key should resolve to the original input
+    ok(result);
+  });
+
+  it('should run metadata generator', async () => {
+    const { runGenerators } = createGenerator(mockInput);
+
+    const result = await runGenerators({
+      ...mockOptions,
+      generators: ['metadata'],
+    });
+
+    // metadata returns an async generator
+    ok(isAsyncGenerator(result));
+  });
+
+  it('should handle generator with dependency', async () => {
+    const { runGenerators } = createGenerator(mockInput);
+
+    // legacy-html depends on metadata
+    const result = await runGenerators({
+      ...mockOptions,
+      generators: ['legacy-html'],
+    });
+
+    // Should complete without error
+    ok(result !== undefined);
+  });
+
+  it('should skip already scheduled generators', async () => {
+    const { runGenerators } = createGenerator(mockInput);
+
+    // Running with ['metadata', 'metadata'] should skip the second
+    const result = await runGenerators({
+      ...mockOptions,
+      generators: ['metadata', 'metadata'],
+    });
+
+    ok(isAsyncGenerator(result));
+  });
+
+  it('should handle multiple generators in sequence', async () => {
+    const { runGenerators } = createGenerator(mockInput);
+
+    // Run metadata twice - the system should skip the already scheduled one
+    // Avoid json-simple since it writes to disk
+    const result = await runGenerators({
+      ...mockOptions,
+      generators: ['metadata'],
+    });
+
+    // Result should be from the last generator
+    ok(result !== undefined);
+  });
+
+  it('should collect async generator results for dependents', async () => {
+    const { runGenerators } = createGenerator(mockInput);
+
+    // legacy-json depends on metadata (async generator)
+    const result = await runGenerators({
+      ...mockOptions,
+      generators: ['legacy-json'],
+    });
+
+    ok(result !== undefined);
+  });
+
+  it('should use multiple threads when specified', async () => {
+    const { runGenerators } = createGenerator(mockInput);
+
+    const result = await runGenerators({
+      ...mockOptions,
+      threads: 4,
+      generators: ['metadata'],
+    });
+
+    ok(isAsyncGenerator(result));
+  });
+
+  it('should pass options to generators', async () => {
+    const { runGenerators } = createGenerator(mockInput);
+
+    const customTypeMap = { TestType: 'https://example.com/TestType' };
+
+    const result = await runGenerators({
+      ...mockOptions,
+      typeMap: customTypeMap,
+      generators: ['metadata'],
+    });
+
+    ok(isAsyncGenerator(result));
+  });
+});