fix: fixed correct awaiting and spawning

Author: ovflowd

src/__tests__/generators.test.mjs

@@ -2,7 +2,6 @@ import { ok, strictEqual } from 'node:assert';
 import { describe, it } from 'node:test';
 
 import createGenerator from '../generators.mjs';
-import { isAsyncGenerator } from '../streaming.mjs';
 
 describe('createGenerator', () => {
   // Simple mock input for testing
@@ -37,101 +36,115 @@ describe('createGenerator', () => {
   it('should return the ast input directly when generators list is empty', async () => {
     const { runGenerators } = createGenerator(mockInput);
 
-    const result = await runGenerators({
+    const results = await runGenerators({
       ...mockOptions,
       generators: ['ast'],
     });
 
-    // The 'ast' key should resolve to the original input
-    ok(result);
+    // Returns array of results, first element is the 'ast' result
+    ok(Array.isArray(results));
+    strictEqual(results.length, 1);
+    ok(results[0]);
   });
 
   it('should run metadata generator', async () => {
     const { runGenerators } = createGenerator(mockInput);
 
-    const result = await runGenerators({
+    const results = await runGenerators({
       ...mockOptions,
       generators: ['metadata'],
     });
 
-    // metadata returns an async generator
-    ok(isAsyncGenerator(result));
+    // Returns array with one element - the collected metadata array
+    ok(Array.isArray(results));
+    strictEqual(results.length, 1);
+    ok(Array.isArray(results[0]));
   });
 
   it('should handle generator with dependency', async () => {
     const { runGenerators } = createGenerator(mockInput);
 
     // legacy-html depends on metadata
-    const result = await runGenerators({
+    const results = await runGenerators({
       ...mockOptions,
       generators: ['legacy-html'],
     });
 
-    // Should complete without error
-    ok(result !== undefined);
+    // Should complete without error - returns array of results
+    ok(Array.isArray(results));
+    strictEqual(results.length, 1);
   });
 
   it('should skip already scheduled generators', async () => {
     const { runGenerators } = createGenerator(mockInput);
 
     // Running with ['metadata', 'metadata'] should skip the second
-    const result = await runGenerators({
+    const results = await runGenerators({
       ...mockOptions,
       generators: ['metadata', 'metadata'],
     });
 
-    ok(isAsyncGenerator(result));
+    // Returns array with two elements (same result cached for both)
+    ok(Array.isArray(results));
+    strictEqual(results.length, 2);
   });
 
   it('should handle multiple generators in sequence', async () => {
     const { runGenerators } = createGenerator(mockInput);
 
-    // Run metadata twice - the system should skip the already scheduled one
-    // Avoid json-simple since it writes to disk
-    const result = await runGenerators({
+    // Run metadata - just one generator
+    const results = await runGenerators({
       ...mockOptions,
       generators: ['metadata'],
     });
 
-    // Result should be from the last generator
-    ok(result !== undefined);
+    // Returns array of results
+    ok(Array.isArray(results));
+    strictEqual(results.length, 1);
   });
 
   it('should collect async generator results for dependents', async () => {
     const { runGenerators } = createGenerator(mockInput);
 
     // legacy-json depends on metadata (async generator)
-    const result = await runGenerators({
+    const results = await runGenerators({
       ...mockOptions,
       generators: ['legacy-json'],
     });
 
-    ok(result !== undefined);
+    ok(Array.isArray(results));
+    strictEqual(results.length, 1);
   });
 
   it('should use multiple threads when specified', async () => {
     const { runGenerators } = createGenerator(mockInput);
 
-    const result = await runGenerators({
+    const results = await runGenerators({
       ...mockOptions,
       threads: 4,
       generators: ['metadata'],
     });
 
-    ok(isAsyncGenerator(result));
+    // Returns array of results
+    ok(Array.isArray(results));
+    strictEqual(results.length, 1);
+    ok(Array.isArray(results[0]));
   });
 
   it('should pass options to generators', async () => {
     const { runGenerators } = createGenerator(mockInput);
 
     const customTypeMap = { TestType: 'https://example.com/TestType' };
 
-    const result = await runGenerators({
+    const results = await runGenerators({
       ...mockOptions,
       typeMap: customTypeMap,
       generators: ['metadata'],
     });
 
-    ok(isAsyncGenerator(result));
+    // Returns array of results
+    ok(Array.isArray(results));
+    strictEqual(results.length, 1);
+    ok(Array.isArray(results[0]));
   });
 });

src/generators.mjs

@@ -120,7 +120,7 @@ const createGenerator = input => {
    * their dependencies to complete.
    *
    * @param {GeneratorOptions} options - Generator runtime options
-   * @returns {Promise<unknown>} Result of the last generator in the pipeline
+   * @returns {Promise<unknown[]>} Results of all requested generators
    */
   const runGenerators = async options => {
     const { generators, threads } = options;
@@ -138,15 +138,31 @@ const createGenerator = input => {
     // Schedule all generators using the shared pool
     scheduleGenerators(options, sharedPool);
 
-    // Wait for the last generator's result
-    const result = await cachedGenerators[generators[generators.length - 1]];
+    // Wait for ALL requested generators to complete (not just the last one)
+    const results = [];
 
-    // Terminate workers after all work is complete
-    await sharedPool.terminate();
+    for (const generatorName of generators) {
+      let result = await cachedGenerators[generatorName];
+
+      // If the generator returns an async generator, consume it
+      // to ensure all side effects (file writes, etc.) complete
+      if (isAsyncGenerator(result)) {
+        generatorsLogger.debug(
+          `Consuming async generator output from "${generatorName}"`
+        );
+
+        result = await streamingCache.getOrCollect(generatorName, result);
+      }
+
+      results.push(result);
+    }
+
+    // Terminate workers after all work is complete (fire-and-forget)
+    sharedPool.terminate();
 
     sharedPool = null;
 
-    return result;
+    return results;
   };
 
   return { runGenerators };

src/generators/metadata/index.mjs

@@ -27,15 +27,18 @@ export default {
    * @param {Partial<Omit<GeneratorOptions, 'worker'>>} options - Serializable options
    * @returns {Promise<ApiDocMetadataEntry[]>} Metadata entries for processed files
    */
-  async processChunk(fullInput, itemIndices, { typeMap }) {
-    const results = [];
-
-    for (const idx of itemIndices) {
-      results.push(...parseApiDoc(fullInput[idx], typeMap));
-    }
-
-    return results;
-  },
+  processChunk: Object.assign(
+    async (fullInput, itemIndices, { typeMap }) => {
+      const results = [];
+
+      for (const idx of itemIndices) {
+        results.push(...parseApiDoc(fullInput[idx], typeMap));
+      }
+
+      return results;
+    },
+    { sliceInput: true } // Only needs individual items, not full context
+  ),
 
   /**
    * @param {Input} inputs

src/generators/types.d.ts

@@ -131,10 +131,20 @@ declare global {
      * @param options - Generator options (without worker, which isn't serializable)
      * @returns Array of results for the processed items
      */
-    processChunk?: (
+    processChunk?: ((
       fullInput: I,
       itemIndices: number[],
       options: Partial<Omit<GeneratorOptions, 'worker'>>
-    ) => Promise<unknown[]>;
+    ) => Promise<unknown[]>) & {
+      /**
+       * When true, only the items at the specified indices are sent to workers
+       * instead of the full input array. This reduces serialization overhead
+       * for generators that don't need full context to process individual items.
+       *
+       * Set this to true when processChunk only accesses `fullInput[idx]` for
+       * each index in itemIndices, and doesn't need the full array for context.
+       */
+      sliceInput?: boolean;
+    };
   }
 }

src/generators/web/index.mjs

@@ -13,6 +13,9 @@ import { processJSXEntries } from './utils/processing.mjs';
  * - Client-side JavaScript with code splitting
  * - Bundled CSS styles
  *
+ * Note: This generator does NOT support streaming/chunked processing because
+ * processJSXEntries needs all entries together to generate code-split bundles.
+ *
  * @type {GeneratorMetadata<Input, string>}
  */
 export default {
@@ -21,69 +24,38 @@ export default {
   description: 'Generates HTML/CSS/JS bundles from JSX AST entries',
   dependsOn: 'jsx-ast',
 
-  /**
-   * Process a chunk of JSX AST entries.
-   * This simply passes through the entries for aggregation in the main generate function.
-   * The actual processing happens in processJSXEntries which needs all entries together.
-   *
-   * @param {import('../jsx-ast/utils/buildContent.mjs').JSXContent[]} fullInput
-   * @param {number[]} itemIndices
-   */
-  processChunk(fullInput, itemIndices) {
-    const results = [];
-
-    for (const idx of itemIndices) {
-      results.push(fullInput[idx]);
-    }
-
-    return results;
-  },
-
   /**
    * Main generation function that processes JSX AST entries into web bundles.
    *
    * @param {import('../jsx-ast/utils/buildContent.mjs').JSXContent[]} entries - JSX AST entries to process.
    * @param {Partial<GeneratorOptions>} options - Generator options.
    * @param {string} [options.output] - Output directory for generated files.
    * @param {string} options.version - Documentation version string.
-   * @returns {AsyncGenerator<Array<import('../jsx-ast/utils/buildContent.mjs').JSXContent>>}
+   * @returns {Promise<import('../jsx-ast/utils/buildContent.mjs').JSXContent[]>}
    */
-  async *generate(entries, { output, version, worker }) {
-    // Start loading template while chunks stream in (parallel I/O)
-    const templatePromise = readFile(
+  async generate(entries, { output, version }) {
+    const template = await readFile(
       new URL('template.html', import.meta.url),
       'utf-8'
     );
 
-    // Collect all chunks as they stream in from jsx-ast
-    const allEntries = [];
+    // Create AST builders for server and client programs
+    const astBuilders = createASTBuilder();
 
-    for await (const chunkResult of worker.stream(entries, entries, {})) {
-      allEntries.push(...chunkResult);
+    // Create require function for resolving external packages in server code
+    const requireFn = createRequire(import.meta.url);
 
-      yield chunkResult;
-    }
+    // Process all entries: convert JSX to HTML/CSS/JS
+    const { results, css, chunks } = await processJSXEntries(
+      entries,
+      template,
+      astBuilders,
+      requireFn,
+      { version }
+    );
 
-    // Now that all chunks are collected, process them together
-    // (processJSXEntries needs all entries to generate code-split bundles)
+    // Process all entries together (required for code-split bundles)
     if (output) {
-      const template = await templatePromise;
-
-      // Create AST builders for server and client programs
-      const astBuilders = createASTBuilder();
-
-      // Create require function for resolving external packages in server code
-      const requireFn = createRequire(import.meta.url);
-
-      // Process all entries: convert JSX to HTML/CSS/JS
-      const { results, css, chunks } = await processJSXEntries(
-        allEntries,
-        template,
-        astBuilders,
-        requireFn,
-        { version }
-      );
-
       // Write HTML files
       for (const { html, api } of results) {
         await writeFile(join(output, `${api}.html`), html, 'utf-8');
@@ -97,5 +69,7 @@ export default {
       // Write CSS bundle
       await writeFile(join(output, 'styles.css'), css, 'utf-8');
     }
+
+    return results;
   },
 };