feat(@angular-devkit/build-angular): support parallel i18n localization

Author: clydin

packages/angular_devkit/build_angular/package.json

@@ -12,6 +12,7 @@
     "@angular-devkit/build-webpack": "0.0.0",
     "@angular-devkit/core": "0.0.0",
     "@babel/core": "7.6.4",
+    "@babel/generator": "7.6.4",
     "@babel/preset-env": "7.6.3",
     "@ngtools/webpack": "0.0.0",
     "ajv": "6.10.2",
@@ -33,6 +34,7 @@
     "less-loader": "5.0.0",
     "license-webpack-plugin": "2.1.3",
     "loader-utils": "1.2.3",
+    "magic-string": "0.25.4",
     "mini-css-extract-plugin": "0.8.0",
     "minimatch": "3.0.4",
     "parse5": "4.0.0",

packages/angular_devkit/build_angular/src/browser/action-executor.ts

@@ -8,9 +8,25 @@
 import JestWorker from 'jest-worker';
 import * as os from 'os';
 import * as path from 'path';
-import { ProcessBundleOptions, ProcessBundleResult } from '../utils/process-bundle';
+import * as v8 from 'v8';
+import { InlineOptions, ProcessBundleOptions, ProcessBundleResult } from '../utils/process-bundle';
 import { BundleActionCache } from './action-cache';
 
+const hasThreadSupport = (() => {
+  try {
+    require('worker_threads');
+
+    return true;
+  } catch {
+    return false;
+  }
+})();
+
+// This is used to normalize serialization messaging across threads and processes
+// Threads use the structured clone algorithm which handles more types
+// Processes use JSON which is much more limited
+const serialize = ((v8 as unknown) as { serialize(value: unknown): Buffer }).serialize;
+
 let workerFile = require.resolve('../utils/process-bundle');
 workerFile =
   path.extname(workerFile) === '.ts'
@@ -41,8 +57,8 @@ export class BundleActionExecutor {
 
     // larger files are processed in a separate process to limit memory usage in the main process
     return (this.largeWorker = new JestWorker(workerFile, {
-      exposedMethods: ['process'],
-      setupArgs: [this.workerOptions],
+      exposedMethods: ['process', 'inlineLocales'],
+      setupArgs: [[...serialize(this.workerOptions)]],
     }));
   }
 
@@ -54,11 +70,10 @@ export class BundleActionExecutor {
     // small files are processed in a limited number of threads to improve speed
     // The limited number also prevents a large increase in memory usage for an otherwise short operation
     return (this.smallWorker = new JestWorker(workerFile, {
-      exposedMethods: ['process'],
-      setupArgs: [this.workerOptions],
+      exposedMethods: ['process', 'inlineLocales'],
+      setupArgs: hasThreadSupport ? [this.workerOptions] : [[...serialize(this.workerOptions)]],
       numWorkers: os.cpus().length < 2 ? 1 : 2,
-      // Will automatically fallback to processes if not supported
-      enableWorkerThreads: true,
+      enableWorkerThreads: hasThreadSupport,
     }));
   }
 
@@ -71,7 +86,7 @@ export class BundleActionExecutor {
     }
   }
 
-  async process(action: ProcessBundleOptions) {
+  async process(action: ProcessBundleOptions): Promise<ProcessBundleResult> {
     const cacheKeys = this.cache.generateCacheKeys(action);
     action.cacheKeys = cacheKeys;
 
@@ -86,10 +101,27 @@ export class BundleActionExecutor {
     return this.executeAction<ProcessBundleResult>('process', action);
   }
 
-  async *processAll(actions: Iterable<ProcessBundleOptions>) {
-    const executions = new Map<Promise<ProcessBundleResult>, Promise<ProcessBundleResult>>();
+  processAll(actions: Iterable<ProcessBundleOptions>): AsyncIterable<ProcessBundleResult> {
+    return BundleActionExecutor.executeAll(actions, action => this.process(action));
+  }
+
+  async inline(
+    action: InlineOptions,
+  ): Promise<{ file: string; diagnostics: { type: string; message: string }[]; count: number; }> {
+    return this.executeAction('inlineLocales', action);
+  }
+
+  inlineAll(actions: Iterable<InlineOptions>) {
+    return BundleActionExecutor.executeAll(actions, action => this.inline(action));
+  }
+
+  private static async *executeAll<I, O>(
+    actions: Iterable<I>,
+    executor: (action: I) => Promise<O>,
+  ): AsyncIterable<O> {
+    const executions = new Map<Promise<O>, Promise<O>>();
     for (const action of actions) {
-      const execution = this.process(action);
+      const execution = executor(action);
       executions.set(
         execution,
         execution.then(result => {
@@ -105,7 +137,7 @@ export class BundleActionExecutor {
     }
   }
 
-  stop() {
+  stop(): void {
     if (this.largeWorker) {
       this.largeWorker.end();
     }

packages/angular_devkit/build_angular/src/browser/index.ts

@@ -6,11 +6,7 @@
  * found in the LICENSE file at https://angular.io/license
  */
 import { BuilderContext, BuilderOutput, createBuilder } from '@angular-devkit/architect';
-import {
-  EmittedFiles,
-  WebpackLoggingCallback,
-  runWebpack,
-} from '@angular-devkit/build-webpack';
+import { EmittedFiles, WebpackLoggingCallback, runWebpack } from '@angular-devkit/build-webpack';
 import { join, json, logging, normalize, tags, virtualFs } from '@angular-devkit/core';
 import { NodeJsSyncHost } from '@angular-devkit/core/node';
 import * as findCacheDirectory from 'find-cache-dir';
@@ -58,6 +54,7 @@ import { copyAssets } from '../utils/copy-assets';
 import { I18nOptions, createI18nOptions } from '../utils/i18n-options';
 import { createTranslationLoader } from '../utils/load-translations';
 import {
+  InlineOptions,
   ProcessBundleFile,
   ProcessBundleOptions,
   ProcessBundleResult,
@@ -447,76 +444,103 @@ export function buildWebpackBrowser(
                 }
 
                 context.logger.info('ES5 bundle generation complete.');
-              } finally {
-                executor.stop();
-              }
-
-              if (i18n.shouldInline) {
-                context.logger.info('Generating localized bundles...');
-
-                const localize = await import('@angular/localize/src/tools/src/translate/main');
-                const localizeDiag = await import('@angular/localize/src/tools/src/diagnostics');
 
-                const diagnostics = new localizeDiag.Diagnostics();
-                const translationFilePaths = [];
-                let handleSourceLocale = false;
-                for (const locale of i18n.inlineLocales) {
-                  if (locale === i18n.sourceLocale) {
-                    handleSourceLocale = true;
-                    continue;
-                  }
-                  translationFilePaths.push(i18n.locales[locale].file);
-                }
+                if (i18n.shouldInline) {
+                  context.logger.info('Generating localized bundles...');
 
-                if (translationFilePaths.length > 0) {
-                  const sourceFilePaths = [];
+                  const inlineActions: InlineOptions[] = [];
+                  const processedFiles = new Set<string>();
                   for (const result of processResults) {
                     if (result.original) {
-                      sourceFilePaths.push(result.original.filename);
+                      inlineActions.push({
+                        filename: path.basename(result.original.filename),
+                        code: fs.readFileSync(result.original.filename, 'utf8'),
+                        outputPath: baseOutputPath,
+                        es5: false,
+                        missingTranslation: options.i18nMissingTranslation,
+                      });
+                      processedFiles.add(result.original.filename);
                     }
                     if (result.downlevel) {
-                      sourceFilePaths.push(result.downlevel.filename);
+                      inlineActions.push({
+                        filename: path.basename(result.downlevel.filename),
+                        code: fs.readFileSync(result.downlevel.filename, 'utf8'),
+                        outputPath: baseOutputPath,
+                        es5: true,
+                        missingTranslation: options.i18nMissingTranslation,
+                      });
+                      processedFiles.add(result.downlevel.filename);
                     }
                   }
+
+                  let hasErrors = false;
                   try {
-                    localize.translateFiles({
-                      // tslint:disable-next-line: no-non-null-assertion
-                      sourceRootPath: webpackStats.outputPath!,
-                      sourceFilePaths,
-                      translationFilePaths,
-                      outputPathFn: (locale, relativePath) =>
-                        path.join(baseOutputPath, locale, relativePath),
-                      diagnostics,
-                      missingTranslation: options.i18nMissingTranslation || 'warning',
-                      sourceLocale: handleSourceLocale ? i18n.sourceLocale : undefined,
-                    });
+                    for (const locale of i18n.inlineLocales) {
+                      const localeOutputPath = path.join(baseOutputPath, locale);
+                      if (!fs.existsSync(localeOutputPath)) {
+                        fs.mkdirSync(localeOutputPath, { recursive: true });
+                      }
+                    }
+
+                    for await (const result of executor.inlineAll(inlineActions)) {
+                      if (options.verbose) {
+                        context.logger.info(
+                          `Localized "${result.file}" [${result.count} translation(s)].`,
+                        );
+                      }
+                      for (const diagnostic of result.diagnostics) {
+                        if (diagnostic.type === 'error') {
+                          hasErrors = true;
+                          context.logger.error(diagnostic.message);
+                        } else {
+                          context.logger.warn(diagnostic.message);
+                        }
+                      }
+                    }
+
+                    // Copy any non-processed files into the output locations
+                    const outputPaths = [...i18n.inlineLocales].map(l =>
+                      path.join(baseOutputPath, l),
+                    );
+                    await copyAssets(
+                      [
+                        {
+                          glob: '**/*',
+                          // tslint:disable-next-line: no-non-null-assertion
+                          input: webpackStats.outputPath!,
+                          output: '',
+                          ignore: [...processedFiles].map(f =>
+                          // tslint:disable-next-line: no-non-null-assertion
+                            path.relative(webpackStats.outputPath!, f),
+                          ),
+                        },
+                      ],
+                      outputPaths,
+                      '',
+                    );
                   } catch (err) {
                     context.logger.error('Localized bundle generation failed: ' + err.message);
 
                     return { success: false };
-                  } finally {
-                    try {
-                      // Remove temporary directory used for i18n processing
-                      // tslint:disable-next-line: no-non-null-assertion
-                      await host.delete(normalize(webpackStats.outputPath!)).toPromise();
-                    } catch {}
                   }
-                }
 
-                context.logger.info(
-                  `Localized bundle generation ${diagnostics.hasErrors ? 'failed' : 'complete'}.`,
-                );
+                  context.logger.info(
+                    `Localized bundle generation ${hasErrors ? 'failed' : 'complete'}.`,
+                  );
 
-                for (const message of diagnostics.messages) {
-                  if (message.type === 'error') {
-                    context.logger.error(message.message);
-                  } else {
-                    context.logger.warn(message.message);
+                  if (hasErrors) {
+                    return { success: false };
                   }
                 }
+              } finally {
+                executor.stop();
 
-                if (diagnostics.hasErrors) {
-                  return { success: false };
+                if (i18n.shouldInline) {
+                  try {
+                    // Remove temporary directory used for i18n processing
+                    // tslint:disable-next-line: no-non-null-assertion
+                    await host.delete(normalize(webpackStats.outputPath!)).toPromise();
+                  } catch {}
                 }
               }
 

packages/angular_devkit/build_angular/src/utils/load-translations.ts

@@ -30,7 +30,7 @@ export async function createTranslationLoader(): Promise<TranslationLoader> {
 
     for (const [format, parser] of Object.entries(parsers)) {
       if (parser.canParse(path, content)) {
-        return { format, translation: parser.parse(path, content) };
+        return { format, translation: parser.parse(path, content).translations };
       }
     }