perf(ci): use bulk command to compare reports for all projects

.gitignore

@@ -44,7 +44,7 @@ testem.log
 Thumbs.db
 
 # generated Code PushUp reports
-**/.code-pushup
+/.code-pushup
 
 # Nx workspace cache
 .nx

e2e/cli-e2e/tests/compare.e2e.test.ts

@@ -39,12 +39,7 @@ describe('CLI compare', () => {
   it('should compare report.json files and create report-diff.json and report-diff.md', async () => {
     await executeProcess({
       command: 'npx',
-      args: [
-        '@code-pushup/cli',
-        'compare',
-        `--before=${path.join('.code-pushup', 'source-report.json')}`,
-        `--after=${path.join('.code-pushup', 'target-report.json')}`,
-      ],
+      args: ['@code-pushup/cli', 'compare'],
       cwd: existingDir,
     });
 

packages/ci/src/lib/cli/commands/compare.ts

@@ -2,26 +2,19 @@ import { DEFAULT_PERSIST_FORMAT } from '@code-pushup/models';
 import { executeProcess, isVerbose } from '@code-pushup/utils';
 import type { CommandContext } from '../context.js';
 
-type CompareOptions = {
-  before: string;
-  after: string;
-  label?: string;
-};
-
 export async function runCompare(
-  { before, after, label }: CompareOptions,
   { bin, config, directory, observer }: CommandContext,
+  { hasFormats }: { hasFormats: boolean },
 ): Promise<void> {
   await executeProcess({
     command: bin,
     args: [
       'compare',
       ...(isVerbose() ? ['--verbose'] : []),
-      `--before=${before}`,
-      `--after=${after}`,
-      ...(label ? [`--label=${label}`] : []),
       ...(config ? [`--config=${config}`] : []),
-      ...DEFAULT_PERSIST_FORMAT.map(format => `--persist.format=${format}`),
+      ...(hasFormats
+        ? []
+        : DEFAULT_PERSIST_FORMAT.map(format => `--persist.format=${format}`)),
     ],
     cwd: directory,
     observer,

packages/ci/src/lib/cli/persist.ts

@@ -9,7 +9,7 @@ import {
   persistConfigSchema,
   uploadConfigSchema,
 } from '@code-pushup/models';
-import { objectFromEntries } from '@code-pushup/utils';
+import { createReportPath, objectFromEntries } from '@code-pushup/utils';
 
 export type EnhancedPersistConfig = Pick<CoreConfig, 'persist' | 'upload'>;
 
@@ -27,12 +27,12 @@ export function persistedFilesFromConfig(
   const dir = path.isAbsolute(outputDir)
     ? outputDir
     : path.join(directory, outputDir);
-  const name = isDiff ? `${filename}-diff` : filename;
+  const suffix = isDiff ? 'diff' : undefined;
 
   return objectFromEntries(
     DEFAULT_PERSIST_FORMAT.map(format => [
       format,
-      path.join(dir, `${name}.${format}`),
+      createReportPath({ outputDir: dir, filename, format, suffix }),
     ]),
   );
 }

packages/ci/src/lib/run-monorepo.ts

@@ -11,6 +11,7 @@ import {
   createCommandContext,
   persistedFilesFromConfig,
   runCollect,
+  runCompare,
   runMergeDiffs,
 } from './cli/index.js';
 import { commentOnPR } from './comment.js';
@@ -29,16 +30,18 @@ import {
 import { saveOutputFiles } from './output-files.js';
 import {
   type BaseReportArgs,
+  type CompareReportsArgs,
   type ReportData,
   type RunEnv,
   checkPrintConfig,
-  compareReports,
   configFromPatterns,
   hasDefaultPersistFormats,
   loadCachedBaseReport,
+  prepareReportFilesToCompare,
   printPersistConfig,
   runInBaseBranch,
   runOnProject,
+  saveDiffFiles,
   saveReportFiles,
 } from './run-utils.js';
 
@@ -226,18 +229,38 @@ async function compareProjectsInBulk(
       ...args,
       prevReport: args.prevReport || collectedPrevReports[args.project.name],
     }))
-    .filter(hasNoNullableProps);
+    .filter(hasNoNullableProps) satisfies CompareReportsArgs[];
 
-  const projectComparisons = Object.fromEntries(
-    await asyncSequential(projectsToCompare, async args => [
-      args.project.name,
-      await compareReports(args),
-    ]),
+  const projectComparisons = await compareManyProjects(
+    projectsToCompare,
+    runManyCommand,
+    env,
   );
 
   return finalizeProjectReports(currProjectReports, projectComparisons);
 }
 
+async function compareManyProjects(
+  projectsToCompare: ExcludeNullableProps<CompareReportsArgs>[],
+  runManyCommand: RunManyCommand,
+  env: RunEnv,
+): Promise<Record<string, ProjectRunResult>> {
+  await Promise.all(projectsToCompare.map(prepareReportFilesToCompare));
+
+  await compareMany(runManyCommand, env, {
+    hasFormats: allProjectsHaveDefaultPersistFormats(projectsToCompare),
+  });
+
+  return Object.fromEntries(
+    await Promise.all(
+      projectsToCompare.map(async args => [
+        args.project.name,
+        await saveDiffFiles(args),
+      ]),
+    ),
+  );
+}
+
 function finalizeProjectReports(
   projectReports: ProjectReport[],
   projectComparisons?: Record<string, ProjectRunResult>,
@@ -352,6 +375,26 @@ async function collectMany(
   );
 }
 
+async function compareMany(
+  runManyCommand: RunManyCommand,
+  env: RunEnv,
+  options: {
+    hasFormats: boolean;
+  },
+): Promise<void> {
+  const { settings } = env;
+  const { hasFormats } = options;
+
+  const ctx: CommandContext = {
+    ...createCommandContext(settings, null),
+    bin: await runManyCommand(),
+  };
+
+  await runCompare(ctx, { hasFormats });
+
+  settings.logger.debug('Compared all project reports');
+}
+
 export function allProjectsHaveDefaultPersistFormats(
   projects: { config: EnhancedPersistConfig }[],
 ): boolean {

packages/ci/src/lib/run-utils.ts

@@ -1,13 +1,20 @@
 /* eslint-disable max-lines */
-import { readFile } from 'node:fs/promises';
+import { readFile, writeFile } from 'node:fs/promises';
+import path from 'node:path';
 import type { SimpleGit } from 'simple-git';
 import {
+  DEFAULT_PERSIST_FILENAME,
   DEFAULT_PERSIST_FORMAT,
+  DEFAULT_PERSIST_OUTPUT_DIR,
   type Report,
   type ReportsDiff,
 } from '@code-pushup/models';
 import {
+  type Diff,
+  createReportPath,
   interpolate,
+  objectFromEntries,
+  readJsonFile,
   removeUndefinedAndEmptyProps,
   stringifyError,
 } from '@code-pushup/utils';
@@ -53,6 +60,7 @@ type NormalizedGitRefs = {
 export type CompareReportsArgs = {
   project: ProjectConfig | null;
   env: RunEnv;
+  ctx: CommandContext;
   base: GitBranch;
   currReport: ReportData<'current'>;
   prevReport: ReportData<'previous'>;
@@ -152,38 +160,91 @@ export async function runOnProject(
     return noDiffOutput;
   }
 
-  const compareArgs = { project, env, base, config, currReport, prevReport };
+  const compareArgs = { ...baseArgs, currReport, prevReport };
   return compareReports(compareArgs);
 }
 
 export async function compareReports(
   args: CompareReportsArgs,
 ): Promise<ProjectRunResult> {
+  const { ctx, env, config } = args;
+  const { logger } = env.settings;
+
+  await prepareReportFilesToCompare(args);
+  await runCompare(ctx, { hasFormats: hasDefaultPersistFormats(config) });
+
+  logger.info('Compared reports and generated diff files');
+
+  return saveDiffFiles(args);
+}
+
+export async function prepareReportFilesToCompare(
+  args: CompareReportsArgs,
+): Promise<Diff<string>> {
+  const { config, project, env, ctx } = args;
+  const {
+    outputDir = DEFAULT_PERSIST_OUTPUT_DIR,
+    filename = DEFAULT_PERSIST_FILENAME,
+  } = config.persist ?? {};
+  const label = project?.name;
+  const { logger } = env.settings;
+
+  const originalReports = await Promise.all(
+    [args.currReport, args.prevReport].map(({ files }) =>
+      readJsonFile<Report>(files.json),
+    ),
+  );
+  const labeledReports = label
+    ? originalReports.map(report => ({ ...report, label }))
+    : originalReports;
+
+  const reportPaths = labeledReports.map((report, idx) => {
+    const key: keyof Diff<string> = idx === 0 ? 'after' : 'before';
+    const filePath = createReportPath({
+      outputDir: path.resolve(ctx.directory, outputDir),
+      filename,
+      format: 'json',
+      suffix: key,
+    });
+    return { key, report, filePath };
+  });
+
+  await Promise.all(
+    reportPaths.map(({ filePath, report }) =>
+      writeFile(filePath, JSON.stringify(report, null, 2)),
+    ),
+  );
+
+  logger.debug(
+    [
+      'Prepared',
+      project && `"${project.name}" project's`,
+      'report files for comparison',
+      `at ${reportPaths.map(({ filePath }) => filePath).join(' and ')}`,
+    ]
+      .filter(Boolean)
+      .join(' '),
+  );
+
+  return objectFromEntries(
+    reportPaths.map(({ key, filePath }) => [key, filePath]),
+  );
+}
+
+export async function saveDiffFiles(args: CompareReportsArgs) {
   const {
     project,
+    ctx,
     env: { settings },
     currReport,
     prevReport,
     config,
   } = args;
-  const logger = settings.logger;
-
-  const ctx = createCommandContext(settings, project);
 
-  await runCompare(
-    {
-      before: prevReport.files.json,
-      after: currReport.files.json,
-      label: project?.name,
-    },
-    ctx,
-  );
   const diffFiles = persistedFilesFromConfig(config, {
     directory: ctx.directory,
     isDiff: true,
   });
-  logger.info('Compared reports and generated diff files');
-  logger.debug(`Generated diff files at ${diffFiles.json} and ${diffFiles.md}`);
 
   return {
     name: projectToName(project),