feat(core): enhance config validation

Author: hanna-skryl

package-lock.json

@@ -43,7 +43,8 @@
     "vscode-material-icons": "^0.1.1",
     "yaml": "^2.5.1",
     "yargs": "^17.7.2",
-    "zod": "^3.23.8"
+    "zod": "^3.23.8",
+    "zod-validation-error": "^3.4.0"
   },
   "devDependencies": {
     "@beaussan/nx-knip": "^0.0.5-15",

package.json

@@ -41,7 +41,8 @@
   "dependencies": {
     "@code-pushup/models": "0.56.0",
     "@code-pushup/utils": "0.56.0",
-    "ansis": "^3.3.0"
+    "ansis": "^3.3.0",
+    "zod-validation-error": "^3.4.0"
   },
   "peerDependencies": {
     "@code-pushup/portal-client": "^0.9.0"

packages/core/package.json

@@ -1,7 +1,7 @@
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { describe, expect } from 'vitest';
-import { readRcByPath } from './read-rc-file.js';
+import { ConfigValidationError, readRcByPath } from './read-rc-file.js';
 
 describe('readRcByPath', () => {
   const configDirPath = join(
@@ -69,7 +69,7 @@ describe('readRcByPath', () => {
   it('should throw if the configuration is empty', async () => {
     await expect(
       readRcByPath(join(configDirPath, 'code-pushup.empty.config.js')),
-    ).rejects.toThrow(`"code": "invalid_type",`);
+    ).rejects.toThrow(expect.any(ConfigValidationError));
   });
 
   it('should throw if the configuration is invalid', async () => {

packages/core/src/lib/implementation/read-rc-file.integration.test.ts

@@ -1,4 +1,10 @@
-import { join } from 'node:path';
+import { bold, red } from 'ansis';
+import path, { join } from 'node:path';
+import {
+  type MessageBuilder,
+  fromError,
+  isZodErrorLike,
+} from 'zod-validation-error';
 import {
   CONFIG_FILE_NAME,
   type CoreConfig,
@@ -7,12 +13,42 @@ import {
 } from '@code-pushup/models';
 import { fileExists, importModule } from '@code-pushup/utils';
 
+function formatErrorPath(errorPath: (string | number)[]): string {
+  return errorPath
+    .map((key, index) => {
+      if (typeof key === 'number') {
+        return `[${key}]`;
+      }
+      return index > 0 ? `.${key}` : key;
+    })
+    .join('');
+}
+
+const coreConfigMessageBuilder: MessageBuilder = issues =>
+  issues
+    .map(issue => {
+      const formattedMessage = red(`${bold(issue.code)}: ${issue.message}`);
+      const formattedPath = formatErrorPath(issue.path);
+      if (formattedPath) {
+        return `Validation error at ${bold(formattedPath)}\n${formattedMessage}\n`;
+      }
+      return `${formattedMessage}\n`;
+    })
+    .join('\n');
+
 export class ConfigPathError extends Error {
   constructor(configPath: string) {
     super(`Provided path '${configPath}' is not valid.`);
   }
 }
 
+export class ConfigValidationError extends Error {
+  constructor(configPath: string, message: string) {
+    const relativePath = path.relative(process.cwd(), configPath);
+    super(`Failed parsing core config in ${bold(relativePath)}.\n\n${message}`);
+  }
+}
+
 export async function readRcByPath(
   filepath: string,
   tsconfig?: string,
@@ -27,7 +63,16 @@ export async function readRcByPath(
 
   const cfg = await importModule({ filepath, tsconfig, format: 'esm' });
 
-  return coreConfigSchema.parse(cfg);
+  try {
+    return coreConfigSchema.parse(cfg);
+  } catch (error) {
+    const validationError = fromError(error, {
+      messageBuilder: coreConfigMessageBuilder,
+    });
+    throw isZodErrorLike(error)
+      ? new ConfigValidationError(filepath, validationError.message)
+      : error;
+  }
 }
 
 export async function autoloadRc(tsconfig?: string): Promise<CoreConfig> {

packages/core/src/lib/implementation/read-rc-file.ts

@@ -23,21 +23,22 @@ export const categoryRefSchema = weightedRefSchema(
 );
 export type CategoryRef = z.infer<typeof categoryRefSchema>;
 
-export const categoryConfigSchema = scorableSchema(
-  'Category with a score calculated from audits and groups from various plugins',
-  categoryRefSchema,
-  getDuplicateRefsInCategoryMetrics,
-  duplicateRefsInCategoryMetricsErrorMsg,
-)
-  .merge(
+export const categoryConfigSchema = z
+  .intersection(
+    scorableSchema(
+      'Category with a score calculated from audits and groups from various plugins',
+      categoryRefSchema,
+      getDuplicateRefsInCategoryMetrics,
+      duplicateRefsInCategoryMetricsErrorMsg,
+    ),
     metaSchema({
       titleDescription: 'Category Title',
       docsUrlDescription: 'Category docs URL',
       descriptionDescription: 'Category description',
       description: 'Meta info for category',
     }),
   )
-  .merge(
+  .and(
     z.object({
       isBinary: z
         .boolean({

packages/models/src/lib/category-config.ts

@@ -129,7 +129,7 @@ describe('categoryConfigSchema', () => {
         title: 'This category is empty for now',
         refs: [],
       } satisfies CategoryConfig),
-    ).toThrow('In a category there has to be at least one ref');
+    ).toThrow('In category in-progress, there has to be at least one ref');
   });
 
   it('should throw for duplicate category references', () => {
@@ -175,7 +175,9 @@ describe('categoryConfigSchema', () => {
           },
         ],
       } satisfies CategoryConfig),
-    ).toThrow('In a category there has to be at least one ref with weight > 0');
+    ).toThrow(
+      'In category informational, there has to be at least one ref with weight > 0. Affected refs: functional/immutable-data, lighthouse-experimental',
+    );
   });
 });
 

packages/models/src/lib/category-config.unit.test.ts

@@ -25,13 +25,16 @@ export const groupMetaSchema = metaSchema({
 });
 export type GroupMeta = z.infer<typeof groupMetaSchema>;
 
-export const groupSchema = scorableSchema(
-  'A group aggregates a set of audits into a single score which can be referenced from a category. ' +
-    'E.g. the group slug "performance" groups audits and can be referenced in a category',
-  groupRefSchema,
-  getDuplicateRefsInGroups,
-  duplicateRefsInGroupsErrorMsg,
-).merge(groupMetaSchema);
+export const groupSchema = z.intersection(
+  scorableSchema(
+    'A group aggregates a set of audits into a single score which can be referenced from a category. ' +
+      'E.g. the group slug "performance" groups audits and can be referenced in a category',
+    groupRefSchema,
+    getDuplicateRefsInGroups,
+    duplicateRefsInGroupsErrorMsg,
+  ),
+  groupMetaSchema,
+);
 export type Group = z.infer<typeof groupSchema>;
 
 export const groupsSchema = z

packages/models/src/lib/group.ts

@@ -69,7 +69,7 @@ describe('groupSchema', () => {
         title: 'Empty group',
         refs: [],
       } satisfies Group),
-    ).toThrow('In a category there has to be at least one ref');
+    ).toThrow('In category empty-group, there has to be at least one ref');
   });
 
   it('should throw for duplicate group references', () => {

packages/models/src/lib/group.unit.test.ts

@@ -38,7 +38,7 @@ export const slugSchema = z
       'The slug has to follow the pattern [0-9a-z] followed by multiple optional groups of -[0-9a-z]. e.g. my-slug',
   })
   .max(MAX_SLUG_LENGTH, {
-    message: `slug can be max ${MAX_SLUG_LENGTH} characters long`,
+    message: `The slug can be max ${MAX_SLUG_LENGTH} characters long`,
   });
 
 /**  Schema for a general description property */
@@ -105,7 +105,7 @@ export function metaSchema(options?: {
 export const filePathSchema = z
   .string()
   .trim()
-  .min(1, { message: 'path is invalid' });
+  .min(1, { message: 'The path is invalid' });
 
 /** Schema for a fileNameSchema */
 export const fileNameSchema = z
@@ -114,7 +114,7 @@ export const fileNameSchema = z
   .regex(filenameRegex, {
     message: `The filename has to be valid`,
   })
-  .min(1, { message: 'file name is invalid' });
+  .min(1, { message: 'The file name is invalid' });
 
 /** Schema for a positiveInt */
 export const positiveIntSchema = z.number().int().positive();
@@ -167,26 +167,43 @@ export function scorableSchema<T extends ReturnType<typeof weightedRefSchema>>(
   duplicateCheckFn: (metrics: z.infer<T>[]) => false | string[],
   duplicateMessageFn: (metrics: z.infer<T>[]) => string,
 ) {
-  return z.object(
-    {
-      slug: slugSchema.describe('Human-readable unique ID, e.g. "performance"'),
-      refs: z
-        .array(refSchema)
-        .min(1)
-        // refs are unique
-        .refine(
-          refs => !duplicateCheckFn(refs),
-          refs => ({
-            message: duplicateMessageFn(refs),
-          }),
-        )
-        // categories weights are correct
-        .refine(hasNonZeroWeightedRef, () => ({
-          message:
-            'In a category there has to be at least one ref with weight > 0',
-        })),
-    },
-    { description },
+  return (
+    z
+      .object(
+        {
+          slug: slugSchema.describe(
+            'Human-readable unique ID, e.g. "performance"',
+          ),
+          refs: z
+            .array(refSchema)
+            .min(1)
+            // refs are unique
+            .refine(
+              refs => !duplicateCheckFn(refs),
+              refs => ({
+                message: duplicateMessageFn(refs),
+              }),
+            ),
+        },
+        { description },
+      )
+      // category weights are correct
+      .superRefine(({ slug, refs }, ctx) => {
+        if (refs.length === 0) {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `In category ${slug}, there has to be at least one ref`,
+            path: ['refs'],
+          });
+        } else if (!hasNonZeroWeightedRef(refs)) {
+          const affectedRefs = refs.map(ref => ref.slug).join(', ');
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `In category ${slug}, there has to be at least one ref with weight > 0. Affected refs: ${affectedRefs}`,
+            path: ['refs'],
+          });
+        }
+      })
   );
 }