Enable liquid batch testing

CHANGELOG.md

@@ -1,6 +1,9 @@
 # Changelog
 
 All notable changes to this project will be documented in this file.
+## [1.49.0] (07/01/2026)
+In this version we are introducing Liquid batch/pattern testing. An extra option was added to the `silverfin run-test`command to run all tests which conatin a common string. 
+To enable it run `silverfin run-test -p "string pattern" -h template_handle`
 
 ## [1.48.0] (25/09/2025)
 In this version we are introducing TAB autocompletion for the CLI commands. It should autocomplete command names, flags, and template handles and names.

bin/cli.js

@@ -459,24 +459,30 @@ program
   .option("--html-preview", "Get a static html of the export-view of the template generated with the Liquid Test data (optional)", false)
   .option("--preview-only", "Skip the checking of the results of the Liquid Test in case you only want to generate a preview template (optional)", false)
   .option("--status", "Only return the status of the test runs as PASSED/FAILED (optional)", false)
+  .option("-p, --pattern <pattern>", "Run all tests that match this pattern (optional)", "")
 
   .action((options) => {
     if (!options.handle && !options.accountTemplate) {
       consola.error("You need to specify either a reconciliation handle or an account template");
       process.exit(1);
     }
 
+    if (options.test && options.pattern) {
+      consola.error("You cannot use both --test and --pattern options at the same time");
+      process.exit(1);
+    }
+
     const templateType = options.handle ? "reconciliationText" : "accountTemplate";
     const templateName = options.handle ? options.handle : options.accountTemplate;
 
     if (options.status) {
-      liquidTestRunner.runTestsStatusOnly(options.firm, templateType, templateName, options.test);
+      liquidTestRunner.runTestsStatusOnly(options.firm, templateType, templateName, options.test, options.pattern);
     } else {
       if (options.previewOnly && !options.htmlInput && !options.htmlPreview) {
         consola.info(`When using "--preview-only" you need to specify at least one of the following options: "--html-input", "--html-preview"`);
         process.exit(1);
       }
-      liquidTestRunner.runTestsWithOutput(options.firm, templateType, templateName, options.test, options.previewOnly, options.htmlInput, options.htmlPreview);
+      liquidTestRunner.runTestsWithOutput(options.firm, templateType, templateName, options.test, options.previewOnly, options.htmlInput, options.htmlPreview, options.pattern);
     }
   });
 
@@ -682,20 +688,26 @@ program
   .option("-t, --test <test-name>", `Specify the name of the test to be run (optional). It has to be used together with "--handle"`, "")
   .option("--html", `Get a html file of the template's input-view generated with the Liquid Test information (optional). It has to be used together with "--handle"`, false)
   .option("--yes", "Skip the prompt confirmation (optional)")
+  .option("-p, --pattern <pattern>", `Run all tests that match this pattern (optional). It has to be used together with "--handle" or "--account-template"`, "")
   .action((options) => {
     cliUtils.checkDefaultFirm(options.firm, firmIdDefault);
     cliUtils.checkUniqueOption(["handle", "updateTemplates", "accountTemplate"], options);
 
+    if (options.test && options.pattern) {
+      consola.error("You cannot use both --test and --pattern options at the same time");
+      process.exit(1);
+    }
+
     if (options.updateTemplates && !options.yes) {
       cliUtils.promptConfirmation();
     }
 
     if (options.accountTemplate) {
-      devMode.watchLiquidTest(options.firm, options.accountTemplate, options.test, options.html, "accountTemplate");
+      devMode.watchLiquidTest(options.firm, options.accountTemplate, options.test, options.html, "accountTemplate", options.pattern);
     }
 
     if (options.handle) {
-      devMode.watchLiquidTest(options.firm, options.handle, options.test, options.html, "reconciliationText");
+      devMode.watchLiquidTest(options.firm, options.handle, options.test, options.html, "reconciliationText", options.pattern);
     }
     if (options.updateTemplates) {
       devMode.watchLiquidFiles(options.firm);

lib/cli/devMode.js

@@ -13,8 +13,9 @@ const chokidar = require("chokidar");
  * @param {String} testName - Test name (empty string to run all tests)
  * @param {boolean} renderInput - Open browser and show the HTML from input view
  * @param {String} templateType - Template type (reconciliationText, accountTemplate)
+ * @param {String} pattern - Pattern to match test names (empty string to run all tests)
  */
-async function watchLiquidTest(firmId, handle, testName, renderInput, templateType) {
+async function watchLiquidTest(firmId, handle, testName, renderInput, templateType, pattern = "") {
   if (templateType !== "reconciliationText" && templateType !== "accountTemplate") {
     consola.error(`Template type is missing or invalid`);
     process.exit(1);
@@ -33,15 +34,15 @@ async function watchLiquidTest(firmId, handle, testName, renderInput, templateTy
   // Watch YAML
   chokidar.watch(filePath).on("change", async () => {
     // Run test
-    await liquidTestRunner.runTestsWithOutput(firmId, templateType, handle, testName, false, renderInput);
+    await liquidTestRunner.runTestsWithOutput(firmId, templateType, handle, testName, false, renderInput, false, pattern);
   });
 
   // Watch liquid files
   const liquidFiles = fsUtils.listExistingRelatedLiquidFiles(firmId, handle, templateType);
   for (const filePath of liquidFiles) {
     chokidar.watch(filePath).on("change", async () => {
       // Run test
-      await liquidTestRunner.runTestsWithOutput(firmId, templateType, handle, testName, false, renderInput);
+      await liquidTestRunner.runTestsWithOutput(firmId, templateType, handle, testName, false, renderInput, false, pattern);
     });
   }
 }

lib/liquidTestRunner.js

@@ -31,7 +31,76 @@ function findTestRows(testContent) {
   return indexes;
 }
 
-function buildTestParams(firmId, templateType, handle, testName = "", renderMode) {
+function filterTestsByPattern(testContent, pattern, testIndexes) {
+  const indexes = testIndexes || findTestRows(testContent);
+  const matchingTests = Object.keys(indexes).filter((testName) => testName.includes(pattern));
+
+  if (matchingTests.length === 0) {
+    return { filteredContent: "", matchingTests: [], lineAdjustments: {} };
+  }
+
+  const testRows = testContent.split("\n");
+
+  const orderedTests = Object.entries(indexes)
+    .map(([name, index]) => ({ name, index }))
+    .sort((a, b) => a.index - b.index);
+
+  const matchingSet = new Set(matchingTests);
+  const segments = [];
+
+  orderedTests.forEach((test, idx) => {
+    if (!matchingSet.has(test.name)) {
+      return;
+    }
+
+    let start = test.index;
+
+    while (start > 0) {
+      const previousLine = testRows[start - 1];
+      const trimmedPrevious = previousLine.trim();
+      if (trimmedPrevious === "" || trimmedPrevious.startsWith("#")) {
+        start -= 1;
+      } else {
+        break;
+      }
+    }
+
+    let end = testRows.length;
+    for (let nextIdx = idx + 1; nextIdx < orderedTests.length; nextIdx++) {
+      const nextTest = orderedTests[nextIdx];
+      if (nextTest.index > test.index) {
+        end = nextTest.index;
+        break;
+      }
+    }
+
+    const segment = testRows.slice(start, end).join("\n").trimEnd();
+    segments.push(segment);
+  });
+
+  const filteredContent = segments.join("\n\n").trim();
+  const orderedMatchingTests = orderedTests.filter((test) => matchingSet.has(test.name)).map((test) => test.name);
+
+  const lineAdjustments = {};
+  if (filteredContent) {
+    const filteredIndexes = findTestRows(filteredContent);
+    orderedMatchingTests.forEach((testName) => {
+      const originalIndex = indexes[testName];
+      const filteredIndex = filteredIndexes[testName];
+      if (typeof originalIndex === "number" && typeof filteredIndex === "number") {
+        lineAdjustments[testName] = originalIndex - filteredIndex;
+      }
+    });
+  }
+
+  return {
+    filteredContent,
+    matchingTests: orderedMatchingTests,
+    lineAdjustments,
+  };
+}
+
+function buildTestParams(firmId, templateType, handle, testName = "", renderMode, pattern = "") {
   let relativePath = `./reconciliation_texts/${handle}`;
 
   if (templateType === "accountTemplate") {
@@ -63,6 +132,29 @@ function buildTestParams(firmId, templateType, handle, testName = "", renderMode
     return false;
   }
 
+  const testIndexes = findTestRows(testContent);
+
+  let finalTests = testContent;
+  let lineAdjustments = {};
+
+  if (pattern) {
+    const { filteredContent, matchingTests, lineAdjustments: patternLineAdjustments } = filterTestsByPattern(testContent, pattern, testIndexes);
+
+    if (!matchingTests.length) {
+      consola.error(`No tests found containing "${pattern}" in their name`);
+      process.exit(1);
+    }
+
+    finalTests = filteredContent;
+    lineAdjustments = patternLineAdjustments;
+    consola.info(
+      `Running ${matchingTests.length} test${matchingTests.length === 1 ? "" : "s"} matching pattern "${pattern}":`
+    );
+    matchingTests.forEach((testName) => {
+      consola.log(`  • ${testName}`);
+    });
+  }
+
   let templateContent;
 
   if (templateType === "accountTemplate") {
@@ -88,20 +180,19 @@ function buildTestParams(firmId, templateType, handle, testName = "", renderMode
 
   const testParams = {
     template: templateContent,
-    tests: testContent,
+    tests: finalTests,
     mode: renderMode,
   };
 
   // Include only one test
   if (testName) {
-    const indexes = findTestRows(testContent);
-    if (!Object.keys(indexes).includes(testName)) {
+    if (!Object.keys(testIndexes).includes(testName)) {
       consola.error(`Test ${testName} not found in YAML`);
       process.exit(1);
     }
-    testParams.test_line = indexes[testName] + 1;
+    testParams.test_line = testIndexes[testName] + 1;
   }
-  return testParams;
+  return { testParams, metadata: { lineAdjustments } };
 }
 
 async function fetchResult(firmId, testRunId, templateType) {
@@ -129,12 +220,14 @@ async function fetchResult(firmId, testRunId, templateType) {
   return testRun;
 }
 
-function listErrors(items, type) {
+function listErrors(items, type, lineAdjustment = 0) {
   const itemsKeys = Object.keys(items);
   consola.log(chalk.red(`${itemsKeys.length} ${type} expectation${itemsKeys.length > 1 ? "s" : ""} failed`));
   itemsKeys.forEach((itemName) => {
     const itemDetails = items[itemName];
-    consola.log(`At line number ${itemDetails.line_number}`);
+    if (typeof itemDetails.line_number === "number") {
+      consola.log(`At line number ${itemDetails.line_number + lineAdjustment}`);
+    }
     let gotDataType = typeof itemDetails.got;
     let expectedDataType = typeof itemDetails.expected;
     let displayedGot = itemDetails.got;
@@ -215,7 +308,7 @@ function checkTestErrorsPresent(testName, testsFeedback) {
   return errorsPresent;
 }
 
-function processTestRunResponse(testRun, previewOnly) {
+function processTestRunResponse(testRun, previewOnly, lineAdjustments = {}) {
   // Possible status: started, completed, test_error, internal_error
   let errorsPresent;
   switch (testRun.status) {
@@ -249,6 +342,7 @@ function processTestRunResponse(testRun, previewOnly) {
           consola.log(chalk.bold(testName));
 
           const testElements = testRun.tests[testName];
+          const lineAdjustment = lineAdjustments[testName] || 0;
 
           // Display success messages of test
           if (testElements.reconciled === null) {
@@ -268,19 +362,21 @@ function processTestRunResponse(testRun, previewOnly) {
           // Reconciled
           if (testElements.reconciled !== null) {
             consola.log(chalk.red("Reconciliation expectation failed"));
-            consola.log(`At line number ${testElements.reconciled.line_number}`);
+            if (typeof testElements.reconciled.line_number === "number") {
+              consola.log(`At line number ${testElements.reconciled.line_number + lineAdjustment}`);
+            }
             consola.log(`got ${chalk.blue.bold(testElements.reconciled.got)} but expected ${chalk.blue.bold(testElements.reconciled.expected)}`);
             consola.log("");
           }
 
           // Results
           if (Object.keys(testElements.results).length > 0) {
-            listErrors(testElements.results, "result");
+            listErrors(testElements.results, "result", lineAdjustment);
           }
 
           // Rollforwards
           if (Object.keys(testElements.rollforwards).length > 0) {
-            listErrors(testElements.rollforwards, "rollforward");
+            listErrors(testElements.rollforwards, "rollforward", lineAdjustment);
           }
         });
         break;
@@ -365,16 +461,18 @@ async function handleHTMLfiles(testName = "", testRun, renderMode) {
 }
 
 // Used by VSCode Extension
-async function runTests(firmId, templateType, handle, testName = "", previewOnly = false, renderMode = "none") {
+async function runTests(firmId, templateType, handle, testName = "", previewOnly = false, renderMode = "none", pattern = "") {
   try {
     if (templateType !== "reconciliationText" && templateType !== "accountTemplate") {
       consola.error(`Template type is missing or invalid`);
       process.exit(1);
     }
 
-    const testParams = buildTestParams(firmId, templateType, handle, testName, renderMode);
+    const buildResult = buildTestParams(firmId, templateType, handle, testName, renderMode, pattern);
+
+    if (!buildResult) return;
 
-    if (!testParams) return;
+    const { testParams, metadata } = buildResult;
 
     let testRun = null;
     let previewRun = null;
@@ -392,24 +490,33 @@ async function runTests(firmId, templateType, handle, testName = "", previewOnly
       testRun = await fetchResult(firmId, testRunId, templateType);
     }
 
-    return { testRun, previewRun };
+    return { testRun, previewRun, metadata };
   } catch (error) {
     errorUtils.errorHandler(error);
   }
 }
 
-async function runTestsWithOutput(firmId, templateType, handle, testName = "", previewOnly = false, htmlInput = false, htmlPreview = false) {
+async function runTestsWithOutput(
+  firmId,
+  templateType,
+  handle,
+  testName = "",
+  previewOnly = false,
+  htmlInput = false,
+  htmlPreview = false,
+  pattern = ""
+) {
   try {
     if (templateType !== "reconciliationText" && templateType !== "accountTemplate") {
       consola.error(`Template type is missing or invalid`);
       process.exit(1);
     }
 
     const renderMode = runTestUtils.checkRenderMode(htmlInput, htmlPreview);
-    const testsRun = await runTests(firmId, templateType, handle, testName, previewOnly, renderMode);
+    const testsRun = await runTests(firmId, templateType, handle, testName, previewOnly, renderMode, pattern);
     if (!testsRun) return;
 
-    processTestRunResponse(testsRun?.testRun || testsRun?.previewRun, previewOnly);
+    processTestRunResponse(testsRun?.testRun || testsRun?.previewRun, previewOnly, testsRun?.metadata?.lineAdjustments || {});
 
     if (testsRun.previewRun && testsRun.previewRun.status !== "test_error" && renderMode !== "none") {
       handleHTMLfiles(testName, testsRun.previewRun, renderMode);
@@ -421,14 +528,14 @@ async function runTestsWithOutput(firmId, templateType, handle, testName = "", p
 
 // RETURN (AND LOG) ONLY PASSED OR FAILED
 // CAN BE USED BY GITHUB ACTIONS
-async function runTestsStatusOnly(firmId, templateType, handle, testName = "") {
+async function runTestsStatusOnly(firmId, templateType, handle, testName = "", pattern = "") {
   if (templateType !== "reconciliationText" && templateType !== "accountTemplate") {
     consola.error(`Template type is missing or invalid`);
     process.exit(1);
   }
 
   let status = "FAILED";
-  const testResult = await runTests(firmId, templateType, handle, testName, false, "none");
+  const testResult = await runTests(firmId, templateType, handle, testName, false, "none", pattern);
 
   if (!testResult) {
     status = "PASSED";