[codex] Add materialized column support

e2e-live/build.test.ts

@@ -59,6 +59,13 @@ async function waitForPipeData(
   return [];
 }
 
+function expectBuildSuccess(buildResult: Awaited<ReturnType<typeof runBuild>>): void {
+  if (!buildResult.success) {
+    throw new Error(buildResult.error ?? "Build failed without an error message.");
+  }
+  expect(buildResult.success).toBe(true);
+}
+
 describeLive("E2E Live: build", () => {
   const config = liveConfig as LiveE2EConfig;
 
@@ -68,7 +75,8 @@ describeLive("E2E Live: build", () => {
   let workspaceId = "";
   let workspaceName = "";
   let workspaceToken = "";
-  let tinybirdBranchName = "";
+  const tinybirdBranchNames: string[] = [];
+  let testBranchIndex = 0;
 
   beforeAll(async () => {
     ensureDistBuild();
@@ -82,7 +90,10 @@ describeLive("E2E Live: build", () => {
   beforeEach(() => {
     tempDir = createTempProjectDir();
     originalEnv = { ...process.env };
-    process.env.GITHUB_REF_NAME = `live-build/${workspaceName}`;
+    testBranchIndex += 1;
+    const testBranchName = `live-build/${workspaceName}/${testBranchIndex}`;
+    process.env.GITHUB_HEAD_REF = testBranchName;
+    process.env.GITHUB_REF_NAME = testBranchName;
     process.env.TINYBIRD_TOKEN = workspaceToken;
   });
 
@@ -92,7 +103,7 @@ describeLive("E2E Live: build", () => {
   });
 
   afterAll(async () => {
-    if (tinybirdBranchName) {
+    for (const tinybirdBranchName of tinybirdBranchNames) {
       try {
         await deleteBranch(
           { baseUrl: config.baseUrl, token: workspaceToken },
@@ -122,11 +133,12 @@ describeLive("E2E Live: build", () => {
     setConfigBaseUrl(tempDir, config.baseUrl);
 
     const buildResult = await runBuild({ cwd: tempDir });
-    expect(buildResult.success).toBe(true);
+    expectBuildSuccess(buildResult);
     expect(buildResult.deploy?.success).toBe(true);
     expect(buildResult.branchInfo?.tinybirdBranch).toBeTruthy();
 
-    tinybirdBranchName = buildResult.branchInfo?.tinybirdBranch ?? "";
+    const tinybirdBranchName = buildResult.branchInfo?.tinybirdBranch ?? "";
+    tinybirdBranchNames.push(tinybirdBranchName);
 
     const branch = await getBranch(
       { baseUrl: config.baseUrl, token: workspaceToken },

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tinybirdco/sdk",
-  "version": "0.0.69",
+  "version": "0.0.70",
   "description": "TypeScript SDK for Tinybird Forward - define datasources and pipes as TypeScript",
   "type": "module",
   "main": "./dist/index.js",

src/generator/datasource.test.ts

@@ -188,6 +188,75 @@ describe('Datasource Generator', () => {
       expect(result.content).toContain("id UUID `json:$.id` DEFAULT generateUUIDv4()");
     });
 
+    it("includes materialized expressions without autogenerated JSON paths", () => {
+      const ds = defineDatasource("test_ds", {
+        schema: {
+          body: t.string(),
+          level: t
+            .string()
+            .lowCardinality()
+            .materialized("JSONExtractString(body, 'level')"),
+        },
+      });
+
+      const result = generateDatasource(ds);
+      expect(result.content).toContain("body String `json:$.body`");
+      expect(result.content).toContain(
+        "level LowCardinality(String) MATERIALIZED JSONExtractString(body, 'level')"
+      );
+      expect(result.content).not.toContain("`json:$.level`");
+    });
+
+    it("generates the full datasource file for a schema with a materialized column", () => {
+      const ds = defineDatasource("raw_logs", {
+        description: "Raw logs with extracted severity",
+        schema: {
+          timestamp: t.dateTime(),
+          body: t.string(),
+          service_name: t.string().jsonPath("$.resource.service.name"),
+          level: t
+            .string()
+            .lowCardinality()
+            .materialized("JSONExtractString(body, 'level')"),
+        },
+        engine: engine.mergeTree({
+          sortingKey: ["service_name", "level", "timestamp"],
+          partitionKey: "toYYYYMM(timestamp)",
+        }),
+      });
+
+      const result = generateDatasource(ds);
+      expect(result.content).toBe(`DESCRIPTION >
+    Raw logs with extracted severity
+
+SCHEMA >
+    timestamp DateTime \`json:$.timestamp\`,
+    body String \`json:$.body\`,
+    service_name String \`json:$.resource.service.name\`,
+    level LowCardinality(String) MATERIALIZED JSONExtractString(body, 'level')
+
+ENGINE "MergeTree"
+ENGINE_PARTITION_KEY "toYYYYMM(timestamp)"
+ENGINE_SORTING_KEY "service_name, level, timestamp"`);
+    });
+
+    it("includes explicit JSON paths on materialized columns", () => {
+      const ds = defineDatasource("test_ds", {
+        schema: {
+          body: t.string(),
+          level: t
+            .string()
+            .materialized("JSONExtractString(body, 'level')")
+            .jsonPath("$.ignored_level"),
+        },
+      });
+
+      const result = generateDatasource(ds);
+      expect(result.content).toContain(
+        "level String `json:$.ignored_level` MATERIALIZED JSONExtractString(body, 'level')"
+      );
+    });
+
     it('formats null default values', () => {
       const ds = defineDatasource('test_ds', {
         schema: {

src/generator/datasource.ts

@@ -99,16 +99,19 @@ function generateColumnLine(
   const modifiers = getModifiersFromValidator(validator);
 
   const parts: string[] = [`    ${columnName} ${tinybirdType}`];
+  const hasMaterializedExpression = typeof modifiers.materializedExpression === "string";
 
   // Add JSON path for Events API ingestion support if enabled
   // Use explicit jsonPath if defined, otherwise default to $.columnName
-  if (includeJsonPaths) {
+  if (includeJsonPaths && (!hasMaterializedExpression || jsonPath !== undefined)) {
     const effectiveJsonPath = jsonPath ?? `$.${columnName}`;
     parts.push(`\`json:${effectiveJsonPath}\``);
   }
 
-  // Add default value/expression if defined
-  if (modifiers.hasDefault) {
+  // Add materialized expression or default value/expression if defined
+  if (hasMaterializedExpression) {
+    parts.push(`MATERIALIZED ${modifiers.materializedExpression}`);
+  } else if (modifiers.hasDefault) {
     if (typeof modifiers.defaultExpression === "string") {
       parts.push(`DEFAULT ${modifiers.defaultExpression}`);
     } else if (modifiers.defaultValue !== undefined) {

src/schema/types.test.ts

@@ -137,6 +137,32 @@ describe("Type Validators (t.*)", () => {
     });
   });
 
+  describe("Materialized expression modifier", () => {
+    it("stores materialized SQL expression in modifiers", () => {
+      const type = t.string().materialized("JSONExtractString(body, 'level')");
+      expect(type._modifiers.materializedExpression).toBe("JSONExtractString(body, 'level')");
+    });
+
+    it("trims materialized SQL expression", () => {
+      const type = t.string().materialized("  lower(service_name)  ");
+      expect(type._modifiers.materializedExpression).toBe("lower(service_name)");
+    });
+
+    it("throws on empty materialized SQL expression", () => {
+      expect(() => t.string().materialized("   ")).toThrow(
+        "Materialized expression cannot be empty."
+      );
+    });
+
+    it("clears default modifiers when materialized is chained after a default", () => {
+      const type = t.string().default("unknown").materialized("lower(name)");
+      expect(type._modifiers.materializedExpression).toBe("lower(name)");
+      expect(type._modifiers.hasDefault).toBeUndefined();
+      expect(type._modifiers.defaultValue).toBeUndefined();
+      expect(type._modifiers.defaultExpression).toBeUndefined();
+    });
+  });
+
   describe("Codec modifier", () => {
     it("sets codec in modifiers", () => {
       const type = t.string().codec("LZ4");

src/schema/types.ts

@@ -52,6 +52,14 @@ export interface TypeValidator<
     TTinybirdType,
     TModifiers & { hasDefault: true; defaultExpression: string }
   >;
+  /** Set a materialized SQL expression for the column */
+  materialized(
+    expression: string,
+  ): TypeValidator<
+    TType,
+    TTinybirdType,
+    TModifiers & { materializedExpression: string }
+  >;
   /** Set a codec for compression */
   codec(
     codec: string,
@@ -68,6 +76,7 @@ export interface TypeModifiers {
   hasDefault?: boolean;
   defaultValue?: unknown;
   defaultExpression?: string;
+  materializedExpression?: string;
   codec?: string;
   jsonPath?: string;
 }
@@ -154,6 +163,7 @@ function createValidator<TType, TTinybirdType extends string>(
         hasDefault: true,
         defaultValue: value,
         defaultExpression: undefined,
+        materializedExpression: undefined,
       }) as TypeValidator<
         TType,
         TTinybirdType,
@@ -171,13 +181,32 @@ function createValidator<TType, TTinybirdType extends string>(
         hasDefault: true,
         defaultValue: undefined,
         defaultExpression: trimmed,
+        materializedExpression: undefined,
       }) as TypeValidator<
         TType,
         TTinybirdType,
         TypeModifiers & { hasDefault: true; defaultExpression: string }
       >;
     },
 
+    materialized(expression: string) {
+      const trimmed = expression.trim();
+      if (!trimmed) {
+        throw new Error("Materialized expression cannot be empty.");
+      }
+      return createValidator<TType, TTinybirdType>(tinybirdType, {
+        ...modifiers,
+        hasDefault: undefined,
+        defaultValue: undefined,
+        defaultExpression: undefined,
+        materializedExpression: trimmed,
+      }) as TypeValidator<
+        TType,
+        TTinybirdType,
+        TypeModifiers & { materializedExpression: string }
+      >;
+    },
+
     codec(codec: string) {
       return createValidator<TType, TTinybirdType>(tinybirdType, {
         ...modifiers,