fix(firestore-bigquery-export): use correct Firestore database ID in document_name when exporting to BigQuery

firestore-bigquery-export/scripts/import/__tests__/getRowsFromDocs.test.ts

@@ -1,4 +1,5 @@
 import { ChangeType } from "@firebaseextensions/firestore-bigquery-change-tracker";
+import { FIRESTORE_DEFAULT_DATABASE } from "../src/config";
 import { getRowsFromDocs } from "../src/helper";
 
 describe("getRowsFromDocs", () => {
@@ -20,6 +21,7 @@ describe("getRowsFromDocs", () => {
       projectId: "test-project",
       sourceCollectionPath: "users",
       queryCollectionGroup: false,
+      firestoreInstanceId: FIRESTORE_DEFAULT_DATABASE,
     } as any;
 
     const beforeTimestamp = new Date().toISOString();
@@ -62,6 +64,7 @@ describe("getRowsFromDocs", () => {
       projectId: "test-project",
       sourceCollectionPath: "regions/{regionId}/countries/{countryId}/cities",
       queryCollectionGroup: false,
+      firestoreInstanceId: FIRESTORE_DEFAULT_DATABASE,
     } as any;
 
     const result = getRowsFromDocs(mockDocs, mockConfig);
@@ -99,6 +102,7 @@ describe("getRowsFromDocs", () => {
       projectId: "test-project",
       sourceCollectionPath: "my/{testId}/collection",
       queryCollectionGroup: false,
+      firestoreInstanceId: FIRESTORE_DEFAULT_DATABASE,
     } as any;
 
     const result = getRowsFromDocs(mockDocs, mockConfig);
@@ -153,6 +157,7 @@ describe("getRowsFromDocs", () => {
       projectId: "test-project",
       sourceCollectionPath: "organizations/{orgId}/users", // Template path
       queryCollectionGroup: true,
+      firestoreInstanceId: FIRESTORE_DEFAULT_DATABASE,
     } as any;
 
     const result = getRowsFromDocs(mockDocs, mockConfig);
@@ -217,6 +222,7 @@ describe("getRowsFromDocs", () => {
       projectId: "test-project",
       sourceCollectionPath: "my/{coolId}/collection",
       queryCollectionGroup: true,
+      firestoreInstanceId: FIRESTORE_DEFAULT_DATABASE,
     } as any;
 
     const myResult = getRowsFromDocs(mockDocs, myConfig);
@@ -233,6 +239,7 @@ describe("getRowsFromDocs", () => {
       projectId: "test-project",
       sourceCollectionPath: "my_other/{coolId}/collection",
       queryCollectionGroup: true,
+      firestoreInstanceId: FIRESTORE_DEFAULT_DATABASE,
     } as any;
 
     const myOtherResult = getRowsFromDocs(mockDocs, myOtherConfig);
@@ -291,6 +298,7 @@ describe("getRowsFromDocs", () => {
       projectId: "test-project",
       sourceCollectionPath: "my_other/{coolId}/collection",
       queryCollectionGroup: true,
+      firestoreInstanceId: FIRESTORE_DEFAULT_DATABASE,
     } as any;
 
     const myOtherResult = getRowsFromDocs(mockDocs, myOtherConfig);
@@ -305,6 +313,7 @@ describe("getRowsFromDocs", () => {
       projectId: "test-project",
       sourceCollectionPath: "my/{coolId}/collection",
       queryCollectionGroup: true,
+      firestoreInstanceId: FIRESTORE_DEFAULT_DATABASE,
     } as any;
 
     const myResult = getRowsFromDocs(mockDocs, myConfig);
@@ -328,4 +337,67 @@ describe("getRowsFromDocs", () => {
       expect(row.documentName).toContain("/my/");
     });
   });
+
+  it("uses non-default Firestore database ID in document_name when specified", () => {
+    const mockDocs = [
+      {
+        id: "doc1",
+        ref: {
+          path: "posts/doc1",
+        },
+        data: () => ({
+          title: "Test Post",
+        }),
+      },
+    ] as any[];
+
+    // Test with non-default database ID
+    const mockConfig = {
+      projectId: "test-project",
+      sourceCollectionPath: "posts",
+      queryCollectionGroup: false,
+      firestoreInstanceId: "alpha",
+    } as any;
+
+    const result = getRowsFromDocs(mockDocs, mockConfig);
+
+    expect(result).toHaveLength(1);
+    expect(result[0].documentName).toBe(
+      "projects/test-project/databases/alpha/documents/posts/doc1"
+    );
+    expect(result[0].documentId).toBe("doc1");
+  });
+
+  it("uses non-default Firestore database ID in collection group queries", () => {
+    const mockDocs = [
+      {
+        id: "doc1",
+        ref: {
+          path: "organizations/org1/posts/doc1",
+        },
+        data: () => ({
+          title: "Test Post",
+        }),
+      },
+    ] as any[];
+
+    // Test with non-default database ID and collection group query
+    const mockConfig = {
+      projectId: "test-project",
+      sourceCollectionPath: "organizations/{orgId}/posts",
+      queryCollectionGroup: true,
+      firestoreInstanceId: "alpha",
+    } as any;
+
+    const result = getRowsFromDocs(mockDocs, mockConfig);
+
+    expect(result).toHaveLength(1);
+    expect(result[0].documentName).toBe(
+      "projects/test-project/databases/alpha/documents/organizations/org1/posts/doc1"
+    );
+    expect(result[0].documentId).toBe("doc1");
+    expect(result[0].pathParams).toEqual({
+      orgId: "org1",
+    });
+  });
 });

firestore-bigquery-export/scripts/import/__tests__/multiThreadCollectionGroupBug.test.ts

@@ -2,6 +2,7 @@ import { runMultiThread } from "../src/run-multi-thread";
 import * as admin from "firebase-admin";
 import { CliConfig } from "../src/types";
 import * as workerpool from "workerpool";
+import { FIRESTORE_DEFAULT_DATABASE } from "../src/config";
 
 // Mock Firebase Admin to avoid credential issues
 jest.mock("firebase-admin", () => {
@@ -124,6 +125,7 @@ describe("Multi-threaded Collection Group Bug Reproduction", () => {
       useEmulator: false,
       rawChangeLogName: "old_data_raw_changelog",
       cursorPositionFile: "/tmp/test_cursor_position",
+      firestoreInstanceId: FIRESTORE_DEFAULT_DATABASE,
     };
   });
 

firestore-bigquery-export/scripts/import/__tests__/runMultiThread.test.ts

@@ -2,6 +2,7 @@ import { runMultiThread } from "../src/run-multi-thread";
 import * as admin from "firebase-admin";
 import { CliConfig } from "../src/types";
 import * as workerpool from "workerpool";
+import { FIRESTORE_DEFAULT_DATABASE } from "../src/config";
 
 // Initialize Firebase Admin (Ensure credentials are set)
 if (admin.apps.length === 0) {
@@ -86,6 +87,7 @@ describe("runMultiThread Partitioning with Firestore", () => {
       useEmulator: false,
       rawChangeLogName: "testTable_raw_changelog",
       cursorPositionFile: "/tmp/test_cursor_position",
+      firestoreInstanceId: FIRESTORE_DEFAULT_DATABASE,
     };
   });
 

firestore-bigquery-export/scripts/import/__tests__/runMultiThreadMock.test.ts

@@ -2,6 +2,7 @@ import { runMultiThread } from "../src/run-multi-thread";
 import * as workerpool from "workerpool";
 import { CliConfig } from "../src/types";
 import { cpus } from "os";
+import { FIRESTORE_DEFAULT_DATABASE } from "../src/config";
 
 // Mock helper functions
 jest.mock("../src/helper", () => ({
@@ -97,6 +98,7 @@ describe("runMultiThread", () => {
       useEmulator: false,
       rawChangeLogName: "testTable_raw_changelog",
       cursorPositionFile: "/tmp/cursor",
+      firestoreInstanceId: FIRESTORE_DEFAULT_DATABASE,
     };
   });
 

firestore-bigquery-export/scripts/import/src/config.ts

@@ -4,6 +4,8 @@ import * as inquirer from "inquirer";
 
 import { CliConfig, CliConfigError } from "./types";
 
+export const FIRESTORE_DEFAULT_DATABASE = "(default)";
+
 const BIGQUERY_VALID_CHARACTERS = /^[a-zA-Z0-9_]+$/;
 // regex of ^[^/]+(/[^/]+/[^/]+)*$
 export const FIRESTORE_VALID_CHARACTERS = new RegExp("^[^/]+(/[^/]+/[^/]+)*$");

firestore-bigquery-export/scripts/import/src/helper.ts

@@ -13,8 +13,6 @@ import * as util from "util";
 
 import { resolveWildcardIds } from "./config";
 
-const FIRESTORE_DEFAULT_DATABASE = "(default)";
-
 // TODO: do we need this logic?
 export const initializeDataSink = async (
   dataSink: FirestoreBigQueryEventHistoryTracker,
@@ -78,7 +76,7 @@ export function getRowsFromDocs(
           rows.push({
             timestamp: new Date().toISOString(), // epoch
             operation: ChangeType.IMPORT,
-            documentName: `projects/${config.projectId}/databases/${FIRESTORE_DEFAULT_DATABASE}/documents/${path}`,
+            documentName: `projects/${config.projectId}/databases/${config.firestoreInstanceId}/documents/${path}`,
             documentId: doc.id,
             // TODO: fix this type
             // @ts-expect-error
@@ -96,7 +94,7 @@ export function getRowsFromDocs(
       return {
         timestamp: new Date().toISOString(), // epoch
         operation: ChangeType.IMPORT,
-        documentName: `projects/${config.projectId}/databases/${FIRESTORE_DEFAULT_DATABASE}/documents/${snapshot.ref.path}`,
+        documentName: `projects/${config.projectId}/databases/${config.firestoreInstanceId}/documents/${snapshot.ref.path}`,
         documentId: snapshot.id,
         pathParams: resolveWildcardIds(
           config.sourceCollectionPath,

storage-resize-images/CHANGELOG.md

@@ -1,3 +1,8 @@
+## Version 0.3.1
+
+fix - add missing recordStartEvent call (#2546)
+feat - add new onStartResize event
+
 ## Version 0.3.0
 
 fix! - remove backfill, due to architectural flaws.

storage-resize-images/extension.yaml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 name: storage-resize-images
-version: 0.3.0
+version: 0.3.1
 specVersion: v1beta
 
 displayName: Resize Images
@@ -426,6 +426,12 @@ events:
     description:
       Occurs when the function is settled. Provides no customized data other
       than the context.
+
+  - type: firebase.extensions.storage-resize-images.v1.onStartResize
+    description:
+      Occurs when an image resize operation completes successfully. This event
+      is only triggered when shouldResize returns true and the resize operation
+      succeeds.
 # Lifecycle events disabled - backfill feature commented out
 # lifecycleEvents:
 #   onInstall:

storage-resize-images/functions/src/events.ts

@@ -60,3 +60,19 @@ export const recordCompletionEvent = async (data: string | object) => {
     data,
   });
 };
+
+export const recordStartResizeEvent = async ({
+  subject,
+  data,
+}: {
+  subject: string;
+  data: string | object;
+}) => {
+  if (!eventChannel) return;
+
+  return eventChannel.publish({
+    type: getEventType("onStartResize"),
+    subject,
+    data,
+  });
+};

storage-resize-images/functions/src/index.ts

@@ -59,6 +59,11 @@ const generateResizedImageHandler = async (
     return;
   }
 
+  await events.recordStartResizeEvent({
+    subject: object.name,
+    data: { input: object },
+  });
+
   const bucket = admin.storage().bucket(object.bucket);
   const filePath = object.name; // File path in the bucket.
   const parsedPath = path.parse(filePath);
@@ -149,6 +154,7 @@ const generateResizedImageHandler = async (
 export const generateResizedImage = functions.storage
   .object()
   .onFinalize(async (object, context) => {
+    await events.recordStartEvent(object);
     await generateResizedImageHandler(object);
     await events.recordCompletionEvent({ context });
   });