Add multimodal support for Gemini tool calls (Blob & Arrays)

README.md

@@ -135,6 +135,10 @@ For more information :
 - [https://ai.google.dev/gemini-api/docs/function-calling](https://ai.google.dev/gemini-api/docs/function-calling)
 - [https://platform.openai.com/docs/guides/gpt/function-calling](https://platform.openai.com/docs/guides/gpt/function-calling)
 
+⚠️ **Limitation when returning files from function calls:** if your function returns one or more files (Blob values),
+the model can read the file content but cannot reliably read or repeat the original file names. If you need deterministic
+file naming in your app, keep the names in your own script-side metadata rather than asking the model to infer them.
+
 ### Enable web browsing (optional)
 
 If you want to allow the chat to perform web searches and fetch web pages, enable browsing on your chat instance:

src/code.gs

@@ -193,12 +193,13 @@ const GenAIApp = (function () {
         // Gemini
         contents.push({
           role: 'user',
-          parts: [{
-            inline_data: {
-              mime_type: fileInfo.mimeType,
-              data: blobToBase64
-            }
-          }]
+          parts: [
+            createGeminiInlinePart(
+              fileInfo.mimeType,
+              blobToBase64,
+              fileInfo.fileName
+            )
+          ]
         });
         return this;
       };
@@ -1479,32 +1480,56 @@ const GenAIApp = (function () {
       }
 
       let functionResponse = _callFunction(functionName, functionArgs, argsOrder);
+
       if (verbose) {
         console.log(`[GenAIApp] - function ${functionName}() called by Gemini.`);
       }
-      if (typeof functionResponse != "string") {
-        if (typeof functionResponse == "object") {
-          functionResponse = JSON.stringify(functionResponse);
-        }
-        else {
-          functionResponse = String(functionResponse);
+
+      let jsonResponse = null;
+      let multimodalParts = [];
+      if (typeof functionResponse !== "string") {
+        if (typeof functionResponse === "object") {
+
+          // handle array of blobs (or mixed arrays)
+          if (Array.isArray(functionResponse)) {
+            if (functionResponse.length > 0 && functionResponse.every(isBlobLike)) {
+              multimodalParts = functionResponse.map(blobToGeminiInlinePart);
+            } else { // non-blob arrays
+              jsonResponse = functionResponse;
+            }
+          } else {// single-object handling
+            // check if response is a blob
+            if (isBlobLike(functionResponse)) {
+              multimodalParts = [blobToGeminiInlinePart(functionResponse)];
+            } else {
+              jsonResponse = functionResponse;
+            }
+          }
+        } else {
+          jsonResponse = String(functionResponse);
         }
+      } else {
+        jsonResponse = functionResponse;
       }
 
-      // Append result of the function execution to contents
-      // https://ai.google.dev/gemini-api/docs/function-calling?example=meeting#step-4
+      // Build the tool response message following Gemini's official function calling spec.
+      // The result must be sent as a role "tool" message containing a functionResponse object.
+      // - JSON payload goes inside `response`
+      // - Binary/multimodal data (PDF, image, etc.) goes inside `parts` using inlineData
+      // https://ai.google.dev/gemini-api/docs/function-calling?example=weather#multimodal
       responseParts.push({
         functionResponse: {
           name: functionName,
-          response: { functionResponse }
+          response: jsonResponse ?? {},
+          ...(multimodalParts.length > 0 && { parts: multimodalParts })
         }
       });
     }
 
     // Append all function results in a single turn
     if (responseParts.length > 0) {
       contents.push({
-        role: 'user',
+        role: "tool",
         parts: responseParts
       });
     }
@@ -1594,6 +1619,7 @@ const GenAIApp = (function () {
             functionResponse = String(functionResponse);
           }
         }
+
         if (verbose) {
           console.log(`[GenAIApp] - function ${functionName}() called by OpenAI.`);
         }
@@ -1743,13 +1769,35 @@ const GenAIApp = (function () {
   });
 
   // OpenAI-only helper for Blob-like values returned by function calling.
+  // Important: even if we send `filename`, current models cannot reliably read/quote
+  // the file names from these tool outputs; they mostly use file content.
   const blobToResponseInputFileContent = (blob) =>
     createOpenAIInputFileContent(
       blob.getContentType(),
       Utilities.base64Encode(blob.getBytes()),
       blob.getName()
     );
 
+  // Gemini-only helper: creates an inlineData part object.
+  const createGeminiInlinePart = (mimeType, base64Data, filename) => ({
+    inlineData: {
+      mimeType: mimeType,
+      displayName: filename,
+      data: base64Data
+    }
+  });
+
+  // Gemini-only helper for Blob-like values returned by function calling.
+  // Same limitation as OpenAI tool outputs: the model can use the file bytes, but
+  // should not be expected to read back the returned file names accurately.
+  const blobToGeminiInlinePart = (blob) =>
+    createGeminiInlinePart(
+      blob.getContentType(),
+      Utilities.base64Encode(blob.getBytes()),
+      blob.getName()
+    );
+
+
   /**
    * Uploads a file to OpenAI and returns the file ID.
    * 

src/testFunctions.gs

@@ -1,13 +1,15 @@
-const GPT_MODEL = "gpt-4.1";
+const GPT_MODEL = "gpt-5.2";
 const REASONING_MODEL = "o4-mini";
-const GEMINI_MODEL = "gemini-2.5-pro";
+const GEMINI_MODEL = "gemini-3-pro-preview";
 
 // Run all tests
 function testAll() {
   testSimpleChatInstance();
   testFunctionCalling();
   testFunctionCallingEndWithResult();
   testFunctionCallingOnlyReturnArguments();
+  testFunctionCallingReturnBlob();
+  testFunctionCallingReturnBlobArray();
   testBrowsing();
   testKnowledgeLink();
   testVision();
@@ -87,6 +89,33 @@ function testFunctionCallingOnlyReturnArguments() {
   });
 }
 
+function testFunctionCallingReturnBlob() {
+  const receiptGenerator = GenAIApp.newFunction()
+    .setName("generateReceipt")
+    .setDescription("Generate a receipt as a PDF file")
+    .addParameter("customerName", "string", "The customer full name")
+    .addParameter("amount", "number", "The billed amount");
+
+  runTestAcrossModels("Function return blob", chat => {
+    chat
+      .addMessage("Create a receipt for Jane Doe with an amount of 42.50 using generateReceipt")
+      .addFunction(receiptGenerator);
+  });
+}
+
+function testFunctionCallingReturnBlobArray() {
+  const packGenerator = GenAIApp.newFunction()
+    .setName("generateWelcomePack")
+    .setDescription("Generate a welcome package as several files")
+    .addParameter("employeeName", "string", "The employee full name");
+
+  runTestAcrossModels("Function return blob array", chat => {
+    chat
+      .addMessage("Generate a welcome pack for Alex Martin using generateWelcomePack")
+      .addFunction(packGenerator);
+  });
+}
+
 function testBrowsing() {
   runTestAcrossModels("Browsing", chat => {
     chat
@@ -128,3 +157,52 @@ function getWeather(cityName) {
   return `The weather in ${cityName} is 19°C today.`;
 }
 
+function _escapeHtml(value) {
+  return String(value)
+    .replace(/&/g, "&")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;");
+}
+
+function _createSimplePdf(fileName, title, lines) {
+  const safeTitle = _escapeHtml(title);
+  const lineHtml = lines
+    .map(line => `<p style="margin: 0 0 6px;">${_escapeHtml(line)}</p>`)
+    .join("");
+
+  const html = `
+    <html>
+      <body style="font-family: Arial, sans-serif; padding: 16px; font-size: 12px;">
+        <h3 style="margin: 0 0 10px;">${safeTitle}</h3>
+        ${lineHtml}
+      </body>
+    </html>
+  `;
+
+  return HtmlService
+    .createHtmlOutput(html)
+    .getBlob()
+    .getAs("application/pdf")
+    .setName(fileName);
+}
+
+function generateReceipt(customerName, amount) {
+  return _createSimplePdf("receipt.pdf", "Receipt", [
+    `Customer: ${customerName}`,
+    `Amount: €${amount}`
+  ]);
+}
+
+function generateWelcomePack(employeeName) {
+  return [
+    _createSimplePdf("welcome.pdf", "Welcome", [
+      `Employee: ${employeeName}`,
+      "We are happy to have you on board."
+    ]),
+    _createSimplePdf("onboarding.pdf", "Onboarding", [
+      `Employee: ${employeeName}`,
+      "Step 1: Meet your team.",
+      "Step 2: Read the onboarding guide."
+    ])
+  ];
+}