fix(storage): resolve transport and retry issues

handwritten/storage/src/file.ts

@@ -1405,7 +1405,12 @@ class File extends ServiceObject<File, FileMetadata> {
     }
 
     if (newFile.encryptionKey !== undefined) {
-      this.setEncryptionKey(newFile.encryptionKey!);
+      headers.set('x-goog-encryption-algorithm', 'AES256');
+      headers.set('x-goog-encryption-key', newFile.encryptionKeyBase64 || '');
+      headers.set(
+        'x-goog-encryption-key-sha256',
+        newFile.encryptionKeyHash || '',
+      );
     } else if (options.destinationKmsKeyName !== undefined) {
       query.destinationKmsKeyName = options.destinationKmsKeyName;
       delete options.destinationKmsKeyName;
@@ -1639,6 +1644,8 @@ class File extends ServiceObject<File, FileMetadata> {
       }
 
       const headers = response.headers;
+      const isStoredCompressed =
+        headers.get('x-goog-stored-content-encoding') === 'gzip';
       const isCompressed = headers.get('content-encoding') === 'gzip';
       const hashes: {crc32c?: string; md5?: string} = {};
 
@@ -1652,7 +1659,7 @@ class File extends ServiceObject<File, FileMetadata> {
 
       const transformStreams: Transform[] = [];
 
-      if (shouldRunValidation) {
+      if (shouldRunValidation && !isStoredCompressed) {
         // The x-goog-hash header should be set with a crc32c and md5 hash.
         // ex: headers.set('x-goog-hash', 'crc32c=xxxx,md5=xxxx')
         if (typeof headers.get('x-goog-hash') === 'string') {
@@ -1721,6 +1728,7 @@ class File extends ServiceObject<File, FileMetadata> {
       const headers = {
         'Accept-Encoding': 'gzip',
         'Cache-Control': 'no-store',
+        ...(this.encryptionKeyHeaders || {}),
       } as Headers;
 
       if (rangeRequest) {
@@ -1735,7 +1743,9 @@ class File extends ServiceObject<File, FileMetadata> {
         headers,
         queryParameters: query as unknown as StorageQueryParameters,
         responseType: 'stream',
-      };
+        decompress: options.decompress,
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      } as any;
 
       if (options[GCCL_GCS_CMD_KEY]) {
         reqOpts[GCCL_GCS_CMD_KEY] = options[GCCL_GCS_CMD_KEY];
@@ -2426,6 +2436,18 @@ class File extends ServiceObject<File, FileMetadata> {
     }
   }
 
+  get encryptionKeyHeaders(): Record<string, string> | undefined {
+    if (!this.encryptionKey) {
+      return undefined;
+    }
+
+    return {
+      'x-goog-encryption-algorithm': 'AES256',
+      'x-goog-encryption-key': this.encryptionKey.toString('base64'),
+      'x-goog-encryption-key-sha256': this.encryptionKeyHash || '',
+    };
+  }
+
   /**
    * The Storage API allows you to use a custom key for server-side encryption.
    *
@@ -4562,6 +4584,14 @@ class File extends ServiceObject<File, FileMetadata> {
       },
     ];
 
+    const headers: Record<string, string> = {};
+    if (this.encryptionKey) {
+      headers['x-goog-encryption-algorithm'] = 'AES256';
+      headers['x-goog-encryption-key'] = this.encryptionKeyBase64!;
+      headers['x-goog-encryption-key-sha256'] = this.encryptionKeyHash!;
+    }
+    reqOpts.headers = headers;
+
     this.storageTransport
       .makeRequest(reqOpts as StorageRequestOptions, (err, body, resp) => {
         if (err) {

handwritten/storage/src/nodejs-common/service-object.ts

@@ -444,16 +444,31 @@ class ServiceObject<T, K extends BaseMetadata> extends EventEmitter {
       url = `${this.parent.baseUrl}/${this.parent.id}${url}`;
     }
 
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const encryptionHeaders = (this as any).encryptionKeyHeaders || {};
+
+    const headers = {
+      ...encryptionHeaders,
+      ...methodConfig.reqOpts?.headers,
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      ...(options as any).headers,
+    };
+
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const query = {...options} as any;
+    delete query.headers;
+
     this.storageTransport
       .makeRequest<K>(
         {
           method: 'GET',
           responseType: 'json',
           url,
           ...methodConfig.reqOpts,
+          headers,
           queryParameters: {
             ...methodConfig.reqOpts?.queryParameters,
-            ...options,
+            ...query,
           },
         },
         (err, data, resp) => {

handwritten/storage/src/storage-transport.ts

@@ -25,13 +25,13 @@ import {
   getModuleFormat,
   getRuntimeTrackingString,
   getUserAgentString,
-} from './util';
+} from './util.js';
 import {randomUUID} from 'crypto';
 // eslint-disable-next-line @typescript-eslint/ban-ts-comment
 // @ts-ignore
 import {getPackageJSON} from './package-json-helper.cjs';
-import {GCCL_GCS_CMD_KEY} from './nodejs-common/util';
-import {RetryOptions} from './storage';
+import {GCCL_GCS_CMD_KEY} from './nodejs-common/util.js';
+import {RETRYABLE_ERR_FN_DEFAULT, RetryOptions} from './storage.js';
 
 export interface StandardStorageQueryParams {
   alt?: 'json' | 'media';
@@ -87,7 +87,6 @@ export interface StorageTransportCallback<T> {
     fullResponse?: GaxiosResponse,
   ): void;
 }
-let projectId: string;
 
 export class StorageTransport {
   authClient: GoogleAuth<AuthClient>;
@@ -113,7 +112,11 @@ export class StorageTransport {
     }
     this.providedUserAgent = options.userAgent;
     this.packageJson = getPackageJSON();
-    this.retryOptions = options.retryOptions;
+    this.retryOptions = {
+      ...options.retryOptions,
+      retryableErrorFn:
+        options.retryOptions?.retryableErrorFn || RETRYABLE_ERR_FN_DEFAULT,
+    };
     this.baseUrl = options.baseUrl;
     this.timeout = options.timeout;
     this.projectId = options.projectId;
@@ -124,76 +127,101 @@ export class StorageTransport {
     reqOpts: StorageRequestOptions,
     callback?: StorageTransportCallback<T>,
   ): Promise<void | T> {
-    const headers = this.#buildRequestHeaders(reqOpts.headers);
-    if (reqOpts[GCCL_GCS_CMD_KEY]) {
-      headers.set(
-        'x-goog-api-client',
-        `${headers.get('x-goog-api-client')} gccl-gcs-cmd/${reqOpts[GCCL_GCS_CMD_KEY]}`,
-      );
+    // Project ID Resolution
+    if (!this.projectId) {
+      this.projectId =
+        reqOpts.projectId || (await this.authClient.getProjectId());
     }
+
+    // Header Construction
+    const headers = this.#prepareHeaders(reqOpts);
+
+    // Interceptor Management
+    this.gaxiosInstance.interceptors.request.clear();
     if (reqOpts.interceptors) {
-      this.gaxiosInstance.interceptors.request.clear();
       for (const inter of reqOpts.interceptors) {
         this.gaxiosInstance.interceptors.request.add(inter);
       }
     }
 
-    try {
-      const getProjectId = async () => {
-        if (reqOpts.projectId) return reqOpts.projectId;
-        projectId = await this.authClient.getProjectId();
-        return projectId;
-      };
-      const _projectId = await getProjectId();
-      if (_projectId) {
-        projectId = _projectId;
-        this.projectId = projectId;
-      }
+    const urlString = reqOpts.url?.toString() || '';
+    const isAbsolute = this.#isValidUrl(urlString);
+
+    // Determine the base URL for the request
+    const requestUrl = isAbsolute
+      ? urlString
+      : new URL(urlString, this.baseUrl).toString();
 
+    try {
       const requestPromise = this.authClient.request<T>({
+        adapter: this.gaxiosInstance.request.bind(this.gaxiosInstance),
         retryConfig: {
           retry: this.retryOptions.maxRetries,
           noResponseRetries: this.retryOptions.maxRetries,
           maxRetryDelay: this.retryOptions.maxRetryDelay,
           retryDelayMultiplier: this.retryOptions.retryDelayMultiplier,
-          shouldRetry: this.retryOptions.retryableErrorFn,
           totalTimeout: this.retryOptions.totalTimeout,
+          shouldRetry: err => !!this.retryOptions.retryableErrorFn?.(err),
         },
         ...reqOpts,
+        params: reqOpts.queryParameters,
+        paramsSerializer: this.#paramsSerializer,
         headers,
-        url: this.#buildUrl(reqOpts.url?.toString(), reqOpts.queryParameters),
+        url: requestUrl,
         timeout: this.timeout,
+        validateStatus: (status: number): boolean => {
+          const isResumable = !!(
+            reqOpts.queryParameters?.uploadType === 'resumable' ||
+            reqOpts.url?.toString().includes('uploadType=resumable')
+          );
+          return (
+            (status >= 200 && status < 300) || (isResumable && status === 308)
+          );
+        },
       });
 
-      return callback
-        ? requestPromise
-            .then(resp => callback(null, resp.data, resp))
-            .catch(err => callback(err, null, err.response))
-        : (requestPromise.then(resp => resp.data) as Promise<T>);
+      // Response Handling
+      const responseHandler = (resp: GaxiosResponse<T>) => {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const data = resp.data as any;
+        if (data !== null && typeof data === 'object') {
+          data.headers = resp.headers;
+          data.status = resp.status;
+          return data;
+        }
+        return resp;
+      };
+
+      if (callback) {
+        requestPromise
+          .then(resp => callback(null, responseHandler(resp), resp))
+          .catch(err => callback(err, null, err.response));
+        return;
+      }
+
+      return requestPromise.then(responseHandler);
     } catch (e) {
       if (callback) return callback(e as GaxiosError);
       throw e;
     }
   }
 
-  #buildUrl(pathUri = '', queryParameters: StorageQueryParameters = {}): URL {
-    if (
-      'project' in queryParameters &&
-      (queryParameters.project !== this.projectId ||
-        queryParameters.project !== projectId)
-    ) {
-      queryParameters.project = this.projectId;
-    }
-    const qp = this.#buildRequestQueryParams(queryParameters);
-    let url: URL;
-    if (this.#isValidUrl(pathUri)) {
-      url = new URL(pathUri);
-    } else {
-      url = new URL(`${this.baseUrl}${pathUri}`);
+  #prepareHeaders(reqOpts: StorageRequestOptions): Record<string, string> {
+    const headersObj = this.#buildRequestHeaders(reqOpts.headers);
+
+    if (reqOpts[GCCL_GCS_CMD_KEY]) {
+      const current = headersObj.get('x-goog-api-client') || '';
+      headersObj.set(
+        'x-goog-api-client',
+        `${current} gccl-gcs-cmd/${reqOpts[GCCL_GCS_CMD_KEY]}`,
+      );
     }
-    url.search = qp;
 
-    return url;
+    const finalHeaders: Record<string, string> = {};
+    headersObj.forEach((v, k) => {
+      finalHeaders[k] = v;
+    });
+    return finalHeaders;
   }
 
   #isValidUrl(url: string): boolean {
@@ -204,32 +232,38 @@ export class StorageTransport {
     }
   }
 
+  /**
+   * Serializes query parameters into a string.
+   * Specifically handles arrays by appending each value individually
+   * to satisfy GCS "repeated key" requirements (e.g., for IAM permissions).
+   */
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  #paramsSerializer = (params: Record<string, any>): string => {
+    const searchParams = new URLSearchParams();
+    for (const [key, value] of Object.entries(params)) {
+      if (value === undefined) continue;
+
+      if (Array.isArray(value)) {
+        value.forEach(v => searchParams.append(key, String(v)));
+      } else {
+        searchParams.set(key, String(value));
+      }
+    }
+    return searchParams.toString();
+  };
+
   #buildRequestHeaders(requestHeaders = {}) {
     const headers = new Headers(requestHeaders);
-
     headers.set('User-Agent', this.#getUserAgentString());
     headers.set(
       'x-goog-api-client',
       `${getRuntimeTrackingString()} gccl/${this.packageJson.version}-${getModuleFormat()} gccl-invocation-id/${randomUUID()}`,
     );
-
     return headers;
   }
 
-  #buildRequestQueryParams(queryParameters: StorageQueryParameters): string {
-    const qp = new URLSearchParams(
-      queryParameters as unknown as Record<string, string>,
-    );
-
-    return qp.toString();
-  }
-
   #getUserAgentString(): string {
-    let userAgent = getUserAgentString();
-    if (this.providedUserAgent) {
-      userAgent = `${this.providedUserAgent} ${userAgent}`;
-    }
-
-    return userAgent;
+    const base = getUserAgentString();
+    return this.providedUserAgent ? `${this.providedUserAgent} ${base}` : base;
   }
 }