Sanitize serialized worker errors

package.json

@@ -44,7 +44,7 @@
         "mocha-multi-reporters": "^1.5.1",
         "mock-fs": "^5.1.2",
         "prettier": "^2.4.1",
-        "protobufjs": "^7.5.5",
+        "protobufjs": "^7.6.0",
         "protobufjs-cli": "^1.2.1",
         "rimraf": "^2.6.3",
         "semver": "^7.3.5",

src/Worker.ts

@@ -7,6 +7,7 @@ import { CreateGrpcEventStream, getConnectionUri } from './GrpcClient';
 import { setupCoreModule } from './setupCoreModule';
 import { setupEventStream } from './setupEventStream';
 import { startBlockedMonitor } from './utils/blockedMonitor';
+import { sanitizeErrorString } from './utils/errorSanitizer';
 import { systemError, systemLog } from './utils/Logger';
 import { isEnvironmentVariableSet } from './utils/util';
 import { worker } from './WorkerContext';
@@ -41,7 +42,7 @@ export function startNodeWorker(args) {
     } catch (err) {
         const error = ensureErrorType(err);
         error.isAzureFunctionsSystemError = true;
-        const message = 'Error creating GRPC event stream: ' + error.message;
+        const message = 'Error creating GRPC event stream: ' + sanitizeErrorString(error.message);
         trySetErrorMessage(error, message);
         throw error;
     }
@@ -60,11 +61,11 @@ export function startNodeWorker(args) {
         const error = ensureErrorType(err);
         let errorMessage: string;
         if (error.isAzureFunctionsSystemError) {
-            errorMessage = `Worker uncaught exception: ${error.stack || err}`;
+            errorMessage = `Worker uncaught exception: ${sanitizeErrorString(error.stack || String(err))}`;
         } else {
-            errorMessage = `Worker uncaught exception (learn more: https://go.microsoft.com/fwlink/?linkid=2097909 ): ${
-                error.stack || err
-            }`;
+            errorMessage = `Worker uncaught exception (learn more: https://go.microsoft.com/fwlink/?linkid=2097909 ): ${sanitizeErrorString(
+                error.stack || String(err)
+            )}`;
         }
 
         systemError(errorMessage);

src/errors.ts

@@ -1,6 +1,8 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License.
 
+import { sanitizeErrorString, stringifySanitizedErrorObject } from './utils/errorSanitizer';
+
 export interface AzFuncError {
     /**
      * System errors can be tracked in our telemetry
@@ -44,19 +46,19 @@ export function ensureErrorType(err: unknown): ValidatedError {
         if (err === undefined || err === null) {
             message = 'Unknown error';
         } else if (typeof err === 'string') {
-            message = err;
+            message = sanitizeErrorString(err);
         } else if (typeof err === 'object') {
-            message = JSON.stringify(err);
+            message = stringifySanitizedErrorObject(err);
         } else {
-            message = String(err);
+            message = sanitizeErrorString(String(err));
         }
         return new Error(message);
     }
 }
 
 export function trySetErrorMessage(err: Error, message: string): void {
     try {
-        err.message = message;
+        err.message = sanitizeErrorString(message);
     } catch {
         // If we can't set the message, we'll keep the error as is
     }

src/setupEventStream.ts

@@ -10,6 +10,7 @@ import { FunctionsMetadataHandler } from './eventHandlers/FunctionsMetadataHandl
 import { InvocationHandler } from './eventHandlers/InvocationHandler';
 import { terminateWorker } from './eventHandlers/terminateWorker';
 import { WorkerInitHandler } from './eventHandlers/WorkerInitHandler';
+import { sanitizeErrorString } from './utils/errorSanitizer';
 import { systemError } from './utils/Logger';
 import { nonNullProp } from './utils/nonNull';
 import { worker } from './WorkerContext';
@@ -98,7 +99,7 @@ async function handleMessage(inMsg: rpc.StreamingMessage): Promise<void> {
         const error = ensureErrorType(err);
         if (error.isAzureFunctionsSystemError && !error.loggedOverRpc) {
             worker.log({
-                message: error.message,
+                message: sanitizeErrorString(error.message),
                 level: rpc.RpcLog.Level.Error,
                 logCategory: rpc.RpcLog.RpcLogCategory.System,
             });
@@ -109,8 +110,8 @@ async function handleMessage(inMsg: rpc.StreamingMessage): Promise<void> {
             response.result = {
                 status: rpc.StatusResult.Status.Failure,
                 exception: {
-                    message: error.message,
-                    stackTrace: error.stack,
+                    message: sanitizeErrorString(error.message),
+                    stackTrace: error.stack ? sanitizeErrorString(error.stack) : error.stack,
                 },
             };
             outMsg[eventHandler.responseName] = response;

src/utils/errorSanitizer.ts

@@ -0,0 +1,93 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License.
+
+const hiddenCredential = '[Hidden Credential]';
+const circularReference = '[Circular]';
+const credentialNameFragments = ['password', 'pwd', 'key', 'secret', 'token', 'sas'];
+const credentialTokens = [
+    'Token=',
+    'DefaultEndpointsProtocol=http',
+    'AccountKey=',
+    'Data Source=',
+    'Server=',
+    'Password=',
+    'pwd=',
+    '&sig=',
+    '&sig=',
+    '?sig=',
+    'SharedAccessKey=',
+    '&code=',
+    '&code=',
+    '?code=',
+    '/code=',
+    'key=',
+];
+const urlCredentialPattern = /\b([a-zA-Z]+):\/\/([^:/\s]+):([^@/\s]+)@([^:/\s]+):([0-9]+)\b/g;
+
+export function sanitizeErrorString(input: string): string {
+    if (!input) {
+        return input;
+    }
+
+    let sanitized = input;
+    for (const token of credentialTokens) {
+        sanitized = replaceCredentialToken(sanitized, token);
+    }
+
+    return sanitized.replace(urlCredentialPattern, hiddenCredential);
+}
+
+export function stringifySanitizedErrorObject(value: object): string {
+    const seen = new WeakSet<object>();
+    return JSON.stringify(value, (key, val: unknown) => {
+        if (isCredentialName(key)) {
+            return hiddenCredential;
+        }
+
+        if (typeof val === 'string') {
+            return sanitizeErrorString(val);
+        }
+
+        if (typeof val === 'bigint') {
+            return val.toString();
+        }
+
+        if (typeof val === 'object' && val !== null) {
+            if (seen.has(val)) {
+                return circularReference;
+            }
+            seen.add(val);
+        }
+
+        return val;
+    });
+}
+
+function replaceCredentialToken(input: string, token: string): string {
+    const lowerInput = input.toLowerCase();
+    const lowerToken = token.toLowerCase();
+    let startIndex = lowerInput.indexOf(lowerToken);
+    if (startIndex === -1) {
+        return input;
+    }
+
+    let sanitized = '';
+    let searchOffset = 0;
+    while (startIndex !== -1) {
+        const credentialEnd = findCredentialEnd(input, startIndex);
+        sanitized += input.substring(searchOffset, startIndex) + hiddenCredential;
+        searchOffset = credentialEnd;
+        startIndex = lowerInput.indexOf(lowerToken, searchOffset);
+    }
+
+    return sanitized + input.substring(searchOffset);
+}
+
+function findCredentialEnd(input: string, startIndex: number): number {
+    const terminatorIndex = input.substring(startIndex).search(/[<"'\r\n]/);
+    return terminatorIndex === -1 ? input.length : startIndex + terminatorIndex;
+}
+
+function isCredentialName(name: string): boolean {
+    return credentialNameFragments.some((fragment) => name.toLowerCase().includes(fragment));
+}