Feat/improved logging (#3833)

* feat(logs): add additional metadata for workflow execution logs

* Revert "Feat(logs) upgrade mothership chat messages to error (#3772)"

This reverts commit 9d1b976.

* Fix lint, address greptile comments

* improvement(sidebar): expand sidebar by hovering and clicking the edge (#3830)

* improvement(sidebar): expand sidebar by hovering and clicking the edge

* improvement(sidebar): add keyboard shortcuts for new workflow/task, center search modal, fix edge ARIA

* improvement(sidebar): use Tooltip.Shortcut for inline shortcut display

* fix(sidebar): change new workflow shortcut from Mod+Shift+W to Mod+Shift+P to avoid browser close-window conflict

* fix(hotkeys): fall back to event.code for international keyboard layout compatibility

* fix(sidebar): guard add-workflow shortcut with canEdit and isCreatingWorkflow checks

* feat(ui): handle image paste (#3826)

* feat(ui): handle image paste

* Fix lint

* Fix type error

---------

Co-authored-by: Theodore Li <theo@sim.ai>

* feat(files): interactive markdown checkbox toggling in preview (#3829)

* feat(files): interactive markdown checkbox toggling in preview

* fix(files): handle ordered-list checkboxes and fix index drift

* lint

* fix(files): remove counter offset that prevented checkbox toggling

* fix(files): apply task-list styling to ordered lists too

* fix(files): render single pass when interactive to avoid index drift

* fix(files): move useMemo above conditional return to fix Rules of Hooks

* fix(files): pass content directly to preview when not streaming to avoid stale frame

* improvement(home): position @ mention popup at caret and fix icon consistency (#3831)

* improvement(home): position @ mention popup at caret and fix icon consistency

* fix(home): pin mirror div to document origin and guard button anchor

* chore(auth): restore hybrid.ts to staging

* improvement(ui): sidebar (#3832)

* Fix logger tests

* Add metadata to mothership logs

---------

Co-authored-by: Theodore Li <theo@sim.ai>
Co-authored-by: Waleed <walif6@gmail.com>
Co-authored-by: Theodore Li <theo@sim.ai>

Author: 3 people

apps/sim/app/api/copilot/chat/route.ts

@@ -15,7 +15,6 @@ import {
   requestChatTitle,
   SSE_RESPONSE_HEADERS,
 } from '@/lib/copilot/chat-streaming'
-import { appendCopilotLogContext } from '@/lib/copilot/logging'
 import { COPILOT_REQUEST_MODES } from '@/lib/copilot/models'
 import { orchestrateCopilotStream } from '@/lib/copilot/orchestrator'
 import { getStreamMeta, readStreamEvents } from '@/lib/copilot/orchestrator/stream/buffer'
@@ -184,36 +183,31 @@ export async function POST(req: NextRequest) {
       const wf = await getWorkflowById(workflowId)
       resolvedWorkspaceId = wf?.workspaceId ?? undefined
     } catch {
-      logger.warn(
-        appendCopilotLogContext('Failed to resolve workspaceId from workflow', {
-          requestId: tracker.requestId,
-          messageId: userMessageId,
-        })
-      )
+      logger
+        .withMetadata({ requestId: tracker.requestId, messageId: userMessageId })
+        .warn('Failed to resolve workspaceId from workflow')
     }
 
     const userMessageIdToUse = userMessageId || crypto.randomUUID()
+    const reqLogger = logger.withMetadata({
+      requestId: tracker.requestId,
+      messageId: userMessageIdToUse,
+    })
     try {
-      logger.error(
-        appendCopilotLogContext('Received chat POST', {
-          requestId: tracker.requestId,
-          messageId: userMessageIdToUse,
-        }),
-        {
-          workflowId,
-          hasContexts: Array.isArray(normalizedContexts),
-          contextsCount: Array.isArray(normalizedContexts) ? normalizedContexts.length : 0,
-          contextsPreview: Array.isArray(normalizedContexts)
-            ? normalizedContexts.map((c: any) => ({
-                kind: c?.kind,
-                chatId: c?.chatId,
-                workflowId: c?.workflowId,
-                executionId: (c as any)?.executionId,
-                label: c?.label,
-              }))
-            : undefined,
-        }
-      )
+      reqLogger.info('Received chat POST', {
+        workflowId,
+        hasContexts: Array.isArray(normalizedContexts),
+        contextsCount: Array.isArray(normalizedContexts) ? normalizedContexts.length : 0,
+        contextsPreview: Array.isArray(normalizedContexts)
+          ? normalizedContexts.map((c: any) => ({
+              kind: c?.kind,
+              chatId: c?.chatId,
+              workflowId: c?.workflowId,
+              executionId: (c as any)?.executionId,
+              label: c?.label,
+            }))
+          : undefined,
+      })
     } catch {}
 
     let currentChat: any = null
@@ -251,40 +245,22 @@ export async function POST(req: NextRequest) {
           actualChatId
         )
         agentContexts = processed
-        logger.error(
-          appendCopilotLogContext('Contexts processed for request', {
-            requestId: tracker.requestId,
-            messageId: userMessageIdToUse,
-          }),
-          {
-            processedCount: agentContexts.length,
-            kinds: agentContexts.map((c) => c.type),
-            lengthPreview: agentContexts.map((c) => c.content?.length ?? 0),
-          }
-        )
+        reqLogger.info('Contexts processed for request', {
+          processedCount: agentContexts.length,
+          kinds: agentContexts.map((c) => c.type),
+          lengthPreview: agentContexts.map((c) => c.content?.length ?? 0),
+        })
         if (
           Array.isArray(normalizedContexts) &&
           normalizedContexts.length > 0 &&
           agentContexts.length === 0
         ) {
-          logger.warn(
-            appendCopilotLogContext(
-              'Contexts provided but none processed. Check executionId for logs contexts.',
-              {
-                requestId: tracker.requestId,
-                messageId: userMessageIdToUse,
-              }
-            )
+          reqLogger.warn(
+            'Contexts provided but none processed. Check executionId for logs contexts.'
           )
         }
       } catch (e) {
-        logger.error(
-          appendCopilotLogContext('Failed to process contexts', {
-            requestId: tracker.requestId,
-            messageId: userMessageIdToUse,
-          }),
-          e
-        )
+        reqLogger.error('Failed to process contexts', e)
       }
     }
 
@@ -313,13 +289,7 @@ export async function POST(req: NextRequest) {
         if (result.status === 'fulfilled' && result.value) {
           agentContexts.push(result.value)
         } else if (result.status === 'rejected') {
-          logger.error(
-            appendCopilotLogContext('Failed to resolve resource attachment', {
-              requestId: tracker.requestId,
-              messageId: userMessageIdToUse,
-            }),
-            result.reason
-          )
+          reqLogger.error('Failed to resolve resource attachment', result.reason)
         }
       }
     }
@@ -358,26 +328,20 @@ export async function POST(req: NextRequest) {
     )
 
     try {
-      logger.error(
-        appendCopilotLogContext('About to call Sim Agent', {
-          requestId: tracker.requestId,
-          messageId: userMessageIdToUse,
-        }),
-        {
-          hasContext: agentContexts.length > 0,
-          contextCount: agentContexts.length,
-          hasFileAttachments: Array.isArray(requestPayload.fileAttachments),
-          messageLength: message.length,
-          mode: effectiveMode,
-          hasTools: Array.isArray(requestPayload.tools),
-          toolCount: Array.isArray(requestPayload.tools) ? requestPayload.tools.length : 0,
-          hasBaseTools: Array.isArray(requestPayload.baseTools),
-          baseToolCount: Array.isArray(requestPayload.baseTools)
-            ? requestPayload.baseTools.length
-            : 0,
-          hasCredentials: !!requestPayload.credentials,
-        }
-      )
+      reqLogger.info('About to call Sim Agent', {
+        hasContext: agentContexts.length > 0,
+        contextCount: agentContexts.length,
+        hasFileAttachments: Array.isArray(requestPayload.fileAttachments),
+        messageLength: message.length,
+        mode: effectiveMode,
+        hasTools: Array.isArray(requestPayload.tools),
+        toolCount: Array.isArray(requestPayload.tools) ? requestPayload.tools.length : 0,
+        hasBaseTools: Array.isArray(requestPayload.baseTools),
+        baseToolCount: Array.isArray(requestPayload.baseTools)
+          ? requestPayload.baseTools.length
+          : 0,
+        hasCredentials: !!requestPayload.credentials,
+      })
     } catch {}
 
     if (stream && actualChatId) {
@@ -521,16 +485,10 @@ export async function POST(req: NextRequest) {
                   .where(eq(copilotChats.id, actualChatId))
               }
             } catch (error) {
-              logger.error(
-                appendCopilotLogContext('Failed to persist chat messages', {
-                  requestId: tracker.requestId,
-                  messageId: userMessageIdToUse,
-                }),
-                {
-                  chatId: actualChatId,
-                  error: error instanceof Error ? error.message : 'Unknown error',
-                }
-              )
+              reqLogger.error('Failed to persist chat messages', {
+                chatId: actualChatId,
+                error: error instanceof Error ? error.message : 'Unknown error',
+              })
             }
           },
         },
@@ -572,19 +530,13 @@ export async function POST(req: NextRequest) {
       provider: typeof requestPayload?.provider === 'string' ? requestPayload.provider : undefined,
     }
 
-    logger.error(
-      appendCopilotLogContext('Non-streaming response from orchestrator', {
-        requestId: tracker.requestId,
-        messageId: userMessageIdToUse,
-      }),
-      {
-        hasContent: !!responseData.content,
-        contentLength: responseData.content?.length || 0,
-        model: responseData.model,
-        provider: responseData.provider,
-        toolCallsCount: responseData.toolCalls?.length || 0,
-      }
-    )
+    reqLogger.info('Non-streaming response from orchestrator', {
+      hasContent: !!responseData.content,
+      contentLength: responseData.content?.length || 0,
+      model: responseData.model,
+      provider: responseData.provider,
+      toolCallsCount: responseData.toolCalls?.length || 0,
+    })
 
     // Save messages if we have a chat
     if (currentChat && responseData.content) {
@@ -617,12 +569,7 @@ export async function POST(req: NextRequest) {
 
       // Start title generation in parallel if this is first message (non-streaming)
       if (actualChatId && !currentChat.title && conversationHistory.length === 0) {
-        logger.error(
-          appendCopilotLogContext('Starting title generation for non-streaming response', {
-            requestId: tracker.requestId,
-            messageId: userMessageIdToUse,
-          })
-        )
+        reqLogger.info('Starting title generation for non-streaming response')
         requestChatTitle({ message, model: selectedModel, provider, messageId: userMessageIdToUse })
           .then(async (title) => {
             if (title) {
@@ -633,22 +580,11 @@ export async function POST(req: NextRequest) {
                   updatedAt: new Date(),
                 })
                 .where(eq(copilotChats.id, actualChatId!))
-              logger.error(
-                appendCopilotLogContext(`Generated and saved title: ${title}`, {
-                  requestId: tracker.requestId,
-                  messageId: userMessageIdToUse,
-                })
-              )
+              reqLogger.info(`Generated and saved title: ${title}`)
             }
           })
           .catch((error) => {
-            logger.error(
-              appendCopilotLogContext('Title generation failed', {
-                requestId: tracker.requestId,
-                messageId: userMessageIdToUse,
-              }),
-              error
-            )
+            reqLogger.error('Title generation failed', error)
           })
       }
 
@@ -662,17 +598,11 @@ export async function POST(req: NextRequest) {
         .where(eq(copilotChats.id, actualChatId!))
     }
 
-    logger.error(
-      appendCopilotLogContext('Returning non-streaming response', {
-        requestId: tracker.requestId,
-        messageId: userMessageIdToUse,
-      }),
-      {
-        duration: tracker.getDuration(),
-        chatId: actualChatId,
-        responseLength: responseData.content?.length || 0,
-      }
-    )
+    reqLogger.info('Returning non-streaming response', {
+      duration: tracker.getDuration(),
+      chatId: actualChatId,
+      responseLength: responseData.content?.length || 0,
+    })
 
     return NextResponse.json({
       success: true,
@@ -696,33 +626,25 @@ export async function POST(req: NextRequest) {
     const duration = tracker.getDuration()
 
     if (error instanceof z.ZodError) {
-      logger.error(
-        appendCopilotLogContext('Validation error', {
-          requestId: tracker.requestId,
-          messageId: pendingChatStreamID ?? undefined,
-        }),
-        {
+      logger
+        .withMetadata({ requestId: tracker.requestId, messageId: pendingChatStreamID ?? undefined })
+        .error('Validation error', {
           duration,
           errors: error.errors,
-        }
-      )
+        })
       return NextResponse.json(
         { error: 'Invalid request data', details: error.errors },
         { status: 400 }
       )
     }
 
-    logger.error(
-      appendCopilotLogContext('Error handling copilot chat', {
-        requestId: tracker.requestId,
-        messageId: pendingChatStreamID ?? undefined,
-      }),
-      {
+    logger
+      .withMetadata({ requestId: tracker.requestId, messageId: pendingChatStreamID ?? undefined })
+      .error('Error handling copilot chat', {
         duration,
         error: error instanceof Error ? error.message : 'Unknown error',
         stack: error instanceof Error ? error.stack : undefined,
-      }
-    )
+      })
 
     return NextResponse.json(
       { error: error instanceof Error ? error.message : 'Internal server error' },
@@ -767,16 +689,13 @@ export async function GET(req: NextRequest) {
             status: meta?.status || 'unknown',
           }
         } catch (err) {
-          logger.warn(
-            appendCopilotLogContext('Failed to read stream snapshot for chat', {
-              messageId: chat.conversationId || undefined,
-            }),
-            {
+          logger
+            .withMetadata({ messageId: chat.conversationId || undefined })
+            .warn('Failed to read stream snapshot for chat', {
               chatId,
               conversationId: chat.conversationId,
               error: err instanceof Error ? err.message : String(err),
-            }
-          )
+            })
         }
       }
 
@@ -795,11 +714,9 @@ export async function GET(req: NextRequest) {
         ...(streamSnapshot ? { streamSnapshot } : {}),
       }
 
-      logger.error(
-        appendCopilotLogContext(`Retrieved chat ${chatId}`, {
-          messageId: chat.conversationId || undefined,
-        })
-      )
+      logger
+        .withMetadata({ messageId: chat.conversationId || undefined })
+        .info(`Retrieved chat ${chatId}`)
       return NextResponse.json({ success: true, chat: transformedChat })
     }