fix(sync-engine): dedup externalIds, enable deletion reconciliation, merge sourceUrl

Three sync engine gaps identified during audit:

1. Duplicate externalId guard: if a connector returns the same externalId
   across pages (pagination overlap), skip the second occurrence to prevent
   unique constraint violations on add and double-uploads on update.

2. Deletion reconciliation: previously required explicit fullSync or
   syncMode='full', meaning docs deleted from the source accumulated in
   the KB forever. Now runs on all non-incremental syncs (which return
   ALL docs). Includes a safety threshold: if >50% of existing docs
   (and >5 docs) would be deleted, skip and warn — protects against
   partial listing failures. Explicit fullSync bypasses the threshold.

3. sourceUrl merge: hydration now picks up sourceUrl from getDocument,
   falling back to the stub's sourceUrl if getDocument doesn't set one.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: waleedlatif1

apps/sim/lib/knowledge/connectors/sync-engine.ts

@@ -403,6 +403,7 @@ export async function executeSync(
 
     const pendingOps: DocOp[] = []
     for (const extDoc of externalDocs) {
+      if (seenExternalIds.has(extDoc.externalId)) continue
       seenExternalIds.add(extDoc.externalId)
 
       if (excludedExternalIds.has(extDoc.externalId)) {
@@ -472,6 +473,7 @@ export async function executeSync(
                 content: fullDoc.content,
                 contentHash: hydratedHash,
                 contentDeferred: false,
+                sourceUrl: fullDoc.sourceUrl ?? op.extDoc.sourceUrl,
                 metadata: { ...op.extDoc.metadata, ...fullDoc.metadata },
               },
             }
@@ -552,15 +554,26 @@ export async function executeSync(
       }
     }
 
-    // Skip deletion reconciliation during incremental syncs — results only contain changed docs
-    if (!isIncremental && (options?.fullSync || connector.syncMode === 'full')) {
+    // Reconcile deletions for non-incremental syncs (which return ALL docs).
+    // Skip for incremental syncs since results only contain changed docs.
+    if (!isIncremental) {
       const removedIds = existingDocs
         .filter((d) => d.externalId && !seenExternalIds.has(d.externalId))
         .map((d) => d.id)
 
       if (removedIds.length > 0) {
-        await hardDeleteDocuments(removedIds, syncLogId)
-        result.docsDeleted += removedIds.length
+        const deletionRatio =
+          existingDocs.length > 0 ? removedIds.length / existingDocs.length : 0
+
+        if (deletionRatio > 0.5 && removedIds.length > 5 && !options?.fullSync) {
+          logger.warn(
+            `Skipping deletion of ${removedIds.length}/${existingDocs.length} docs — exceeds safety threshold. Trigger a full sync to force cleanup.`,
+            { connectorId, deletionRatio: Math.round(deletionRatio * 100) }
+          )
+        } else {
+          await hardDeleteDocuments(removedIds, syncLogId)
+          result.docsDeleted += removedIds.length
+        }
       }
     }