Skip to content
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 19 additions & 21 deletions apps/sim/lib/knowledge/connectors/sync-engine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -401,8 +401,6 @@ export async function executeSync(

const seenExternalIds = new Set<string>()

const pendingProcessing: DocumentData[] = []

const pendingOps: DocOp[] = []
for (const extDoc of externalDocs) {
seenExternalIds.add(extDoc.externalId)
Expand Down Expand Up @@ -508,10 +506,11 @@ export async function executeSync(
})
)

const batchDocs: DocumentData[] = []
for (let j = 0; j < settled.length; j++) {
const outcome = settled[j]
if (outcome.status === 'fulfilled') {
pendingProcessing.push(outcome.value)
batchDocs.push(outcome.value)
if (batch[j].type === 'add') result.docsAdded++
else result.docsUpdated++
} else {
Expand All @@ -524,6 +523,23 @@ export async function executeSync(
})
}
}

if (batchDocs.length > 0) {
try {
await processDocumentsWithQueue(
batchDocs,
connector.knowledgeBaseId,
{},
crypto.randomUUID()
)
} catch (error) {
logger.warn('Failed to enqueue batch for processing — will retry on next sync', {
connectorId,
count: batchDocs.length,
error: error instanceof Error ? error.message : String(error),
})
}
}
}

// Skip deletion reconciliation during incremental syncs — results only contain changed docs
Expand Down Expand Up @@ -608,24 +624,6 @@ export async function executeSync(
}
}

// Enqueue all added/updated documents for processing in a single batch
if (pendingProcessing.length > 0) {
try {
await processDocumentsWithQueue(
pendingProcessing,
connector.knowledgeBaseId,
{},
crypto.randomUUID()
)
} catch (error) {
logger.warn('Failed to enqueue documents for processing — will retry on next sync', {
connectorId,
count: pendingProcessing.length,
error: error instanceof Error ? error.message : String(error),
})
}
}

await completeSyncLog(syncLogId, 'completed', result)

const [{ count: actualDocCount }] = await db
Expand Down
Loading