Skip to content

Commit b84aef3

Browse files
author
StackMemory Bot (CLI)
committed
fix(provenant): add rate limiting, pagination fixes, and PROVENANT_DB env var
Linear adapter: - Add retry-after handling for 429 rate limits (3 retries, exponential) - Increase comment fetch limit to 250 per issue (was default ~50) Slack adapter: - Add retry-after handling for 429 and ok:false ratelimited errors - Paginate thread replies (was single call, truncated at 200) CLI: - Support PROVENANT_DB env var as default for --db flag across all commands
1 parent 4e3900e commit b84aef3

3 files changed

Lines changed: 131 additions & 55 deletions

File tree

‎packages/provenant/src/adapters/linear.ts‎

Lines changed: 36 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ const ISSUES_QUERY = `
167167
createdAt
168168
updatedAt
169169
url
170-
comments {
170+
comments(first: 250) {
171171
nodes {
172172
id
173173
body
@@ -204,7 +204,7 @@ const ISSUES_QUERY_ALL = `
204204
createdAt
205205
updatedAt
206206
url
207-
comments {
207+
comments(first: 250) {
208208
nodes {
209209
id
210210
body
@@ -372,20 +372,41 @@ export class LinearAdapter implements SourceAdapter {
372372
query: string,
373373
variables: Record<string, unknown>
374374
): Promise<T> {
375-
const response = await fetch(this.baseUrl, {
376-
method: 'POST',
377-
headers: {
378-
'Content-Type': 'application/json',
379-
Authorization: this.apiKey,
380-
},
381-
body: JSON.stringify({ query, variables }),
382-
});
375+
const maxRetries = 3;
376+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
377+
const response = await fetch(this.baseUrl, {
378+
method: 'POST',
379+
headers: {
380+
'Content-Type': 'application/json',
381+
Authorization: this.apiKey,
382+
},
383+
body: JSON.stringify({ query, variables }),
384+
});
383385

384-
if (!response.ok) {
385-
const body = await response.text();
386-
throw new Error(`Linear API error ${response.status}: ${body}`);
387-
}
386+
if (response.status === 429) {
387+
if (attempt >= maxRetries) {
388+
throw new Error('Linear API rate limited after max retries');
389+
}
390+
const retryAfter = response.headers.get('retry-after');
391+
const waitMs = retryAfter
392+
? parseInt(retryAfter, 10) * 1000
393+
: (attempt + 1) * 2000;
394+
console.warn(`[provenant] Linear rate limited, waiting ${waitMs}ms...`);
395+
await sleep(waitMs);
396+
continue;
397+
}
388398

389-
return response.json() as Promise<T>;
399+
if (!response.ok) {
400+
const body = await response.text();
401+
throw new Error(`Linear API error ${response.status}: ${body}`);
402+
}
403+
404+
return response.json() as Promise<T>;
405+
}
406+
throw new Error('Linear API request failed after retries');
390407
}
391408
}
409+
410+
function sleep(ms: number): Promise<void> {
411+
return new Promise((resolve) => setTimeout(resolve, ms));
412+
}

‎packages/provenant/src/adapters/slack.ts‎

Lines changed: 80 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -315,18 +315,29 @@ export class SlackAdapter implements SourceAdapter {
315315
threadTs: string,
316316
oldestTs: string
317317
): Promise<SlackMessage[]> {
318-
const params = new URLSearchParams({
319-
channel: channelId,
320-
ts: threadTs,
321-
oldest: oldestTs,
322-
limit: '200',
323-
});
324-
325-
const data = await this.api<{
326-
messages: SlackMessage[];
327-
}>('conversations.replies', params);
328-
329-
return data.messages;
318+
const messages: SlackMessage[] = [];
319+
let cursor: string | undefined;
320+
321+
do {
322+
const params = new URLSearchParams({
323+
channel: channelId,
324+
ts: threadTs,
325+
oldest: oldestTs,
326+
limit: '200',
327+
});
328+
if (cursor) params.set('cursor', cursor);
329+
330+
const data = await this.api<{
331+
messages: SlackMessage[];
332+
has_more?: boolean;
333+
response_metadata?: { next_cursor?: string };
334+
}>('conversations.replies', params);
335+
336+
messages.push(...data.messages);
337+
cursor = data.response_metadata?.next_cursor || undefined;
338+
} while (cursor);
339+
340+
return messages;
330341
}
331342

332343
private async resolveUser(userId: string): Promise<string> {
@@ -347,26 +358,68 @@ export class SlackAdapter implements SourceAdapter {
347358
}
348359

349360
private async api<T>(method: string, params: URLSearchParams): Promise<T> {
350-
const url = `${this.baseUrl}/${method}?${params.toString()}`;
351-
const response = await fetch(url, {
352-
headers: { Authorization: `Bearer ${this.token}` },
353-
});
354-
355-
if (!response.ok) {
356-
throw new Error(
357-
`Slack API error ${response.status}: ${await response.text()}`
358-
);
359-
}
361+
const maxRetries = 3;
362+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
363+
const url = `${this.baseUrl}/${method}?${params.toString()}`;
364+
const response = await fetch(url, {
365+
headers: { Authorization: `Bearer ${this.token}` },
366+
});
360367

361-
const json = (await response.json()) as { ok: boolean; error?: string } & T;
362-
if (!json.ok) {
363-
throw new Error(`Slack API error: ${json.error}`);
364-
}
368+
if (response.status === 429) {
369+
if (attempt >= maxRetries) {
370+
throw new Error(
371+
`Slack API rate limited (${method}) after max retries`
372+
);
373+
}
374+
const retryAfter = response.headers.get('retry-after');
375+
const waitMs = retryAfter
376+
? parseInt(retryAfter, 10) * 1000
377+
: (attempt + 1) * 3000;
378+
console.warn(
379+
`[provenant] Slack rate limited (${method}), waiting ${waitMs}ms...`
380+
);
381+
await sleep(waitMs);
382+
continue;
383+
}
365384

366-
return json as T;
385+
if (!response.ok) {
386+
throw new Error(
387+
`Slack API error ${response.status}: ${await response.text()}`
388+
);
389+
}
390+
391+
const json = (await response.json()) as {
392+
ok: boolean;
393+
error?: string;
394+
} & T;
395+
if (!json.ok) {
396+
// Slack returns 200 with ok:false and retry_after for rate limits
397+
if (json.error === 'ratelimited') {
398+
if (attempt >= maxRetries) {
399+
throw new Error(
400+
`Slack API rate limited (${method}) after max retries`
401+
);
402+
}
403+
const waitMs = (attempt + 1) * 3000;
404+
console.warn(
405+
`[provenant] Slack rate limited (${method}), waiting ${waitMs}ms...`
406+
);
407+
await sleep(waitMs);
408+
continue;
409+
}
410+
throw new Error(`Slack API error: ${json.error}`);
411+
}
412+
413+
return json as T;
414+
}
415+
throw new Error(`Slack API request failed (${method}) after retries`);
367416
}
368417
}
369418

370419
function parseSlackTs(ts: string): number {
371420
return Math.floor(parseFloat(ts) * 1000);
372421
}
422+
423+
function sleep(ms: number): Promise<void> {
424+
return new Promise((resolve) => setTimeout(resolve, ms));
425+
}

‎packages/provenant/src/cli/index.ts‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ import {
1818
import { serve } from './commands/serve.js';
1919
import { calibrate } from './commands/calibrate.js';
2020

21+
const DB_DEFAULT = process.env['PROVENANT_DB'] || '.provenant/graph.db';
22+
2123
const program = new Command();
2224

2325
program
@@ -33,13 +35,13 @@ program
3335
.option('-r, --reasoning <text>', 'Why this decision was made')
3436
.option('--source-url <url>', 'URL evidence for this decision')
3537
.option('--source-file <path>', 'File path evidence for this decision')
36-
.option('--db <path>', 'Database path', '.provenant/graph.db')
38+
.option('--db <path>', 'Database path (or set PROVENANT_DB)', DB_DEFAULT)
3739
.action(logDecision);
3840

3941
program
4042
.command('status')
4143
.description('Show graph status')
42-
.option('--db <path>', 'Database path', '.provenant/graph.db')
44+
.option('--db <path>', 'Database path (or set PROVENANT_DB)', DB_DEFAULT)
4345
.action(status);
4446

4547
program
@@ -49,7 +51,7 @@ program
4951
'-s, --source <system>',
5052
'Source adapter (e.g. linear, slack)'
5153
)
52-
.option('--db <path>', 'Database path', '.provenant/graph.db')
54+
.option('--db <path>', 'Database path (or set PROVENANT_DB)', DB_DEFAULT)
5355
.option('--dry-run', 'Score and classify without writing', false)
5456
.action(runIngest);
5557

@@ -60,7 +62,7 @@ program
6062
.option('-a, --actor <name>', 'Filter by actor')
6163
.option('-s, --since <date>', 'Only include nodes after this date')
6264
.option('-m, --model <model>', 'Claude model to use', 'claude-sonnet-4-6')
63-
.option('--db <path>', 'Database path', '.provenant/graph.db')
65+
.option('--db <path>', 'Database path (or set PROVENANT_DB)', DB_DEFAULT)
6466
.action(runQuery);
6567

6668
program
@@ -70,7 +72,7 @@ program
7072
.argument('<node_b>', 'Second node ID (or prefix)')
7173
.option('-w, --winner <id>', 'Winning node ID (or prefix)')
7274
.option('-d, --dismiss', 'Dismiss as noise', false)
73-
.option('--db <path>', 'Database path', '.provenant/graph.db')
75+
.option('--db <path>', 'Database path (or set PROVENANT_DB)', DB_DEFAULT)
7476
.action(resolve);
7577

7678
// Review queue subcommands
@@ -80,29 +82,29 @@ review
8082
.command('list')
8183
.description('List pending review queue items')
8284
.option('-l, --limit <n>', 'Max items to show', '20')
83-
.option('--db <path>', 'Database path', '.provenant/graph.db')
85+
.option('--db <path>', 'Database path (or set PROVENANT_DB)', DB_DEFAULT)
8486
.action(reviewList);
8587

8688
review
8789
.command('approve')
8890
.description('Approve a queue item → promote to node')
8991
.argument('<id>', 'Queue item ID (or prefix)')
90-
.option('--db <path>', 'Database path', '.provenant/graph.db')
92+
.option('--db <path>', 'Database path (or set PROVENANT_DB)', DB_DEFAULT)
9193
.action(reviewApprove);
9294

9395
review
9496
.command('dismiss')
9597
.description('Dismiss a queue item')
9698
.argument('<id>', 'Queue item ID (or prefix)')
97-
.option('--db <path>', 'Database path', '.provenant/graph.db')
99+
.option('--db <path>', 'Database path (or set PROVENANT_DB)', DB_DEFAULT)
98100
.action(reviewDismiss);
99101

100102
review
101103
.command('expire')
102104
.description(
103105
'Process expired queue items (auto-promote >=0.55, discard rest)'
104106
)
105-
.option('--db <path>', 'Database path', '.provenant/graph.db')
107+
.option('--db <path>', 'Database path (or set PROVENANT_DB)', DB_DEFAULT)
106108
.action(reviewExpire);
107109

108110
// Log-override subcommands
@@ -114,23 +116,23 @@ logOverride
114116
.command('list')
115117
.description('List unresolved rejection log entries')
116118
.option('-l, --limit <n>', 'Max items to show', '20')
117-
.option('--db <path>', 'Database path', '.provenant/graph.db')
119+
.option('--db <path>', 'Database path (or set PROVENANT_DB)', DB_DEFAULT)
118120
.action(logOverrideList);
119121

120122
logOverride
121123
.command('resolve')
122124
.description('Resolve a rejection by adding reasoning')
123125
.argument('<id>', 'Rejection ID (or prefix)')
124126
.requiredOption('-r, --reasoning <text>', 'Resolution reasoning')
125-
.option('--db <path>', 'Database path', '.provenant/graph.db')
127+
.option('--db <path>', 'Database path (or set PROVENANT_DB)', DB_DEFAULT)
126128
.action(logOverrideResolve);
127129

128130
// REST API server
129131
program
130132
.command('serve')
131133
.description('Start the REST API server')
132134
.option('-p, --port <port>', 'Port to listen on', '3847')
133-
.option('--db <path>', 'Database path', '.provenant/graph.db')
135+
.option('--db <path>', 'Database path (or set PROVENANT_DB)', DB_DEFAULT)
134136
.action(serve);
135137

136138
// Shadow mode calibration
@@ -139,7 +141,7 @@ program
139141
.description(
140142
'Re-score existing nodes to calibrate confidence thresholds (shadow mode)'
141143
)
142-
.option('--db <path>', 'Database path', '.provenant/graph.db')
144+
.option('--db <path>', 'Database path (or set PROVENANT_DB)', DB_DEFAULT)
143145
.option('--since <date>', 'Only calibrate nodes after this date')
144146
.option('--auto-accept <threshold>', 'Auto-accept threshold to test', '0.7')
145147
.option('--review <threshold>', 'Review threshold to test', '0.4')

0 commit comments

Comments
 (0)