feat(phase5): epistemic confidence model

CHANGELOG.md

@@ -2,6 +2,46 @@
 
 All notable changes to MemForge are documented here.
 
+## [Unreleased] — Epistemic Confidence Model + Memory Sentiment Tagging + Adaptive Sleep Intelligence
+
+### Added (Epistemic Confidence Model — F1)
+
+- **Epistemic Confidence Model (F1)** — calibrated uncertainty levels on warm-tier
+  memories. New columns on `warm_tier`: `epistemic_status TEXT NOT NULL DEFAULT 'provisional'`,
+  `evidence_count INTEGER NOT NULL DEFAULT 1`, `last_corroborated_at TIMESTAMPTZ`.
+  Index `warm_tier_epistemic_idx` on `(agent_id, epistemic_status)`.
+  New types `EpistemicStatus` (`established | provisional | contested | deprecated | inferred`)
+  and `EpistemicFilter` (`only_established | include_provisional | include_contested | all`)
+  in `src/types.ts`. New Zod schema `EpistemicFilterSchema` in `src/schemas.ts`.
+
+- **`GET /memory/:id/epistemic`** — returns counts of warm-tier memories per
+  `epistemic_status`. All five values always present, defaulting to 0.
+  New `getEpistemicProfile(agentId)` method on `MemoryManager`.
+  Client SDK methods `epistemicProfile(agentId)` on `MemForgeClient` and
+  `ResilientMemForgeClient`. Python SDK `epistemic_profile(agent_id)` on
+  `MemForgeClient`. OpenAPI entry added.
+
+- **Epistemic query filter** — `query()` now accepts `epistemic?: EpistemicFilter`.
+  REST `GET /memory/:id/query` accepts `?epistemic=` query param validated against
+  `EpistemicFilterSchema`. `QueryResult` now carries `epistemic_status` and
+  `evidence_count` fields. TypeScript SDK `query()` accepts `epistemic` option.
+  Python SDK `query()` accepts `epistemic` kwarg.
+
+- **Sleep Phase 5.12: Epistemic Promotion** — new `phaseEpistemicPromotion(agentId)`
+  method on `SleepCycleEngine`, wired into `run()` between Phase 5.10 and Phase 5.8.
+  Promotes `provisional → established` when `evidence_count >= 3` AND the memory
+  has been retrieved positively from at least 2 distinct namespaces in `retrieval_log`.
+  Demotes `established → provisional` when `staleness_score > 0.7` and not accessed
+  in 30 days. Stamps `last_corroborated_at` on promotion. Return count exposed as
+  `epistemic_promoted` on `SleepCycleResult`.
+
+- **MCP tools** — `memforge_certainty` (query with epistemic filter) and
+  `memforge_epistemic_profile` (get status counts) added to `src/mcp.ts` and
+  `src/tool-definitions.ts`.
+
+- **Migration** — `schema/migration-v3.9.sql` (idempotent `IF NOT EXISTS`).
+  `schema/schema.sql` updated as canonical from-scratch schema.
+
 ## [Unreleased] — Memory Sentiment Tagging + Adaptive Sleep Intelligence
 
 ### Added

package.json

@@ -45,10 +45,11 @@
     "test:dreams-compat": "node --import tsx/esm --test tests/dreams-compat.test.ts",
     "test:dreams-anthropic": "node --import tsx/esm --test tests/dreams-anthropic.test.ts",
     "test:dreams-bridge": "node --import tsx/esm --test tests/dreams-bridge.test.ts",
-    "test": "node --import tsx/esm --test --test-concurrency=1 tests/integration.test.ts tests/llm-paths.test.ts tests/http-api.test.ts tests/cache.test.ts tests/embedding-migration.test.ts tests/outcome-revision.test.ts tests/reflection-revision.test.ts tests/selective-forgetting.test.ts tests/multi-device.test.ts tests/dream-runs.test.ts tests/dreams-compat.test.ts tests/dreams-anthropic.test.ts tests/dreams-bridge.test.ts tests/sentiment-tagging.test.ts tests/adaptive-sleep.test.ts",
+    "test": "node --import tsx/esm --test --test-concurrency=1 tests/integration.test.ts tests/llm-paths.test.ts tests/http-api.test.ts tests/cache.test.ts tests/embedding-migration.test.ts tests/outcome-revision.test.ts tests/reflection-revision.test.ts tests/selective-forgetting.test.ts tests/multi-device.test.ts tests/dream-runs.test.ts tests/dreams-compat.test.ts tests/dreams-anthropic.test.ts tests/dreams-bridge.test.ts tests/sentiment-tagging.test.ts tests/adaptive-sleep.test.ts tests/epistemic-confidence.test.ts",
     "test:multi-device": "node --import tsx/esm --test tests/multi-device.test.ts",
     "test:sentiment-tagging": "node --import tsx/esm --test tests/sentiment-tagging.test.ts",
     "test:adaptive-sleep": "node --import tsx/esm --test tests/adaptive-sleep.test.ts",
+    "test:epistemic-confidence": "node --import tsx/esm --test tests/epistemic-confidence.test.ts",
     "benchmark:longmemeval": "node --import tsx/esm benchmarks/longmemeval/run.ts",
     "benchmark:download": "node --import tsx/esm benchmarks/longmemeval/download.ts",
     "benchmark:ingest": "node --import tsx/esm benchmarks/longmemeval/ingest.ts",

python/memforge/client.py

@@ -126,8 +126,15 @@ async def query(
         decay: float | None = None,
         max_tokens: int | None = None,
         namespace: str | None = None,
+        epistemic: str | None = None,
     ) -> list[QueryResult]:
-        """Search warm-tier memory."""
+        """Search warm-tier memory.
+
+        Args:
+            epistemic: Restrict results by calibrated uncertainty level.
+                One of 'only_established', 'include_provisional',
+                'include_contested', or 'all'. Defaults to no filter.
+        """
         params: dict[str, Any] = {"q": q, "limit": limit}
         if mode:
             params["mode"] = mode
@@ -141,6 +148,8 @@ async def query(
             params["max_tokens"] = max_tokens
         if namespace:
             params["namespace"] = namespace
+        if epistemic:
+            params["epistemic"] = epistemic
         raw = await self._get(f"/memory/{agent_id}/query", params)
         return [QueryResult(**r) for r in raw] if isinstance(raw, list) else []
 
@@ -270,6 +279,17 @@ async def memory_health(self, agent_id: str) -> MemoryHealth:
         raw = await self._get(f"/memory/{agent_id}/health")
         return MemoryHealth(**raw)
 
+    # ── Epistemic Confidence Model (v3.9) ─────────────────────────────────
+
+    async def epistemic_profile(self, agent_id: str) -> dict[str, int]:
+        """Return the count of warm-tier memories per epistemic_status.
+
+        All five status values (established, provisional, contested,
+        deprecated, inferred) are always present, defaulting to 0.
+        """
+        raw = await self._get(f"/memory/{agent_id}/epistemic")
+        return raw if isinstance(raw, dict) else {}
+
     async def resume(self, agent_id: str, limit: int = 5, namespace: str | None = None) -> ResumeContext:
         """Get session resumption context bundle."""
         params: dict[str, Any] = {"limit": limit}

schema/migration-v3.9.sql

@@ -0,0 +1,32 @@
+-- MemForge — Migration v3.9: Epistemic Confidence Model
+--
+-- Feature 1 of the Phase 5 Autonomous Knowledge Architecture split.
+--
+-- Adds calibrated uncertainty levels to warm-tier memories:
+--   established   — corroborated by multiple positive retrievals across sessions
+--   provisional   — default; accepted but not yet confirmed
+--   contested     — contradicted by a conflicting memory
+--   inferred      — derived by the sleep cycle, not directly observed
+--   deprecated    — superseded or stale; retained for audit purposes
+--
+-- Sleep Phase 5.12 (phaseEpistemicPromotion) runs each cycle and automatically
+-- promotes provisional → established when evidence_count >= 3 and the memory
+-- has been retrieved positively from at least 2 distinct namespaces.
+-- It also demotes established → provisional when staleness_score > 0.7 and
+-- the row has not been accessed in 30 days.
+--
+-- Apply: psql "$DATABASE_URL" -f schema/migration-v3.9.sql
+
+BEGIN;
+
+-- ─── Feature 1: Epistemic Confidence Model ──────────────────────────────────
+
+ALTER TABLE warm_tier
+  ADD COLUMN IF NOT EXISTS epistemic_status     TEXT    NOT NULL DEFAULT 'provisional',
+  ADD COLUMN IF NOT EXISTS evidence_count       INTEGER NOT NULL DEFAULT 1,
+  ADD COLUMN IF NOT EXISTS last_corroborated_at TIMESTAMPTZ;
+
+CREATE INDEX IF NOT EXISTS warm_tier_epistemic_idx
+  ON warm_tier (agent_id, epistemic_status);
+
+COMMIT;

schema/schema.sql

@@ -63,6 +63,7 @@ CREATE INDEX IF NOT EXISTS hot_tier_session_idx      ON hot_tier (agent_id, name
 -- v2.6: surprise_score, staleness_score, last_corroborated
 -- v3.1: namespace
 -- v3.8: context_signals (merged from contributing hot rows at consolidation time)
+-- v3.9: epistemic_status, evidence_count, last_corroborated_at (epistemic confidence model)
 -- ─────────────────────────────────────────────────────────────────────────────
 CREATE TABLE IF NOT EXISTS warm_tier (
   id                          BIGSERIAL   PRIMARY KEY,
@@ -110,7 +111,13 @@ CREATE TABLE IF NOT EXISTS warm_tier (
   -- per-session tracking, distinct from the literal 'default' session.
   session_id                  TEXT,
   -- Sentiment tagging (v3.8) — merged from contributing hot rows: urgency=max, others=majority
-  context_signals             JSONB       NOT NULL DEFAULT '{}'
+  context_signals             JSONB       NOT NULL DEFAULT '{}',
+  -- Epistemic confidence model (v3.9) — calibrated uncertainty level for this memory
+  epistemic_status            TEXT        NOT NULL DEFAULT 'provisional',
+  -- Number of positive retrieval events corroborating this memory
+  evidence_count              INTEGER     NOT NULL DEFAULT 1,
+  -- Timestamp of the most recent promotion to 'established' by Phase 5.12
+  last_corroborated_at        TIMESTAMPTZ
 );
 
 CREATE INDEX IF NOT EXISTS warm_tier_agent_id_idx      ON warm_tier (agent_id);
@@ -130,6 +137,7 @@ CREATE INDEX IF NOT EXISTS warm_tier_time_idx       ON warm_tier (agent_id, time
 CREATE INDEX IF NOT EXISTS warm_tier_importance_idx ON warm_tier (agent_id, importance DESC);
 CREATE INDEX IF NOT EXISTS warm_tier_namespace_idx  ON warm_tier (agent_id, namespace);
 CREATE INDEX IF NOT EXISTS warm_tier_session_idx    ON warm_tier (agent_id, namespace, session_id) WHERE session_id IS NOT NULL;
+CREATE INDEX IF NOT EXISTS warm_tier_epistemic_idx  ON warm_tier (agent_id, epistemic_status);
 
 -- ─────────────────────────────────────────────────────────────────────────────
 -- cold_tier — archived / cleared memory (audit trail, never hard-deleted)

src/app.ts

@@ -18,7 +18,7 @@ import {
   httpRequestDurationSeconds,
 } from './metrics.js';
 import { bearerAuth, requireScope, getClientId } from './auth.js';
-import { NamespaceSchema, AddMemorySchema, ConsolidateSchema, SleepSchema, ColdTierSearchSchema, ColdTierRestoreSchema, ConfigReloadSchema, CreateDreamRunSchema, ListDreamRunsQuerySchema, AnthropicDreamCreateSchema, AnthropicPushSchema, AnthropicPullSchema } from './schemas.js';
+import { NamespaceSchema, AddMemorySchema, ConsolidateSchema, SleepSchema, ColdTierSearchSchema, ColdTierRestoreSchema, ConfigReloadSchema, CreateDreamRunSchema, ListDreamRunsQuerySchema, AnthropicDreamCreateSchema, AnthropicPushSchema, AnthropicPullSchema, EpistemicFilterSchema } from './schemas.js';
 import { reloadConfig } from './config.js';
 import {
   cacheGet,
@@ -432,7 +432,7 @@ export function createApp(deps: AppDependencies): express.Express {
   });
 
   /**
-   * GET /memory/:agentId/query?q=<text>[&limit=<n>][&mode=keyword|semantic|hybrid][&after=<iso>][&before=<iso>][&decay=<rate>]
+   * GET /memory/:agentId/query?q=<text>[&limit=<n>][&mode=keyword|semantic|hybrid][&after=<iso>][&before=<iso>][&decay=<rate>][&epistemic=only_established|include_provisional|include_contested|all]
    */
   app.get('/memory/:agentId/query', requireScope('memforge:read'), async (req: Request, res: Response) => {
     const q = qstr(req.query['q']);
@@ -443,6 +443,7 @@ export function createApp(deps: AppDependencies): express.Express {
     const decay = qstr(req.query['decay']);
     const maxTokens = qstr(req.query['max_tokens']);
     const rawNamespace = qstr(req.query['namespace']);
+    const rawEpistemic = qstr(req.query['epistemic']);
 
     if (!q) {
       fail(res, 400, '"q" query param (string) is required');
@@ -493,6 +494,16 @@ export function createApp(deps: AppDependencies): express.Express {
       namespace = nsResult.data;
     }
 
+    let epistemic: import('./types.js').EpistemicFilter | undefined;
+    if (rawEpistemic !== undefined) {
+      const epistemicResult = EpistemicFilterSchema.safeParse(rawEpistemic);
+      if (!epistemicResult.success) {
+        fail(res, 400, `Invalid epistemic filter: must be one of only_established, include_provisional, include_contested, all`);
+        return;
+      }
+      epistemic = epistemicResult.data;
+    }
+
     let agentId: string;
     try {
       agentId = getAgentId(req);
@@ -501,8 +512,8 @@ export function createApp(deps: AppDependencies): express.Express {
       return;
     }
 
-    // Cache key includes all query parameters (including max_tokens to prevent budget mismatch)
-    const cacheKeySuffix = `${mode ?? 'auto'}:${after ?? ''}:${before ?? ''}:${decay ?? ''}:${maxTokensNum ?? ''}:${namespace ?? ''}`;
+    // Cache key includes all query parameters (including epistemic filter to prevent result mismatch)
+    const cacheKeySuffix = `${mode ?? 'auto'}:${after ?? ''}:${before ?? ''}:${decay ?? ''}:${maxTokensNum ?? ''}:${namespace ?? ''}:${epistemic ?? ''}`;
     const key = searchKey(agentId, `${q}:${cacheKeySuffix}`, limitNum);
     const cached = await cacheGet(key);
     if (cached !== null) {
@@ -523,6 +534,7 @@ export function createApp(deps: AppDependencies): express.Express {
         decayRate,
         maxTokens: maxTokensNum,
         namespace,
+        epistemic,
       });
       void cacheSet(key, results, 'search');
       ok(res, results);
@@ -868,6 +880,29 @@ export function createApp(deps: AppDependencies): express.Express {
     }
   });
 
+  // ─── Epistemic Confidence Model (v3.9) ──────────────────────────────────────
+
+  /**
+   * GET /memory/:agentId/epistemic
+   *
+   * Returns counts of warm-tier memories by epistemic_status.
+   * All five status values (established, provisional, contested, deprecated,
+   * inferred) are always present, defaulting to 0 when empty.
+   */
+  app.get('/memory/:agentId/epistemic', requireScope('memforge:read'), async (req: Request, res: Response) => {
+    try {
+      const profile = await manager.getEpistemicProfile(getAgentId(req));
+      ok(res, profile);
+    } catch (err) {
+      const e = err as Error;
+      if (e instanceof TypeError) {
+        fail(res, 400, e.message);
+      } else {
+        fail(res, 500, e.message);
+      }
+    }
+  });
+
   // ─── Dream runs (Claude Dreaming compatibility, v3.6) ────────────────────
   // Async sleep-cycle job model — first-class run records with status polling
   // and cancellation. The synchronous /sleep route is kept for back-compat;

src/client.ts

@@ -120,6 +120,8 @@ export class MemForgeClient {
     before?: string;
     decay?: number;
     namespace?: string;
+    /** Restrict results by epistemic confidence level (v3.9). */
+    epistemic?: 'only_established' | 'include_provisional' | 'include_contested' | 'all';
   }): Promise<QueryResult[]> {
     const params = new URLSearchParams({ q: options.q });
     if (options.limit !== undefined) params.set('limit', String(options.limit));
@@ -128,6 +130,7 @@ export class MemForgeClient {
     if (options.before) params.set('before', options.before);
     if (options.decay !== undefined) params.set('decay', String(options.decay));
     if (options.namespace) params.set('namespace', options.namespace);
+    if (options.epistemic) params.set('epistemic', options.epistemic);
     return this.get<QueryResult[]>(`/memory/${enc(agentId)}/query?${params}`);
   }
 
@@ -363,6 +366,16 @@ export class MemForgeClient {
     return this.get<MemoryHealth>(`/memory/${enc(agentId)}/health`);
   }
 
+  // ─── Epistemic Confidence Model (v3.9) ───────────────────────────────────
+
+  /**
+   * Returns the count of warm-tier memories per epistemic_status.
+   * All five status values are always present, defaulting to 0 when empty.
+   */
+  async epistemicProfile(agentId: string): Promise<Record<string, number>> {
+    return this.get<Record<string, number>>(`/memory/${enc(agentId)}/epistemic`);
+  }
+
   /** Generate a session resumption context for an agent. */
   async resume(agentId: string, limit?: number, namespace?: string): Promise<ResumeContext> {
     const params = new URLSearchParams();
@@ -629,7 +642,7 @@ export class ResilientMemForgeClient {
     return this.safe('add', () => this.client.add(agentId, content, metadata, namespace, sessionId), null);
   }
 
-  async query(agentId: string, options: { q: string; limit?: number; mode?: QueryMode; after?: string; before?: string; decay?: number; namespace?: string }): Promise<QueryResult[]> {
+  async query(agentId: string, options: Parameters<MemForgeClient['query']>[1]): Promise<QueryResult[]> {
     return this.safe('query', () => this.client.query(agentId, options), []);
   }
 
@@ -677,6 +690,10 @@ export class ResilientMemForgeClient {
     return this.safe('memoryHealth', () => this.client.memoryHealth(agentId), null);
   }
 
+  async epistemicProfile(agentId: string): Promise<Record<string, number> | null> {
+    return this.safe('epistemicProfile', () => this.client.epistemicProfile(agentId), null);
+  }
+
   async resume(agentId: string, limit?: number, namespace?: string): Promise<ResumeContext | null> {
     return this.safe('resume', () => this.client.resume(agentId, limit, namespace), null);
   }