Feat/litellm unified client

Makefile

@@ -118,6 +118,10 @@ migrate-upgrade:
 	export FLASK_APP="$(CURDIR)/cre.py" 
 	flask db upgrade  
 
+alembic-guardrail:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	python scripts/check_alembic_revision_guardrail.py
+
 migrate-downgrade:
 	[ -d "./venv" ] && . ./venv/bin/activate &&\
 	export FLASK_APP="$(CURDIR)/cre.py"

Procfile

@@ -1,2 +1,3 @@
+release: python scripts/check_alembic_revision_guardrail.py
 web: gunicorn cre:app
 worker: FLASK_APP=`pwd`/cre.py python cre.py --start_worker

README.md

@@ -143,15 +143,30 @@ To run only missing gap-analysis pair backfill (without starting Flask), use:
 RUN_COUNT=8 bash scripts/backfill_gap_analysis.sh
 ```
 
-To sync local Postgres data into a Heroku app (staging or prod), use:
+### Production DB Operations (opencreorg)
+
+Prefer the dedicated scripts in `scripts/db/` for production operations. These scripts enforce safety guards and always capture a fresh backup before DB changes.
+
+- Backup only:
+  - `APP_NAME=opencreorg scripts/db/backup-opencreorg.sh`
+- Sync local Postgres to Heroku:
+  - `APP_NAME=opencreorg SOURCE_DB_URL="postgresql://cre:password@127.0.0.1:5432/cre" scripts/db/sync-local-to-opencreorg.sh`
+- Targeted SQL surgery:
+  - `APP_NAME=opencreorg scripts/db/surgery-opencreorg.sh --sql-file ./tmp/change.sql`
+
+For destructive surgery (`DELETE`, `DROP`, `TRUNCATE`, irreversible `ALTER`), use:
 
 ```bash
-APP_NAME=stagingopencreorg \
-SOURCE_DB_URL="postgresql://cre:password@127.0.0.1:5432/cre" \
-SYNC_TABLES=gap_analysis \
-bash scripts/push-local-postgres-to-heroku.sh --gap_analysis
+APP_NAME=opencreorg \
+CONFIRM_DESTRUCTIVE=I_UNDERSTAND_OPENCREORG_PROD_DB_DESTRUCTIVE_ACTION \
+scripts/db/surgery-opencreorg.sh --sql-file ./tmp/destructive-change.sql --destructive
 ```
 
+Runbooks:
+
+- `docs/runbooks/opencreorg-db-sync-and-surgery.md`
+- `docs/runbooks/opencreorg-db-destructive-ops-checklist.md`
+
 Environment variables for app to connect to neo4jDB (default):
 
 * `NEO4J_URL` (neo4j//neo4j:password@localhost:7687)
@@ -268,13 +283,45 @@ Then edit `.env` and provide values appropriate for your environment.
 * Neo4j: `NEO4J_URL`
 * Redis: `REDIS_HOST`, `REDIS_PORT`, `REDIS_URL`, `REDIS_NO_SSL`
 * Flask: `FLASK_CONFIG`, `INSECURE_REQUESTS`
-* Embeddings: `NO_GEN_EMBEDDINGS`
+* Embeddings: `NO_GEN_EMBEDDINGS`, `CRE_EMBED_MODEL`, `CRE_EMBED_EXPECTED_DIM`, `CRE_VALIDATE_EMBED_DIM_ON_INIT`
+* LLM models/retries: `CRE_LLM_CHAT_MODEL`, `CRE_EMBED_ALIGN_MODEL`, `CRE_LLM_MAX_RETRIES`, `CRE_LLM_RETRY_SLEEP_SECONDS`
+* Provider credentials: `OPENAI_API_KEY`, `GEMINI_API_KEY`, `GCP_NATIVE`
 * Google Auth: `GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET`, `GOOGLE_SECRET_JSON`, `LOGIN_ALLOWED_DOMAINS`
 * GCP: `GCP_NATIVE`
 * Spreadsheet Auth: `OpenCRE_gspread_Auth`
 
 See `.env.example` for full list and defaults.
 
+### LiteLLM backend (optional)
+
+OpenCRE uses LiteLLM for LLM calls. Configure models and provider credentials via environment variables.
+
+Recommended minimal example:
+
+```bash
+# Chat / completion models (LiteLLM model strings)
+CRE_LLM_CHAT_MODEL=gemini/gemini-2.5-flash
+CRE_EMBED_ALIGN_MODEL=gemini/gemini-2.5-flash
+
+# Embedding model used for persisted vectors
+CRE_EMBED_MODEL=gemini/gemini-embedding-001
+CRE_EMBED_EXPECTED_DIM=3072
+CRE_VALIDATE_EMBED_DIM_ON_INIT=1
+
+# Retry policy
+CRE_LLM_MAX_RETRIES=2
+CRE_LLM_RETRY_SLEEP_SECONDS=15
+
+# Provider credential (example for Gemini)
+GEMINI_API_KEY=your-key
+```
+
+Notes:
+
+* Treat changes to `CRE_EMBED_MODEL` or `CRE_EMBED_EXPECTED_DIM` as a data migration event (usually requires re-embedding).
+* `CRE_EMBED_EXPECTED_DIM` is a safety guard: writes fail fast on dimension mismatch.
+* Keep chat/alignment models and embedding model independently configurable; only embeddings must remain dimension-compatible with stored vectors.
+
 You can run the containers with:
 
 ```bash

application/database/db.py

@@ -188,6 +188,8 @@ class Embeddings(BaseModel):  # type: ignore
 
     embeddings_url = sqla.Column(sqla.String, nullable=True, default=None)
     embeddings_content = sqla.Column(sqla.String, nullable=True, default=None)
+    embedding_model_id = sqla.Column(sqla.String, nullable=True, default=None)
+    embedding_dim = sqla.Column(sqla.Integer, nullable=True, default=None)
 
 
 class GapAnalysisResults(BaseModel):
@@ -2286,6 +2288,18 @@ def add_embedding(
         For nodes, ``embeddings_url`` is the resolved URL used for fetch/embed alignment
         (may include a fragment). When ``None``, defaults to ``db_object.link`` (importer hyperlink).
         """
+        expected_dim_raw = (os.environ.get("CRE_EMBED_EXPECTED_DIM", "") or "").strip()
+        embedding_model_id = (
+            os.environ.get("CRE_EMBED_MODEL", "") or ""
+        ).strip() or "gemini/gemini-embedding-001"
+        embedding_dim = len(embeddings)
+        if expected_dim_raw:
+            expected_dim = int(expected_dim_raw)
+            if len(embeddings) != expected_dim:
+                raise ValueError(
+                    f"embedding dimension mismatch for {db_object.id}: "
+                    f"expected {expected_dim}, got {len(embeddings)}"
+                )
         existing = self.get_embedding(db_object.id)
         embeddings_str = ",".join([str(e) for e in embeddings])
         resolved_node_url: Optional[str] = None
@@ -2302,6 +2316,8 @@ def add_embedding(
                     cre_id=db_object.id,
                     doc_type=cre_defs.Credoctypes.CRE.value,
                     embeddings_content=embedding_text,
+                    embedding_model_id=embedding_model_id,
+                    embedding_dim=embedding_dim,
                 )
             else:
                 emb = Embeddings(
@@ -2310,6 +2326,8 @@ def add_embedding(
                     doc_type=db_object.ntype,
                     embeddings_content=embedding_text,
                     embeddings_url=resolved_node_url,
+                    embedding_model_id=embedding_model_id,
+                    embedding_dim=embedding_dim,
                 )
             self.session.add(emb)
             self.session.commit()
@@ -2318,6 +2336,8 @@ def add_embedding(
             logger.debug(f"knew of embedding for object {db_object.id} ,updating")
             existing[0].embeddings = embeddings_str
             existing[0].embeddings_content = embedding_text
+            existing[0].embedding_model_id = embedding_model_id
+            existing[0].embedding_dim = embedding_dim
             if doctype != cre_defs.Credoctypes.CRE:
                 if embeddings_url is not None:
                     existing[0].embeddings_url = embeddings_url
@@ -2327,6 +2347,57 @@ def add_embedding(
 
             return existing
 
+    def assert_embedding_contract(
+        self,
+        *,
+        expected_model_id: Optional[str],
+        expected_dim: Optional[int],
+    ) -> None:
+        """
+        Validate persisted embedding metadata consistency.
+
+        - Fails when multiple dimensions are stored.
+        - Fails when metadata is missing or mismatched against expected model/dimension.
+        """
+        rows = self.session.query(
+            Embeddings.embedding_dim, Embeddings.embedding_model_id
+        ).all()
+        if not rows:
+            return
+
+        dims = {int(r[0]) for r in rows if r[0] is not None}
+        model_ids = {str(r[1]) for r in rows if r[1]}
+        has_missing_dim = any(r[0] is None for r in rows)
+        has_missing_model = any(not r[1] for r in rows)
+
+        if len(dims) > 1:
+            raise RuntimeError(
+                f"multiple embedding dimensions detected in DB: {sorted(dims)}"
+            )
+        if len(model_ids) > 1:
+            raise RuntimeError(
+                f"multiple embedding models detected in DB: {sorted(model_ids)}"
+            )
+
+        if has_missing_dim or has_missing_model:
+            raise RuntimeError(
+                "embedding metadata missing in DB; run metadata migration/backfill"
+            )
+
+        if expected_dim is not None and dims:
+            db_dim = next(iter(dims))
+            if db_dim != expected_dim:
+                raise RuntimeError(
+                    f"DB embedding dim {db_dim} does not match expected dim {expected_dim}"
+                )
+
+        if expected_model_id and model_ids:
+            db_model = next(iter(model_ids))
+            if db_model != expected_model_id:
+                raise RuntimeError(
+                    f"DB embedding model {db_model} does not match expected model {expected_model_id}"
+                )
+
     def gap_analysis_exists(self, cache_key) -> bool:
         row = (
             self.session.query(GapAnalysisResults)

application/prompt_client/llm_error_utils.py

@@ -0,0 +1,29 @@
+from typing import Any
+
+
+def is_rate_limit_error(err: BaseException) -> bool:
+    msg = str(err).lower()
+    if "rate limit" in msg or "too many requests" in msg:
+        return True
+    if "resource exhausted" in msg or "quota" in msg or "exceeded quota" in msg:
+        return True
+    if "429" in msg:
+        return True
+
+    status = (
+        getattr(err, "status", None)
+        or getattr(err, "status_code", None)
+        or getattr(err, "http_status", None)
+        or getattr(err, "code", None)
+    )
+    if status == 429:
+        return True
+
+    if isinstance(getattr(err, "args", None), tuple):
+        # Some SDKs nest details in args[0]/args[1].
+        nested: Any = err.args[0] if err.args else None
+        if isinstance(nested, dict):
+            code = nested.get("code") or nested.get("status_code")
+            if code == 429:
+                return True
+    return False