Add TiDB storage backend

.gitignore

@@ -10,3 +10,4 @@ pdfs/
 docs/rendered/
 discussions/
 .DS_Store
+.gstack/

Cargo.toml

@@ -7,6 +7,7 @@ members = [
     "crates/engine",
     "crates/storage",
     "crates/storage-postgres",
+    "crates/storage-tidb",
     "crates/auth",
     "crates/server",
     "crates/bin",
@@ -24,6 +25,7 @@ extenddb-core = { path = "crates/core" }
 extenddb-engine = { path = "crates/engine" }
 extenddb-storage = { path = "crates/storage" }
 extenddb-storage-postgres = { path = "crates/storage-postgres" }
+extenddb-storage-tidb = { path = "crates/storage-tidb" }
 extenddb-auth = { path = "crates/auth" }
 extenddb-server = { path = "crates/server" }
 
@@ -51,7 +53,7 @@ hyper = { version = "1" }
 inventory = "0.3"
 
 # Database
-sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "postgres", "json", "time", "uuid", "bigdecimal"] }
+sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "postgres", "mysql", "json", "time", "uuid", "bigdecimal"] }
 
 # Crypto & checksums
 crc32fast = "1"
@@ -85,6 +87,8 @@ metrics-exporter-prometheus = "0.16"
 # Config
 clap = { version = "4", features = ["derive"] }
 config = "0.14"
+percent-encoding = "2"
+url = "2"
 
 # Security
 zeroize = { version = "1.8", features = ["derive"] }

README.md

@@ -9,7 +9,7 @@ A DynamoDB-compatible API adapter, ExtendDB speaks the DynamoDB wire protocol 
 - **Local development** — run DynamoDB workloads on your laptop with zero cloud dependency
 - **CI/CD pipelines** — deterministic integration tests against a DynamoDB-compatible backend
 - **Self-hosted deployments** — run DynamoDB workloads on your own infrastructure (on-premises, private cloud, edge)
-- **Multi-cloud** — use DynamoDB semantics on any cloud that runs PostgreSQL
+- **Multi-cloud** — use DynamoDB semantics on any cloud that runs a supported storage backend
 - **Air-gapped environments** — DynamoDB functionality with no internet connectivity
 
 ## Features
@@ -21,7 +21,7 @@ A DynamoDB-compatible API adapter, ExtendDB speaks the DynamoDB wire protocol 
 - CSRF protection, security headers, session management
 - Prometheus-compatible metrics endpoint
 - Daemon mode with syslog logging
-- PostgreSQL storage — use standard backup, replication, and HA tools
+- Pluggable storage backends — PostgreSQL by default, TiDB as an optional in-tree backend
 
 ## Quick Start
 
@@ -50,7 +50,7 @@ scripts/install-macos.sh   # macOS
 ## Prerequisites
 
 - Rust 1.85+ (`rustup update`)
-- PostgreSQL 14+ (see `docs/local-postgres-setup.md`)
+- A supported storage backend: PostgreSQL 14+ by default, or TiDB when building with the `tidb` feature
 - Python 3.10+ (for test suites and documentation)
 
 ### Python Environment
@@ -168,6 +168,7 @@ crates/
   engine/           — operation handlers
   storage/          — storage trait definitions
   storage-postgres/ — PostgreSQL backend
+  storage-tidb/     — TiDB backend
   auth/             — SigV4 verification, IAM policy engine
   server/           — HTTP server, management API, web console
   bin/              — CLI, config, daemon lifecycle

crates/bin/Cargo.toml

@@ -14,12 +14,14 @@ path = "src/main.rs"
 [features]
 default = ["postgres"]
 postgres = ["extenddb-storage-postgres"]
+tidb = ["extenddb-storage-tidb"]
 
 [dependencies]
 extenddb-core = { workspace = true }
 extenddb-engine = { workspace = true }
 extenddb-storage = { workspace = true }
 extenddb-storage-postgres = { workspace = true, optional = true }
+extenddb-storage-tidb = { workspace = true, optional = true }
 extenddb-auth = { workspace = true }
 extenddb-server = { workspace = true }
 tokio = { workspace = true }

crates/bin/src/cmd_destroy.rs

@@ -6,6 +6,7 @@
 //! Reads config, enumerates tables, requires `--yes` to confirm, drops both databases.
 
 use clap::Args;
+use extenddb_storage::bootstrapper::BootstrapOptions;
 
 use crate::config;
 
@@ -16,13 +17,13 @@ pub struct DestroyArgs {
     #[arg(short, long, default_value = "extenddb.toml")]
     config: String,
 
-    /// PostgreSQL admin user (for DROP DATABASE)
-    #[arg(long, default_value_t = config::whoami("postgres"))]
-    pg_user: String,
+    /// Storage admin user (for DROP DATABASE)
+    #[arg(long = "storage-admin-user")]
+    storage_admin_user: Option<String>,
 
-    /// PostgreSQL admin password
-    #[arg(long)]
-    pg_pass: Option<String>,
+    /// Storage admin password
+    #[arg(long = "storage-admin-password")]
+    storage_admin_password: Option<String>,
 
     /// Confirm destruction (required, no interactive prompt)
     #[arg(long)]
@@ -40,16 +41,23 @@ pub async fn run(args: DestroyArgs) -> anyhow::Result<()> {
     let app_config = config::load(&args.config)?;
     let backend = &app_config.storage._backend;
 
-    // Collect CLI args for backend-specific parsing
-    let cli_args: Vec<String> = std::env::args().collect();
+    let bootstrap_options = BootstrapOptions {
+        admin_user: args.storage_admin_user.clone(),
+        admin_password: args.storage_admin_password.clone(),
+        ..BootstrapOptions::default()
+    };
 
     println!("=== extenddb destroy ===");
     println!("Config:           {}", args.config);
     println!();
 
     // Create bootstrap store for catalog queries and database teardown.
-    let bootstrap =
-        extenddb_storage::bootstrapper::create_bootstrapper(backend, &args.config, &cli_args).await;
+    let bootstrap = extenddb_storage::bootstrapper::create_bootstrapper(
+        backend,
+        &args.config,
+        bootstrap_options.clone(),
+    )
+    .await;
 
     let mut data_db = String::new();
 
@@ -92,8 +100,7 @@ pub async fn run(args: DestroyArgs) -> anyhow::Result<()> {
     }
 
     // For drop, we need a fresh bootstrap store connected as admin (not to the
-    // catalog DB we're about to drop). The existing bootstrap store's admin pool
-    // connects to the `postgres` database, so we can reuse it.
+    // catalog DB we're about to drop).
     if !data_db.is_empty() {
         // Defense-in-depth: validate even though this came from the catalog.
         config::validate_identifier(backend, &data_db, "data database name")?;
@@ -102,10 +109,13 @@ pub async fn run(args: DestroyArgs) -> anyhow::Result<()> {
     // Reconnect as admin for DDL operations (the catalog pool must be dropped
     // before we can DROP DATABASE).
     drop(bootstrap);
-    let bootstrap =
-        extenddb_storage::bootstrapper::create_bootstrapper(backend, &args.config, &cli_args)
-            .await
-            .map_err(|e| anyhow::anyhow!("Cannot connect as admin: {e:?}"))?;
+    let bootstrap = extenddb_storage::bootstrapper::create_bootstrapper(
+        backend,
+        &args.config,
+        bootstrap_options,
+    )
+    .await
+    .map_err(|e| anyhow::anyhow!("Cannot connect as admin: {e:?}"))?;
 
     bootstrap
         .drop_databases(&data_db)

crates/bin/src/cmd_init.rs

@@ -9,15 +9,16 @@
 use std::path::Path;
 
 use clap::Args;
+use extenddb_storage::bootstrapper::BootstrapOptions;
 
 use crate::config;
 use crate::init_helpers::{generate_config, generate_tls_cert_if_needed};
 
 #[derive(Args)]
 #[allow(clippy::doc_markdown)] // Clap help text, not rustdoc
 pub struct InitArgs {
-    /// Storage backend (postgres) (default: postgres)
-    #[arg(long, default_value = "postgres")]
+    /// Storage backend name
+    #[arg(long)]
     backend: Option<String>,
 
     /// Data database name (default: extenddb)
@@ -28,21 +29,21 @@ pub struct InitArgs {
     #[arg(long)]
     catalog_db: Option<String>,
 
-    /// PostgreSQL host
-    #[arg(long)]
-    pg_host: Option<String>,
+    /// Storage host
+    #[arg(long = "storage-host")]
+    storage_host: Option<String>,
 
-    /// PostgreSQL port
-    #[arg(long)]
-    pg_port: Option<u16>,
+    /// Storage port
+    #[arg(long = "storage-port")]
+    storage_port: Option<u16>,
 
-    /// PostgreSQL admin user (for CREATE DATABASE)
-    #[arg(long)]
-    pg_user: Option<String>,
+    /// Storage admin user (for CREATE DATABASE)
+    #[arg(long = "storage-admin-user")]
+    storage_admin_user: Option<String>,
 
-    /// PostgreSQL admin password (required for remote/Aurora connections).
-    #[arg(long)]
-    pg_pass: Option<String>,
+    /// Storage admin password (required for remote connections).
+    #[arg(long = "storage-admin-password")]
+    storage_admin_password: Option<String>,
 
     /// extenddb application user
     #[arg(long)]
@@ -114,14 +115,14 @@ fn discover_docs_dir() -> Option<String> {
 
 /// Returns exit code: 0 = success, 255 = existing config preserved.
 pub async fn run(args: InitArgs) -> anyhow::Result<u8> {
-    // Determine backend: CLI flag > config file > default
+    // Determine backend: CLI flag > config file > compiled default.
     let backend = if let Some(ref b) = args.backend {
         b.clone()
     } else if Path::new(&args.config).exists() {
         let app_config = config::load(&args.config)?;
         app_config.storage._backend
     } else {
-        "postgres".to_owned()
+        config::default_backend()
     };
 
     println!("=== extenddb init (backend: {backend}) ===");
@@ -136,14 +137,23 @@ pub async fn run(args: InitArgs) -> anyhow::Result<u8> {
         return Ok(255);
     }
 
-    // Collect CLI args for backend-specific parsing
-    let cli_args: Vec<String> = std::env::args().collect();
-
     // Create bootstrapper via registry (no hardcoded match!)
-    let bootstrapper =
-        extenddb_storage::bootstrapper::create_bootstrapper(&backend, &args.config, &cli_args)
-            .await
-            .map_err(|e| anyhow::anyhow!("{e:?}"))?;
+    let bootstrapper = extenddb_storage::bootstrapper::create_bootstrapper(
+        &backend,
+        &args.config,
+        BootstrapOptions {
+            storage_host: args.storage_host.clone(),
+            storage_port: args.storage_port,
+            admin_user: args.storage_admin_user.clone(),
+            admin_password: args.storage_admin_password.clone(),
+            data_db: args.data_db.clone(),
+            catalog_db: args.catalog_db.clone(),
+            app_user: args.extenddb_user.clone(),
+            app_password: args.extenddb_pass.clone(),
+        },
+    )
+    .await
+    .map_err(|e| anyhow::anyhow!("{e:?}"))?;
 
     // Ensure application user exists.
     bootstrapper
@@ -234,9 +244,10 @@ pub async fn run(args: InitArgs) -> anyhow::Result<u8> {
         );
     }
 
-    // Extract bind_addr from CLI args
-    let bind_addr =
-        extract_arg(&cli_args, "--bind-addr").unwrap_or_else(|| "127.0.0.1".to_string());
+    let bind_addr = args
+        .bind_addr
+        .clone()
+        .unwrap_or_else(|| "127.0.0.1".to_string());
 
     // Generate self-signed TLS certificate if not already present.
     // Include the server bind address as a SAN so the cert matches the URL.
@@ -261,16 +272,17 @@ pub async fn run(args: InitArgs) -> anyhow::Result<u8> {
     if Path::new(config_path).exists() {
         std::fs::remove_file(config_path)?;
     }
-    generate_config(config_path, &catalog_url, &bind_addr, docs_dir.as_deref())?;
+    generate_config(
+        config_path,
+        &backend,
+        &catalog_url,
+        &bind_addr,
+        docs_dir.as_deref(),
+    )?;
 
     println!(
         "\n=== extenddb init complete ===\nStart the server with: extenddb serve --config {config_path}"
     );
 
     Ok(0)
 }
-
-/// Extract a CLI argument value by flag name.
-fn extract_arg(args: &[String], flag: &str) -> Option<String> {
-    args.windows(2).find(|w| w[0] == flag).map(|w| w[1].clone())
-}