sea-orm integration + rust-lightning persitence layer

.gitignore

@@ -7,7 +7,10 @@
 /tmp
 
 # will have compiled files and executables
-/target
+**/target
+
+# migration crate is a subpackage; its lock file is autogenerated
+migration/Cargo.lock
 
 # These are backup files generated by rustfmt
 **/*.rs.bk

.gitmodules

@@ -1,5 +1,5 @@
 [submodule "rust-lightning"]
 	path = rust-lightning
-	url = https://github.com/RGB-Tools/rust-lightning.git
-	branch = rgb
+	url = https://github.com/dcorral/rust-lightning.git
+	branch = persistence-layer
 	shallow = true

Cargo.toml

@@ -15,6 +15,7 @@ axum-extra = "0.9.4"
 # axum-macros = "0.4.2"  # uncomment to use debug_handler
 baid58 = "0.4.4"
 base64 = "0.22.1"
+bincode = "1"
 biscuit-auth = "6.0.0"
 bitcoin = "0.32"
 bitcoin-bech32 = "0.13"
@@ -40,6 +41,12 @@ rgb-lib = { version = "0.3.0-beta.5", features = [
     "electrum",
     "esplora",
 ] }
+rln-migration = { path = "migration" }
+sea-orm = { version = "1.1.19", default-features = false, features = [
+    "macros",
+    "runtime-tokio-rustls",
+    "sqlx-sqlite",
+] }
 scrypt = "0.11.0"
 serde = { version = "^1.0", features = ["derive"] }
 serde_json = "1.0"

migration/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "rln-migration"
+version = "0.1.0"
+edition = "2021"
+publish = false
+
+[lib]
+name = "rln_migration"
+path = "src/lib.rs"
+
+[dependencies]
+sea-orm-migration = { version = "1.1.19", default-features = false, features = [
+    "cli",
+    "runtime-tokio-rustls",
+    "sqlx-postgres",
+    "sqlx-sqlite",
+] }
+tokio = { version = "1", features = ["rt", "macros"] }

migration/README.md

@@ -0,0 +1,57 @@
+# DB migrations
+
+Every time a change to a DB object or table is needed, a migration has to be
+created.
+
+In rgb-lightning-node we use sea-orm tools to handle the DB and its migrations.
+
+To generate new migrations the [sea-orm-cli] tool is needed. You should install
+the same version that has been previously used. You can find this in the
+`src/database/entities/mod.rs` file, where the first line will specify `//!
+SeaORM Entity. Generated by sea-orm-codegen <VERSION>`. Install it with:
+```sh
+cargo install sea-orm-cli --version <VERSION>
+```
+
+Then, to generate a new migration file, run:
+```sh
+sea-orm-cli migrate generate <migration_name>
+```
+
+This command will create a new file where you'll find the `up` and `down`
+methods (see `migration/src/m20250127_000001_init_db.rs` for an example). These
+methods will be empty and will need to be implemented in order to give
+instructions on how to respectively update and revert the new changes.
+
+Once the migration file is ready, you'll need to run a local postgres DB and
+use it to refresh the migration and generate entities with `sea-orm-cli`. This
+is accomplished with:
+```sh
+docker pull postgres:latest
+
+docker run -p 127.0.0.1:5432:5432/tcp --name migration-postgres \
+    -e POSTGRES_PASSWORD=mysecretpassword -d postgres
+
+DATABASE_URL=postgres://postgres:mysecretpassword@localhost:5432 \
+    sea-orm-cli migrate up
+
+DATABASE_URL=postgres://postgres:mysecretpassword@localhost:5432 \
+    sea-orm-cli migrate refresh
+
+DATABASE_URL=postgres://postgres:mysecretpassword@localhost:5432/postgres \
+    sea-orm-cli generate entity -o src/database/entities --expanded-format
+
+docker rm -f migration-postgres
+```
+
+The command to generate entities will apply some unwanted changes, for example
+it will change the enum fields to integers and will remove some extra `derive`s
+that we manually added. Those changes will need to be discarded, so please be
+sure to add only the code that is related to the new changes you just applied.
+To do this we suggest to first refresh the migration and generate entities with
+`sea-orm-cli` on the branch you are about to apply the DB changes on. The
+generated diff will only include unwanted changes, so they can be used as a
+reference to revert them.
+
+
+[sea-orm-cli]: https://github.com/SeaQL/sea-orm/tree/master/sea-orm-cli

migration/src/lib.rs

@@ -0,0 +1,12 @@
+pub use sea_orm_migration::prelude::*;
+
+mod m20250127_000001_init_db;
+
+pub struct Migrator;
+
+#[async_trait::async_trait]
+impl MigratorTrait for Migrator {
+    fn migrations() -> Vec<Box<dyn MigrationTrait>> {
+        vec![Box::new(m20250127_000001_init_db::Migration)]
+    }
+}

migration/src/m20250127_000001_init_db.rs

@@ -0,0 +1,165 @@
+use sea_orm_migration::{prelude::*, schema::*};
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .create_table(
+                Table::create()
+                    .table(Mnemonic::Table)
+                    .if_not_exists()
+                    .col(
+                        ColumnDef::new(Mnemonic::Idx)
+                            .integer()
+                            .not_null()
+                            .primary_key(),
+                    )
+                    .col(string(Mnemonic::EncryptedMnemonic))
+                    .col(big_unsigned(Mnemonic::CreatedAt))
+                    .col(big_unsigned(Mnemonic::UpdatedAt))
+                    .to_owned(),
+            )
+            .await?;
+
+        manager
+            .create_table(
+                Table::create()
+                    .table(KvStore::Table)
+                    .if_not_exists()
+                    .col(string(KvStore::PrimaryNamespace))
+                    .col(string(KvStore::SecondaryNamespace))
+                    .col(string(KvStore::Key))
+                    .col(blob(KvStore::Value))
+                    .primary_key(
+                        Index::create()
+                            .col(KvStore::PrimaryNamespace)
+                            .col(KvStore::SecondaryNamespace)
+                            .col(KvStore::Key),
+                    )
+                    .to_owned(),
+            )
+            .await?;
+
+        manager
+            .create_table(
+                Table::create()
+                    .table(Config::Table)
+                    .if_not_exists()
+                    .col(
+                        ColumnDef::new(Config::Key)
+                            .string()
+                            .not_null()
+                            .primary_key(),
+                    )
+                    .col(ColumnDef::new(Config::Value).string().not_null())
+                    .col(ColumnDef::new(Config::CreatedAt).big_integer().not_null())
+                    .col(ColumnDef::new(Config::UpdatedAt).big_integer().not_null())
+                    .to_owned(),
+            )
+            .await?;
+
+        manager
+            .create_table(
+                Table::create()
+                    .table(RevokedToken::Table)
+                    .if_not_exists()
+                    .col(
+                        ColumnDef::new(RevokedToken::TokenId)
+                            .string()
+                            .not_null()
+                            .primary_key(),
+                    )
+                    .col(
+                        ColumnDef::new(RevokedToken::RevokedAt)
+                            .big_integer()
+                            .not_null(),
+                    )
+                    .to_owned(),
+            )
+            .await?;
+
+        manager
+            .create_table(
+                Table::create()
+                    .table(ChannelPeer::Table)
+                    .if_not_exists()
+                    .col(
+                        ColumnDef::new(ChannelPeer::Pubkey)
+                            .string()
+                            .not_null()
+                            .primary_key(),
+                    )
+                    .col(ColumnDef::new(ChannelPeer::Address).string().not_null())
+                    .col(
+                        ColumnDef::new(ChannelPeer::CreatedAt)
+                            .big_integer()
+                            .not_null(),
+                    )
+                    .to_owned(),
+            )
+            .await
+    }
+
+    async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .drop_table(Table::drop().table(ChannelPeer::Table).to_owned())
+            .await?;
+        manager
+            .drop_table(Table::drop().table(RevokedToken::Table).to_owned())
+            .await?;
+        manager
+            .drop_table(Table::drop().table(Config::Table).to_owned())
+            .await?;
+        manager
+            .drop_table(Table::drop().table(KvStore::Table).to_owned())
+            .await?;
+        manager
+            .drop_table(Table::drop().table(Mnemonic::Table).to_owned())
+            .await
+    }
+}
+
+#[derive(DeriveIden)]
+enum Mnemonic {
+    Table,
+    Idx,
+    EncryptedMnemonic,
+    CreatedAt,
+    UpdatedAt,
+}
+
+#[derive(DeriveIden)]
+enum KvStore {
+    Table,
+    PrimaryNamespace,
+    SecondaryNamespace,
+    Key,
+    Value,
+}
+
+#[derive(DeriveIden)]
+enum Config {
+    Table,
+    Key,
+    Value,
+    CreatedAt,
+    UpdatedAt,
+}
+
+#[derive(DeriveIden)]
+enum RevokedToken {
+    Table,
+    TokenId,
+    RevokedAt,
+}
+
+#[derive(DeriveIden)]
+enum ChannelPeer {
+    Table,
+    Pubkey,
+    Address,
+    CreatedAt,
+}

migration/src/main.rs

@@ -0,0 +1,6 @@
+use sea_orm_migration::prelude::*;
+
+#[tokio::main]
+async fn main() {
+    cli::run_cli(rln_migration::Migrator).await;
+}