Skip to content

chore: store large numbers as text in sqlite #45

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
SwenSchaeferjohann wants to merge 4 commits into from
Closed

chore: store large numbers as text in sqlite #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2d4a193
parse ctoken accounts explicitly
SwenSchaeferjohann Nov 23, 2025
21c40cb
bump
SwenSchaeferjohann Nov 23, 2025
3dfde4f
TEXT for sqlite
SwenSchaeferjohann Nov 27, 2025
743bebe
clean
SwenSchaeferjohann Nov 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,6 @@ test-ledger/
minio
test.db
docker-compose.yml

.cursor
**/photon.log
2 changes: 1 addition & 1 deletion Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ name = "photon-indexer"
publish = true
readme = "README.md"
repository = "https://github.com/helius-labs/photon"
version = "0.51.0"
version = "0.51.1"

[[bin]]
name = "photon"
Expand Down
40 changes: 28 additions & 12 deletions src/api/method/get_compressed_account_balance.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
use super::super::error::PhotonApiError;
use super::utils::CompressedAccountRequest;
use super::utils::{parse_decimal, AccountBalanceResponse, AccountDataTable, LamportModel};
use super::utils::{
is_sqlite, parse_balance_string, parse_decimal, AccountBalanceResponse, AccountDataTable,
LamportModel, LamportModelString,
};
use crate::common::typedefs::context::Context;
use crate::common::typedefs::unsigned_integer::UnsignedInteger;
use crate::dao::generated::accounts;
use sea_orm::{DatabaseConnection, EntityTrait, QueryFilter, QuerySelect};
use sqlx::types::Decimal;

pub async fn get_compressed_account_balance(
conn: &DatabaseConnection,
Expand All @@ -14,18 +16,32 @@ pub async fn get_compressed_account_balance(
let context = Context::extract(conn).await?;
let id = request.parse_id()?;

let balance = accounts::Entity::find()
.select_only()
.column(accounts::Column::Lamports)
.filter(id.filter(AccountDataTable::Accounts))
.into_model::<LamportModel>()
.one(conn)
.await?
.map(|x| x.lamports)
.unwrap_or(Decimal::from(0));
let balance = if is_sqlite(conn) {
accounts::Entity::find()
.select_only()
.column(accounts::Column::Lamports)
.filter(id.filter(AccountDataTable::Accounts))
.into_model::<LamportModelString>()
.one(conn)
.await?
.map(|x| parse_balance_string(&x.lamports))
.transpose()?
.unwrap_or(0)
} else {
accounts::Entity::find()
.select_only()
.column(accounts::Column::Lamports)
.filter(id.filter(AccountDataTable::Accounts))
.into_model::<LamportModel>()
.one(conn)
.await?
.map(|x| parse_decimal(x.lamports))
.transpose()?
.unwrap_or(0)
};

Ok(AccountBalanceResponse {
value: UnsignedInteger(parse_decimal(balance)?),
value: UnsignedInteger(balance),
context,
})
}
38 changes: 27 additions & 11 deletions src/api/method/get_compressed_balance_by_owner.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
use super::super::error::PhotonApiError;
use super::utils::{parse_decimal, AccountBalanceResponse, LamportModel};
use super::utils::{
is_sqlite, parse_balance_string, parse_decimal, AccountBalanceResponse, LamportModel,
LamportModelString,
};
use crate::common::typedefs::context::Context;
use crate::common::typedefs::serializable_pubkey::SerializablePubkey;
use crate::common::typedefs::unsigned_integer::UnsignedInteger;
Expand All @@ -21,16 +24,29 @@ pub async fn get_compressed_balance_by_owner(
let context = Context::extract(conn).await?;
let owner = request.owner;

let balances = owner_balances::Entity::find()
.select_only()
.column(owner_balances::Column::Lamports)
.filter(owner_balances::Column::Owner.eq::<Vec<u8>>(owner.into()))
.into_model::<LamportModel>()
.all(conn)
.await?
.iter()
.map(|x| parse_decimal(x.lamports))
.collect::<Result<Vec<u64>, PhotonApiError>>()?;
let balances = if is_sqlite(conn) {
owner_balances::Entity::find()
.select_only()
.column(owner_balances::Column::Lamports)
.filter(owner_balances::Column::Owner.eq::<Vec<u8>>(owner.into()))
.into_model::<LamportModelString>()
.all(conn)
.await?
.iter()
.map(|x| parse_balance_string(&x.lamports))
.collect::<Result<Vec<u64>, PhotonApiError>>()?
} else {
owner_balances::Entity::find()
.select_only()
.column(owner_balances::Column::Lamports)
.filter(owner_balances::Column::Owner.eq::<Vec<u8>>(owner.into()))
.into_model::<LamportModel>()
.all(conn)
.await?
.iter()
.map(|x| parse_decimal(x.lamports))
.collect::<Result<Vec<u64>, PhotonApiError>>()?
};

let total_balance = balances.iter().sum::<u64>();

Expand Down
59 changes: 44 additions & 15 deletions src/api/method/get_compressed_mint_token_holders.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,10 @@ use crate::common::typedefs::unsigned_integer::UnsignedInteger;
use crate::dao::generated::token_owner_balances;

use super::super::error::PhotonApiError;
use super::utils::{parse_decimal, PAGE_LIMIT};
use super::utils::{
is_sqlite, parse_balance_string, parse_decimal, OwnerBalanceModel, OwnerBalanceModelString,
PAGE_LIMIT,
};

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
pub struct OwnerBalance {
Expand Down Expand Up @@ -78,21 +81,47 @@ pub async fn get_compressed_mint_token_holders(
}
let limit = limit.map(|l| l.value()).unwrap_or(PAGE_LIMIT);

let items = token_owner_balances::Entity::find()
.filter(filter)
.order_by_desc(token_owner_balances::Column::Amount)
.order_by_desc(token_owner_balances::Column::Owner)
.limit(limit)
.all(conn)
.await?
.drain(..)
.map(|token_owner_balance| {
Ok(OwnerBalance {
owner: token_owner_balance.owner.try_into()?,
balance: UnsignedInteger(parse_decimal(token_owner_balance.amount)?),
let items = if is_sqlite(conn) {
token_owner_balances::Entity::find()
.select_only()
.column(token_owner_balances::Column::Owner)
.column(token_owner_balances::Column::Amount)
.filter(filter)
.order_by_desc(token_owner_balances::Column::Amount)
.order_by_desc(token_owner_balances::Column::Owner)
.limit(limit)
.into_model::<OwnerBalanceModelString>()
.all(conn)
.await?
.drain(..)
.map(|m| {
Ok(OwnerBalance {
owner: m.owner.try_into()?,
balance: UnsignedInteger(parse_balance_string(&m.amount)?),
})
})
})
.collect::<Result<Vec<OwnerBalance>, PhotonApiError>>()?;
.collect::<Result<Vec<OwnerBalance>, PhotonApiError>>()?
} else {
token_owner_balances::Entity::find()
.select_only()
.column(token_owner_balances::Column::Owner)
.column(token_owner_balances::Column::Amount)
.filter(filter)
.order_by_desc(token_owner_balances::Column::Amount)
.order_by_desc(token_owner_balances::Column::Owner)
.limit(limit)
.into_model::<OwnerBalanceModel>()
.all(conn)
.await?
.drain(..)
.map(|m| {
Ok(OwnerBalance {
owner: m.owner.try_into()?,
balance: UnsignedInteger(parse_decimal(m.amount)?),
})
})
.collect::<Result<Vec<OwnerBalance>, PhotonApiError>>()?
};

let mut cursor = items.last().map(|item| {
Base58String({
Expand Down
42 changes: 29 additions & 13 deletions src/api/method/get_compressed_token_account_balance.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,11 @@ use sea_orm::{DatabaseConnection, EntityTrait, QueryFilter, QuerySelect};
use serde::{Deserialize, Serialize};

use super::super::error::PhotonApiError;
use super::utils::{parse_decimal, AccountDataTable};
use super::utils::{BalanceModel, CompressedAccountRequest};
use super::utils::{
is_sqlite, parse_balance_string, parse_decimal, AccountDataTable, BalanceModel,
BalanceModelString, CompressedAccountRequest,
};
use crate::common::typedefs::context::Context;
use sqlx::types::Decimal;
use utoipa::ToSchema;

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
Expand All @@ -32,19 +33,34 @@ pub async fn get_compressed_token_account_balance(
) -> Result<GetCompressedTokenAccountBalanceResponse, PhotonApiError> {
let context = Context::extract(conn).await?;
let id = request.parse_id()?;
let balance = token_accounts::Entity::find()
.select_only()
.column(token_accounts::Column::Amount)
.filter(id.filter(AccountDataTable::TokenAccounts))
.into_model::<BalanceModel>()
.one(conn)
.await?
.map(|x| x.amount)
.unwrap_or(Decimal::from(0));

let balance = if is_sqlite(conn) {
token_accounts::Entity::find()
.select_only()
.column(token_accounts::Column::Amount)
.filter(id.filter(AccountDataTable::TokenAccounts))
.into_model::<BalanceModelString>()
.one(conn)
.await?
.map(|x| parse_balance_string(&x.amount))
.transpose()?
.unwrap_or(0)
} else {
token_accounts::Entity::find()
.select_only()
.column(token_accounts::Column::Amount)
.filter(id.filter(AccountDataTable::TokenAccounts))
.into_model::<BalanceModel>()
.one(conn)
.await?
.map(|x| parse_decimal(x.amount))
.transpose()?
.unwrap_or(0)
};

Ok(GetCompressedTokenAccountBalanceResponse {
value: TokenAccountBalance {
amount: UnsignedInteger(parse_decimal(balance)?),
amount: UnsignedInteger(balance),
},
context,
})
Expand Down
56 changes: 42 additions & 14 deletions src/api/method/get_compressed_token_balances_by_owner.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,10 @@ use crate::common::typedefs::unsigned_integer::UnsignedInteger;
use crate::dao::generated::token_owner_balances;

use super::super::error::PhotonApiError;
use super::utils::{parse_decimal, PAGE_LIMIT};
use super::utils::{
is_sqlite, parse_balance_string, parse_decimal, MintBalanceModel, MintBalanceModelString,
PAGE_LIMIT,
};

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
pub struct TokenBalance {
Expand Down Expand Up @@ -79,20 +82,45 @@ pub async fn get_compressed_token_balances_by_owner(
}
let limit = limit.map(|l| l.value()).unwrap_or(PAGE_LIMIT);

let items = token_owner_balances::Entity::find()
.filter(filter)
.order_by_asc(token_owner_balances::Column::Mint)
.limit(limit)
.all(conn)
.await?
.drain(..)
.map(|token_owner_balance| {
Ok(TokenBalance {
mint: token_owner_balance.mint.try_into()?,
balance: UnsignedInteger(parse_decimal(token_owner_balance.amount)?),
let items = if is_sqlite(conn) {
token_owner_balances::Entity::find()
.select_only()
.column(token_owner_balances::Column::Mint)
.column(token_owner_balances::Column::Amount)
.filter(filter)
.order_by_asc(token_owner_balances::Column::Mint)
.limit(limit)
.into_model::<MintBalanceModelString>()
.all(conn)
.await?
.drain(..)
.map(|m| {
Ok(TokenBalance {
mint: m.mint.try_into()?,
balance: UnsignedInteger(parse_balance_string(&m.amount)?),
})
})
})
.collect::<Result<Vec<TokenBalance>, PhotonApiError>>()?;
.collect::<Result<Vec<TokenBalance>, PhotonApiError>>()?
} else {
token_owner_balances::Entity::find()
.select_only()
.column(token_owner_balances::Column::Mint)
.column(token_owner_balances::Column::Amount)
.filter(filter)
.order_by_asc(token_owner_balances::Column::Mint)
.limit(limit)
.into_model::<MintBalanceModel>()
.all(conn)
.await?
.drain(..)
.map(|m| {
Ok(TokenBalance {
mint: m.mint.try_into()?,
balance: UnsignedInteger(parse_decimal(m.amount)?),
})
})
.collect::<Result<Vec<TokenBalance>, PhotonApiError>>()?
};

let mut cursor = items.last().map(|item| {
Base58String({
Expand Down
Loading
Loading