Skip to content

fix out of memory #131

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
siovanus wants to merge 1 commit into
base: master
Choose a base branch
from
Open

fix out of memory #131

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 22 additions & 82 deletions core/store/ledgerstore/block_store.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,23 +23,24 @@ import (
"encoding/binary"
"fmt"

"io"

"github.com/polynetwork/poly/common"
"github.com/polynetwork/poly/common/serialization"
scom "github.com/polynetwork/poly/core/store/common"
"github.com/polynetwork/poly/core/store/leveldbstore"
"github.com/polynetwork/poly/core/types"
"io"
)

//Block store save the data of block & transaction
// Block store save the data of block & transaction
type BlockStore struct {
enableCache bool //Is enable lru cache
dbDir string //The path of store file
cache *BlockCache //The cache of block, if have.
store *leveldbstore.LevelDBStore //block store handler
}

//NewBlockStore return the block store instance
// NewBlockStore return the block store instance
func NewBlockStore(dbDir string, enableCache bool) (*BlockStore, error) {
var cache *BlockCache
var err error
Expand All @@ -63,12 +64,12 @@ func NewBlockStore(dbDir string, enableCache bool) (*BlockStore, error) {
return blockStore, nil
}

//NewBatch start a commit batch
// NewBatch start a commit batch
func (this *BlockStore) NewBatch() {
this.store.NewBatch()
}

//SaveBlock persist block to store
// SaveBlock persist block to store
func (this *BlockStore) SaveBlock(block *types.Block) error {
if this.enableCache {
this.cache.AddBlock(block)
Expand All @@ -89,7 +90,7 @@ func (this *BlockStore) SaveBlock(block *types.Block) error {
return nil
}

//ContainBlock return the block specified by block hash save in store
// ContainBlock return the block specified by block hash save in store
func (this *BlockStore) ContainBlock(blockHash common.Uint256) (bool, error) {
if this.enableCache {
if this.cache.ContainBlock(blockHash) {
Expand All @@ -107,7 +108,7 @@ func (this *BlockStore) ContainBlock(blockHash common.Uint256) (bool, error) {
return true, nil
}

//GetBlock return block by block hash
// GetBlock return block by block hash
func (this *BlockStore) GetBlock(blockHash common.Uint256) (*types.Block, error) {
var block *types.Block
if this.enableCache {
Expand Down Expand Up @@ -165,7 +166,7 @@ func (this *BlockStore) loadHeaderWithTx(blockHash common.Uint256) (*types.Heade
return header, txHashes, nil
}

//SaveHeader persist block header to store
// SaveHeader persist block header to store
func (this *BlockStore) SaveHeader(block *types.Block) error {
blockHash := block.Hash()
key := this.getHeaderKey(blockHash)
Expand All @@ -180,7 +181,7 @@ func (this *BlockStore) SaveHeader(block *types.Block) error {
return nil
}

//GetHeader return the header specified by block hash
// GetHeader return the header specified by block hash
func (this *BlockStore) GetHeader(blockHash common.Uint256) (*types.Header, error) {
if this.enableCache {
block := this.cache.GetBlock(blockHash)
Expand All @@ -206,7 +207,7 @@ func (this *BlockStore) loadHeader(blockHash common.Uint256) (*types.Header, err
return header, nil
}

//GetCurrentBlock return the current block hash and current block height
// GetCurrentBlock return the current block hash and current block height
func (this *BlockStore) GetCurrentBlock() (common.Uint256, uint32, error) {
key := this.getCurrentBlockKey()
data, err := this.store.Get(key)
Expand All @@ -226,7 +227,7 @@ func (this *BlockStore) GetCurrentBlock() (common.Uint256, uint32, error) {
return blockHash, height, nil
}

//SaveCurrentBlock persist the current block height and current block hash to store
// SaveCurrentBlock persist the current block height and current block hash to store
func (this *BlockStore) SaveCurrentBlock(height uint32, blockHash common.Uint256) error {
key := this.getCurrentBlockKey()
value := bytes.NewBuffer(nil)
Expand All @@ -236,52 +237,7 @@ func (this *BlockStore) SaveCurrentBlock(height uint32, blockHash common.Uint256
return nil
}

//GetHeaderIndexList return the head index store in header index list
func (this *BlockStore) GetHeaderIndexList() (map[uint32]common.Uint256, error) {
result := make(map[uint32]common.Uint256)
iter := this.store.NewIterator([]byte{byte(scom.IX_HEADER_HASH_LIST)})
defer iter.Release()
for iter.Next() {
startCount, err := this.getStartHeightByHeaderIndexKey(iter.Key())
if err != nil {
return nil, fmt.Errorf("getStartHeightByHeaderIndexKey error %s", err)
}
reader := bytes.NewReader(iter.Value())
count, err := serialization.ReadUint32(reader)
if err != nil {
return nil, fmt.Errorf("serialization.ReadUint32 count error %s", err)
}
for i := uint32(0); i < count; i++ {
height := startCount + i
blockHash := common.Uint256{}
err = blockHash.Deserialize(reader)
if err != nil {
return nil, fmt.Errorf("blockHash.Deserialize error %s", err)
}
result[height] = blockHash
}
}
if err := iter.Error(); err != nil {
return nil, err
}
return result, nil
}

//SaveHeaderIndexList persist header index list to store
func (this *BlockStore) SaveHeaderIndexList(startIndex uint32, indexList []common.Uint256) error {
indexKey := this.getHeaderIndexListKey(startIndex)
indexSize := uint32(len(indexList))
value := bytes.NewBuffer(nil)
serialization.WriteUint32(value, indexSize)
for _, hash := range indexList {
hash.Serialize(value)
}

this.store.BatchPut(indexKey, value.Bytes())
return nil
}

//GetBlockHash return block hash by block height
// GetBlockHash return block hash by block height
func (this *BlockStore) GetBlockHash(height uint32) (common.Uint256, error) {
key := this.getBlockHashKey(height)
value, err := this.store.Get(key)
Expand All @@ -295,13 +251,13 @@ func (this *BlockStore) GetBlockHash(height uint32) (common.Uint256, error) {
return blockHash, nil
}

//SaveBlockHash persist block height and block hash to store
// SaveBlockHash persist block height and block hash to store
func (this *BlockStore) SaveBlockHash(height uint32, blockHash common.Uint256) {
key := this.getBlockHashKey(height)
this.store.BatchPut(key, blockHash.ToArray())
}

//SaveTransaction persist transaction to store
// SaveTransaction persist transaction to store
func (this *BlockStore) SaveTransaction(tx *types.Transaction, height uint32) error {
if this.enableCache {
this.cache.AddTransaction(tx, height)
Expand All @@ -323,7 +279,7 @@ func (this *BlockStore) putTransaction(tx *types.Transaction, height uint32) err
return nil
}

//GetTransaction return transaction by transaction hash
// GetTransaction return transaction by transaction hash
func (this *BlockStore) GetTransaction(txHash common.Uint256) (*types.Transaction, uint32, error) {
if this.enableCache {
tx, height := this.cache.GetTransaction(txHash)
Expand Down Expand Up @@ -364,7 +320,7 @@ func (this *BlockStore) loadTransaction(txHash common.Uint256) (*types.Transacti
return tx, height, nil
}

//IsContainTransaction return whether the transaction is in store
// IsContainTransaction return whether the transaction is in store
func (this *BlockStore) ContainTransaction(txHash common.Uint256) (bool, error) {
key := this.getTransactionKey(txHash)

Expand All @@ -383,7 +339,7 @@ func (this *BlockStore) ContainTransaction(txHash common.Uint256) (bool, error)
return true, nil
}

//GetVersion return the version of store
// GetVersion return the version of store
func (this *BlockStore) GetVersion() (byte, error) {
key := this.getVersionKey()
value, err := this.store.Get(key)
Expand All @@ -394,13 +350,13 @@ func (this *BlockStore) GetVersion() (byte, error) {
return reader.ReadByte()
}

//SaveVersion persist version to store
// SaveVersion persist version to store
func (this *BlockStore) SaveVersion(ver byte) error {
key := this.getVersionKey()
return this.store.Put(key, []byte{ver})
}

//ClearAll clear all the data of block store
// ClearAll clear all the data of block store
func (this *BlockStore) ClearAll() error {
this.NewBatch()
iter := this.store.NewIterator(nil)
Expand All @@ -414,12 +370,12 @@ func (this *BlockStore) ClearAll() error {
return this.CommitTo()
}

//CommitTo commit the batch to store
// CommitTo commit the batch to store
func (this *BlockStore) CommitTo() error {
return this.store.BatchCommit()
}

//Close block store
// Close block store
func (this *BlockStore) Close() error {
return this.store.Close()
}
Expand Down Expand Up @@ -457,19 +413,3 @@ func (this *BlockStore) getBlockMerkleTreeKey() []byte {
func (this *BlockStore) getVersionKey() []byte {
return []byte{byte(scom.SYS_VERSION)}
}

func (this *BlockStore) getHeaderIndexListKey(startHeight uint32) []byte {
key := bytes.NewBuffer(nil)
key.WriteByte(byte(scom.IX_HEADER_HASH_LIST))
serialization.WriteUint32(key, startHeight)
return key.Bytes()
}

func (this *BlockStore) getStartHeightByHeaderIndexKey(key []byte) (uint32, error) {
reader := bytes.NewReader(key[1:])
height, err := serialization.ReadUint32(reader)
if err != nil {
return 0, err
}
return height, nil
}
54 changes: 3 additions & 51 deletions core/store/ledgerstore/block_store_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,14 @@ package ledgerstore
import (
"crypto/sha256"
"fmt"
"testing"
"time"

"github.com/ontio/ontology-crypto/keypair"
"github.com/polynetwork/poly/account"
"github.com/polynetwork/poly/common"
"github.com/polynetwork/poly/core/payload"
"github.com/polynetwork/poly/core/types"
"testing"
"time"
)

func TestVersion(t *testing.T) {
Expand Down Expand Up @@ -165,55 +166,6 @@ func TestSaveTransaction(t *testing.T) {
}
}

func TestHeaderIndexList(t *testing.T) {
testBlockStore.NewBatch()
startHeight := uint32(0)
size := uint32(100)
indexMap := make(map[uint32]common.Uint256, size)
indexList := make([]common.Uint256, 0)
for i := startHeight; i < size; i++ {
hash := common.Uint256(sha256.Sum256([]byte(fmt.Sprintf("%v", i))))
indexMap[i] = hash
indexList = append(indexList, hash)
}
err := testBlockStore.SaveHeaderIndexList(startHeight, indexList)
if err != nil {
t.Errorf("SaveHeaderIndexList error %s", err)
return
}
startHeight = uint32(100)
size = uint32(100)
indexMap = make(map[uint32]common.Uint256, size)
for i := startHeight; i < size; i++ {
hash := common.Uint256(sha256.Sum256([]byte(fmt.Sprintf("%v", i))))
indexMap[i] = hash
indexList = append(indexList, hash)
}
err = testBlockStore.CommitTo()
if err != nil {
t.Errorf("CommitTo error %s", err)
return
}

totalMap, err := testBlockStore.GetHeaderIndexList()
if err != nil {
t.Errorf("GetHeaderIndexList error %s", err)
return
}

for height, hash := range indexList {
h, ok := totalMap[uint32(height)]
if !ok {
t.Errorf("TestHeaderIndexList failed height:%d hash not exist", height)
return
}
if hash != h {
t.Errorf("TestHeaderIndexList failed height:%d hash %x != %x", height, hash, h)
return
}
}
}

func TestSaveHeader(t *testing.T) {
acc1 := account.NewAccount("")
acc2 := account.NewAccount("")
Expand Down
Loading