Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion component/block_cache/block_cache_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3027,7 +3027,9 @@ func (suite *blockCacheTestSuite) TestReadWriteBlockInParallel() {
// TODO: Uncomment when stream is deprecated
// func (suite *blockCacheTestSuite) TestZZZZZStreamToBlockCacheConfig() {

// free := memory.FreeMemory()
// v, err := mem.VirtualMemory()
// suite.assert.Nil(err)
// free := v.Free
// maxbuffers := max(1, free/_1MB-1)
// common.IsStream = true
// config := fmt.Sprintf("read-only: true\n\nstream:\n block-size-mb: 2\n max-buffers: %d\n buffer-size-mb: 1\n", maxbuffers)
Expand Down
9 changes: 7 additions & 2 deletions component/block_cache/stream.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ package block_cache
// "github.com/Seagate/cloudfuse/common/log"
// "github.com/Seagate/cloudfuse/internal"

// "github.com/pbnjay/memory"
// "github.com/shirou/gopsutil/v4/mem"
// )

// type Stream struct {
Expand Down Expand Up @@ -91,7 +91,12 @@ package block_cache
// }
// }

// if uint64((conf.BufferSize*conf.CachedObjLimit)*mb) > memory.FreeMemory() {
// v, err := mem.VirtualMemory()
// if err != nil {
// return err
// }

// if uint64((conf.BufferSize*conf.CachedObjLimit)*mb) > v.Free {
// log.Err(
// "Stream::Configure : config error, not enough free memory for provided configuration",
// )
Expand Down
16 changes: 13 additions & 3 deletions component/stream/read_write.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,7 @@ import (
"github.com/Seagate/cloudfuse/common/log"
"github.com/Seagate/cloudfuse/internal"
"github.com/Seagate/cloudfuse/internal/handlemap"

"github.com/pbnjay/memory"
"github.com/shirou/gopsutil/v4/mem"
)

type ReadWriteCache struct {
Expand Down Expand Up @@ -447,7 +446,18 @@ func (rw *ReadWriteCache) createHandleCache(handle *handlemap.Handle) error {
handle.CacheObj.BlockOffsetList = offsets
// if its a small file then download the file in its entirety if there is memory available, otherwise stream only
if handle.CacheObj.HasNoBlocks() {
if uint64(atomic.LoadInt64(&handle.Size)) > memory.FreeMemory() {
v, err := mem.VirtualMemory()
if err != nil {
log.Warn(
"ReadWriteCache::createHandleCache : unable to read system memory info for %s [%v]; switching handle to stream-only mode",
handle.Path,
err,
)
handle.CacheObj.StreamOnly = true
return nil
}

if uint64(atomic.LoadInt64(&handle.Size)) > v.Free {
handle.CacheObj.StreamOnly = true
return nil
}
Expand Down
16 changes: 13 additions & 3 deletions component/stream/read_write_filename.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,7 @@ import (
"github.com/Seagate/cloudfuse/common/log"
"github.com/Seagate/cloudfuse/internal"
"github.com/Seagate/cloudfuse/internal/handlemap"

"github.com/pbnjay/memory"
"github.com/shirou/gopsutil/v4/mem"
)

type ReadWriteFilenameCache struct {
Expand Down Expand Up @@ -389,7 +388,18 @@ func (rw *ReadWriteFilenameCache) createFileCache(handle *handlemap.Handle) erro
atomic.StoreInt64(&handle.CacheObj.Size, handle.Size)
handle.CacheObj.Mtime = handle.Mtime
if handle.CacheObj.HasNoBlocks() {
if uint64(atomic.LoadInt64(&handle.Size)) > memory.FreeMemory() {
v, err := mem.VirtualMemory()
if err != nil {
log.Warn(
"ReadWriteFilenameCache::createFileCache : unable to read system memory info for %s [%v]; switching handle to stream-only mode",
handle.Path,
err,
)
handle.CacheObj.StreamOnly = true
return nil
}

if uint64(atomic.LoadInt64(&handle.Size)) > v.Free {
handle.CacheObj.StreamOnly = true
return nil
}
Expand Down
12 changes: 9 additions & 3 deletions component/stream/stream.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,7 @@ import (
"github.com/Seagate/cloudfuse/common/log"
"github.com/Seagate/cloudfuse/internal"
"github.com/Seagate/cloudfuse/internal/handlemap"

"github.com/pbnjay/memory"
"github.com/shirou/gopsutil/v4/mem"
)

type Stream struct {
Expand Down Expand Up @@ -100,7 +99,14 @@ func (st *Stream) Configure(_ bool) error {
return fmt.Errorf("config error in %s [%s]", st.Name(), err.Error())
}

if uint64((conf.BufferSize*conf.CachedObjLimit)*mb) > memory.FreeMemory() {
v, err := mem.VirtualMemory()
if err != nil {
log.Warn(
"Stream::Configure : unable to read system memory info [%v]; falling back to stream-only mode",
err,
)
st.StreamOnly = true
} else if uint64((conf.BufferSize*conf.CachedObjLimit)*mb) > v.Free {
log.Err(
"Stream::Configure : config error, not enough free memory for provided configuration",
)
Expand Down
1 change: 0 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ require (
github.com/go-viper/mapstructure/v2 v2.5.0
github.com/montanaflynn/stats v0.7.1
github.com/netresearch/go-cron v0.9.1
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
github.com/radovskyb/watcher v1.0.7
github.com/rivo/tview v0.42.0
github.com/sevlyar/go-daemon v0.1.7-0.20251110065050-63665fab0d07
Expand Down
2 changes: 0 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,6 @@ github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8
github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
github.com/netresearch/go-cron v0.9.1 h1:lEjuHz4oJmbR622XTAHcHH0Ekec0tjwMLQyQWQm7UDQ=
github.com/netresearch/go-cron v0.9.1/go.mod h1:oRPUA7fHC/ul86n+d3SdUD54cEuHIuCLiFJCua5a5/E=
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
Expand Down
Loading