mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-23 06:20:48 +02:00
filer leveldb2: tunable cache, buffer, block and table sizes (#11927)
* filer leveldb2: read cache and buffer sizes from filer.toml The leveldb2 store hardcoded its block cache (32MiB) and write buffer (16MiB) and offered no way to tune the table block size or compaction table size. Optional block_cache_size_mb, write_buffer_size_mb, block_size_kb and compaction_table_size_mb keys under [leveldb2] now override them; zero or negative values keep the previous defaults. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * filer leveldb2: document tuning options in scaffold filer.toml Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * filer leveldb2: saturate size options instead of overflowing int The option fields take int, so a MiB config value past math.MaxInt/MiB (e.g. 2048 on 32-bit) wrapped negative and goleveldb would panic sizing the memtable. Clamp the product instead of the input. * filer leveldb2: reject size options too large for int bytes Clamping a MiB setting at math.MaxInt silently hands goleveldb a different size than configured; fail Initialize on values that cannot fit instead. * filer leveldb3: same tunable cache, buffer, block and table sizes Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * filer leveldb2/3: exercise oversized-option rejection on 64-bit too The tests skipped whenever int was 64-bit, but write_buffer_size_mb values above MaxInt/1MiB still overflow there, so the rejection path had no coverage on the primary platform. Use MaxInt/1MiB + 1 so the rejection triggers on every platform. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Chris LuandDevin authored and GitHub committed
1 parent
1451096c17
commit
0b197867a1
5 files changed
+306
-21
No files matched your search
@@ -33,12 +33,22 @@ recursive_delete = false
|
||||
# faster than previous leveldb, recommended.
|
||||
enabled = true
|
||||
dir = "./filerldb2" # directory to store level db files
|
||||
# optional tuning, applied to each of the 8 partitioned stores
|
||||
# block_cache_size_mb = 32 # sorted-table read cache
|
||||
# write_buffer_size_mb = 16 # memtable size before flushing to disk
|
||||
# block_size_kb = 4 # uncompressed block ('page') size inside table files
|
||||
# compaction_table_size_mb = 2 # target size of generated table files
|
||||
|
||||
[leveldb3]
|
||||
# similar to leveldb2.
|
||||
# each bucket has its own meta store.
|
||||
enabled = false
|
||||
dir = "./filerldb3" # directory to store level db files
|
||||
# optional tuning, applied to the default store and each bucket store
|
||||
# block_cache_size_mb = 32 # sorted-table read cache
|
||||
# write_buffer_size_mb = 16 # memtable size before flushing to disk
|
||||
# block_size_kb = 4 # uncompressed block ('page') size inside table files
|
||||
# compaction_table_size_mb = 2 # target size of generated table files
|
||||
|
||||
[rocksdb]
|
||||
# local on disk, similar to leveldb
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb"
|
||||
@@ -28,6 +29,11 @@ type LevelDB2Store struct {
|
||||
dbs []*leveldb.DB
|
||||
dbCount int
|
||||
ReadOnly bool
|
||||
|
||||
blockCacheSizeMB int
|
||||
writeBufferSizeMB int
|
||||
blockSizeKB int
|
||||
compactionTableSizeMB int
|
||||
}
|
||||
|
||||
func (store *LevelDB2Store) GetName() string {
|
||||
@@ -36,9 +42,54 @@ func (store *LevelDB2Store) GetName() string {
|
||||
|
||||
func (store *LevelDB2Store) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
|
||||
dir := configuration.GetString(prefix + "dir")
|
||||
store.blockCacheSizeMB = configuration.GetInt(prefix + "block_cache_size_mb")
|
||||
store.writeBufferSizeMB = configuration.GetInt(prefix + "write_buffer_size_mb")
|
||||
store.blockSizeKB = configuration.GetInt(prefix + "block_size_kb")
|
||||
store.compactionTableSizeMB = configuration.GetInt(prefix + "compaction_table_size_mb")
|
||||
if err := store.validateSizeOptions(); err != nil {
|
||||
return err
|
||||
}
|
||||
return store.initialize(dir, 8)
|
||||
}
|
||||
|
||||
// validateSizeOptions rejects sizes too large to convert to bytes without
|
||||
// overflowing int on this platform.
|
||||
func (store *LevelDB2Store) validateSizeOptions() error {
|
||||
for name, kv := range map[string][2]int{
|
||||
"block_cache_size_mb": {store.blockCacheSizeMB, 1 << 20},
|
||||
"write_buffer_size_mb": {store.writeBufferSizeMB, 1 << 20},
|
||||
"block_size_kb": {store.blockSizeKB, 1 << 10},
|
||||
"compaction_table_size_mb": {store.compactionTableSizeMB, 1 << 20},
|
||||
} {
|
||||
if kv[0] > math.MaxInt/kv[1] {
|
||||
return fmt.Errorf("leveldb2 option %s=%d overflows int", name, kv[0])
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *LevelDB2Store) buildOptions() *opt.Options {
|
||||
opts := &opt.Options{
|
||||
BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB
|
||||
WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB
|
||||
Filter: filter.NewBloomFilter(8), // false positive rate 0.02
|
||||
ReadOnly: store.ReadOnly,
|
||||
}
|
||||
if store.blockCacheSizeMB > 0 {
|
||||
opts.BlockCacheCapacity = store.blockCacheSizeMB * 1024 * 1024
|
||||
}
|
||||
if store.writeBufferSizeMB > 0 {
|
||||
opts.WriteBuffer = store.writeBufferSizeMB * 1024 * 1024
|
||||
}
|
||||
if store.blockSizeKB > 0 {
|
||||
opts.BlockSize = store.blockSizeKB * 1024
|
||||
}
|
||||
if store.compactionTableSizeMB > 0 {
|
||||
opts.CompactionTableSize = store.compactionTableSizeMB * 1024 * 1024
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) {
|
||||
glog.Infof("filer store leveldb2 dir: %s", dir)
|
||||
os.MkdirAll(dir, 0755)
|
||||
@@ -46,12 +97,7 @@ func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) {
|
||||
return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
|
||||
}
|
||||
|
||||
opts := &opt.Options{
|
||||
BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB
|
||||
WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB
|
||||
Filter: filter.NewBloomFilter(8), // false positive rate 0.02
|
||||
ReadOnly: store.ReadOnly,
|
||||
}
|
||||
opts := store.buildOptions()
|
||||
|
||||
for d := 0; d < dbCount; d++ {
|
||||
dbFolder := fmt.Sprintf("%s/%02d", dir, d)
|
||||
|
||||
@@ -2,6 +2,8 @@ package leveldb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
@@ -62,6 +64,93 @@ func TestCreateAndFind(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
type fakeConfiguration map[string]interface{}
|
||||
|
||||
func (c fakeConfiguration) GetString(key string) string {
|
||||
value, _ := c[key].(string)
|
||||
return value
|
||||
}
|
||||
|
||||
func (c fakeConfiguration) GetBool(key string) bool {
|
||||
value, _ := c[key].(bool)
|
||||
return value
|
||||
}
|
||||
|
||||
func (c fakeConfiguration) GetInt(key string) int {
|
||||
value, _ := c[key].(int)
|
||||
return value
|
||||
}
|
||||
|
||||
func (c fakeConfiguration) GetStringSlice(key string) []string {
|
||||
value, _ := c[key].([]string)
|
||||
return value
|
||||
}
|
||||
|
||||
func (c fakeConfiguration) SetDefault(key string, value interface{}) {
|
||||
if _, found := c[key]; !found {
|
||||
c[key] = value
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitializeOptions(t *testing.T) {
|
||||
store := &LevelDB2Store{}
|
||||
if err := store.Initialize(fakeConfiguration{
|
||||
"leveldb2.dir": t.TempDir(),
|
||||
"leveldb2.block_cache_size_mb": 64,
|
||||
"leveldb2.write_buffer_size_mb": 32,
|
||||
"leveldb2.block_size_kb": 16,
|
||||
"leveldb2.compaction_table_size_mb": 8,
|
||||
}, "leveldb2."); err != nil {
|
||||
t.Fatalf("initialize: %v", err)
|
||||
}
|
||||
defer store.Shutdown()
|
||||
|
||||
opts := store.buildOptions()
|
||||
if opts.BlockCacheCapacity != 64*1024*1024 {
|
||||
t.Errorf("block cache %d, want 64MiB", opts.BlockCacheCapacity)
|
||||
}
|
||||
if opts.WriteBuffer != 32*1024*1024 {
|
||||
t.Errorf("write buffer %d, want 32MiB", opts.WriteBuffer)
|
||||
}
|
||||
if opts.BlockSize != 16*1024 {
|
||||
t.Errorf("block size %d, want 16KiB", opts.BlockSize)
|
||||
}
|
||||
if opts.CompactionTableSize != 8*1024*1024 {
|
||||
t.Errorf("compaction table size %d, want 8MiB", opts.CompactionTableSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitializeDefaultOptions(t *testing.T) {
|
||||
store := &LevelDB2Store{}
|
||||
if err := store.Initialize(fakeConfiguration{
|
||||
"leveldb2.dir": t.TempDir(),
|
||||
"leveldb2.block_cache_size_mb": -1,
|
||||
"leveldb2.write_buffer_size_mb": 0,
|
||||
}, "leveldb2."); err != nil {
|
||||
t.Fatalf("initialize: %v", err)
|
||||
}
|
||||
defer store.Shutdown()
|
||||
|
||||
opts := store.buildOptions()
|
||||
if opts.BlockCacheCapacity != 32*1024*1024 {
|
||||
t.Errorf("block cache %d, want the 32MiB default", opts.BlockCacheCapacity)
|
||||
}
|
||||
if opts.WriteBuffer != 16*1024*1024 {
|
||||
t.Errorf("write buffer %d, want the 16MiB default", opts.WriteBuffer)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitializeRejectsOversizedOption(t *testing.T) {
|
||||
store := &LevelDB2Store{}
|
||||
err := store.Initialize(fakeConfiguration{
|
||||
"leveldb2.dir": t.TempDir(),
|
||||
"leveldb2.write_buffer_size_mb": math.MaxInt/(1<<20) + 1,
|
||||
}, "leveldb2.")
|
||||
if err == nil || !strings.Contains(err.Error(), "write_buffer_size_mb") {
|
||||
t.Fatalf("expected overflow rejection, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyRoot(t *testing.T) {
|
||||
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
|
||||
dir := t.TempDir()
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -35,6 +36,11 @@ type LevelDB3Store struct {
|
||||
dbs map[string]*leveldb.DB
|
||||
dbsLock sync.RWMutex
|
||||
ReadOnly bool
|
||||
|
||||
blockCacheSizeMB int
|
||||
writeBufferSizeMB int
|
||||
blockSizeKB int
|
||||
compactionTableSizeMB int
|
||||
}
|
||||
|
||||
func (store *LevelDB3Store) GetName() string {
|
||||
@@ -43,9 +49,58 @@ func (store *LevelDB3Store) GetName() string {
|
||||
|
||||
func (store *LevelDB3Store) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
|
||||
dir := configuration.GetString(prefix + "dir")
|
||||
store.blockCacheSizeMB = configuration.GetInt(prefix + "block_cache_size_mb")
|
||||
store.writeBufferSizeMB = configuration.GetInt(prefix + "write_buffer_size_mb")
|
||||
store.blockSizeKB = configuration.GetInt(prefix + "block_size_kb")
|
||||
store.compactionTableSizeMB = configuration.GetInt(prefix + "compaction_table_size_mb")
|
||||
if err := store.validateSizeOptions(); err != nil {
|
||||
return err
|
||||
}
|
||||
return store.initialize(dir)
|
||||
}
|
||||
|
||||
// validateSizeOptions rejects sizes too large to convert to bytes without
|
||||
// overflowing int on this platform.
|
||||
func (store *LevelDB3Store) validateSizeOptions() error {
|
||||
for name, kv := range map[string][2]int{
|
||||
"block_cache_size_mb": {store.blockCacheSizeMB, 1 << 20},
|
||||
"write_buffer_size_mb": {store.writeBufferSizeMB, 1 << 20},
|
||||
"block_size_kb": {store.blockSizeKB, 1 << 10},
|
||||
"compaction_table_size_mb": {store.compactionTableSizeMB, 1 << 20},
|
||||
} {
|
||||
if kv[0] > math.MaxInt/kv[1] {
|
||||
return fmt.Errorf("leveldb3 option %s=%d overflows int", name, kv[0])
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *LevelDB3Store) buildOptions(name string) *opt.Options {
|
||||
opts := &opt.Options{
|
||||
BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB
|
||||
WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB
|
||||
Filter: filter.NewBloomFilter(8),
|
||||
ReadOnly: store.ReadOnly,
|
||||
}
|
||||
if name != DEFAULT {
|
||||
opts.BlockCacheCapacity = 16 * 1024 * 1024 // default value is 8MiB
|
||||
opts.WriteBuffer = 8 * 1024 * 1024 // default value is 4MiB
|
||||
}
|
||||
if store.blockCacheSizeMB > 0 {
|
||||
opts.BlockCacheCapacity = store.blockCacheSizeMB * 1024 * 1024
|
||||
}
|
||||
if store.writeBufferSizeMB > 0 {
|
||||
opts.WriteBuffer = store.writeBufferSizeMB * 1024 * 1024
|
||||
}
|
||||
if store.blockSizeKB > 0 {
|
||||
opts.BlockSize = store.blockSizeKB * 1024
|
||||
}
|
||||
if store.compactionTableSizeMB > 0 {
|
||||
opts.CompactionTableSize = store.compactionTableSizeMB * 1024 * 1024
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
func (store *LevelDB3Store) initialize(dir string) (err error) {
|
||||
glog.Infof("filer store leveldb3 dir: %s", dir)
|
||||
os.MkdirAll(dir, 0755)
|
||||
@@ -65,21 +120,7 @@ func (store *LevelDB3Store) initialize(dir string) (err error) {
|
||||
}
|
||||
|
||||
func (store *LevelDB3Store) loadDB(name string) (*leveldb.DB, error) {
|
||||
bloom := filter.NewBloomFilter(8) // false positive rate 0.02
|
||||
opts := &opt.Options{
|
||||
BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB
|
||||
WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB
|
||||
Filter: bloom,
|
||||
ReadOnly: store.ReadOnly,
|
||||
}
|
||||
if name != DEFAULT {
|
||||
opts = &opt.Options{
|
||||
BlockCacheCapacity: 16 * 1024 * 1024, // default value is 8MiB
|
||||
WriteBuffer: 8 * 1024 * 1024, // default value is 4MiB
|
||||
Filter: bloom,
|
||||
ReadOnly: store.ReadOnly,
|
||||
}
|
||||
}
|
||||
opts := store.buildOptions(name)
|
||||
|
||||
dbFolder := fmt.Sprintf("%s/%s", store.dir, name)
|
||||
os.MkdirAll(dbFolder, 0755)
|
||||
|
||||
@@ -2,6 +2,8 @@ package leveldb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
@@ -62,6 +64,103 @@ func TestCreateAndFind(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
type fakeConfiguration map[string]interface{}
|
||||
|
||||
func (c fakeConfiguration) GetString(key string) string {
|
||||
value, _ := c[key].(string)
|
||||
return value
|
||||
}
|
||||
|
||||
func (c fakeConfiguration) GetBool(key string) bool {
|
||||
value, _ := c[key].(bool)
|
||||
return value
|
||||
}
|
||||
|
||||
func (c fakeConfiguration) GetInt(key string) int {
|
||||
value, _ := c[key].(int)
|
||||
return value
|
||||
}
|
||||
|
||||
func (c fakeConfiguration) GetStringSlice(key string) []string {
|
||||
value, _ := c[key].([]string)
|
||||
return value
|
||||
}
|
||||
|
||||
func (c fakeConfiguration) SetDefault(key string, value interface{}) {
|
||||
if _, found := c[key]; !found {
|
||||
c[key] = value
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitializeOptions(t *testing.T) {
|
||||
store := &LevelDB3Store{}
|
||||
if err := store.Initialize(fakeConfiguration{
|
||||
"leveldb3.dir": t.TempDir(),
|
||||
"leveldb3.block_cache_size_mb": 64,
|
||||
"leveldb3.write_buffer_size_mb": 32,
|
||||
"leveldb3.block_size_kb": 16,
|
||||
"leveldb3.compaction_table_size_mb": 8,
|
||||
}, "leveldb3."); err != nil {
|
||||
t.Fatalf("initialize: %v", err)
|
||||
}
|
||||
defer store.Shutdown()
|
||||
|
||||
for _, name := range []string{DEFAULT, "bucket-a"} {
|
||||
opts := store.buildOptions(name)
|
||||
if opts.BlockCacheCapacity != 64*1024*1024 {
|
||||
t.Errorf("%s block cache %d, want 64MiB", name, opts.BlockCacheCapacity)
|
||||
}
|
||||
if opts.WriteBuffer != 32*1024*1024 {
|
||||
t.Errorf("%s write buffer %d, want 32MiB", name, opts.WriteBuffer)
|
||||
}
|
||||
if opts.BlockSize != 16*1024 {
|
||||
t.Errorf("%s block size %d, want 16KiB", name, opts.BlockSize)
|
||||
}
|
||||
if opts.CompactionTableSize != 8*1024*1024 {
|
||||
t.Errorf("%s compaction table size %d, want 8MiB", name, opts.CompactionTableSize)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitializeDefaultOptions(t *testing.T) {
|
||||
store := &LevelDB3Store{}
|
||||
if err := store.Initialize(fakeConfiguration{
|
||||
"leveldb3.dir": t.TempDir(),
|
||||
"leveldb3.block_cache_size_mb": -1,
|
||||
"leveldb3.write_buffer_size_mb": 0,
|
||||
}, "leveldb3."); err != nil {
|
||||
t.Fatalf("initialize: %v", err)
|
||||
}
|
||||
defer store.Shutdown()
|
||||
|
||||
opts := store.buildOptions(DEFAULT)
|
||||
if opts.BlockCacheCapacity != 32*1024*1024 {
|
||||
t.Errorf("block cache %d, want the 32MiB default", opts.BlockCacheCapacity)
|
||||
}
|
||||
if opts.WriteBuffer != 16*1024*1024 {
|
||||
t.Errorf("write buffer %d, want the 16MiB default", opts.WriteBuffer)
|
||||
}
|
||||
|
||||
bucketOpts := store.buildOptions("bucket-a")
|
||||
if bucketOpts.BlockCacheCapacity != 16*1024*1024 {
|
||||
t.Errorf("bucket block cache %d, want the 16MiB default", bucketOpts.BlockCacheCapacity)
|
||||
}
|
||||
if bucketOpts.WriteBuffer != 8*1024*1024 {
|
||||
t.Errorf("bucket write buffer %d, want the 8MiB default", bucketOpts.WriteBuffer)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitializeRejectsOversizedOption(t *testing.T) {
|
||||
store := &LevelDB3Store{}
|
||||
err := store.Initialize(fakeConfiguration{
|
||||
"leveldb3.dir": t.TempDir(),
|
||||
"leveldb3.write_buffer_size_mb": math.MaxInt/(1<<20) + 1,
|
||||
}, "leveldb3.")
|
||||
if err == nil || !strings.Contains(err.Error(), "write_buffer_size_mb") {
|
||||
t.Fatalf("expected overflow rejection, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyRoot(t *testing.T) {
|
||||
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
|
||||
dir := t.TempDir()
|
||||
|
||||
Reference in new issue
Block a user