From b9899d09ab6c032b54fe4a09e9389dff28d8a7f8 Mon Sep 17 00:00:00 2001 From: Martin Kobetic Date: Fri, 9 May 2014 20:50:55 +0000 Subject: [PATCH 1/9] first part --- bucket.go | 43 ++++++++++++++++++++++++++++++++----- cmd/bolt/main.go | 8 +++++++ cmd/bolt/stats.go | 54 +++++++++++++++++++++++++++++++++++++++++++++++ page.go | 4 ++-- 4 files changed, 102 insertions(+), 7 deletions(-) create mode 100644 cmd/bolt/stats.go diff --git a/bucket.go b/bucket.go index 43204f7..41156d7 100644 --- a/bucket.go +++ b/bucket.go @@ -129,16 +129,23 @@ func (b *Bucket) Bucket(name []byte) *Bucket { } // Otherwise create a bucket and cache it. - var child = newBucket(b.tx) - child.bucket = &bucket{} - *child.bucket = *(*bucket)(unsafe.Pointer(&v[0])) - b.buckets[string(name)] = &child + var child = b.openBucket(v) + b.buckets[string(name)] = child // Save a reference to the inline page if the bucket is inline. if child.root == 0 { child.page = (*page)(unsafe.Pointer(&v[bucketHeaderSize])) } + return child +} + +// Helper method that re-interprets a sub-bucket value +// from a parent into a Bucket +func (b *Bucket) openBucket(value []byte) *Bucket { + var child = newBucket(b.tx) + child.bucket = &bucket{} + *child.bucket = *(*bucket)(unsafe.Pointer(&value[0])) return &child } @@ -354,7 +361,7 @@ func (b *Bucket) ForEach(fn func(k, v []byte) error) error { // Stat returns stats on a bucket. func (b *Bucket) Stats() BucketStats { - var s BucketStats + var s, subStats BucketStats pageSize := b.tx.db.pageSize b.forEachPage(func(p *page, depth int) { if (p.flags & leafPageFlag) != 0 { @@ -365,6 +372,13 @@ func (b *Bucket) Stats() BucketStats { used += int(lastElement.pos + lastElement.ksize + lastElement.vsize) s.LeafInuse += used s.LeafOverflowN += int(p.overflow) + + // Recurse into sub-buckets + for _, e := range p.leafPageElements() { + if e.flags&bucketLeafFlag != 0 { + subStats.Add(b.openBucket(e.value()).Stats()) + } + } } else if (p.flags & branchPageFlag) != 0 { s.BranchPageN++ lastElement := p.branchPageElement(p.count - 1) @@ -380,6 +394,10 @@ func (b *Bucket) Stats() BucketStats { }) s.BranchAlloc = (s.BranchPageN + s.BranchOverflowN) * pageSize s.LeafAlloc = (s.LeafPageN + s.LeafOverflowN) * pageSize + + // add the max depth of sub-buckets to get total nested depth + s.Depth += subStats.Depth + s.Add(subStats) return s } @@ -643,6 +661,21 @@ type BucketStats struct { LeafInuse int // bytes actually used for leaf data } +func (s *BucketStats) Add(other BucketStats) { + s.BranchPageN += other.BranchPageN + s.BranchOverflowN += other.BranchOverflowN + s.LeafPageN += other.LeafPageN + s.LeafOverflowN += other.LeafOverflowN + s.KeyN += s.KeyN + if s.Depth < other.Depth { + s.Depth = other.Depth + } + s.BranchAlloc += other.BranchAlloc + s.BranchInuse += other.BranchInuse + s.LeafAlloc += other.LeafAlloc + s.LeafInuse += other.LeafInuse +} + // cloneBytes returns a copy of a given slice. func cloneBytes(v []byte) []byte { var clone = make([]byte, len(v)) diff --git a/cmd/bolt/main.go b/cmd/bolt/main.go index 3397042..659f1c3 100644 --- a/cmd/bolt/main.go +++ b/cmd/bolt/main.go @@ -91,6 +91,14 @@ func NewApp() *cli.App { Check(path) }, }, + { + Name: "stats", + Usage: "Retrieve statistics for a bucket (aggregated recursively)", + Action: func(c *cli.Context) { + path, name := c.Args().Get(0), c.Args().Get(1) + Stats(path, name) + }, + }, { Name: "bench", Usage: "Performs a synthetic benchmark", diff --git a/cmd/bolt/stats.go b/cmd/bolt/stats.go new file mode 100644 index 0000000..da344d0 --- /dev/null +++ b/cmd/bolt/stats.go @@ -0,0 +1,54 @@ +package main + +import ( + "os" + + "github.com/boltdb/bolt" +) + +// Keys retrieves a list of keys for a given bucket. +func Stats(path, name string) { + if _, err := os.Stat(path); os.IsNotExist(err) { + fatal(err) + return + } + + db, err := bolt.Open(path, 0600) + if err != nil { + fatal(err) + return + } + defer db.Close() + + err = db.View(func(tx *bolt.Tx) error { + // Find bucket. + b := tx.Bucket([]byte(name)) + if b == nil { + fatalf("bucket not found: %s", name) + return nil + } + + // Iterate over each key. + s := b.Stats() + println("Page count statistics") + printf("\tNumber of logical branch pages: %d\n", s.BranchPageN) + printf("\tNumber of physical branch overflow pages: %d\n", s.BranchOverflowN) + printf("\tNumber of logical leaf pages: %d\n", s.LeafPageN) + printf("\tNumber of physical leaf overflow pages: %d\n", s.LeafOverflowN) + + println("Tree statistics") + printf("\tNumber of keys/value pairs: %d\n", s.KeyN) + printf("\tNumber of levels in B+tree: %d\n", s.Depth) + + println("Page size utilization") + printf("\tBytes allocated for physical branch pages: %d\n", s.BranchAlloc) + printf("\tBytes actually used for branch data: %d\n", s.BranchInuse) + printf("\tBytes allocated for physical leaf pages: %d\n", s.LeafAlloc) + printf("\tBytes actually used for leaf data: %d\n", s.LeafInuse) + return nil + }) + if err != nil { + fatal(err) + return + } +} diff --git a/page.go b/page.go index 56cf064..40e2421 100644 --- a/page.go +++ b/page.go @@ -63,7 +63,7 @@ func (p *page) leafPageElement(index uint16) *leafPageElement { // leafPageElements retrieves a list of leaf nodes. func (p *page) leafPageElements() []leafPageElement { - return ((*[maxNodesPerPage]leafPageElement)(unsafe.Pointer(&p.ptr)))[:] + return ((*[maxNodesPerPage]leafPageElement)(unsafe.Pointer(&p.ptr)))[:p.count] } // branchPageElement retrieves the branch node by index @@ -73,7 +73,7 @@ func (p *page) branchPageElement(index uint16) *branchPageElement { // branchPageElements retrieves a list of branch nodes. func (p *page) branchPageElements() []branchPageElement { - return ((*[maxNodesPerPage]branchPageElement)(unsafe.Pointer(&p.ptr)))[:] + return ((*[maxNodesPerPage]branchPageElement)(unsafe.Pointer(&p.ptr)))[:p.count] } // dump writes n bytes of the page to STDERR as hex output. From c4ad027df774d84c29c9cc53436071c87138c60d Mon Sep 17 00:00:00 2001 From: Martin Kobetic Date: Fri, 9 May 2014 18:16:38 +0000 Subject: [PATCH 2/9] aggregate bucket stats recursively and add stats to cmd --- bucket.go | 12 ++++++++---- bucket_test.go | 44 ++++++++++++++++++++++++++++++++++++++++++ cmd/bolt/main.go | 2 +- cmd/bolt/stats.go | 28 ++++++++++++++++----------- cmd/bolt/stats_test.go | 37 +++++++++++++++++++++++++++++++++++ 5 files changed, 107 insertions(+), 16 deletions(-) create mode 100644 cmd/bolt/stats_test.go diff --git a/bucket.go b/bucket.go index 41156d7..6f7bbd2 100644 --- a/bucket.go +++ b/bucket.go @@ -366,6 +366,9 @@ func (b *Bucket) Stats() BucketStats { b.forEachPage(func(p *page, depth int) { if (p.flags & leafPageFlag) != 0 { s.LeafPageN++ + if p.count == 0 { + return + } s.KeyN += int(p.count) lastElement := p.leafPageElement(p.count - 1) used := pageHeaderSize + (leafPageElementSize * int(p.count-1)) @@ -373,9 +376,10 @@ func (b *Bucket) Stats() BucketStats { s.LeafInuse += used s.LeafOverflowN += int(p.overflow) - // Recurse into sub-buckets - for _, e := range p.leafPageElements() { - if e.flags&bucketLeafFlag != 0 { + // Collect stats from sub-buckets + for i := uint16(0); i < p.count; i++ { + e := p.leafPageElement(i) + if (e.flags & bucketLeafFlag) != 0 { subStats.Add(b.openBucket(e.value()).Stats()) } } @@ -666,7 +670,7 @@ func (s *BucketStats) Add(other BucketStats) { s.BranchOverflowN += other.BranchOverflowN s.LeafPageN += other.LeafPageN s.LeafOverflowN += other.LeafOverflowN - s.KeyN += s.KeyN + s.KeyN += other.KeyN if s.Depth < other.Depth { s.Depth = other.Depth } diff --git a/bucket_test.go b/bucket_test.go index b87031b..626927a 100644 --- a/bucket_test.go +++ b/bucket_test.go @@ -628,6 +628,50 @@ func TestBucket_Stats_Small(t *testing.T) { }) } +// Ensure a bucket can calculate stats. +func TestBucket_Stats_Nested(t *testing.T) { + + withOpenDB(func(db *DB, path string) { + db.Update(func(tx *Tx) error { + b, err := tx.CreateBucket([]byte("foo")) + assert.NoError(t, err) + for i := 0; i < 100; i++ { + b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) + } + bar, err := b.CreateBucket([]byte("bar")) + assert.NoError(t, err) + for i := 0; i < 10; i++ { + bar.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) + } + baz, err := b.CreateBucket([]byte("baz")) + assert.NoError(t, err) + for i := 0; i < 10; i++ { + baz.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) + } + return nil + }) + mustCheck(db) + db.View(func(tx *Tx) error { + b := tx.Bucket([]byte("foo")) + stats := b.Stats() + assert.Equal(t, stats.BranchPageN, 0) + assert.Equal(t, stats.BranchOverflowN, 0) + assert.Equal(t, stats.LeafPageN, 3) + assert.Equal(t, stats.LeafOverflowN, 0) + assert.Equal(t, stats.KeyN, 122) + assert.Equal(t, stats.Depth, 2) + if os.Getpagesize() != 4096 { + // Incompatible page size + assert.Equal(t, stats.BranchInuse, 0) + assert.Equal(t, stats.BranchAlloc, 0) + assert.Equal(t, stats.LeafInuse, 38) + assert.Equal(t, stats.LeafAlloc, 4096) + } + return nil + }) + }) +} + // Ensure a large bucket can calculate stats. func TestBucket_Stats_Large(t *testing.T) { if testing.Short() { diff --git a/cmd/bolt/main.go b/cmd/bolt/main.go index 659f1c3..302fe09 100644 --- a/cmd/bolt/main.go +++ b/cmd/bolt/main.go @@ -93,7 +93,7 @@ func NewApp() *cli.App { }, { Name: "stats", - Usage: "Retrieve statistics for a bucket (aggregated recursively)", + Usage: "Aggregate statistics for all buckets matching specified prefix", Action: func(c *cli.Context) { path, name := c.Args().Get(0), c.Args().Get(1) Stats(path, name) diff --git a/cmd/bolt/stats.go b/cmd/bolt/stats.go index da344d0..6f8505a 100644 --- a/cmd/bolt/stats.go +++ b/cmd/bolt/stats.go @@ -1,13 +1,14 @@ package main import ( + "bytes" "os" "github.com/boltdb/bolt" ) -// Keys retrieves a list of keys for a given bucket. -func Stats(path, name string) { +// Collect stats for all top level buckets matching the prefix. +func Stats(path, prefix string) { if _, err := os.Stat(path); os.IsNotExist(err) { fatal(err) return @@ -21,15 +22,18 @@ func Stats(path, name string) { defer db.Close() err = db.View(func(tx *bolt.Tx) error { - // Find bucket. - b := tx.Bucket([]byte(name)) - if b == nil { - fatalf("bucket not found: %s", name) + var s bolt.BucketStats + var count int + var prefix = []byte(prefix) + tx.ForEach(func(name []byte, b *bolt.Bucket) error { + if bytes.HasPrefix(name, prefix) { + s.Add(b.Stats()) + count += 1 + } return nil - } + }) + printf("Aggregate statistics for %d buckets\n\n", count) - // Iterate over each key. - s := b.Stats() println("Page count statistics") printf("\tNumber of logical branch pages: %d\n", s.BranchPageN) printf("\tNumber of physical branch overflow pages: %d\n", s.BranchOverflowN) @@ -42,9 +46,11 @@ func Stats(path, name string) { println("Page size utilization") printf("\tBytes allocated for physical branch pages: %d\n", s.BranchAlloc) - printf("\tBytes actually used for branch data: %d\n", s.BranchInuse) + percentage := int(float32(s.BranchInuse) * 100.0 / float32(s.BranchAlloc)) + printf("\tBytes actually used for branch data: %d (%d%%)\n", s.BranchInuse, percentage) printf("\tBytes allocated for physical leaf pages: %d\n", s.LeafAlloc) - printf("\tBytes actually used for leaf data: %d\n", s.LeafInuse) + percentage = int(float32(s.LeafInuse) * 100.0 / float32(s.LeafAlloc)) + printf("\tBytes actually used for leaf data: %d (%d%%)\n", s.LeafInuse, percentage) return nil }) if err != nil { diff --git a/cmd/bolt/stats_test.go b/cmd/bolt/stats_test.go new file mode 100644 index 0000000..9662f09 --- /dev/null +++ b/cmd/bolt/stats_test.go @@ -0,0 +1,37 @@ +package main_test + +import ( + "testing" + + "github.com/boltdb/bolt" + . "github.com/boltdb/bolt/cmd/bolt" + "github.com/stretchr/testify/assert" +) + +func TestStats(t *testing.T) { + SetTestMode(true) + open(func(db *bolt.DB, path string) { + db.Update(func(tx *bolt.Tx) error { + tx.CreateBucket([]byte("foo")) + tx.CreateBucket([]byte("bar")) + tx.CreateBucket([]byte("baz")) + return nil + }) + db.Close() + output := run("stats", path, "b") + assert.Equal(t, "Aggregate statistics for 2 buckets\n\n"+ + "Page count statistics\n"+ + "\tNumber of logical branch pages: 0\n"+ + "\tNumber of physical branch overflow pages: 0\n"+ + "\tNumber of logical leaf pages: 2\n"+ + "\tNumber of physical leaf overflow pages: 0\n"+ + "Tree statistics\n"+ + "\tNumber of keys/value pairs: 0\n"+ + "\tNumber of levels in B+tree: 0\n"+ + "Page size utilization\n"+ + "\tBytes allocated for physical branch pages: 0\n"+ + "\tBytes actually used for branch data: 0\n"+ + "\tBytes allocated for physical leaf pages: 8192\n"+ + "\tBytes actually used for leaf data: 0", output) + }) +} From deffc06a05af5b39f8b06c29768d2300fedbcc76 Mon Sep 17 00:00:00 2001 From: Martin Kobetic Date: Mon, 12 May 2014 17:29:16 +0000 Subject: [PATCH 3/9] fix inline bucket stats --- bucket.go | 32 +++++++++++++--- bucket_test.go | 102 +++++++++++++++++++++++++++---------------------- 2 files changed, 83 insertions(+), 51 deletions(-) diff --git a/bucket.go b/bucket.go index 6f7bbd2..a2463a3 100644 --- a/bucket.go +++ b/bucket.go @@ -132,11 +132,6 @@ func (b *Bucket) Bucket(name []byte) *Bucket { var child = b.openBucket(v) b.buckets[string(name)] = child - // Save a reference to the inline page if the bucket is inline. - if child.root == 0 { - child.page = (*page)(unsafe.Pointer(&v[bucketHeaderSize])) - } - return child } @@ -146,6 +141,12 @@ func (b *Bucket) openBucket(value []byte) *Bucket { var child = newBucket(b.tx) child.bucket = &bucket{} *child.bucket = *(*bucket)(unsafe.Pointer(&value[0])) + + // Save a reference to the inline page if the bucket is inline. + if child.root == 0 { + child.page = (*page)(unsafe.Pointer(&value[bucketHeaderSize])) + } + return &child } @@ -363,8 +364,18 @@ func (b *Bucket) ForEach(fn func(k, v []byte) error) error { func (b *Bucket) Stats() BucketStats { var s, subStats BucketStats pageSize := b.tx.db.pageSize + s.BucketN += 1 + if b.root == 0 { + s.InlineBucketN += 1 + } b.forEachPage(func(p *page, depth int) { - if (p.flags & leafPageFlag) != 0 { + if b.root == 0 { // inline bucket + s.KeyN += int(p.count) + lastElement := p.leafPageElement(p.count - 1) + used := bucketHeaderSize + pageHeaderSize + (leafPageElementSize * int(p.count-1)) + used += int(lastElement.pos + lastElement.ksize + lastElement.vsize) + s.InlineBucketInuse += used + } else if (p.flags & leafPageFlag) != 0 { s.LeafPageN++ if p.count == 0 { return @@ -663,6 +674,11 @@ type BucketStats struct { BranchInuse int // bytes actually used for branch data LeafAlloc int // bytes allocated for physical leaf pages LeafInuse int // bytes actually used for leaf data + + // Bucket statistics + BucketN int // total number of buckets including the top bucket + InlineBucketN int // total number on inlined buckets + InlineBucketInuse int // bytes used for inlined buckets (also accounted for in LeafInuse) } func (s *BucketStats) Add(other BucketStats) { @@ -678,6 +694,10 @@ func (s *BucketStats) Add(other BucketStats) { s.BranchInuse += other.BranchInuse s.LeafAlloc += other.LeafAlloc s.LeafInuse += other.LeafInuse + + s.BucketN += other.BucketN + s.InlineBucketN += other.InlineBucketN + s.InlineBucketInuse += other.InlineBucketInuse } // cloneBytes returns a copy of a given slice. diff --git a/bucket_test.go b/bucket_test.go index 626927a..b53a74b 100644 --- a/bucket_test.go +++ b/bucket_test.go @@ -576,19 +576,22 @@ func TestBucket_Stats(t *testing.T) { db.View(func(tx *Tx) error { b := tx.Bucket([]byte("woojits")) stats := b.Stats() - assert.Equal(t, stats.BranchPageN, 1) - assert.Equal(t, stats.BranchOverflowN, 0) - assert.Equal(t, stats.LeafPageN, 6) - assert.Equal(t, stats.LeafOverflowN, 2) - assert.Equal(t, stats.KeyN, 501) - assert.Equal(t, stats.Depth, 2) - if os.Getpagesize() != 4096 { + assert.Equal(t, 1, stats.BranchPageN, "BranchPageN") + assert.Equal(t, 0, stats.BranchOverflowN, "BranchOverflowN") + assert.Equal(t, 6, stats.LeafPageN, "LeafPageN") + assert.Equal(t, 2, stats.LeafOverflowN, "LeafOverflowN") + assert.Equal(t, 501, stats.KeyN, "KeyN") + assert.Equal(t, 2, stats.Depth, "Depth") + assert.Equal(t, 125, stats.BranchInuse, "BranchInuse") + assert.Equal(t, 20908, stats.LeafInuse, "LeafInuse") + if os.Getpagesize() == 4096 { // Incompatible page size - assert.Equal(t, stats.BranchInuse, 125) - assert.Equal(t, stats.BranchAlloc, 4096) - assert.Equal(t, stats.LeafInuse, 20908) - assert.Equal(t, stats.LeafAlloc, 32768) + assert.Equal(t, 4096, stats.BranchAlloc, "BranchAlloc") + assert.Equal(t, 32768, stats.LeafAlloc, "LeafAlloc") } + assert.Equal(t, 1, stats.BucketN, "BucketN") + assert.Equal(t, 0, stats.InlineBucketN, "InlineBucketN") + assert.Equal(t, 0, stats.InlineBucketInuse, "InlineBucketInuse") return nil }) }) @@ -610,19 +613,22 @@ func TestBucket_Stats_Small(t *testing.T) { db.View(func(tx *Tx) error { b := tx.Bucket([]byte("whozawhats")) stats := b.Stats() - assert.Equal(t, 0, stats.BranchPageN) - assert.Equal(t, 0, stats.BranchOverflowN) - assert.Equal(t, 1, stats.LeafPageN) - assert.Equal(t, 0, stats.LeafOverflowN) - assert.Equal(t, 1, stats.KeyN) - assert.Equal(t, 1, stats.Depth) - if os.Getpagesize() != 4096 { + assert.Equal(t, 0, stats.BranchPageN, "BranchPageN") + assert.Equal(t, 0, stats.BranchOverflowN, "BranchOverflowN") + assert.Equal(t, 0, stats.LeafPageN, "LeafPageN") + assert.Equal(t, 0, stats.LeafOverflowN, "LeafOverflowN") + assert.Equal(t, 1, stats.KeyN, "KeyN") + assert.Equal(t, 1, stats.Depth, "Depth") + assert.Equal(t, 0, stats.BranchInuse, "BranchInuse") + assert.Equal(t, 0, stats.LeafInuse, "LeafInuse") + if os.Getpagesize() == 4096 { // Incompatible page size - assert.Equal(t, 0, stats.BranchInuse) - assert.Equal(t, 0, stats.BranchAlloc) - assert.Equal(t, 38, stats.LeafInuse) - assert.Equal(t, 4096, stats.LeafAlloc) + assert.Equal(t, 0, stats.BranchAlloc, "BranchAlloc") + assert.Equal(t, 0, stats.LeafAlloc, "LeafAlloc") } + assert.Equal(t, 1, stats.BucketN, "BucketN") + assert.Equal(t, 1, stats.InlineBucketN, "InlineBucketN") + assert.Equal(t, bucketHeaderSize+pageHeaderSize+leafPageElementSize+6, stats.InlineBucketInuse, "InlineBucketInuse") return nil }) }) @@ -643,7 +649,7 @@ func TestBucket_Stats_Nested(t *testing.T) { for i := 0; i < 10; i++ { bar.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) } - baz, err := b.CreateBucket([]byte("baz")) + baz, err := bar.CreateBucket([]byte("baz")) assert.NoError(t, err) for i := 0; i < 10; i++ { baz.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) @@ -654,19 +660,22 @@ func TestBucket_Stats_Nested(t *testing.T) { db.View(func(tx *Tx) error { b := tx.Bucket([]byte("foo")) stats := b.Stats() - assert.Equal(t, stats.BranchPageN, 0) - assert.Equal(t, stats.BranchOverflowN, 0) - assert.Equal(t, stats.LeafPageN, 3) - assert.Equal(t, stats.LeafOverflowN, 0) - assert.Equal(t, stats.KeyN, 122) - assert.Equal(t, stats.Depth, 2) - if os.Getpagesize() != 4096 { + assert.Equal(t, 0, stats.BranchPageN, "BranchPageN") + assert.Equal(t, 0, stats.BranchOverflowN, "BranchOverflowN") + assert.Equal(t, 2, stats.LeafPageN, "LeafPageN") + assert.Equal(t, 0, stats.LeafOverflowN, "LeafOverflowN") + assert.Equal(t, 122, stats.KeyN, "KeyN") + assert.Equal(t, 3, stats.Depth, "Depth") + assert.Equal(t, 0, stats.BranchInuse, "BranchInuse") + assert.Equal(t, 2474, stats.LeafInuse, "LeafInuse") + if os.Getpagesize() == 4096 { // Incompatible page size - assert.Equal(t, stats.BranchInuse, 0) - assert.Equal(t, stats.BranchAlloc, 0) - assert.Equal(t, stats.LeafInuse, 38) - assert.Equal(t, stats.LeafAlloc, 4096) + assert.Equal(t, 0, stats.BranchAlloc, "BranchAlloc") + assert.Equal(t, 8192, stats.LeafAlloc, "LeafAlloc") } + assert.Equal(t, 3, stats.BucketN, "BucketN") + assert.Equal(t, 1, stats.InlineBucketN, "InlineBucketN") + assert.Equal(t, 212, stats.InlineBucketInuse, "InlineBucketInuse") return nil }) }) @@ -696,19 +705,22 @@ func TestBucket_Stats_Large(t *testing.T) { db.View(func(tx *Tx) error { b := tx.Bucket([]byte("widgets")) stats := b.Stats() - assert.Equal(t, 19, stats.BranchPageN) - assert.Equal(t, 0, stats.BranchOverflowN) - assert.Equal(t, 1291, stats.LeafPageN) - assert.Equal(t, 0, stats.LeafOverflowN) - assert.Equal(t, 100000, stats.KeyN) - assert.Equal(t, 3, stats.Depth) - if os.Getpagesize() != 4096 { + assert.Equal(t, 19, stats.BranchPageN, "BranchPageN") + assert.Equal(t, 0, stats.BranchOverflowN, "BranchOverflowN") + assert.Equal(t, 1291, stats.LeafPageN, "LeafPageN") + assert.Equal(t, 0, stats.LeafOverflowN, "LeafOverflowN") + assert.Equal(t, 100000, stats.KeyN, "KeyN") + assert.Equal(t, 3, stats.Depth, "Depth") + assert.Equal(t, 27007, stats.BranchInuse, "BranchInuse") + assert.Equal(t, 2598436, stats.LeafInuse, "LeafInuse") + if os.Getpagesize() == 4096 { // Incompatible page size - assert.Equal(t, 27289, stats.BranchInuse) - assert.Equal(t, 61440, stats.BranchAlloc) - assert.Equal(t, 2598276, stats.LeafInuse) - assert.Equal(t, 5246976, stats.LeafAlloc) + assert.Equal(t, 77824, stats.BranchAlloc, "BranchAlloc") + assert.Equal(t, 5287936, stats.LeafAlloc, "LeafAlloc") } + assert.Equal(t, 1, stats.BucketN, "BucketN") + assert.Equal(t, 0, stats.InlineBucketN, "InlineBucketN") + assert.Equal(t, 0, stats.InlineBucketInuse, "InlineBucketInuse") return nil }) }) From 0dbfa3f08a0d2657cc7fc9b7b46af1f0aa271d45 Mon Sep 17 00:00:00 2001 From: Martin Kobetic Date: Mon, 12 May 2014 18:22:45 +0000 Subject: [PATCH 4/9] merge inline branch into leaf branch --- bucket.go | 37 +++++++++++++++++-------------------- bucket_test.go | 40 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 54 insertions(+), 23 deletions(-) diff --git a/bucket.go b/bucket.go index a2463a3..fdc9852 100644 --- a/bucket.go +++ b/bucket.go @@ -369,29 +369,26 @@ func (b *Bucket) Stats() BucketStats { s.InlineBucketN += 1 } b.forEachPage(func(p *page, depth int) { - if b.root == 0 { // inline bucket + if (p.flags & leafPageFlag) != 0 { s.KeyN += int(p.count) - lastElement := p.leafPageElement(p.count - 1) - used := bucketHeaderSize + pageHeaderSize + (leafPageElementSize * int(p.count-1)) - used += int(lastElement.pos + lastElement.ksize + lastElement.vsize) - s.InlineBucketInuse += used - } else if (p.flags & leafPageFlag) != 0 { - s.LeafPageN++ - if p.count == 0 { - return + used := pageHeaderSize + (leafPageElementSize * int(p.count)) + if p.count != 0 { + lastElement := p.leafPageElement(p.count - 1) + used += int(lastElement.pos + lastElement.ksize + lastElement.vsize) } - s.KeyN += int(p.count) - lastElement := p.leafPageElement(p.count - 1) - used := pageHeaderSize + (leafPageElementSize * int(p.count-1)) - used += int(lastElement.pos + lastElement.ksize + lastElement.vsize) - s.LeafInuse += used - s.LeafOverflowN += int(p.overflow) + if b.root == 0 { + s.InlineBucketInuse += used + } else { + s.LeafPageN++ + s.LeafInuse += used + s.LeafOverflowN += int(p.overflow) - // Collect stats from sub-buckets - for i := uint16(0); i < p.count; i++ { - e := p.leafPageElement(i) - if (e.flags & bucketLeafFlag) != 0 { - subStats.Add(b.openBucket(e.value()).Stats()) + // Collect stats from sub-buckets + for i := uint16(0); i < p.count; i++ { + e := p.leafPageElement(i) + if (e.flags & bucketLeafFlag) != 0 { + subStats.Add(b.openBucket(e.value()).Stats()) + } } } } else if (p.flags & branchPageFlag) != 0 { diff --git a/bucket_test.go b/bucket_test.go index b53a74b..11d1d77 100644 --- a/bucket_test.go +++ b/bucket_test.go @@ -583,7 +583,7 @@ func TestBucket_Stats(t *testing.T) { assert.Equal(t, 501, stats.KeyN, "KeyN") assert.Equal(t, 2, stats.Depth, "Depth") assert.Equal(t, 125, stats.BranchInuse, "BranchInuse") - assert.Equal(t, 20908, stats.LeafInuse, "LeafInuse") + assert.Equal(t, 21004, stats.LeafInuse, "LeafInuse") if os.Getpagesize() == 4096 { // Incompatible page size assert.Equal(t, 4096, stats.BranchAlloc, "BranchAlloc") @@ -634,6 +634,40 @@ func TestBucket_Stats_Small(t *testing.T) { }) } +func TestBucket_Stats_EmptyBucket(t *testing.T) { + + withOpenDB(func(db *DB, path string) { + db.Update(func(tx *Tx) error { + // Add a bucket that fits on a single root leaf. + _, err := tx.CreateBucket([]byte("whozawhats")) + assert.NoError(t, err) + return nil + }) + mustCheck(db) + db.View(func(tx *Tx) error { + b := tx.Bucket([]byte("whozawhats")) + stats := b.Stats() + assert.Equal(t, 0, stats.BranchPageN, "BranchPageN") + assert.Equal(t, 0, stats.BranchOverflowN, "BranchOverflowN") + assert.Equal(t, 0, stats.LeafPageN, "LeafPageN") + assert.Equal(t, 0, stats.LeafOverflowN, "LeafOverflowN") + assert.Equal(t, 0, stats.KeyN, "KeyN") + assert.Equal(t, 1, stats.Depth, "Depth") + assert.Equal(t, 0, stats.BranchInuse, "BranchInuse") + assert.Equal(t, 0, stats.LeafInuse, "LeafInuse") + if os.Getpagesize() == 4096 { + // Incompatible page size + assert.Equal(t, 0, stats.BranchAlloc, "BranchAlloc") + assert.Equal(t, 0, stats.LeafAlloc, "LeafAlloc") + } + assert.Equal(t, 1, stats.BucketN, "BucketN") + assert.Equal(t, 1, stats.InlineBucketN, "InlineBucketN") + assert.Equal(t, pageHeaderSize, stats.InlineBucketInuse, "InlineBucketInuse") + return nil + }) + }) +} + // Ensure a bucket can calculate stats. func TestBucket_Stats_Nested(t *testing.T) { @@ -667,7 +701,7 @@ func TestBucket_Stats_Nested(t *testing.T) { assert.Equal(t, 122, stats.KeyN, "KeyN") assert.Equal(t, 3, stats.Depth, "Depth") assert.Equal(t, 0, stats.BranchInuse, "BranchInuse") - assert.Equal(t, 2474, stats.LeafInuse, "LeafInuse") + assert.Equal(t, 2506, stats.LeafInuse, "LeafInuse") if os.Getpagesize() == 4096 { // Incompatible page size assert.Equal(t, 0, stats.BranchAlloc, "BranchAlloc") @@ -712,7 +746,7 @@ func TestBucket_Stats_Large(t *testing.T) { assert.Equal(t, 100000, stats.KeyN, "KeyN") assert.Equal(t, 3, stats.Depth, "Depth") assert.Equal(t, 27007, stats.BranchInuse, "BranchInuse") - assert.Equal(t, 2598436, stats.LeafInuse, "LeafInuse") + assert.Equal(t, 2619092, stats.LeafInuse, "LeafInuse") if os.Getpagesize() == 4096 { // Incompatible page size assert.Equal(t, 77824, stats.BranchAlloc, "BranchAlloc") From 0716cdaec8e4a0d12085cda61d58ae275028670b Mon Sep 17 00:00:00 2001 From: Martin Kobetic Date: Mon, 12 May 2014 19:33:53 +0000 Subject: [PATCH 5/9] tweaks --- bucket.go | 3 ++- bucket_test.go | 21 +++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/bucket.go b/bucket.go index fdc9852..4d45b38 100644 --- a/bucket.go +++ b/bucket.go @@ -371,8 +371,9 @@ func (b *Bucket) Stats() BucketStats { b.forEachPage(func(p *page, depth int) { if (p.flags & leafPageFlag) != 0 { s.KeyN += int(p.count) - used := pageHeaderSize + (leafPageElementSize * int(p.count)) + used := pageHeaderSize if p.count != 0 { + used += leafPageElementSize * int(p.count-1) lastElement := p.leafPageElement(p.count - 1) used += int(lastElement.pos + lastElement.ksize + lastElement.vsize) } diff --git a/bucket_test.go b/bucket_test.go index 11d1d77..0be02e5 100644 --- a/bucket_test.go +++ b/bucket_test.go @@ -560,15 +560,15 @@ func TestBucket_Put_KeyTooLarge(t *testing.T) { // Ensure a bucket can calculate stats. func TestBucket_Stats(t *testing.T) { withOpenDB(func(db *DB, path string) { + big_key := []byte("really-big-value") db.Update(func(tx *Tx) error { // Add bucket with fewer keys but one big value. - _, err := tx.CreateBucket([]byte("woojits")) + b, err := tx.CreateBucket([]byte("woojits")) assert.NoError(t, err) - b := tx.Bucket([]byte("woojits")) for i := 0; i < 500; i++ { b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) } - b.Put([]byte("really-big-value"), []byte(strings.Repeat("*", 10000))) + b.Put(big_key, []byte(strings.Repeat("*", 10000))) return nil }) @@ -583,7 +583,9 @@ func TestBucket_Stats(t *testing.T) { assert.Equal(t, 501, stats.KeyN, "KeyN") assert.Equal(t, 2, stats.Depth, "Depth") assert.Equal(t, 125, stats.BranchInuse, "BranchInuse") - assert.Equal(t, 21004, stats.LeafInuse, "LeafInuse") + used := pageHeaderSize + 501*leafPageElementSize + used += 10*2 + 90*4 + 400*6 + len(big_key) + 10000 + assert.Equal(t, used, stats.LeafInuse, "LeafInuse") if os.Getpagesize() == 4096 { // Incompatible page size assert.Equal(t, 4096, stats.BranchAlloc, "BranchAlloc") @@ -628,7 +630,7 @@ func TestBucket_Stats_Small(t *testing.T) { } assert.Equal(t, 1, stats.BucketN, "BucketN") assert.Equal(t, 1, stats.InlineBucketN, "InlineBucketN") - assert.Equal(t, bucketHeaderSize+pageHeaderSize+leafPageElementSize+6, stats.InlineBucketInuse, "InlineBucketInuse") + assert.Equal(t, pageHeaderSize+leafPageElementSize+6, stats.InlineBucketInuse, "InlineBucketInuse") return nil }) }) @@ -701,7 +703,10 @@ func TestBucket_Stats_Nested(t *testing.T) { assert.Equal(t, 122, stats.KeyN, "KeyN") assert.Equal(t, 3, stats.Depth, "Depth") assert.Equal(t, 0, stats.BranchInuse, "BranchInuse") - assert.Equal(t, 2506, stats.LeafInuse, "LeafInuse") + baz := pageHeaderSize + 10*leafPageElementSize + 10*2 + foo := pageHeaderSize + 101*leafPageElementSize + 10*2 + 90*4 + 3 + bucketHeaderSize + bar := pageHeaderSize + 11*leafPageElementSize + 10*2 + 3 + bucketHeaderSize + baz + assert.Equal(t, foo+bar, stats.LeafInuse, "LeafInuse") if os.Getpagesize() == 4096 { // Incompatible page size assert.Equal(t, 0, stats.BranchAlloc, "BranchAlloc") @@ -709,7 +714,7 @@ func TestBucket_Stats_Nested(t *testing.T) { } assert.Equal(t, 3, stats.BucketN, "BucketN") assert.Equal(t, 1, stats.InlineBucketN, "InlineBucketN") - assert.Equal(t, 212, stats.InlineBucketInuse, "InlineBucketInuse") + assert.Equal(t, baz, stats.InlineBucketInuse, "InlineBucketInuse") return nil }) }) @@ -746,7 +751,7 @@ func TestBucket_Stats_Large(t *testing.T) { assert.Equal(t, 100000, stats.KeyN, "KeyN") assert.Equal(t, 3, stats.Depth, "Depth") assert.Equal(t, 27007, stats.BranchInuse, "BranchInuse") - assert.Equal(t, 2619092, stats.LeafInuse, "LeafInuse") + assert.Equal(t, 2598436, stats.LeafInuse, "LeafInuse") if os.Getpagesize() == 4096 { // Incompatible page size assert.Equal(t, 77824, stats.BranchAlloc, "BranchAlloc") From 4dbd354a8d9b4dae839de6e24707017af432ba2f Mon Sep 17 00:00:00 2001 From: Martin Kobetic Date: Mon, 12 May 2014 20:46:01 +0000 Subject: [PATCH 6/9] add inline stats to cmd --- cmd/bolt/stats.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmd/bolt/stats.go b/cmd/bolt/stats.go index 6f8505a..6968273 100644 --- a/cmd/bolt/stats.go +++ b/cmd/bolt/stats.go @@ -51,6 +51,14 @@ func Stats(path, prefix string) { printf("\tBytes allocated for physical leaf pages: %d\n", s.LeafAlloc) percentage = int(float32(s.LeafInuse) * 100.0 / float32(s.LeafAlloc)) printf("\tBytes actually used for leaf data: %d (%d%%)\n", s.LeafInuse, percentage) + + println("Bucket statistics ") + printf("\tTotal number of buckets: %d\n", s.BucketN) + percentage = int(float32(s.InlineBucketN) * 100.0 / float32(s.BucketN)) + printf("\tTotal number on inlined buckets: %d (%d%%)\n", s.InlineBucketN, percentage) + percentage = int(float32(s.InlineBucketInuse) * 100.0 / float32(s.LeafInuse)) + printf("\tBytes used for inlined buckets: %d (%d%%)\n", s.InlineBucketInuse, percentage) + return nil }) if err != nil { From 51568b52df23cf3c000f6cfe998daeb2b739b6f4 Mon Sep 17 00:00:00 2001 From: Martin Kobetic Date: Mon, 12 May 2014 20:47:03 +0000 Subject: [PATCH 7/9] undo page.go changes --- page.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/page.go b/page.go index 40e2421..56cf064 100644 --- a/page.go +++ b/page.go @@ -63,7 +63,7 @@ func (p *page) leafPageElement(index uint16) *leafPageElement { // leafPageElements retrieves a list of leaf nodes. func (p *page) leafPageElements() []leafPageElement { - return ((*[maxNodesPerPage]leafPageElement)(unsafe.Pointer(&p.ptr)))[:p.count] + return ((*[maxNodesPerPage]leafPageElement)(unsafe.Pointer(&p.ptr)))[:] } // branchPageElement retrieves the branch node by index @@ -73,7 +73,7 @@ func (p *page) branchPageElement(index uint16) *branchPageElement { // branchPageElements retrieves a list of branch nodes. func (p *page) branchPageElements() []branchPageElement { - return ((*[maxNodesPerPage]branchPageElement)(unsafe.Pointer(&p.ptr)))[:p.count] + return ((*[maxNodesPerPage]branchPageElement)(unsafe.Pointer(&p.ptr)))[:] } // dump writes n bytes of the page to STDERR as hex output. From 6eaeb31424c5cf7dae9dff104b33d166b9f4cdc3 Mon Sep 17 00:00:00 2001 From: Martin Kobetic Date: Tue, 13 May 2014 17:28:17 +0000 Subject: [PATCH 8/9] address review comments --- bucket.go | 26 ++++++++++++++++++++++--- cmd/bolt/stats.go | 17 +++++++++++++---- cmd/bolt/stats_test.go | 43 +++++++++++++++++++++++++++++++++--------- 3 files changed, 70 insertions(+), 16 deletions(-) diff --git a/bucket.go b/bucket.go index 4d45b38..96218f1 100644 --- a/bucket.go +++ b/bucket.go @@ -371,23 +371,36 @@ func (b *Bucket) Stats() BucketStats { b.forEachPage(func(p *page, depth int) { if (p.flags & leafPageFlag) != 0 { s.KeyN += int(p.count) + // used totals the used bytes for the page used := pageHeaderSize if p.count != 0 { + // If page has any elements, add all element headers. used += leafPageElementSize * int(p.count-1) + // Add all element key, value sizes. + // The computation takes advantage of the fact that the position + // of the last element's key/value equals to the total of the sizes + // of all previous elements' keys and values. + // It also includes the last element's header. lastElement := p.leafPageElement(p.count - 1) used += int(lastElement.pos + lastElement.ksize + lastElement.vsize) } if b.root == 0 { + // For inlined bucket just update the inline stats s.InlineBucketInuse += used } else { + // For non-inlined bucket update all the leaf stats s.LeafPageN++ s.LeafInuse += used s.LeafOverflowN += int(p.overflow) - // Collect stats from sub-buckets + // Collect stats from sub-buckets. + // Do that by iterating over all element headers + // looking for the ones with the bucketLeafFlag. for i := uint16(0); i < p.count; i++ { e := p.leafPageElement(i) if (e.flags & bucketLeafFlag) != 0 { + // For any bucket element, open the element value + // and recursively call Stats on the contained bucket. subStats.Add(b.openBucket(e.value()).Stats()) } } @@ -395,21 +408,28 @@ func (b *Bucket) Stats() BucketStats { } else if (p.flags & branchPageFlag) != 0 { s.BranchPageN++ lastElement := p.branchPageElement(p.count - 1) + // used totals the used bytes for the page + // Add header and all element headers. used := pageHeaderSize + (branchPageElementSize * int(p.count-1)) + // Add size of all keys and values. + // Again, use the fact that last element's position equals to + // the total of key, value sizes of all previous elements. used += int(lastElement.pos + lastElement.ksize) s.BranchInuse += used s.BranchOverflowN += int(p.overflow) } - + // Keep track of maximum page depth. if depth+1 > s.Depth { s.Depth = (depth + 1) } }) + // Alloc stats can be computed from page counts and pageSize. s.BranchAlloc = (s.BranchPageN + s.BranchOverflowN) * pageSize s.LeafAlloc = (s.LeafPageN + s.LeafOverflowN) * pageSize - // add the max depth of sub-buckets to get total nested depth + // Add the max depth of sub-buckets to get total nested depth. s.Depth += subStats.Depth + // Add the stats for all sub-buckets s.Add(subStats) return s } diff --git a/cmd/bolt/stats.go b/cmd/bolt/stats.go index 6968273..54a0f44 100644 --- a/cmd/bolt/stats.go +++ b/cmd/bolt/stats.go @@ -46,17 +46,26 @@ func Stats(path, prefix string) { println("Page size utilization") printf("\tBytes allocated for physical branch pages: %d\n", s.BranchAlloc) - percentage := int(float32(s.BranchInuse) * 100.0 / float32(s.BranchAlloc)) + var percentage int + if s.BranchAlloc != 0 { + percentage = int(float32(s.BranchInuse) * 100.0 / float32(s.BranchAlloc)) + } printf("\tBytes actually used for branch data: %d (%d%%)\n", s.BranchInuse, percentage) printf("\tBytes allocated for physical leaf pages: %d\n", s.LeafAlloc) - percentage = int(float32(s.LeafInuse) * 100.0 / float32(s.LeafAlloc)) + percentage = 0 + if s.LeafAlloc != 0 { + percentage = int(float32(s.LeafInuse) * 100.0 / float32(s.LeafAlloc)) + } printf("\tBytes actually used for leaf data: %d (%d%%)\n", s.LeafInuse, percentage) - println("Bucket statistics ") + println("Bucket statistics") printf("\tTotal number of buckets: %d\n", s.BucketN) percentage = int(float32(s.InlineBucketN) * 100.0 / float32(s.BucketN)) printf("\tTotal number on inlined buckets: %d (%d%%)\n", s.InlineBucketN, percentage) - percentage = int(float32(s.InlineBucketInuse) * 100.0 / float32(s.LeafInuse)) + percentage = 0 + if s.LeafInuse != 0 { + percentage = int(float32(s.InlineBucketInuse) * 100.0 / float32(s.LeafInuse)) + } printf("\tBytes used for inlined buckets: %d (%d%%)\n", s.InlineBucketInuse, percentage) return nil diff --git a/cmd/bolt/stats_test.go b/cmd/bolt/stats_test.go index 9662f09..2ad5d51 100644 --- a/cmd/bolt/stats_test.go +++ b/cmd/bolt/stats_test.go @@ -1,6 +1,8 @@ package main_test import ( + "os" + "strconv" "testing" "github.com/boltdb/bolt" @@ -9,12 +11,31 @@ import ( ) func TestStats(t *testing.T) { + if os.Getpagesize() != 4096 { + t.Skip() + } SetTestMode(true) open(func(db *bolt.DB, path string) { db.Update(func(tx *bolt.Tx) error { - tx.CreateBucket([]byte("foo")) - tx.CreateBucket([]byte("bar")) - tx.CreateBucket([]byte("baz")) + b, err := tx.CreateBucket([]byte("foo")) + if err != nil { + return err + } + for i := 0; i < 10; i++ { + b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) + } + b, err = tx.CreateBucket([]byte("bar")) + if err != nil { + return err + } + for i := 0; i < 100; i++ { + b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) + } + b, err = tx.CreateBucket([]byte("baz")) + if err != nil { + return err + } + b.Put([]byte("key"), []byte("value")) return nil }) db.Close() @@ -23,15 +44,19 @@ func TestStats(t *testing.T) { "Page count statistics\n"+ "\tNumber of logical branch pages: 0\n"+ "\tNumber of physical branch overflow pages: 0\n"+ - "\tNumber of logical leaf pages: 2\n"+ + "\tNumber of logical leaf pages: 1\n"+ "\tNumber of physical leaf overflow pages: 0\n"+ "Tree statistics\n"+ - "\tNumber of keys/value pairs: 0\n"+ - "\tNumber of levels in B+tree: 0\n"+ + "\tNumber of keys/value pairs: 101\n"+ + "\tNumber of levels in B+tree: 1\n"+ "Page size utilization\n"+ "\tBytes allocated for physical branch pages: 0\n"+ - "\tBytes actually used for branch data: 0\n"+ - "\tBytes allocated for physical leaf pages: 8192\n"+ - "\tBytes actually used for leaf data: 0", output) + "\tBytes actually used for branch data: 0 (0%)\n"+ + "\tBytes allocated for physical leaf pages: 4096\n"+ + "\tBytes actually used for leaf data: 1996 (48%)\n"+ + "Bucket statistics\n"+ + "\tTotal number of buckets: 2\n"+ + "\tTotal number on inlined buckets: 1 (50%)\n"+ + "\tBytes used for inlined buckets: 40 (2%)", output) }) } From e728eb90da02e301aecbcabff96e691efcfe019c Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Wed, 14 May 2014 12:06:31 -0600 Subject: [PATCH 9/9] Minor stats fixes. --- bucket.go | 10 +++++++++- bucket_test.go | 44 +++++++++++++++++++++++++++++++++----------- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/bucket.go b/bucket.go index 96218f1..877c310 100644 --- a/bucket.go +++ b/bucket.go @@ -371,11 +371,14 @@ func (b *Bucket) Stats() BucketStats { b.forEachPage(func(p *page, depth int) { if (p.flags & leafPageFlag) != 0 { s.KeyN += int(p.count) + // used totals the used bytes for the page used := pageHeaderSize + if p.count != 0 { // If page has any elements, add all element headers. used += leafPageElementSize * int(p.count-1) + // Add all element key, value sizes. // The computation takes advantage of the fact that the position // of the last element's key/value equals to the total of the sizes @@ -384,6 +387,7 @@ func (b *Bucket) Stats() BucketStats { lastElement := p.leafPageElement(p.count - 1) used += int(lastElement.pos + lastElement.ksize + lastElement.vsize) } + if b.root == 0 { // For inlined bucket just update the inline stats s.InlineBucketInuse += used @@ -408,9 +412,11 @@ func (b *Bucket) Stats() BucketStats { } else if (p.flags & branchPageFlag) != 0 { s.BranchPageN++ lastElement := p.branchPageElement(p.count - 1) + // used totals the used bytes for the page // Add header and all element headers. used := pageHeaderSize + (branchPageElementSize * int(p.count-1)) + // Add size of all keys and values. // Again, use the fact that last element's position equals to // the total of key, value sizes of all previous elements. @@ -418,11 +424,13 @@ func (b *Bucket) Stats() BucketStats { s.BranchInuse += used s.BranchOverflowN += int(p.overflow) } + // Keep track of maximum page depth. if depth+1 > s.Depth { s.Depth = (depth + 1) } }) + // Alloc stats can be computed from page counts and pageSize. s.BranchAlloc = (s.BranchPageN + s.BranchOverflowN) * pageSize s.LeafAlloc = (s.LeafPageN + s.LeafOverflowN) * pageSize @@ -568,7 +576,7 @@ func (b *Bucket) maxInlineBucketSize() int { func (b *Bucket) write() []byte { // Allocate the appropriate size. var n = b.rootNode - var value = make([]byte, bucketHeaderSize+pageHeaderSize+n.size()) + var value = make([]byte, bucketHeaderSize+n.size()) // Write a bucket header. var bucket = (*bucket)(unsafe.Pointer(&value[0])) diff --git a/bucket_test.go b/bucket_test.go index 0be02e5..f6bd13f 100644 --- a/bucket_test.go +++ b/bucket_test.go @@ -566,7 +566,7 @@ func TestBucket_Stats(t *testing.T) { b, err := tx.CreateBucket([]byte("woojits")) assert.NoError(t, err) for i := 0; i < 500; i++ { - b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) + b.Put([]byte(fmt.Sprintf("%03d", i)), []byte(strconv.Itoa(i))) } b.Put(big_key, []byte(strings.Repeat("*", 10000))) @@ -582,15 +582,24 @@ func TestBucket_Stats(t *testing.T) { assert.Equal(t, 2, stats.LeafOverflowN, "LeafOverflowN") assert.Equal(t, 501, stats.KeyN, "KeyN") assert.Equal(t, 2, stats.Depth, "Depth") - assert.Equal(t, 125, stats.BranchInuse, "BranchInuse") - used := pageHeaderSize + 501*leafPageElementSize - used += 10*2 + 90*4 + 400*6 + len(big_key) + 10000 - assert.Equal(t, used, stats.LeafInuse, "LeafInuse") + + branchInuse := pageHeaderSize // branch page header + branchInuse += 6 * branchPageElementSize // branch elements + branchInuse += 6 * 3 // branch keys (6 3-byte keys) + assert.Equal(t, branchInuse, stats.BranchInuse, "BranchInuse") + + leafInuse := 6 * pageHeaderSize // leaf page header + leafInuse += 501 * leafPageElementSize // leaf elements + leafInuse += 500*3 + len(big_key) // leaf keys + leafInuse += 1*10 + 2*90 + 3*400 + 10000 // leaf values + assert.Equal(t, leafInuse, stats.LeafInuse, "LeafInuse") + if os.Getpagesize() == 4096 { // Incompatible page size assert.Equal(t, 4096, stats.BranchAlloc, "BranchAlloc") assert.Equal(t, 32768, stats.LeafAlloc, "LeafAlloc") } + assert.Equal(t, 1, stats.BucketN, "BucketN") assert.Equal(t, 0, stats.InlineBucketN, "InlineBucketN") assert.Equal(t, 0, stats.InlineBucketInuse, "InlineBucketInuse") @@ -672,13 +681,12 @@ func TestBucket_Stats_EmptyBucket(t *testing.T) { // Ensure a bucket can calculate stats. func TestBucket_Stats_Nested(t *testing.T) { - withOpenDB(func(db *DB, path string) { db.Update(func(tx *Tx) error { b, err := tx.CreateBucket([]byte("foo")) assert.NoError(t, err) for i := 0; i < 100; i++ { - b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) + b.Put([]byte(fmt.Sprintf("%02d", i)), []byte(fmt.Sprintf("%02d", i))) } bar, err := b.CreateBucket([]byte("bar")) assert.NoError(t, err) @@ -692,7 +700,9 @@ func TestBucket_Stats_Nested(t *testing.T) { } return nil }) + mustCheck(db) + db.View(func(tx *Tx) error { b := tx.Bucket([]byte("foo")) stats := b.Stats() @@ -703,10 +713,22 @@ func TestBucket_Stats_Nested(t *testing.T) { assert.Equal(t, 122, stats.KeyN, "KeyN") assert.Equal(t, 3, stats.Depth, "Depth") assert.Equal(t, 0, stats.BranchInuse, "BranchInuse") - baz := pageHeaderSize + 10*leafPageElementSize + 10*2 - foo := pageHeaderSize + 101*leafPageElementSize + 10*2 + 90*4 + 3 + bucketHeaderSize - bar := pageHeaderSize + 11*leafPageElementSize + 10*2 + 3 + bucketHeaderSize + baz - assert.Equal(t, foo+bar, stats.LeafInuse, "LeafInuse") + + foo := pageHeaderSize // foo + foo += 101 * leafPageElementSize // foo leaf elements + foo += 100*2 + 100*2 // foo leaf key/values + foo += 3 + bucketHeaderSize // foo -> bar key/value + + bar := pageHeaderSize // bar + bar += 11 * leafPageElementSize // bar leaf elements + bar += 10 + 10 // bar leaf key/values + bar += 3 + bucketHeaderSize // bar -> baz key/value + + baz := pageHeaderSize // baz (inline) + baz += 10 * leafPageElementSize // baz leaf elements + baz += 10 + 10 // baz leaf key/values + + assert.Equal(t, foo+bar+baz, stats.LeafInuse, "LeafInuse") if os.Getpagesize() == 4096 { // Incompatible page size assert.Equal(t, 0, stats.BranchAlloc, "BranchAlloc")