From 50e04a29aeebe461fe9d00ec780246ac3a222c09 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Thu, 8 May 2014 08:43:18 -0600 Subject: [PATCH 1/4] Add 'bolt info'. --- cmd/bolt/info.go | 26 ++++++++++++++++++++++++++ cmd/bolt/info_test.go | 32 ++++++++++++++++++++++++++++++++ cmd/bolt/main.go | 8 ++++++++ 3 files changed, 66 insertions(+) create mode 100644 cmd/bolt/info.go create mode 100644 cmd/bolt/info_test.go diff --git a/cmd/bolt/info.go b/cmd/bolt/info.go new file mode 100644 index 0000000..1e9e0d8 --- /dev/null +++ b/cmd/bolt/info.go @@ -0,0 +1,26 @@ +package main + +import ( + "os" + + "github.com/boltdb/bolt" +) + +// Info prints basic information about a database. +func Info(path string) { + if _, err := os.Stat(path); os.IsNotExist(err) { + fatal(err) + return + } + + db, err := bolt.Open(path, 0600) + if err != nil { + fatal(err) + return + } + defer db.Close() + + // Print basic database info. + var info = db.Info() + printf("Page Size: %d", info.PageSize) +} diff --git a/cmd/bolt/info_test.go b/cmd/bolt/info_test.go new file mode 100644 index 0000000..668cc61 --- /dev/null +++ b/cmd/bolt/info_test.go @@ -0,0 +1,32 @@ +package main_test + +import ( + "testing" + + "github.com/boltdb/bolt" + . "github.com/boltdb/bolt/cmd/bolt" + "github.com/stretchr/testify/assert" +) + +// Ensure that a database info can be printed. +func TestInfo(t *testing.T) { + SetTestMode(true) + open(func(db *bolt.DB, path string) { + db.Update(func(tx *bolt.Tx) error { + tx.CreateBucket([]byte("widgets")) + b := tx.Bucket([]byte("widgets")) + b.Put([]byte("foo"), []byte("0000")) + return nil + }) + db.Close() + output := run("info", path) + assert.Equal(t, `Page Size: 4096`, output) + }) +} + +// Ensure that an error is reported if the database is not found. +func TestInfo_NotFound(t *testing.T) { + SetTestMode(true) + output := run("info", "no/such/db") + assert.Equal(t, "stat no/such/db: no such file or directory", output) +} diff --git a/cmd/bolt/main.go b/cmd/bolt/main.go index 1af2636..3397042 100644 --- a/cmd/bolt/main.go +++ b/cmd/bolt/main.go @@ -25,6 +25,14 @@ func NewApp() *cli.App { app.Usage = "BoltDB toolkit" app.Version = fmt.Sprintf("0.1.0 (%s %s)", branch, commit) app.Commands = []cli.Command{ + { + Name: "info", + Usage: "Print basic information about a database", + Action: func(c *cli.Context) { + path := c.Args().Get(0) + Info(path) + }, + }, { Name: "get", Usage: "Retrieve a value for given key in a bucket", From d279ea44cedc355c5dc574d2fd6edfb5e2088410 Mon Sep 17 00:00:00 2001 From: Martin Kobetic Date: Fri, 9 May 2014 13:35:00 +0000 Subject: [PATCH 2/4] add asserts for detecting pgid high watermark overflow --- bucket.go | 1 + cmd/bolt/stats.go | 54 +++++++++++++++++++++++++++++++++++++++++++++++ db.go | 4 ++++ node.go | 4 ++++ 4 files changed, 63 insertions(+) create mode 100644 cmd/bolt/stats.go diff --git a/bucket.go b/bucket.go index 25ad1ba..a57f456 100644 --- a/bucket.go +++ b/bucket.go @@ -476,6 +476,7 @@ func (b *Bucket) spill() error { b.rootNode = b.rootNode.root() // Update the root node for this bucket. + _assert(b.rootNode.pgid < b.tx.meta.pgid, "pgid (%d) above high water mark (%d)", b.rootNode.pgid, b.tx.meta.pgid) b.root = b.rootNode.pgid return nil diff --git a/cmd/bolt/stats.go b/cmd/bolt/stats.go new file mode 100644 index 0000000..da344d0 --- /dev/null +++ b/cmd/bolt/stats.go @@ -0,0 +1,54 @@ +package main + +import ( + "os" + + "github.com/boltdb/bolt" +) + +// Keys retrieves a list of keys for a given bucket. +func Stats(path, name string) { + if _, err := os.Stat(path); os.IsNotExist(err) { + fatal(err) + return + } + + db, err := bolt.Open(path, 0600) + if err != nil { + fatal(err) + return + } + defer db.Close() + + err = db.View(func(tx *bolt.Tx) error { + // Find bucket. + b := tx.Bucket([]byte(name)) + if b == nil { + fatalf("bucket not found: %s", name) + return nil + } + + // Iterate over each key. + s := b.Stats() + println("Page count statistics") + printf("\tNumber of logical branch pages: %d\n", s.BranchPageN) + printf("\tNumber of physical branch overflow pages: %d\n", s.BranchOverflowN) + printf("\tNumber of logical leaf pages: %d\n", s.LeafPageN) + printf("\tNumber of physical leaf overflow pages: %d\n", s.LeafOverflowN) + + println("Tree statistics") + printf("\tNumber of keys/value pairs: %d\n", s.KeyN) + printf("\tNumber of levels in B+tree: %d\n", s.Depth) + + println("Page size utilization") + printf("\tBytes allocated for physical branch pages: %d\n", s.BranchAlloc) + printf("\tBytes actually used for branch data: %d\n", s.BranchInuse) + printf("\tBytes allocated for physical leaf pages: %d\n", s.LeafAlloc) + printf("\tBytes actually used for leaf data: %d\n", s.LeafInuse) + return nil + }) + if err != nil { + fatal(err) + return + } +} diff --git a/db.go b/db.go index d96a161..0b74621 100644 --- a/db.go +++ b/db.go @@ -705,6 +705,10 @@ func (m *meta) copy(dest *meta) { // write writes the meta onto a page. func (m *meta) write(p *page) { + + _assert(m.root.root < m.pgid, "root bucket pgid (%d) above high water mark (%d)", m.root.root, m.pgid) + _assert(m.freelist < m.pgid, "freelist pgid (%d) above high water mark (%d)", m.freelist, m.pgid) + // Page id is either going to be 0 or 1 which we can determine by the transaction ID. p.id = pgid(m.txid % 2) p.flags |= metaPageFlag diff --git a/node.go b/node.go index 1bc889d..14bf136 100644 --- a/node.go +++ b/node.go @@ -98,6 +98,8 @@ func (n *node) prevSibling() *node { // put inserts a key/value. func (n *node) put(oldKey, newKey, value []byte, pgid pgid, flags uint32) { + _assert(pgid < n.bucket.tx.meta.pgid, "pgid (%d) above high water mark (%d)", pgid, n.bucket.tx.meta.pgid) + // Find insertion index. index := sort.Search(len(n.inodes), func(i int) bool { return bytes.Compare(n.inodes[i].key, oldKey) != -1 }) @@ -278,6 +280,7 @@ func (n *node) spill() error { } // Write the node. + _assert(p.id < tx.meta.pgid, "pgid (%d) above high water mark (%d)", p.id, tx.meta.pgid) node.pgid = p.id node.write(p) @@ -307,6 +310,7 @@ func (n *node) spill() error { } // Write the new root. + _assert(p.id < tx.meta.pgid, "pgid (%d) above high water mark (%d)", p.id, tx.meta.pgid) parent.pgid = p.id parent.write(p) } From 3ffcee742c104f84321cd6c08eaa449c65114395 Mon Sep 17 00:00:00 2001 From: Martin Kobetic Date: Fri, 9 May 2014 13:38:11 +0000 Subject: [PATCH 3/4] remove stray file --- cmd/bolt/stats.go | 54 ----------------------------------------------- 1 file changed, 54 deletions(-) delete mode 100644 cmd/bolt/stats.go diff --git a/cmd/bolt/stats.go b/cmd/bolt/stats.go deleted file mode 100644 index da344d0..0000000 --- a/cmd/bolt/stats.go +++ /dev/null @@ -1,54 +0,0 @@ -package main - -import ( - "os" - - "github.com/boltdb/bolt" -) - -// Keys retrieves a list of keys for a given bucket. -func Stats(path, name string) { - if _, err := os.Stat(path); os.IsNotExist(err) { - fatal(err) - return - } - - db, err := bolt.Open(path, 0600) - if err != nil { - fatal(err) - return - } - defer db.Close() - - err = db.View(func(tx *bolt.Tx) error { - // Find bucket. - b := tx.Bucket([]byte(name)) - if b == nil { - fatalf("bucket not found: %s", name) - return nil - } - - // Iterate over each key. - s := b.Stats() - println("Page count statistics") - printf("\tNumber of logical branch pages: %d\n", s.BranchPageN) - printf("\tNumber of physical branch overflow pages: %d\n", s.BranchOverflowN) - printf("\tNumber of logical leaf pages: %d\n", s.LeafPageN) - printf("\tNumber of physical leaf overflow pages: %d\n", s.LeafOverflowN) - - println("Tree statistics") - printf("\tNumber of keys/value pairs: %d\n", s.KeyN) - printf("\tNumber of levels in B+tree: %d\n", s.Depth) - - println("Page size utilization") - printf("\tBytes allocated for physical branch pages: %d\n", s.BranchAlloc) - printf("\tBytes actually used for branch data: %d\n", s.BranchInuse) - printf("\tBytes allocated for physical leaf pages: %d\n", s.LeafAlloc) - printf("\tBytes actually used for leaf data: %d\n", s.LeafInuse) - return nil - }) - if err != nil { - fatal(err) - return - } -} From c595561faae918374670ed667c64a7b95bbde9df Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Fri, 9 May 2014 07:55:01 -0600 Subject: [PATCH 4/4] Fix node unit tests. --- node_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/node_test.go b/node_test.go index f639376..1393e2a 100644 --- a/node_test.go +++ b/node_test.go @@ -9,7 +9,7 @@ import ( // Ensure that a node can insert a key/value. func TestNode_put(t *testing.T) { - n := &node{inodes: make(inodes, 0)} + n := &node{inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{meta: &meta{pgid: 1}}}} n.put([]byte("baz"), []byte("baz"), []byte("2"), 0, 0) n.put([]byte("foo"), []byte("foo"), []byte("0"), 0, 0) n.put([]byte("bar"), []byte("bar"), []byte("1"), 0, 0) @@ -58,7 +58,7 @@ func TestNode_read_LeafPage(t *testing.T) { // Ensure that a node can serialize into a leaf page. func TestNode_write_LeafPage(t *testing.T) { // Create a node. - n := &node{isLeaf: true, inodes: make(inodes, 0)} + n := &node{isLeaf: true, inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{meta: &meta{pgid: 1}}}} n.put([]byte("susy"), []byte("susy"), []byte("que"), 0, 0) n.put([]byte("ricki"), []byte("ricki"), []byte("lake"), 0, 0) n.put([]byte("john"), []byte("john"), []byte("johnson"), 0, 0) @@ -85,7 +85,7 @@ func TestNode_write_LeafPage(t *testing.T) { // Ensure that a node can split into appropriate subgroups. func TestNode_split(t *testing.T) { // Create a node. - n := &node{inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{}}} + n := &node{inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{meta: &meta{pgid: 1}}}} n.put([]byte("00000001"), []byte("00000001"), []byte("0123456701234567"), 0, 0) n.put([]byte("00000002"), []byte("00000002"), []byte("0123456701234567"), 0, 0) n.put([]byte("00000003"), []byte("00000003"), []byte("0123456701234567"), 0, 0) @@ -104,7 +104,7 @@ func TestNode_split(t *testing.T) { // Ensure that a page with the minimum number of inodes just returns a single node. func TestNode_split_MinKeys(t *testing.T) { // Create a node. - n := &node{inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{}}} + n := &node{inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{meta: &meta{pgid: 1}}}} n.put([]byte("00000001"), []byte("00000001"), []byte("0123456701234567"), 0, 0) n.put([]byte("00000002"), []byte("00000002"), []byte("0123456701234567"), 0, 0) @@ -116,7 +116,7 @@ func TestNode_split_MinKeys(t *testing.T) { // Ensure that a node that has keys that all fit on a page just returns one leaf. func TestNode_split_SinglePage(t *testing.T) { // Create a node. - n := &node{inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{}}} + n := &node{inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{meta: &meta{pgid: 1}}}} n.put([]byte("00000001"), []byte("00000001"), []byte("0123456701234567"), 0, 0) n.put([]byte("00000002"), []byte("00000002"), []byte("0123456701234567"), 0, 0) n.put([]byte("00000003"), []byte("00000003"), []byte("0123456701234567"), 0, 0)