diff --git a/bolt.go b/bolt.go index f258731..538c5a9 100644 --- a/bolt.go +++ b/bolt.go @@ -15,6 +15,14 @@ func Open(path string, mode os.FileMode) (*DB, error) { return db, nil } +// ErrorList represents a slice of errors. +type ErrorList []error + +// Error returns a readable count of the errors in the list. +func (l ErrorList) Error() string { + return fmt.Sprintf("%d errors occurred", len(l)) +} + // _assert will panic with a given formatted message if the given condition is false. func _assert(condition bool, msg string, v ...interface{}) { if !condition { diff --git a/bucket_test.go b/bucket_test.go index 5f54b17..e2eb659 100644 --- a/bucket_test.go +++ b/bucket_test.go @@ -271,6 +271,7 @@ func TestBucketStat(t *testing.T) { return nil }) + mustCheck(db) db.View(func(tx *Tx) error { b := tx.Bucket("widgets") stat := b.Stat() diff --git a/cmd/bolt/check.go b/cmd/bolt/check.go new file mode 100644 index 0000000..ec2ea69 --- /dev/null +++ b/cmd/bolt/check.go @@ -0,0 +1,34 @@ +package main + +import ( + "os" + + "github.com/boltdb/bolt" +) + +// Check performs a consistency check on the database and prints any errors found. +func Check(path string) { + if _, err := os.Stat(path); os.IsNotExist(err) { + fatal(err) + return + } + + db, err := bolt.Open(path, 0600) + if err != nil { + fatal(err) + return + } + defer db.Close() + + // Perform consistency check. + if err := db.Check(); err != nil { + if errors, ok := err.(bolt.ErrorList); ok { + for _, err := range errors { + println(err) + } + } + fatalln(err) + return + } + println("OK") +} diff --git a/cmd/bolt/main.go b/cmd/bolt/main.go index 6b28060..bc5fadb 100644 --- a/cmd/bolt/main.go +++ b/cmd/bolt/main.go @@ -61,6 +61,14 @@ func NewApp() *cli.App { Pages(path) }, }, + { + Name: "check", + Usage: "Performs a consistency check on the database", + Action: func(c *cli.Context) { + path := c.Args().Get(0) + Check(path) + }, + }, } return app } diff --git a/db.go b/db.go index cbcc322..7ad9ca6 100644 --- a/db.go +++ b/db.go @@ -523,6 +523,63 @@ func (db *DB) Stat() (*Stat, error) { return s, nil } +// Check performs several consistency checks on the database. +// An error is returned if any inconsistency is found. +func (db *DB) Check() error { + return db.Update(func(tx *Tx) error { + var errors ErrorList + + // Track every reachable page. + reachable := make(map[pgid]*page) + reachable[0] = tx.page(0) // meta0 + reachable[1] = tx.page(1) // meta1 + reachable[tx.meta.buckets] = tx.page(tx.meta.buckets) + reachable[tx.meta.freelist] = tx.page(tx.meta.freelist) + + // Check each reachable page within each bucket. + for _, bucket := range tx.Buckets() { + // warnf("[bucket] %s", bucket.name) + tx.forEachPage(bucket.root, 0, func(p *page, _ int) { + // Ensure each page is only referenced once. + for i := pgid(0); i <= pgid(p.overflow); i++ { + var id = p.id + i + if _, ok := reachable[id]; ok { + errors = append(errors, fmt.Errorf("page %d: multiple references", int(id))) + } + reachable[id] = p + } + + // Retrieve page info. + info, err := tx.Page(int(p.id)) + // warnf("[page] %d + %d (%s)", p.id, p.overflow, info.Type) + if err != nil { + errors = append(errors, err) + } else if info == nil { + errors = append(errors, fmt.Errorf("page %d: out of bounds: %d", int(p.id), int(tx.meta.pgid))) + } else if info.Type != "branch" && info.Type != "leaf" { + errors = append(errors, fmt.Errorf("page %d: invalid type: %s", int(p.id), info.Type)) + } + }) + } + + // Ensure all pages below high water mark are either reachable or freed. + for i := pgid(0); i < tx.meta.pgid; i++ { + _, isReachable := reachable[i] + if !isReachable && !db.freelist.isFree(i) { + errors = append(errors, fmt.Errorf("page %d: unreachable unfreed", int(i))) + } + } + + // TODO(benbjohnson): Ensure that only one buckets page exists. + + if len(errors) > 0 { + return errors + } + + return nil + }) +} + // page retrieves a page reference from the mmap based on the current page size. func (db *DB) page(id pgid) *page { pos := id * pgid(db.pageSize) diff --git a/db_test.go b/db_test.go index a2df9d3..487b968 100644 --- a/db_test.go +++ b/db_test.go @@ -78,7 +78,8 @@ func TestDBMetaInitWriteError(t *testing.T) { // Ensure that a database that is too small returns an error. func TestDBFileTooSmall(t *testing.T) { - withOpenDB(func(db *DB, path string) { + withDB(func(db *DB, path string) { + assert.NoError(t, db.Open(path, 0666)) db.Close() // corrupt the database @@ -130,6 +131,7 @@ func TestDBBeginRW(t *testing.T) { assert.NoError(t, err) assert.Equal(t, tx.DB(), db) assert.Equal(t, tx.Writable(), true) + assert.NoError(t, tx.Commit()) }) } @@ -382,9 +384,28 @@ func withOpenDB(fn func(*DB, string)) { } defer db.Close() fn(db, path) + + // Check database consistency after every test. + mustCheck(db) }) } +// mustCheck runs a consistency check on the database and panics if any errors are found. +func mustCheck(db *DB) { + if err := db.Check(); err != nil { + // Copy db off first. + db.CopyFile("/tmp/check.db", 0600) + + if errors, ok := err.(ErrorList); ok { + for _, err := range errors { + warn(err) + } + } + warn(err) + panic("check failure: see /tmp/check.db") + } +} + func trunc(b []byte, length int) []byte { if length < len(b) { return b[:length] diff --git a/node.go b/node.go index 51be690..0a4c7af 100644 --- a/node.go +++ b/node.go @@ -258,6 +258,7 @@ func (n *node) rebalance() { // Remove old child. child.parent = nil delete(n.tx.nodes, child.pgid) + child.free() } return @@ -318,6 +319,7 @@ func (n *node) rebalance() { n.inodes = append(n.inodes, target.inodes...) n.parent.del(target.key) delete(n.tx.nodes, target.pgid) + target.free() } else { // Reparent all child nodes being moved. for _, inode := range n.inodes { @@ -331,6 +333,7 @@ func (n *node) rebalance() { n.parent.del(n.key) n.parent.put(target.key, target.inodes[0].key, nil, target.pgid) delete(n.tx.nodes, n.pgid) + n.free() } // Either this node or the target node was deleted from the parent so rebalance it. @@ -357,6 +360,13 @@ func (n *node) dereference() { } } +// free adds the node's underlying page to the freelist. +func (n *node) free() { + if n.pgid != 0 { + n.tx.db.freelist.free(n.tx.id(), n.tx.page(n.pgid)) + } +} + // nodesByDepth sorts a list of branches by deepest first. type nodesByDepth []*node