From d0e8a99e30eebd30669908bbe57c969e8921ca6c Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Tue, 14 Apr 2015 16:32:20 -0600 Subject: [PATCH 1/4] Refactor bolt CLI. --- cmd/bolt/bench.go | 425 ----------------- cmd/bolt/buckets.go | 33 -- cmd/bolt/buckets_test.go | 31 -- cmd/bolt/check.go | 47 -- cmd/bolt/get.go | 45 -- cmd/bolt/get_test.go | 54 --- cmd/bolt/info.go | 26 -- cmd/bolt/info_test.go | 31 -- cmd/bolt/keys.go | 41 -- cmd/bolt/keys_test.go | 42 -- cmd/bolt/main.go | 951 ++++++++++++++++++++++++++++++++------- cmd/bolt/main_test.go | 164 +++++-- cmd/bolt/pages.go | 57 --- cmd/bolt/stats.go | 77 ---- cmd/bolt/stats_test.go | 61 --- 15 files changed, 913 insertions(+), 1172 deletions(-) delete mode 100644 cmd/bolt/bench.go delete mode 100644 cmd/bolt/buckets.go delete mode 100644 cmd/bolt/buckets_test.go delete mode 100644 cmd/bolt/check.go delete mode 100644 cmd/bolt/get.go delete mode 100644 cmd/bolt/get_test.go delete mode 100644 cmd/bolt/info.go delete mode 100644 cmd/bolt/info_test.go delete mode 100644 cmd/bolt/keys.go delete mode 100644 cmd/bolt/keys_test.go delete mode 100644 cmd/bolt/pages.go delete mode 100644 cmd/bolt/stats.go delete mode 100644 cmd/bolt/stats_test.go diff --git a/cmd/bolt/bench.go b/cmd/bolt/bench.go deleted file mode 100644 index 3ade8b8..0000000 --- a/cmd/bolt/bench.go +++ /dev/null @@ -1,425 +0,0 @@ -package main - -import ( - "encoding/binary" - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "math/rand" - "os" - "runtime" - "runtime/pprof" - "time" - - "github.com/boltdb/bolt" -) - -// File handlers for the various profiles. -var cpuprofile, memprofile, blockprofile *os.File - -var benchBucketName = []byte("bench") - -// Bench executes a customizable, synthetic benchmark against Bolt. -func Bench(options *BenchOptions) { - var results BenchResults - - // Validate options. - if options.BatchSize == 0 { - options.BatchSize = options.Iterations - } else if options.Iterations%options.BatchSize != 0 { - fatal("number of iterations must be divisible by the batch size") - } - - // Generate temp path if one is not passed in. - path := options.Path - if path == "" { - path = tempfile() - } - - if options.Clean { - defer os.Remove(path) - } else { - println("work:", path) - } - - // Create database. - db, err := bolt.Open(path, 0600, nil) - if err != nil { - fatal(err) - return - } - db.NoSync = options.NoSync - defer db.Close() - - // Enable streaming stats. - if options.StatsInterval > 0 { - go printStats(db, options.StatsInterval) - } - - // Start profiling for writes. - if options.ProfileMode == "rw" || options.ProfileMode == "w" { - benchStartProfiling(options) - } - - // Write to the database. - if err := benchWrite(db, options, &results); err != nil { - fatal("bench: write: ", err) - } - - // Stop profiling for writes only. - if options.ProfileMode == "w" { - benchStopProfiling() - } - - // Start profiling for reads. - if options.ProfileMode == "r" { - benchStartProfiling(options) - } - - // Read from the database. - if err := benchRead(db, options, &results); err != nil { - fatal("bench: read: ", err) - } - - // Stop profiling for writes only. - if options.ProfileMode == "rw" || options.ProfileMode == "r" { - benchStopProfiling() - } - - // Print results. - fmt.Fprintf(os.Stderr, "# Write\t%v\t(%v/op)\t(%v op/sec)\n", results.WriteDuration, results.WriteOpDuration(), results.WriteOpsPerSecond()) - fmt.Fprintf(os.Stderr, "# Read\t%v\t(%v/op)\t(%v op/sec)\n", results.ReadDuration, results.ReadOpDuration(), results.ReadOpsPerSecond()) - fmt.Fprintln(os.Stderr, "") -} - -// Writes to the database. -func benchWrite(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - var err error - var t = time.Now() - - switch options.WriteMode { - case "seq": - err = benchWriteSequential(db, options, results) - case "rnd": - err = benchWriteRandom(db, options, results) - case "seq-nest": - err = benchWriteSequentialNested(db, options, results) - case "rnd-nest": - err = benchWriteRandomNested(db, options, results) - default: - return fmt.Errorf("invalid write mode: %s", options.WriteMode) - } - - results.WriteDuration = time.Since(t) - - return err -} - -func benchWriteSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - var i = uint32(0) - return benchWriteWithSource(db, options, results, func() uint32 { i++; return i }) -} - -func benchWriteRandom(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - r := rand.New(rand.NewSource(time.Now().UnixNano())) - return benchWriteWithSource(db, options, results, func() uint32 { return r.Uint32() }) -} - -func benchWriteSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - var i = uint32(0) - return benchWriteNestedWithSource(db, options, results, func() uint32 { i++; return i }) -} - -func benchWriteRandomNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - r := rand.New(rand.NewSource(time.Now().UnixNano())) - return benchWriteNestedWithSource(db, options, results, func() uint32 { return r.Uint32() }) -} - -func benchWriteWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error { - results.WriteOps = options.Iterations - - for i := 0; i < options.Iterations; i += options.BatchSize { - err := db.Update(func(tx *bolt.Tx) error { - b, _ := tx.CreateBucketIfNotExists(benchBucketName) - b.FillPercent = options.FillPercent - - for j := 0; j < options.BatchSize; j++ { - var key = make([]byte, options.KeySize) - var value = make([]byte, options.ValueSize) - binary.BigEndian.PutUint32(key, keySource()) - if err := b.Put(key, value); err != nil { - return err - } - } - - return nil - }) - if err != nil { - return err - } - } - return nil -} - -func benchWriteNestedWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error { - results.WriteOps = options.Iterations - - for i := 0; i < options.Iterations; i += options.BatchSize { - err := db.Update(func(tx *bolt.Tx) error { - top, _ := tx.CreateBucketIfNotExists(benchBucketName) - top.FillPercent = options.FillPercent - - var name = make([]byte, options.KeySize) - binary.BigEndian.PutUint32(name, keySource()) - b, _ := top.CreateBucketIfNotExists(name) - b.FillPercent = options.FillPercent - - for j := 0; j < options.BatchSize; j++ { - var key = make([]byte, options.KeySize) - var value = make([]byte, options.ValueSize) - binary.BigEndian.PutUint32(key, keySource()) - if err := b.Put(key, value); err != nil { - return err - } - } - - return nil - }) - if err != nil { - return err - } - } - return nil -} - -// Reads from the database. -func benchRead(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - var err error - var t = time.Now() - - switch options.ReadMode { - case "seq": - if options.WriteMode == "seq-nest" || options.WriteMode == "rnd-nest" { - err = benchReadSequentialNested(db, options, results) - } else { - err = benchReadSequential(db, options, results) - } - default: - return fmt.Errorf("invalid read mode: %s", options.ReadMode) - } - - results.ReadDuration = time.Since(t) - - return err -} - -func benchReadSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - return db.View(func(tx *bolt.Tx) error { - var t = time.Now() - - for { - c := tx.Bucket(benchBucketName).Cursor() - var count int - for k, v := c.First(); k != nil; k, v = c.Next() { - if v == nil { - return errors.New("invalid value") - } - count++ - } - - if options.WriteMode == "seq" && count != options.Iterations { - return fmt.Errorf("read seq: iter mismatch: expected %d, got %d", options.Iterations, count) - } - - results.ReadOps += count - - // Make sure we do this for at least a second. - if time.Since(t) >= time.Second { - break - } - } - - return nil - }) -} - -func benchReadSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - return db.View(func(tx *bolt.Tx) error { - var t = time.Now() - - for { - var count int - var top = tx.Bucket(benchBucketName) - top.ForEach(func(name, _ []byte) error { - c := top.Bucket(name).Cursor() - for k, v := c.First(); k != nil; k, v = c.Next() { - if v == nil { - return errors.New("invalid value") - } - count++ - } - return nil - }) - - if options.WriteMode == "seq-nest" && count != options.Iterations { - return fmt.Errorf("read seq-nest: iter mismatch: expected %d, got %d", options.Iterations, count) - } - - results.ReadOps += count - - // Make sure we do this for at least a second. - if time.Since(t) >= time.Second { - break - } - } - - return nil - }) -} - -// Starts all profiles set on the options. -func benchStartProfiling(options *BenchOptions) { - var err error - - // Start CPU profiling. - if options.CPUProfile != "" { - cpuprofile, err = os.Create(options.CPUProfile) - if err != nil { - fatalf("bench: could not create cpu profile %q: %v", options.CPUProfile, err) - } - pprof.StartCPUProfile(cpuprofile) - } - - // Start memory profiling. - if options.MemProfile != "" { - memprofile, err = os.Create(options.MemProfile) - if err != nil { - fatalf("bench: could not create memory profile %q: %v", options.MemProfile, err) - } - runtime.MemProfileRate = 4096 - } - - // Start fatal profiling. - if options.BlockProfile != "" { - blockprofile, err = os.Create(options.BlockProfile) - if err != nil { - fatalf("bench: could not create block profile %q: %v", options.BlockProfile, err) - } - runtime.SetBlockProfileRate(1) - } -} - -// Stops all profiles. -func benchStopProfiling() { - if cpuprofile != nil { - pprof.StopCPUProfile() - cpuprofile.Close() - cpuprofile = nil - } - - if memprofile != nil { - pprof.Lookup("heap").WriteTo(memprofile, 0) - memprofile.Close() - memprofile = nil - } - - if blockprofile != nil { - pprof.Lookup("block").WriteTo(blockprofile, 0) - blockprofile.Close() - blockprofile = nil - runtime.SetBlockProfileRate(0) - } -} - -// Continuously prints stats on the database at given intervals. -func printStats(db *bolt.DB, interval time.Duration) { - var prevStats = db.Stats() - var encoder = json.NewEncoder(os.Stdout) - - for { - // Wait for the stats interval. - time.Sleep(interval) - - // Retrieve new stats and find difference from previous iteration. - var stats = db.Stats() - var diff = stats.Sub(&prevStats) - - // Print as JSON to STDOUT. - if err := encoder.Encode(diff); err != nil { - fatal(err) - } - - // Save stats for next iteration. - prevStats = stats - } -} - -// BenchOptions represents the set of options that can be passed to Bench(). -type BenchOptions struct { - ProfileMode string - WriteMode string - ReadMode string - Iterations int - BatchSize int - KeySize int - ValueSize int - CPUProfile string - MemProfile string - BlockProfile string - StatsInterval time.Duration - FillPercent float64 - NoSync bool - Clean bool - Path string -} - -// BenchResults represents the performance results of the benchmark. -type BenchResults struct { - WriteOps int - WriteDuration time.Duration - ReadOps int - ReadDuration time.Duration -} - -// Returns the duration for a single write operation. -func (r *BenchResults) WriteOpDuration() time.Duration { - if r.WriteOps == 0 { - return 0 - } - return r.WriteDuration / time.Duration(r.WriteOps) -} - -// Returns average number of write operations that can be performed per second. -func (r *BenchResults) WriteOpsPerSecond() int { - var op = r.WriteOpDuration() - if op == 0 { - return 0 - } - return int(time.Second) / int(op) -} - -// Returns the duration for a single read operation. -func (r *BenchResults) ReadOpDuration() time.Duration { - if r.ReadOps == 0 { - return 0 - } - return r.ReadDuration / time.Duration(r.ReadOps) -} - -// Returns average number of read operations that can be performed per second. -func (r *BenchResults) ReadOpsPerSecond() int { - var op = r.ReadOpDuration() - if op == 0 { - return 0 - } - return int(time.Second) / int(op) -} - -// tempfile returns a temporary file path. -func tempfile() string { - f, _ := ioutil.TempFile("", "bolt-bench-") - f.Close() - os.Remove(f.Name()) - return f.Name() -} diff --git a/cmd/bolt/buckets.go b/cmd/bolt/buckets.go deleted file mode 100644 index 68e7dde..0000000 --- a/cmd/bolt/buckets.go +++ /dev/null @@ -1,33 +0,0 @@ -package main - -import ( - "os" - - "github.com/boltdb/bolt" -) - -// Buckets prints a list of all buckets. -func Buckets(path string) { - if _, err := os.Stat(path); os.IsNotExist(err) { - fatal(err) - return - } - - db, err := bolt.Open(path, 0600, nil) - if err != nil { - fatal(err) - return - } - defer db.Close() - - err = db.View(func(tx *bolt.Tx) error { - return tx.ForEach(func(name []byte, _ *bolt.Bucket) error { - println(string(name)) - return nil - }) - }) - if err != nil { - fatal(err) - return - } -} diff --git a/cmd/bolt/buckets_test.go b/cmd/bolt/buckets_test.go deleted file mode 100644 index d5050fd..0000000 --- a/cmd/bolt/buckets_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package main_test - -import ( - "testing" - - "github.com/boltdb/bolt" - . "github.com/boltdb/bolt/cmd/bolt" -) - -// Ensure that a list of buckets can be retrieved. -func TestBuckets(t *testing.T) { - SetTestMode(true) - open(func(db *bolt.DB, path string) { - db.Update(func(tx *bolt.Tx) error { - tx.CreateBucket([]byte("woojits")) - tx.CreateBucket([]byte("widgets")) - tx.CreateBucket([]byte("whatchits")) - return nil - }) - db.Close() - output := run("buckets", path) - equals(t, "whatchits\nwidgets\nwoojits", output) - }) -} - -// Ensure that an error is reported if the database is not found. -func TestBucketsDBNotFound(t *testing.T) { - SetTestMode(true) - output := run("buckets", "no/such/db") - equals(t, "stat no/such/db: no such file or directory", output) -} diff --git a/cmd/bolt/check.go b/cmd/bolt/check.go deleted file mode 100644 index 125f2b8..0000000 --- a/cmd/bolt/check.go +++ /dev/null @@ -1,47 +0,0 @@ -package main - -import ( - "os" - - "github.com/boltdb/bolt" -) - -// Check performs a consistency check on the database and prints any errors found. -func Check(path string) { - if _, err := os.Stat(path); os.IsNotExist(err) { - fatal(err) - return - } - - db, err := bolt.Open(path, 0600, nil) - if err != nil { - fatal(err) - return - } - defer db.Close() - - // Perform consistency check. - _ = db.View(func(tx *bolt.Tx) error { - var count int - ch := tx.Check() - loop: - for { - select { - case err, ok := <-ch: - if !ok { - break loop - } - println(err) - count++ - } - } - - // Print summary of errors. - if count > 0 { - fatalf("%d errors found", count) - } else { - println("OK") - } - return nil - }) -} diff --git a/cmd/bolt/get.go b/cmd/bolt/get.go deleted file mode 100644 index 90e0c1d..0000000 --- a/cmd/bolt/get.go +++ /dev/null @@ -1,45 +0,0 @@ -package main - -import ( - "os" - - "github.com/boltdb/bolt" -) - -// Get retrieves the value for a given bucket/key. -func Get(path, name, key string) { - if _, err := os.Stat(path); os.IsNotExist(err) { - fatal(err) - return - } - - db, err := bolt.Open(path, 0600, nil) - if err != nil { - fatal(err) - return - } - defer db.Close() - - err = db.View(func(tx *bolt.Tx) error { - // Find bucket. - b := tx.Bucket([]byte(name)) - if b == nil { - fatalf("bucket not found: %s", name) - return nil - } - - // Find value for a given key. - value := b.Get([]byte(key)) - if value == nil { - fatalf("key not found: %s", key) - return nil - } - - println(string(value)) - return nil - }) - if err != nil { - fatal(err) - return - } -} diff --git a/cmd/bolt/get_test.go b/cmd/bolt/get_test.go deleted file mode 100644 index 8acd0f4..0000000 --- a/cmd/bolt/get_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package main_test - -import ( - "testing" - - "github.com/boltdb/bolt" - . "github.com/boltdb/bolt/cmd/bolt" -) - -// Ensure that a value can be retrieved from the CLI. -func TestGet(t *testing.T) { - SetTestMode(true) - open(func(db *bolt.DB, path string) { - db.Update(func(tx *bolt.Tx) error { - tx.CreateBucket([]byte("widgets")) - tx.Bucket([]byte("widgets")).Put([]byte("foo"), []byte("bar")) - return nil - }) - db.Close() - output := run("get", path, "widgets", "foo") - equals(t, "bar", output) - }) -} - -// Ensure that an error is reported if the database is not found. -func TestGetDBNotFound(t *testing.T) { - SetTestMode(true) - output := run("get", "no/such/db", "widgets", "foo") - equals(t, "stat no/such/db: no such file or directory", output) -} - -// Ensure that an error is reported if the bucket is not found. -func TestGetBucketNotFound(t *testing.T) { - SetTestMode(true) - open(func(db *bolt.DB, path string) { - db.Close() - output := run("get", path, "widgets", "foo") - equals(t, "bucket not found: widgets", output) - }) -} - -// Ensure that an error is reported if the key is not found. -func TestGetKeyNotFound(t *testing.T) { - SetTestMode(true) - open(func(db *bolt.DB, path string) { - db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucket([]byte("widgets")) - return err - }) - db.Close() - output := run("get", path, "widgets", "foo") - equals(t, "key not found: foo", output) - }) -} diff --git a/cmd/bolt/info.go b/cmd/bolt/info.go deleted file mode 100644 index cb01e38..0000000 --- a/cmd/bolt/info.go +++ /dev/null @@ -1,26 +0,0 @@ -package main - -import ( - "os" - - "github.com/boltdb/bolt" -) - -// Info prints basic information about a database. -func Info(path string) { - if _, err := os.Stat(path); os.IsNotExist(err) { - fatal(err) - return - } - - db, err := bolt.Open(path, 0600, nil) - if err != nil { - fatal(err) - return - } - defer db.Close() - - // Print basic database info. - var info = db.Info() - printf("Page Size: %d\n", info.PageSize) -} diff --git a/cmd/bolt/info_test.go b/cmd/bolt/info_test.go deleted file mode 100644 index dab74f6..0000000 --- a/cmd/bolt/info_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package main_test - -import ( - "testing" - - "github.com/boltdb/bolt" - . "github.com/boltdb/bolt/cmd/bolt" -) - -// Ensure that a database info can be printed. -func TestInfo(t *testing.T) { - SetTestMode(true) - open(func(db *bolt.DB, path string) { - db.Update(func(tx *bolt.Tx) error { - tx.CreateBucket([]byte("widgets")) - b := tx.Bucket([]byte("widgets")) - b.Put([]byte("foo"), []byte("0000")) - return nil - }) - db.Close() - output := run("info", path) - equals(t, `Page Size: 4096`, output) - }) -} - -// Ensure that an error is reported if the database is not found. -func TestInfo_NotFound(t *testing.T) { - SetTestMode(true) - output := run("info", "no/such/db") - equals(t, "stat no/such/db: no such file or directory", output) -} diff --git a/cmd/bolt/keys.go b/cmd/bolt/keys.go deleted file mode 100644 index d4bb3c3..0000000 --- a/cmd/bolt/keys.go +++ /dev/null @@ -1,41 +0,0 @@ -package main - -import ( - "os" - - "github.com/boltdb/bolt" -) - -// Keys retrieves a list of keys for a given bucket. -func Keys(path, name string) { - if _, err := os.Stat(path); os.IsNotExist(err) { - fatal(err) - return - } - - db, err := bolt.Open(path, 0600, nil) - if err != nil { - fatal(err) - return - } - defer db.Close() - - err = db.View(func(tx *bolt.Tx) error { - // Find bucket. - b := tx.Bucket([]byte(name)) - if b == nil { - fatalf("bucket not found: %s", name) - return nil - } - - // Iterate over each key. - return b.ForEach(func(key, _ []byte) error { - println(string(key)) - return nil - }) - }) - if err != nil { - fatal(err) - return - } -} diff --git a/cmd/bolt/keys_test.go b/cmd/bolt/keys_test.go deleted file mode 100644 index 0cc4e0c..0000000 --- a/cmd/bolt/keys_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package main_test - -import ( - "testing" - - "github.com/boltdb/bolt" - . "github.com/boltdb/bolt/cmd/bolt" -) - -// Ensure that a list of keys can be retrieved for a given bucket. -func TestKeys(t *testing.T) { - SetTestMode(true) - open(func(db *bolt.DB, path string) { - db.Update(func(tx *bolt.Tx) error { - tx.CreateBucket([]byte("widgets")) - tx.Bucket([]byte("widgets")).Put([]byte("0002"), []byte("")) - tx.Bucket([]byte("widgets")).Put([]byte("0001"), []byte("")) - tx.Bucket([]byte("widgets")).Put([]byte("0003"), []byte("")) - return nil - }) - db.Close() - output := run("keys", path, "widgets") - equals(t, "0001\n0002\n0003", output) - }) -} - -// Ensure that an error is reported if the database is not found. -func TestKeysDBNotFound(t *testing.T) { - SetTestMode(true) - output := run("keys", "no/such/db", "widgets") - equals(t, "stat no/such/db: no such file or directory", output) -} - -// Ensure that an error is reported if the bucket is not found. -func TestKeysBucketNotFound(t *testing.T) { - SetTestMode(true) - open(func(db *bolt.DB, path string) { - db.Close() - output := run("keys", path, "widgets") - equals(t, "bucket not found: widgets", output) - }) -} diff --git a/cmd/bolt/main.go b/cmd/bolt/main.go index 0372e19..7fcf530 100644 --- a/cmd/bolt/main.go +++ b/cmd/bolt/main.go @@ -2,199 +2,834 @@ package main import ( "bytes" + "encoding/binary" + "errors" + "flag" "fmt" - "log" + "io" + "io/ioutil" + "math/rand" "os" + "runtime" + "runtime/pprof" + "strconv" + "strings" "time" "github.com/boltdb/bolt" - "github.com/codegangsta/cli" ) -var branch, commit string +var ( + // ErrCommandRequired is returned when a CLI command is not specified. + ErrCommandRequired = errors.New("command required") + + // ErrUnknownCommand is returned when a CLI command is not specified. + ErrUnknownCommand = errors.New("unknown command") + + // ErrPathRequired is returned when the path to a Bolt database is not specified. + ErrPathRequired = errors.New("path required") + + // ErrFileNotFound is returned when a Bolt database does not exist. + ErrFileNotFound = errors.New("file not found") + + // ErrInvalidValue is returned when a benchmark reads an unexpected value. + ErrInvalidValue = errors.New("invalid value") + + // ErrCorrupt is returned when a checking a data file finds errors. + ErrCorrupt = errors.New("invalid value") + + // ErrNonDivisibleBatchSize is returned when the batch size can't be evenly + // divided by the iteration count. + ErrNonDivisibleBatchSize = errors.New("number of iterations must be divisible by the batch size") +) func main() { - log.SetFlags(0) - NewApp().Run(os.Args) + m := NewMain() + if err := m.Run(os.Args[1:]...); err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } } -// NewApp creates an Application instance. -func NewApp() *cli.App { - app := cli.NewApp() - app.Name = "bolt" - app.Usage = "BoltDB toolkit" - app.Version = fmt.Sprintf("0.1.0 (%s %s)", branch, commit) - app.Commands = []cli.Command{ - { - Name: "info", - Usage: "Print basic information about a database", - Action: func(c *cli.Context) { - path := c.Args().Get(0) - Info(path) - }, - }, - { - Name: "get", - Usage: "Retrieve a value for given key in a bucket", - Action: func(c *cli.Context) { - path, name, key := c.Args().Get(0), c.Args().Get(1), c.Args().Get(2) - Get(path, name, key) - }, - }, - { - Name: "keys", - Usage: "Retrieve a list of all keys in a bucket", - Action: func(c *cli.Context) { - path, name := c.Args().Get(0), c.Args().Get(1) - Keys(path, name) - }, - }, - { - Name: "buckets", - Usage: "Retrieves a list of all buckets", - Action: func(c *cli.Context) { - path := c.Args().Get(0) - Buckets(path) - }, - }, - { - Name: "pages", - Usage: "Dumps page information for a database", - Action: func(c *cli.Context) { - path := c.Args().Get(0) - Pages(path) - }, - }, - { - Name: "check", - Usage: "Performs a consistency check on the database", - Action: func(c *cli.Context) { - path := c.Args().Get(0) - Check(path) - }, - }, - { - Name: "stats", - Usage: "Aggregate statistics for all buckets matching specified prefix", - Action: func(c *cli.Context) { - path, name := c.Args().Get(0), c.Args().Get(1) - Stats(path, name) - }, - }, - { - Name: "bench", - Usage: "Performs a synthetic benchmark", - Flags: []cli.Flag{ - &cli.StringFlag{Name: "profile-mode", Value: "rw", Usage: "Profile mode"}, - &cli.StringFlag{Name: "write-mode", Value: "seq", Usage: "Write mode"}, - &cli.StringFlag{Name: "read-mode", Value: "seq", Usage: "Read mode"}, - &cli.IntFlag{Name: "count", Value: 1000, Usage: "Item count"}, - &cli.IntFlag{Name: "batch-size", Usage: "Write batch size"}, - &cli.IntFlag{Name: "key-size", Value: 8, Usage: "Key size"}, - &cli.IntFlag{Name: "value-size", Value: 32, Usage: "Value size"}, - &cli.StringFlag{Name: "cpuprofile", Usage: "CPU profile output path"}, - &cli.StringFlag{Name: "memprofile", Usage: "Memory profile output path"}, - &cli.StringFlag{Name: "blockprofile", Usage: "Block profile output path"}, - &cli.StringFlag{Name: "stats-interval", Value: "0s", Usage: "Continuous stats interval"}, - &cli.Float64Flag{Name: "fill-percent", Value: bolt.DefaultFillPercent, Usage: "Fill percentage"}, - &cli.BoolFlag{Name: "no-sync", Usage: "Skip fsync on every commit"}, - &cli.BoolFlag{Name: "work", Usage: "Print the temp db and do not delete on exit"}, - &cli.StringFlag{Name: "path", Usage: "Path to database to use"}, - }, - Action: func(c *cli.Context) { - statsInterval, err := time.ParseDuration(c.String("stats-interval")) - if err != nil { - fatal(err) +// Main represents the main program execution. +type Main struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewMain returns a new instance of Main connect to the standard input/output. +func NewMain() *Main { + return &Main{ + Stdin: os.Stdin, + Stdout: os.Stdout, + Stderr: os.Stderr, + } +} + +// Run executes the program. +func (m *Main) Run(args ...string) error { + // Require a command at the beginning. + if len(args) == 0 || strings.HasPrefix(args[0], "-") { + return ErrCommandRequired + } + + // Execute command. + switch args[0] { + case "bench": + return newBenchCommand(m).Run(args[1:]...) + case "check": + return newCheckCommand(m).Run(args[1:]...) + case "info": + return newInfoCommand(m).Run(args[1:]...) + case "pages": + return newPagesCommand(m).Run(args[1:]...) + case "stats": + return newStatsCommand(m).Run(args[1:]...) + default: + return ErrUnknownCommand + } +} + +// CheckCommand represents the "check" command execution. +type CheckCommand struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewCheckCommand returns a CheckCommand. +func newCheckCommand(m *Main) *CheckCommand { + return &CheckCommand{ + Stdin: m.Stdin, + Stdout: m.Stdout, + Stderr: m.Stderr, + } +} + +// Run executes the command. +func (cmd *CheckCommand) Run(args ...string) error { + // Parse flags. + fs := flag.NewFlagSet("", flag.ContinueOnError) + if err := fs.Parse(args); err != nil { + return err + } + + // Require database path. + path := fs.Arg(0) + if path == "" { + return ErrPathRequired + } else if _, err := os.Stat(path); os.IsNotExist(err) { + return ErrFileNotFound + } + + // Open database. + db, err := bolt.Open(path, 0666, nil) + if err != nil { + return err + } + defer db.Close() + + // Perform consistency check. + return db.View(func(tx *bolt.Tx) error { + var count int + ch := tx.Check() + loop: + for { + select { + case err, ok := <-ch: + if !ok { + break loop } + fmt.Fprintln(cmd.Stdout, err) + count++ + } + } - Bench(&BenchOptions{ - ProfileMode: c.String("profile-mode"), - WriteMode: c.String("write-mode"), - ReadMode: c.String("read-mode"), - Iterations: c.Int("count"), - BatchSize: c.Int("batch-size"), - KeySize: c.Int("key-size"), - ValueSize: c.Int("value-size"), - CPUProfile: c.String("cpuprofile"), - MemProfile: c.String("memprofile"), - BlockProfile: c.String("blockprofile"), - StatsInterval: statsInterval, - FillPercent: c.Float64("fill-percent"), - NoSync: c.Bool("no-sync"), - Clean: !c.Bool("work"), - Path: c.String("path"), - }) - }, - }} - return app + // Print summary of errors. + if count > 0 { + fmt.Fprintf(cmd.Stdout, "%d errors found\n", count) + return ErrCorrupt + } + + // Notify user that database is valid. + fmt.Fprintln(cmd.Stdout, "OK") + return nil + }) } -var logger = log.New(os.Stderr, "", 0) -var logBuffer *bytes.Buffer +// InfoCommand represents the "info" command execution. +type InfoCommand struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} -func print(v ...interface{}) { - if testMode { - logger.Print(v...) +// NewInfoCommand returns a InfoCommand. +func newInfoCommand(m *Main) *InfoCommand { + return &InfoCommand{ + Stdin: m.Stdin, + Stdout: m.Stdout, + Stderr: m.Stderr, + } +} + +// Run executes the command. +func (cmd *InfoCommand) Run(args ...string) error { + // Parse flags. + fs := flag.NewFlagSet("", flag.ContinueOnError) + if err := fs.Parse(args); err != nil { + return err + } + + // Require database path. + path := fs.Arg(0) + if path == "" { + return ErrPathRequired + } else if _, err := os.Stat(path); os.IsNotExist(err) { + return ErrFileNotFound + } + + // Open the database. + db, err := bolt.Open(path, 0666, nil) + if err != nil { + return err + } + defer db.Close() + + // Print basic database info. + info := db.Info() + fmt.Fprintf(cmd.Stdout, "Page Size: %d\n", info.PageSize) + + return nil +} + +// PagesCommand represents the "pages" command execution. +type PagesCommand struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewPagesCommand returns a PagesCommand. +func newPagesCommand(m *Main) *PagesCommand { + return &PagesCommand{ + Stdin: m.Stdin, + Stdout: m.Stdout, + Stderr: m.Stderr, + } +} + +// Run executes the command. +func (cmd *PagesCommand) Run(args ...string) error { + // Parse flags. + fs := flag.NewFlagSet("", flag.ContinueOnError) + if err := fs.Parse(args); err != nil { + return err + } + + // Require database path. + path := fs.Arg(0) + if path == "" { + return ErrPathRequired + } else if _, err := os.Stat(path); os.IsNotExist(err) { + return ErrFileNotFound + } + + // Open database. + db, err := bolt.Open(path, 0666, nil) + if err != nil { + return err + } + defer func() { _ = db.Close() }() + + // Write header. + fmt.Fprintln(cmd.Stdout, "ID TYPE ITEMS OVRFLW") + fmt.Fprintln(cmd.Stdout, "======== ========== ====== ======") + + return db.Update(func(tx *bolt.Tx) error { + var id int + for { + p, err := tx.Page(id) + if err != nil { + return &PageError{ID: id, Err: err} + } else if p == nil { + break + } + + // Only display count and overflow if this is a non-free page. + var count, overflow string + if p.Type != "free" { + count = strconv.Itoa(p.Count) + if p.OverflowCount > 0 { + overflow = strconv.Itoa(p.OverflowCount) + } + } + + // Print table row. + fmt.Fprintf(cmd.Stdout, "%-8d %-10s %-6s %-6s\n", p.ID, p.Type, count, overflow) + + // Move to the next non-overflow page. + id += 1 + if p.Type != "free" { + id += p.OverflowCount + } + } + return nil + }) +} + +// StatsCommand represents the "stats" command execution. +type StatsCommand struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewStatsCommand returns a StatsCommand. +func newStatsCommand(m *Main) *StatsCommand { + return &StatsCommand{ + Stdin: m.Stdin, + Stdout: m.Stdout, + Stderr: m.Stderr, + } +} + +// Run executes the command. +func (cmd *StatsCommand) Run(args ...string) error { + // Parse flags. + fs := flag.NewFlagSet("", flag.ContinueOnError) + if err := fs.Parse(args); err != nil { + return err + } + + // Require database path. + path, prefix := fs.Arg(0), fs.Arg(1) + if path == "" { + return ErrPathRequired + } else if _, err := os.Stat(path); os.IsNotExist(err) { + return ErrFileNotFound + } + + // Open database. + db, err := bolt.Open(path, 0666, nil) + if err != nil { + return err + } + defer db.Close() + + return db.View(func(tx *bolt.Tx) error { + var s bolt.BucketStats + var count int + if err := tx.ForEach(func(name []byte, b *bolt.Bucket) error { + if bytes.HasPrefix(name, []byte(prefix)) { + s.Add(b.Stats()) + count += 1 + } + return nil + }); err != nil { + return err + } + + fmt.Fprintf(cmd.Stdout, "Aggregate statistics for %d buckets\n\n", count) + + fmt.Fprintln(cmd.Stdout, "Page count statistics") + fmt.Fprintf(cmd.Stdout, "\tNumber of logical branch pages: %d\n", s.BranchPageN) + fmt.Fprintf(cmd.Stdout, "\tNumber of physical branch overflow pages: %d\n", s.BranchOverflowN) + fmt.Fprintf(cmd.Stdout, "\tNumber of logical leaf pages: %d\n", s.LeafPageN) + fmt.Fprintf(cmd.Stdout, "\tNumber of physical leaf overflow pages: %d\n", s.LeafOverflowN) + + fmt.Fprintln(cmd.Stdout, "Tree statistics") + fmt.Fprintf(cmd.Stdout, "\tNumber of keys/value pairs: %d\n", s.KeyN) + fmt.Fprintf(cmd.Stdout, "\tNumber of levels in B+tree: %d\n", s.Depth) + + fmt.Fprintln(cmd.Stdout, "Page size utilization") + fmt.Fprintf(cmd.Stdout, "\tBytes allocated for physical branch pages: %d\n", s.BranchAlloc) + var percentage int + if s.BranchAlloc != 0 { + percentage = int(float32(s.BranchInuse) * 100.0 / float32(s.BranchAlloc)) + } + fmt.Fprintf(cmd.Stdout, "\tBytes actually used for branch data: %d (%d%%)\n", s.BranchInuse, percentage) + fmt.Fprintf(cmd.Stdout, "\tBytes allocated for physical leaf pages: %d\n", s.LeafAlloc) + percentage = 0 + if s.LeafAlloc != 0 { + percentage = int(float32(s.LeafInuse) * 100.0 / float32(s.LeafAlloc)) + } + fmt.Fprintf(cmd.Stdout, "\tBytes actually used for leaf data: %d (%d%%)\n", s.LeafInuse, percentage) + + fmt.Fprintln(cmd.Stdout, "Bucket statistics") + fmt.Fprintf(cmd.Stdout, "\tTotal number of buckets: %d\n", s.BucketN) + percentage = int(float32(s.InlineBucketN) * 100.0 / float32(s.BucketN)) + fmt.Fprintf(cmd.Stdout, "\tTotal number on inlined buckets: %d (%d%%)\n", s.InlineBucketN, percentage) + percentage = 0 + if s.LeafInuse != 0 { + percentage = int(float32(s.InlineBucketInuse) * 100.0 / float32(s.LeafInuse)) + } + fmt.Fprintf(cmd.Stdout, "\tBytes used for inlined buckets: %d (%d%%)\n", s.InlineBucketInuse, percentage) + + return nil + }) +} + +var benchBucketName = []byte("bench") + +// BenchCommand represents the "bench" command execution. +type BenchCommand struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewBenchCommand returns a BenchCommand using the +func newBenchCommand(m *Main) *BenchCommand { + return &BenchCommand{ + Stdin: m.Stdin, + Stdout: m.Stdout, + Stderr: m.Stderr, + } +} + +// Run executes the "bench" command. +func (cmd *BenchCommand) Run(args ...string) error { + // Parse CLI arguments. + options, err := cmd.ParseFlags(args) + if err != nil { + return err + } + + // Remove path if "-work" is not set. Otherwise keep path. + if options.Work { + fmt.Fprintf(cmd.Stdout, "work: %s\n", options.Path) } else { - fmt.Print(v...) + defer os.Remove(options.Path) + } + + // Create database. + db, err := bolt.Open(options.Path, 0666, nil) + if err != nil { + return err + } + db.NoSync = options.NoSync + defer db.Close() + + // Write to the database. + var results BenchResults + if err := cmd.runWrites(db, options, &results); err != nil { + return fmt.Errorf("write: ", err) + } + + // Read from the database. + if err := cmd.runReads(db, options, &results); err != nil { + return fmt.Errorf("bench: read: %s", err) + } + + // Print results. + fmt.Fprintf(os.Stderr, "# Write\t%v\t(%v/op)\t(%v op/sec)\n", results.WriteDuration, results.WriteOpDuration(), results.WriteOpsPerSecond()) + fmt.Fprintf(os.Stderr, "# Read\t%v\t(%v/op)\t(%v op/sec)\n", results.ReadDuration, results.ReadOpDuration(), results.ReadOpsPerSecond()) + fmt.Fprintln(os.Stderr, "") + return nil +} + +// ParseFlags parses the command line flags. +func (cmd *BenchCommand) ParseFlags(args []string) (*BenchOptions, error) { + var options BenchOptions + + // Parse flagset. + fs := flag.NewFlagSet("", flag.ContinueOnError) + fs.StringVar(&options.ProfileMode, "profile-mode", "rw", "") + fs.StringVar(&options.WriteMode, "write-mode", "seq", "") + fs.StringVar(&options.ReadMode, "read-mode", "seq", "") + fs.IntVar(&options.Iterations, "count", 1000, "") + fs.IntVar(&options.BatchSize, "batch-size", 0, "") + fs.IntVar(&options.KeySize, "key-size", 8, "") + fs.IntVar(&options.ValueSize, "value-size", 32, "") + fs.StringVar(&options.CPUProfile, "cpuprofile", "", "") + fs.StringVar(&options.MemProfile, "memprofile", "", "") + fs.StringVar(&options.BlockProfile, "blockprofile", "", "") + fs.StringVar(&options.BlockProfile, "blockprofile", "", "") + fs.Float64Var(&options.FillPercent, "fill-percent", bolt.DefaultFillPercent, "") + fs.BoolVar(&options.NoSync, "no-sync", false, "") + fs.BoolVar(&options.Work, "work", false, "") + fs.StringVar(&options.Path, "path", "", "") + fs.SetOutput(cmd.Stderr) + if err := fs.Parse(args); err != nil { + return nil, err + } + + // Set batch size to iteration size if not set. + // Require that batch size can be evenly divided by the iteration count. + if options.BatchSize == 0 { + options.BatchSize = options.Iterations + } else if options.Iterations%options.BatchSize != 0 { + return nil, ErrNonDivisibleBatchSize + } + + // Generate temp path if one is not passed in. + if options.Path == "" { + f, err := ioutil.TempFile("", "bolt-bench-") + if err != nil { + return nil, fmt.Errorf("temp file: %s", err) + } + f.Close() + os.Remove(f.Name()) + options.Path = f.Name() + } + + return &options, nil +} + +// Writes to the database. +func (cmd *BenchCommand) runWrites(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + // Start profiling for writes. + if options.ProfileMode == "rw" || options.ProfileMode == "w" { + cmd.startProfiling(options) + } + + t := time.Now() + + var err error + switch options.WriteMode { + case "seq": + err = cmd.runWritesSequential(db, options, results) + case "rnd": + err = cmd.runWritesRandom(db, options, results) + case "seq-nest": + err = cmd.runWritesSequentialNested(db, options, results) + case "rnd-nest": + err = cmd.runWritesRandomNested(db, options, results) + default: + return fmt.Errorf("invalid write mode: %s", options.WriteMode) + } + + // Save time to write. + results.WriteDuration = time.Since(t) + + // Stop profiling for writes only. + if options.ProfileMode == "w" { + cmd.stopProfiling() + } + + return err +} + +func (cmd *BenchCommand) runWritesSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + var i = uint32(0) + return cmd.runWritesWithSource(db, options, results, func() uint32 { i++; return i }) +} + +func (cmd *BenchCommand) runWritesRandom(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + return cmd.runWritesWithSource(db, options, results, func() uint32 { return r.Uint32() }) +} + +func (cmd *BenchCommand) runWritesSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + var i = uint32(0) + return cmd.runWritesWithSource(db, options, results, func() uint32 { i++; return i }) +} + +func (cmd *BenchCommand) runWritesRandomNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + return cmd.runWritesWithSource(db, options, results, func() uint32 { return r.Uint32() }) +} + +func (cmd *BenchCommand) runWritesWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error { + results.WriteOps = options.Iterations + + for i := 0; i < options.Iterations; i += options.BatchSize { + if err := db.Update(func(tx *bolt.Tx) error { + b, _ := tx.CreateBucketIfNotExists(benchBucketName) + b.FillPercent = options.FillPercent + + for j := 0; j < options.BatchSize; j++ { + key := make([]byte, options.KeySize) + value := make([]byte, options.ValueSize) + + // Write key as uint32. + binary.BigEndian.PutUint32(key, keySource()) + + // Insert key/value. + if err := b.Put(key, value); err != nil { + return err + } + } + + return nil + }); err != nil { + return err + } + } + return nil +} + +func (cmd *BenchCommand) runWritesNestedWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error { + results.WriteOps = options.Iterations + + for i := 0; i < options.Iterations; i += options.BatchSize { + if err := db.Update(func(tx *bolt.Tx) error { + top, err := tx.CreateBucketIfNotExists(benchBucketName) + if err != nil { + return err + } + top.FillPercent = options.FillPercent + + // Create bucket key. + name := make([]byte, options.KeySize) + binary.BigEndian.PutUint32(name, keySource()) + + // Create bucket. + b, err := top.CreateBucketIfNotExists(name) + if err != nil { + return err + } + b.FillPercent = options.FillPercent + + for j := 0; j < options.BatchSize; j++ { + var key = make([]byte, options.KeySize) + var value = make([]byte, options.ValueSize) + + // Generate key as uint32. + binary.BigEndian.PutUint32(key, keySource()) + + // Insert value into subbucket. + if err := b.Put(key, value); err != nil { + return err + } + } + + return nil + }); err != nil { + return err + } + } + return nil +} + +// Reads from the database. +func (cmd *BenchCommand) runReads(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + // Start profiling for reads. + if options.ProfileMode == "r" { + cmd.startProfiling(options) + } + + t := time.Now() + + var err error + switch options.ReadMode { + case "seq": + switch options.WriteMode { + case "seq-nest", "rnd-nest": + err = cmd.runReadsSequentialNested(db, options, results) + default: + err = cmd.runReadsSequential(db, options, results) + } + default: + return fmt.Errorf("invalid read mode: %s", options.ReadMode) + } + + // Save read time. + results.ReadDuration = time.Since(t) + + // Stop profiling for reads. + if options.ProfileMode == "rw" || options.ProfileMode == "r" { + cmd.stopProfiling() + } + + return err +} + +func (cmd *BenchCommand) runReadsSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + return db.View(func(tx *bolt.Tx) error { + t := time.Now() + + for { + var count int + + c := tx.Bucket(benchBucketName).Cursor() + for k, v := c.First(); k != nil; k, v = c.Next() { + if v == nil { + return errors.New("invalid value") + } + count++ + } + + if options.WriteMode == "seq" && count != options.Iterations { + return fmt.Errorf("read seq: iter mismatch: expected %d, got %d", options.Iterations, count) + } + + results.ReadOps += count + + // Make sure we do this for at least a second. + if time.Since(t) >= time.Second { + break + } + } + + return nil + }) +} + +func (cmd *BenchCommand) runReadsSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + return db.View(func(tx *bolt.Tx) error { + t := time.Now() + + for { + var count int + var top = tx.Bucket(benchBucketName) + if err := top.ForEach(func(name, _ []byte) error { + c := top.Bucket(name).Cursor() + for k, v := c.First(); k != nil; k, v = c.Next() { + if v == nil { + return ErrInvalidValue + } + count++ + } + return nil + }); err != nil { + return err + } + + if options.WriteMode == "seq-nest" && count != options.Iterations { + return fmt.Errorf("read seq-nest: iter mismatch: expected %d, got %d", options.Iterations, count) + } + + results.ReadOps += count + + // Make sure we do this for at least a second. + if time.Since(t) >= time.Second { + break + } + } + + return nil + }) +} + +// File handlers for the various profiles. +var cpuprofile, memprofile, blockprofile *os.File + +// Starts all profiles set on the options. +func (cmd *BenchCommand) startProfiling(options *BenchOptions) { + var err error + + // Start CPU profiling. + if options.CPUProfile != "" { + cpuprofile, err = os.Create(options.CPUProfile) + if err != nil { + fmt.Fprintf(cmd.Stderr, "bench: could not create cpu profile %q: %v\n", options.CPUProfile, err) + os.Exit(1) + } + pprof.StartCPUProfile(cpuprofile) + } + + // Start memory profiling. + if options.MemProfile != "" { + memprofile, err = os.Create(options.MemProfile) + if err != nil { + fmt.Fprintf(cmd.Stderr, "bench: could not create memory profile %q: %v\n", options.MemProfile, err) + os.Exit(1) + } + runtime.MemProfileRate = 4096 + } + + // Start fatal profiling. + if options.BlockProfile != "" { + blockprofile, err = os.Create(options.BlockProfile) + if err != nil { + fmt.Fprintf(cmd.Stderr, "bench: could not create block profile %q: %v\n", options.BlockProfile, err) + os.Exit(1) + } + runtime.SetBlockProfileRate(1) } } -func printf(format string, v ...interface{}) { - if testMode { - logger.Printf(format, v...) - } else { - fmt.Printf(format, v...) +// Stops all profiles. +func (cmd *BenchCommand) stopProfiling() { + if cpuprofile != nil { + pprof.StopCPUProfile() + cpuprofile.Close() + cpuprofile = nil + } + + if memprofile != nil { + pprof.Lookup("heap").WriteTo(memprofile, 0) + memprofile.Close() + memprofile = nil + } + + if blockprofile != nil { + pprof.Lookup("block").WriteTo(blockprofile, 0) + blockprofile.Close() + blockprofile = nil + runtime.SetBlockProfileRate(0) } } -func println(v ...interface{}) { - if testMode { - logger.Println(v...) - } else { - fmt.Println(v...) - } +// BenchOptions represents the set of options that can be passed to "bolt bench". +type BenchOptions struct { + ProfileMode string + WriteMode string + ReadMode string + Iterations int + BatchSize int + KeySize int + ValueSize int + CPUProfile string + MemProfile string + BlockProfile string + StatsInterval time.Duration + FillPercent float64 + NoSync bool + Work bool + Path string } -func fatal(v ...interface{}) { - logger.Print(v...) - if !testMode { - os.Exit(1) - } +// BenchResults represents the performance results of the benchmark. +type BenchResults struct { + WriteOps int + WriteDuration time.Duration + ReadOps int + ReadDuration time.Duration } -func fatalf(format string, v ...interface{}) { - logger.Printf(format, v...) - if !testMode { - os.Exit(1) +// Returns the duration for a single write operation. +func (r *BenchResults) WriteOpDuration() time.Duration { + if r.WriteOps == 0 { + return 0 } + return r.WriteDuration / time.Duration(r.WriteOps) } -func fatalln(v ...interface{}) { - logger.Println(v...) - if !testMode { - os.Exit(1) +// Returns average number of write operations that can be performed per second. +func (r *BenchResults) WriteOpsPerSecond() int { + var op = r.WriteOpDuration() + if op == 0 { + return 0 } + return int(time.Second) / int(op) } -// LogBuffer returns the contents of the log. -// This only works while the CLI is in test mode. -func LogBuffer() string { - if logBuffer != nil { - return logBuffer.String() +// Returns the duration for a single read operation. +func (r *BenchResults) ReadOpDuration() time.Duration { + if r.ReadOps == 0 { + return 0 } - return "" + return r.ReadDuration / time.Duration(r.ReadOps) } -var testMode bool - -// SetTestMode sets whether the CLI is running in test mode and resets the logger. -func SetTestMode(value bool) { - testMode = value - if testMode { - logBuffer = bytes.NewBuffer(nil) - logger = log.New(logBuffer, "", 0) - } else { - logger = log.New(os.Stderr, "", 0) +// Returns average number of read operations that can be performed per second. +func (r *BenchResults) ReadOpsPerSecond() int { + var op = r.ReadOpDuration() + if op == 0 { + return 0 } + return int(time.Second) / int(op) +} + +type PageError struct { + ID int + Err error +} + +func (e *PageError) Error() string { + return fmt.Sprintf("page error: id=%d, err=%s", e.ID, e.Err) } diff --git a/cmd/bolt/main_test.go b/cmd/bolt/main_test.go index 4448d6e..b9e8c67 100644 --- a/cmd/bolt/main_test.go +++ b/cmd/bolt/main_test.go @@ -1,69 +1,145 @@ package main_test import ( - "fmt" + "bytes" "io/ioutil" "os" - "path/filepath" - "reflect" - "runtime" - "strings" + "strconv" "testing" "github.com/boltdb/bolt" - . "github.com/boltdb/bolt/cmd/bolt" + "github.com/boltdb/bolt/cmd/bolt" ) -// open creates and opens a Bolt database in the temp directory. -func open(fn func(*bolt.DB, string)) { - path := tempfile() - defer os.RemoveAll(path) +// Ensure the "info" command can print information about a database. +func TestInfoCommand_Run(t *testing.T) { + db := MustOpen(0666, nil) + db.DB.Close() + defer db.Close() - db, err := bolt.Open(path, 0600, nil) - if err != nil { - panic("db open error: " + err.Error()) + // Run the info command. + m := NewMain() + if err := m.Run("info", db.Path); err != nil { + t.Fatal(err) } - fn(db, path) } -// run executes a command against the CLI and returns the output. -func run(args ...string) string { - args = append([]string{"bolt"}, args...) - NewApp().Run(args) - return strings.TrimSpace(LogBuffer()) +// Ensure the "stats" command can execute correctly. +func TestStatsCommand_Run(t *testing.T) { + // Ignore + if os.Getpagesize() != 4096 { + t.Skip("system does not use 4KB page size") + } + + db := MustOpen(0666, nil) + defer db.Close() + + if err := db.Update(func(tx *bolt.Tx) error { + // Create "foo" bucket. + b, err := tx.CreateBucket([]byte("foo")) + if err != nil { + return err + } + for i := 0; i < 10; i++ { + if err := b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))); err != nil { + return err + } + } + + // Create "bar" bucket. + b, err = tx.CreateBucket([]byte("bar")) + if err != nil { + return err + } + for i := 0; i < 100; i++ { + if err := b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))); err != nil { + return err + } + } + + // Create "baz" bucket. + b, err = tx.CreateBucket([]byte("baz")) + if err != nil { + return err + } + if err := b.Put([]byte("key"), []byte("value")); err != nil { + return err + } + + return nil + }); err != nil { + t.Fatal(err) + } + db.DB.Close() + + // Generate expected result. + exp := "Aggregate statistics for 3 buckets\n\n" + + "Page count statistics\n" + + "\tNumber of logical branch pages: 0\n" + + "\tNumber of physical branch overflow pages: 0\n" + + "\tNumber of logical leaf pages: 1\n" + + "\tNumber of physical leaf overflow pages: 0\n" + + "Tree statistics\n" + + "\tNumber of keys/value pairs: 111\n" + + "\tNumber of levels in B+tree: 1\n" + + "Page size utilization\n" + + "\tBytes allocated for physical branch pages: 0\n" + + "\tBytes actually used for branch data: 0 (0%)\n" + + "\tBytes allocated for physical leaf pages: 4096\n" + + "\tBytes actually used for leaf data: 1996 (48%)\n" + + "Bucket statistics\n" + + "\tTotal number of buckets: 3\n" + + "\tTotal number on inlined buckets: 2 (66%)\n" + + "\tBytes used for inlined buckets: 236 (11%)\n" + + // Run the command. + m := NewMain() + if err := m.Run("stats", db.Path); err != nil { + t.Fatal(err) + } else if m.Stdout.String() != exp { + t.Fatalf("unexpected stdout:\n\n%s", m.Stdout.String()) + } } -// tempfile returns a temporary file path. -func tempfile() string { +// Main represents a test wrapper for main.Main that records output. +type Main struct { + *main.Main + Stdin bytes.Buffer + Stdout bytes.Buffer + Stderr bytes.Buffer +} + +// NewMain returns a new instance of Main. +func NewMain() *Main { + m := &Main{Main: main.NewMain()} + m.Main.Stdin = &m.Stdin + m.Main.Stdout = &m.Stdout + m.Main.Stderr = &m.Stderr + return m +} + +// MustOpen creates a Bolt database in a temporary location. +func MustOpen(mode os.FileMode, options *bolt.Options) *DB { + // Create temporary path. f, _ := ioutil.TempFile("", "bolt-") f.Close() os.Remove(f.Name()) - return f.Name() -} -// assert fails the test if the condition is false. -func assert(tb testing.TB, condition bool, msg string, v ...interface{}) { - if !condition { - _, file, line, _ := runtime.Caller(1) - fmt.Printf("\033[31m%s:%d: "+msg+"\033[39m\n\n", append([]interface{}{filepath.Base(file), line}, v...)...) - tb.FailNow() - } -} - -// ok fails the test if an err is not nil. -func ok(tb testing.TB, err error) { + db, err := bolt.Open(f.Name(), mode, options) if err != nil { - _, file, line, _ := runtime.Caller(1) - fmt.Printf("\033[31m%s:%d: unexpected error: %s\033[39m\n\n", filepath.Base(file), line, err.Error()) - tb.FailNow() + panic(err.Error()) } + return &DB{DB: db, Path: f.Name()} } -// equals fails the test if exp is not equal to act. -func equals(tb testing.TB, exp, act interface{}) { - if !reflect.DeepEqual(exp, act) { - _, file, line, _ := runtime.Caller(1) - fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act) - tb.FailNow() - } +// DB is a test wrapper for bolt.DB. +type DB struct { + *bolt.DB + Path string +} + +// Close closes and removes the database. +func (db *DB) Close() error { + defer os.Remove(db.Path) + return db.DB.Close() } diff --git a/cmd/bolt/pages.go b/cmd/bolt/pages.go deleted file mode 100644 index ec1c4b4..0000000 --- a/cmd/bolt/pages.go +++ /dev/null @@ -1,57 +0,0 @@ -package main - -import ( - "os" - "strconv" - - "github.com/boltdb/bolt" -) - -// Pages prints a list of all pages in a database. -func Pages(path string) { - if _, err := os.Stat(path); os.IsNotExist(err) { - fatal(err) - return - } - - db, err := bolt.Open(path, 0600, nil) - if err != nil { - fatal(err) - return - } - defer db.Close() - - println("ID TYPE ITEMS OVRFLW") - println("======== ========== ====== ======") - - db.Update(func(tx *bolt.Tx) error { - var id int - for { - p, err := tx.Page(id) - if err != nil { - fatalf("page error: %d: %s", id, err) - } else if p == nil { - break - } - - // Only display count and overflow if this is a non-free page. - var count, overflow string - if p.Type != "free" { - count = strconv.Itoa(p.Count) - if p.OverflowCount > 0 { - overflow = strconv.Itoa(p.OverflowCount) - } - } - - // Print table row. - printf("%-8d %-10s %-6s %-6s\n", p.ID, p.Type, count, overflow) - - // Move to the next non-overflow page. - id += 1 - if p.Type != "free" { - id += p.OverflowCount - } - } - return nil - }) -} diff --git a/cmd/bolt/stats.go b/cmd/bolt/stats.go deleted file mode 100644 index b5d0083..0000000 --- a/cmd/bolt/stats.go +++ /dev/null @@ -1,77 +0,0 @@ -package main - -import ( - "bytes" - "os" - - "github.com/boltdb/bolt" -) - -// Collect stats for all top level buckets matching the prefix. -func Stats(path, prefix string) { - if _, err := os.Stat(path); os.IsNotExist(err) { - fatal(err) - return - } - - db, err := bolt.Open(path, 0600, nil) - if err != nil { - fatal(err) - return - } - defer db.Close() - - err = db.View(func(tx *bolt.Tx) error { - var s bolt.BucketStats - var count int - var prefix = []byte(prefix) - tx.ForEach(func(name []byte, b *bolt.Bucket) error { - if bytes.HasPrefix(name, prefix) { - s.Add(b.Stats()) - count += 1 - } - return nil - }) - printf("Aggregate statistics for %d buckets\n\n", count) - - println("Page count statistics") - printf("\tNumber of logical branch pages: %d\n", s.BranchPageN) - printf("\tNumber of physical branch overflow pages: %d\n", s.BranchOverflowN) - printf("\tNumber of logical leaf pages: %d\n", s.LeafPageN) - printf("\tNumber of physical leaf overflow pages: %d\n", s.LeafOverflowN) - - println("Tree statistics") - printf("\tNumber of keys/value pairs: %d\n", s.KeyN) - printf("\tNumber of levels in B+tree: %d\n", s.Depth) - - println("Page size utilization") - printf("\tBytes allocated for physical branch pages: %d\n", s.BranchAlloc) - var percentage int - if s.BranchAlloc != 0 { - percentage = int(float32(s.BranchInuse) * 100.0 / float32(s.BranchAlloc)) - } - printf("\tBytes actually used for branch data: %d (%d%%)\n", s.BranchInuse, percentage) - printf("\tBytes allocated for physical leaf pages: %d\n", s.LeafAlloc) - percentage = 0 - if s.LeafAlloc != 0 { - percentage = int(float32(s.LeafInuse) * 100.0 / float32(s.LeafAlloc)) - } - printf("\tBytes actually used for leaf data: %d (%d%%)\n", s.LeafInuse, percentage) - - println("Bucket statistics") - printf("\tTotal number of buckets: %d\n", s.BucketN) - percentage = int(float32(s.InlineBucketN) * 100.0 / float32(s.BucketN)) - printf("\tTotal number on inlined buckets: %d (%d%%)\n", s.InlineBucketN, percentage) - percentage = 0 - if s.LeafInuse != 0 { - percentage = int(float32(s.InlineBucketInuse) * 100.0 / float32(s.LeafInuse)) - } - printf("\tBytes used for inlined buckets: %d (%d%%)\n", s.InlineBucketInuse, percentage) - - return nil - }) - if err != nil { - fatal(err) - return - } -} diff --git a/cmd/bolt/stats_test.go b/cmd/bolt/stats_test.go deleted file mode 100644 index 44ed434..0000000 --- a/cmd/bolt/stats_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package main_test - -import ( - "os" - "strconv" - "testing" - - "github.com/boltdb/bolt" - . "github.com/boltdb/bolt/cmd/bolt" -) - -func TestStats(t *testing.T) { - if os.Getpagesize() != 4096 { - t.Skip() - } - SetTestMode(true) - open(func(db *bolt.DB, path string) { - db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("foo")) - if err != nil { - return err - } - for i := 0; i < 10; i++ { - b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) - } - b, err = tx.CreateBucket([]byte("bar")) - if err != nil { - return err - } - for i := 0; i < 100; i++ { - b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) - } - b, err = tx.CreateBucket([]byte("baz")) - if err != nil { - return err - } - b.Put([]byte("key"), []byte("value")) - return nil - }) - db.Close() - output := run("stats", path, "b") - equals(t, "Aggregate statistics for 2 buckets\n\n"+ - "Page count statistics\n"+ - "\tNumber of logical branch pages: 0\n"+ - "\tNumber of physical branch overflow pages: 0\n"+ - "\tNumber of logical leaf pages: 1\n"+ - "\tNumber of physical leaf overflow pages: 0\n"+ - "Tree statistics\n"+ - "\tNumber of keys/value pairs: 101\n"+ - "\tNumber of levels in B+tree: 1\n"+ - "Page size utilization\n"+ - "\tBytes allocated for physical branch pages: 0\n"+ - "\tBytes actually used for branch data: 0 (0%)\n"+ - "\tBytes allocated for physical leaf pages: 4096\n"+ - "\tBytes actually used for leaf data: 1996 (48%)\n"+ - "Bucket statistics\n"+ - "\tTotal number of buckets: 2\n"+ - "\tTotal number on inlined buckets: 1 (50%)\n"+ - "\tBytes used for inlined buckets: 40 (2%)", output) - }) -} From dd542876faf001930412c80ab8866226992c4f3c Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Wed, 15 Apr 2015 15:21:50 -0600 Subject: [PATCH 2/4] Add improved CLI documentation. --- cmd/bolt/main.go | 127 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 123 insertions(+), 4 deletions(-) diff --git a/cmd/bolt/main.go b/cmd/bolt/main.go index 7fcf530..2ca878c 100644 --- a/cmd/bolt/main.go +++ b/cmd/bolt/main.go @@ -20,8 +20,9 @@ import ( ) var ( - // ErrCommandRequired is returned when a CLI command is not specified. - ErrCommandRequired = errors.New("command required") + // ErrUsage is returned when a usage message was printed and the process + // should simply exit with an error. + ErrUsage = errors.New("usage") // ErrUnknownCommand is returned when a CLI command is not specified. ErrUnknownCommand = errors.New("unknown command") @@ -45,7 +46,9 @@ var ( func main() { m := NewMain() - if err := m.Run(os.Args[1:]...); err != nil { + if err := m.Run(os.Args[1:]...); err == ErrUsage { + os.Exit(2) + } else if err != nil { fmt.Println(err.Error()) os.Exit(1) } @@ -71,11 +74,15 @@ func NewMain() *Main { func (m *Main) Run(args ...string) error { // Require a command at the beginning. if len(args) == 0 || strings.HasPrefix(args[0], "-") { - return ErrCommandRequired + fmt.Fprintln(m.Stderr, m.Usage()) + return ErrUsage } // Execute command. switch args[0] { + case "help": + fmt.Fprintln(m.Stderr, m.Usage()) + return ErrUsage case "bench": return newBenchCommand(m).Run(args[1:]...) case "check": @@ -91,6 +98,28 @@ func (m *Main) Run(args ...string) error { } } +// Usage returns the help message. +func (m *Main) Usage() string { + return strings.TrimLeft(` +Bolt is a tool for inspecting bolt databases. + +Usage: + + bolt command [arguments] + +The commands are: + + bench run synthetic benchmark against bolt + check verifies integrity of bolt database + info print basic info + help print this screen + pages print list of pages with their types + stats iterate over all pages and generate usage stats + +Use "bolt [command] -h" for more information about a command. +`, "\n") +} + // CheckCommand represents the "check" command execution. type CheckCommand struct { Stdin io.Reader @@ -111,8 +140,12 @@ func newCheckCommand(m *Main) *CheckCommand { func (cmd *CheckCommand) Run(args ...string) error { // Parse flags. fs := flag.NewFlagSet("", flag.ContinueOnError) + help := fs.Bool("h", false, "") if err := fs.Parse(args); err != nil { return err + } else if *help { + fmt.Fprintln(cmd.Stderr, cmd.Usage()) + return ErrUsage } // Require database path. @@ -158,6 +191,20 @@ func (cmd *CheckCommand) Run(args ...string) error { }) } +// Usage returns the help message. +func (cmd *CheckCommand) Usage() string { + return strings.TrimLeft(` +usage: bolt check PATH + +Check opens a database at PATH and runs an exhaustive check to verify that +all pages are accessible or are marked as freed. It also verifies that no +pages are double referenced. + +Verification errors will stream out as they are found and the process will +return after all pages have been checked. +`, "\n") +} + // InfoCommand represents the "info" command execution. type InfoCommand struct { Stdin io.Reader @@ -178,8 +225,12 @@ func newInfoCommand(m *Main) *InfoCommand { func (cmd *InfoCommand) Run(args ...string) error { // Parse flags. fs := flag.NewFlagSet("", flag.ContinueOnError) + help := fs.Bool("h", false, "") if err := fs.Parse(args); err != nil { return err + } else if *help { + fmt.Fprintln(cmd.Stderr, cmd.Usage()) + return ErrUsage } // Require database path. @@ -204,6 +255,15 @@ func (cmd *InfoCommand) Run(args ...string) error { return nil } +// Usage returns the help message. +func (cmd *InfoCommand) Usage() string { + return strings.TrimLeft(` +usage: bolt info PATH + +Info prints basic information about the Bolt database at PATH. +`, "\n") +} + // PagesCommand represents the "pages" command execution. type PagesCommand struct { Stdin io.Reader @@ -224,8 +284,12 @@ func newPagesCommand(m *Main) *PagesCommand { func (cmd *PagesCommand) Run(args ...string) error { // Parse flags. fs := flag.NewFlagSet("", flag.ContinueOnError) + help := fs.Bool("h", false, "") if err := fs.Parse(args); err != nil { return err + } else if *help { + fmt.Fprintln(cmd.Stderr, cmd.Usage()) + return ErrUsage } // Require database path. @@ -279,6 +343,21 @@ func (cmd *PagesCommand) Run(args ...string) error { }) } +// Usage returns the help message. +func (cmd *PagesCommand) Usage() string { + return strings.TrimLeft(` +usage: bolt pages PATH + +Pages prints a table of pages with their type (meta, leaf, branch, freelist). +Leaf and branch pages will show a key count in the "items" column while the +freelist will show the number of free pages in the "items" column. + +The "overflow" column shows the number of blocks that the page spills over +into. Normally there is no overflow but large keys and values can cause +a single page to take up multiple blocks. +`, "\n") +} + // StatsCommand represents the "stats" command execution. type StatsCommand struct { Stdin io.Reader @@ -299,8 +378,12 @@ func newStatsCommand(m *Main) *StatsCommand { func (cmd *StatsCommand) Run(args ...string) error { // Parse flags. fs := flag.NewFlagSet("", flag.ContinueOnError) + help := fs.Bool("h", false, "") if err := fs.Parse(args); err != nil { return err + } else if *help { + fmt.Fprintln(cmd.Stderr, cmd.Usage()) + return ErrUsage } // Require database path. @@ -371,6 +454,42 @@ func (cmd *StatsCommand) Run(args ...string) error { }) } +// Usage returns the help message. +func (cmd *StatsCommand) Usage() string { + return strings.TrimLeft(` +usage: bolt stats PATH + +Stats performs an extensive search of the database to track every page +reference. It starts at the current meta page and recursively iterates +through every accessible bucket. + +The following errors can be reported: + + already freed + The page is referenced more than once in the freelist. + + unreachable unfreed + The page is not referenced by a bucket or in the freelist. + + reachable freed + The page is referenced by a bucket but is also in the freelist. + + out of bounds + A page is referenced that is above the high water mark. + + multiple references + A page is referenced by more than one other page. + + invalid type + The page type is not "meta", "leaf", "branch", or "freelist". + +No errors should occur in your database. However, if for some reason you +experience corruption, please submit a ticket to the Bolt project page: + + https://github.com/boltdb/bolt/issues +`, "\n") +} + var benchBucketName = []byte("bench") // BenchCommand represents the "bench" command execution. From 3ad30436da1357ba57bcf5ea3b50cc76f2672a79 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Thu, 16 Apr 2015 11:13:34 -0600 Subject: [PATCH 3/4] Add 'bolt dump' command. --- cmd/bolt/main.go | 117 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/cmd/bolt/main.go b/cmd/bolt/main.go index 2ca878c..246d04c 100644 --- a/cmd/bolt/main.go +++ b/cmd/bolt/main.go @@ -42,6 +42,9 @@ var ( // ErrNonDivisibleBatchSize is returned when the batch size can't be evenly // divided by the iteration count. ErrNonDivisibleBatchSize = errors.New("number of iterations must be divisible by the batch size") + + // ErrPageIDRequired is returned when a required page id is not specified. + ErrPageIDRequired = errors.New("page id required") ) func main() { @@ -87,6 +90,8 @@ func (m *Main) Run(args ...string) error { return newBenchCommand(m).Run(args[1:]...) case "check": return newCheckCommand(m).Run(args[1:]...) + case "dump": + return newDumpCommand(m).Run(args[1:]...) case "info": return newInfoCommand(m).Run(args[1:]...) case "pages": @@ -264,6 +269,118 @@ Info prints basic information about the Bolt database at PATH. `, "\n") } +// DumpCommand represents the "dump" command execution. +type DumpCommand struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// newDumpCommand returns a DumpCommand. +func newDumpCommand(m *Main) *DumpCommand { + return &DumpCommand{ + Stdin: m.Stdin, + Stdout: m.Stdout, + Stderr: m.Stderr, + } +} + +// Run executes the command. +func (cmd *DumpCommand) Run(args ...string) error { + // Parse flags. + fs := flag.NewFlagSet("", flag.ContinueOnError) + help := fs.Bool("h", false, "") + pageID := fs.Int("page", -1, "") + if err := fs.Parse(args); err != nil { + return err + } else if *help { + fmt.Fprintln(cmd.Stderr, cmd.Usage()) + return ErrUsage + } + + // Require database path and page id. + path := fs.Arg(0) + if path == "" { + return ErrPathRequired + } else if _, err := os.Stat(path); os.IsNotExist(err) { + return ErrFileNotFound + } else if *pageID == -1 { + return ErrPageIDRequired + } + + // Open database to retrieve page size. + db, err := bolt.Open(path, 0666, nil) + if err != nil { + return err + } + pageSize := db.Info().PageSize + _ = db.Close() + + // Open database file handler. + f, err := os.Open(path) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + + // Print page to stdout. + return cmd.PrintPage(cmd.Stdout, f, *pageID, pageSize) +} + +// PrintPage prints a given page as hexidecimal. +func (cmd *DumpCommand) PrintPage(w io.Writer, r io.ReaderAt, pageID int, pageSize int) error { + const bytesPerLineN = 16 + + // Read page into buffer. + buf := make([]byte, pageSize) + addr := pageID * pageSize + if n, err := r.ReadAt(buf, int64(addr)); err != nil { + return err + } else if n != pageSize { + return io.ErrUnexpectedEOF + } + + // Write out to writer in 16-byte lines. + var prev []byte + var skipped bool + for offset := 0; offset < pageSize; offset += bytesPerLineN { + // Retrieve current 16-byte line. + line := buf[offset : offset+bytesPerLineN] + isLastLine := (offset == (pageSize - bytesPerLineN)) + + // If it's the same as the previous line then print a skip. + if bytes.Equal(line, prev) && !isLastLine { + if !skipped { + fmt.Fprintf(w, "%07x *\n", addr+offset) + skipped = true + } + } else { + // Print line as hexadecimal in 2-byte groups. + fmt.Fprintf(w, "%07x %04x %04x %04x %04x %04x %04x %04x %04x\n", addr+offset, + line[0:2], line[2:4], line[4:6], line[6:8], + line[8:10], line[10:12], line[12:14], line[14:16], + ) + + skipped = false + } + + // Save the previous line. + prev = line + } + fmt.Fprint(w, "\n") + + return nil +} + +// Usage returns the help message. +func (cmd *DumpCommand) Usage() string { + return strings.TrimLeft(` +usage: bolt dump -page PAGEID PATH + +Dump prints a hexidecimal dump of a single page. +`, "\n") +} + // PagesCommand represents the "pages" command execution. type PagesCommand struct { Stdin io.Reader From 73a3aa1ccc3b318b699591f5893d9f410945a5e9 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Fri, 17 Apr 2015 13:28:32 -0600 Subject: [PATCH 4/4] Add 'bolt page' command. --- cmd/bolt/main.go | 473 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 466 insertions(+), 7 deletions(-) diff --git a/cmd/bolt/main.go b/cmd/bolt/main.go index 246d04c..2c4c414 100644 --- a/cmd/bolt/main.go +++ b/cmd/bolt/main.go @@ -15,6 +15,9 @@ import ( "strconv" "strings" "time" + "unicode" + "unicode/utf8" + "unsafe" "github.com/boltdb/bolt" ) @@ -45,8 +48,17 @@ var ( // ErrPageIDRequired is returned when a required page id is not specified. ErrPageIDRequired = errors.New("page id required") + + // ErrPageNotFound is returned when specifying a page above the high water mark. + ErrPageNotFound = errors.New("page not found") + + // ErrPageFreed is returned when reading a page that has already been freed. + ErrPageFreed = errors.New("page freed") ) +// PageHeaderSize represents the size of the bolt.page header. +const PageHeaderSize = 16 + func main() { m := NewMain() if err := m.Run(os.Args[1:]...); err == ErrUsage { @@ -94,6 +106,8 @@ func (m *Main) Run(args ...string) error { return newDumpCommand(m).Run(args[1:]...) case "info": return newInfoCommand(m).Run(args[1:]...) + case "page": + return newPageCommand(m).Run(args[1:]...) case "pages": return newPagesCommand(m).Run(args[1:]...) case "stats": @@ -290,7 +304,6 @@ func (cmd *DumpCommand) Run(args ...string) error { // Parse flags. fs := flag.NewFlagSet("", flag.ContinueOnError) help := fs.Bool("h", false, "") - pageID := fs.Int("page", -1, "") if err := fs.Parse(args); err != nil { return err } else if *help { @@ -304,17 +317,21 @@ func (cmd *DumpCommand) Run(args ...string) error { return ErrPathRequired } else if _, err := os.Stat(path); os.IsNotExist(err) { return ErrFileNotFound - } else if *pageID == -1 { + } + + // Read page ids. + pageIDs, err := atois(fs.Args()[1:]) + if err != nil { + return err + } else if len(pageIDs) == 0 { return ErrPageIDRequired } // Open database to retrieve page size. - db, err := bolt.Open(path, 0666, nil) + pageSize, err := ReadPageSize(path) if err != nil { return err } - pageSize := db.Info().PageSize - _ = db.Close() // Open database file handler. f, err := os.Open(path) @@ -323,8 +340,20 @@ func (cmd *DumpCommand) Run(args ...string) error { } defer func() { _ = f.Close() }() - // Print page to stdout. - return cmd.PrintPage(cmd.Stdout, f, *pageID, pageSize) + // Print each page listed. + for i, pageID := range pageIDs { + // Print a separator. + if i > 0 { + fmt.Fprintln(cmd.Stdout, "===============================================\n") + } + + // Print page to stdout. + if err := cmd.PrintPage(cmd.Stdout, f, pageID, pageSize); err != nil { + return err + } + } + + return nil } // PrintPage prints a given page as hexidecimal. @@ -381,6 +410,243 @@ Dump prints a hexidecimal dump of a single page. `, "\n") } +// PageCommand represents the "page" command execution. +type PageCommand struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// newPageCommand returns a PageCommand. +func newPageCommand(m *Main) *PageCommand { + return &PageCommand{ + Stdin: m.Stdin, + Stdout: m.Stdout, + Stderr: m.Stderr, + } +} + +// Run executes the command. +func (cmd *PageCommand) Run(args ...string) error { + // Parse flags. + fs := flag.NewFlagSet("", flag.ContinueOnError) + help := fs.Bool("h", false, "") + if err := fs.Parse(args); err != nil { + return err + } else if *help { + fmt.Fprintln(cmd.Stderr, cmd.Usage()) + return ErrUsage + } + + // Require database path and page id. + path := fs.Arg(0) + if path == "" { + return ErrPathRequired + } else if _, err := os.Stat(path); os.IsNotExist(err) { + return ErrFileNotFound + } + + // Read page ids. + pageIDs, err := atois(fs.Args()[1:]) + if err != nil { + return err + } else if len(pageIDs) == 0 { + return ErrPageIDRequired + } + + // Open database file handler. + f, err := os.Open(path) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + + // Print each page listed. + for i, pageID := range pageIDs { + // Print a separator. + if i > 0 { + fmt.Fprintln(cmd.Stdout, "===============================================\n") + } + + // Retrieve page info and page size. + p, buf, err := ReadPage(path, pageID) + if err != nil { + return err + } + + // Print basic page info. + fmt.Fprintf(cmd.Stdout, "Page ID: %d\n", p.id) + fmt.Fprintf(cmd.Stdout, "Page Type: %s\n", p.Type()) + fmt.Fprintf(cmd.Stdout, "Total Size: %d bytes\n", len(buf)) + + // Print type-specific data. + switch p.Type() { + case "meta": + err = cmd.PrintMeta(cmd.Stdout, buf) + case "leaf": + err = cmd.PrintLeaf(cmd.Stdout, buf) + case "branch": + err = cmd.PrintBranch(cmd.Stdout, buf) + case "freelist": + err = cmd.PrintFreelist(cmd.Stdout, buf) + } + if err != nil { + return err + } + } + + return nil +} + +// PrintMeta prints the data from the meta page. +func (cmd *PageCommand) PrintMeta(w io.Writer, buf []byte) error { + m := (*meta)(unsafe.Pointer(&buf[PageHeaderSize])) + fmt.Fprintf(w, "Version: %d\n", m.version) + fmt.Fprintf(w, "Page Size: %d bytes\n", m.pageSize) + fmt.Fprintf(w, "Flags: %08x\n", m.flags) + fmt.Fprintf(w, "Root: \n", m.root.root) + fmt.Fprintf(w, "Freelist: \n", m.freelist) + fmt.Fprintf(w, "HWM: \n", m.pgid) + fmt.Fprintf(w, "Txn ID: %d\n", m.txid) + fmt.Fprintf(w, "Checksum: %016x\n", m.checksum) + fmt.Fprintf(w, "\n") + return nil +} + +// PrintLeaf prints the data for a leaf page. +func (cmd *PageCommand) PrintLeaf(w io.Writer, buf []byte) error { + p := (*page)(unsafe.Pointer(&buf[0])) + + // Print number of items. + fmt.Fprintf(w, "Item Count: %d\n", p.count) + fmt.Fprintf(w, "\n") + + // Print each key/value. + for i := uint16(0); i < p.count; i++ { + e := p.leafPageElement(i) + + // Format key as string. + var k string + if isPrintable(string(e.key())) { + k = fmt.Sprintf("%q", string(e.key())) + } else { + k = fmt.Sprintf("%x", string(e.key())) + } + + // Format value as string. + var v string + if (e.flags & uint32(bucketLeafFlag)) != 0 { + b := (*bucket)(unsafe.Pointer(&e.value()[0])) + v = fmt.Sprintf("", b.root, b.sequence) + } else if isPrintable(string(e.value())) { + k = fmt.Sprintf("%q", string(e.value())) + } else { + k = fmt.Sprintf("%x", string(e.value())) + } + + fmt.Fprintf(w, "%s: %s\n", k, v) + } + fmt.Fprintf(w, "\n") + return nil +} + +// PrintBranch prints the data for a leaf page. +func (cmd *PageCommand) PrintBranch(w io.Writer, buf []byte) error { + p := (*page)(unsafe.Pointer(&buf[0])) + + // Print number of items. + fmt.Fprintf(w, "Item Count: %d\n", p.count) + fmt.Fprintf(w, "\n") + + // Print each key/value. + for i := uint16(0); i < p.count; i++ { + e := p.branchPageElement(i) + + // Format key as string. + var k string + if isPrintable(string(e.key())) { + k = fmt.Sprintf("%q", string(e.key())) + } else { + k = fmt.Sprintf("%x", string(e.key())) + } + + fmt.Fprintf(w, "%s: \n", k, e.pgid) + } + fmt.Fprintf(w, "\n") + return nil +} + +// PrintFreelist prints the data for a freelist page. +func (cmd *PageCommand) PrintFreelist(w io.Writer, buf []byte) error { + p := (*page)(unsafe.Pointer(&buf[0])) + + // Print number of items. + fmt.Fprintf(w, "Item Count: %d\n", p.count) + fmt.Fprintf(w, "\n") + + // Print each page in the freelist. + ids := (*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)) + for i := uint16(0); i < p.count; i++ { + fmt.Fprintf(w, "%d\n", ids[i]) + } + fmt.Fprintf(w, "\n") + return nil +} + +// PrintPage prints a given page as hexidecimal. +func (cmd *PageCommand) PrintPage(w io.Writer, r io.ReaderAt, pageID int, pageSize int) error { + const bytesPerLineN = 16 + + // Read page into buffer. + buf := make([]byte, pageSize) + addr := pageID * pageSize + if n, err := r.ReadAt(buf, int64(addr)); err != nil { + return err + } else if n != pageSize { + return io.ErrUnexpectedEOF + } + + // Write out to writer in 16-byte lines. + var prev []byte + var skipped bool + for offset := 0; offset < pageSize; offset += bytesPerLineN { + // Retrieve current 16-byte line. + line := buf[offset : offset+bytesPerLineN] + isLastLine := (offset == (pageSize - bytesPerLineN)) + + // If it's the same as the previous line then print a skip. + if bytes.Equal(line, prev) && !isLastLine { + if !skipped { + fmt.Fprintf(w, "%07x *\n", addr+offset) + skipped = true + } + } else { + // Print line as hexadecimal in 2-byte groups. + fmt.Fprintf(w, "%07x %04x %04x %04x %04x %04x %04x %04x %04x\n", addr+offset, + line[0:2], line[2:4], line[4:6], line[6:8], + line[8:10], line[10:12], line[12:14], line[14:16], + ) + + skipped = false + } + + // Save the previous line. + prev = line + } + fmt.Fprint(w, "\n") + + return nil +} + +// Usage returns the help message. +func (cmd *PageCommand) Usage() string { + return strings.TrimLeft(` +usage: bolt page -page PATH pageid [pageid...] + +Page prints one or more pages in human readable format. +`, "\n") +} + // PagesCommand represents the "pages" command execution. type PagesCommand struct { Stdin io.Reader @@ -1069,3 +1335,196 @@ type PageError struct { func (e *PageError) Error() string { return fmt.Sprintf("page error: id=%d, err=%s", e.ID, e.Err) } + +// isPrintable returns true if the string is valid unicode and contains only printable runes. +func isPrintable(s string) bool { + if !utf8.ValidString(s) { + return false + } + for _, ch := range s { + if !unicode.IsPrint(ch) { + return false + } + } + return true +} + +// ReadPage reads page info & full page data from a path. +// This is not transactionally safe. +func ReadPage(path string, pageID int) (*page, []byte, error) { + // Find page size. + pageSize, err := ReadPageSize(path) + if err != nil { + return nil, nil, fmt.Errorf("read page size: %s", err) + } + + // Open database file. + f, err := os.Open(path) + if err != nil { + return nil, nil, err + } + defer f.Close() + + // Read one block into buffer. + buf := make([]byte, pageSize) + if n, err := f.ReadAt(buf, int64(pageID*pageSize)); err != nil { + return nil, nil, err + } else if n != len(buf) { + return nil, nil, io.ErrUnexpectedEOF + } + + // Determine total number of blocks. + p := (*page)(unsafe.Pointer(&buf[0])) + overflowN := p.overflow + + // Re-read entire page (with overflow) into buffer. + buf = make([]byte, (int(overflowN)+1)*pageSize) + if n, err := f.ReadAt(buf, int64(pageID*pageSize)); err != nil { + return nil, nil, err + } else if n != len(buf) { + return nil, nil, io.ErrUnexpectedEOF + } + p = (*page)(unsafe.Pointer(&buf[0])) + + return p, buf, nil +} + +// ReadPageSize reads page size a path. +// This is not transactionally safe. +func ReadPageSize(path string) (int, error) { + // Open database file. + f, err := os.Open(path) + if err != nil { + return 0, err + } + defer f.Close() + + // Read 4KB chunk. + buf := make([]byte, 4096) + if _, err := io.ReadFull(f, buf); err != nil { + return 0, err + } + + // Read page size from metadata. + m := (*meta)(unsafe.Pointer(&buf[PageHeaderSize])) + return int(m.pageSize), nil +} + +// atois parses a slice of strings into integers. +func atois(strs []string) ([]int, error) { + var a []int + for _, str := range strs { + i, err := strconv.Atoi(str) + if err != nil { + return nil, err + } + a = append(a, i) + } + return a, nil +} + +// DO NOT EDIT. Copied from the "bolt" package. +const maxAllocSize = 0xFFFFFFF + +// DO NOT EDIT. Copied from the "bolt" package. +const ( + branchPageFlag = 0x01 + leafPageFlag = 0x02 + metaPageFlag = 0x04 + freelistPageFlag = 0x10 +) + +// DO NOT EDIT. Copied from the "bolt" package. +const bucketLeafFlag = 0x01 + +// DO NOT EDIT. Copied from the "bolt" package. +type pgid uint64 + +// DO NOT EDIT. Copied from the "bolt" package. +type txid uint64 + +// DO NOT EDIT. Copied from the "bolt" package. +type meta struct { + magic uint32 + version uint32 + pageSize uint32 + flags uint32 + root bucket + freelist pgid + pgid pgid + txid txid + checksum uint64 +} + +// DO NOT EDIT. Copied from the "bolt" package. +type bucket struct { + root pgid + sequence uint64 +} + +// DO NOT EDIT. Copied from the "bolt" package. +type page struct { + id pgid + flags uint16 + count uint16 + overflow uint32 + ptr uintptr +} + +// DO NOT EDIT. Copied from the "bolt" package. +func (p *page) Type() string { + if (p.flags & branchPageFlag) != 0 { + return "branch" + } else if (p.flags & leafPageFlag) != 0 { + return "leaf" + } else if (p.flags & metaPageFlag) != 0 { + return "meta" + } else if (p.flags & freelistPageFlag) != 0 { + return "freelist" + } + return fmt.Sprintf("unknown<%02x>", p.flags) +} + +// DO NOT EDIT. Copied from the "bolt" package. +func (p *page) leafPageElement(index uint16) *leafPageElement { + n := &((*[0x7FFFFFF]leafPageElement)(unsafe.Pointer(&p.ptr)))[index] + return n +} + +// DO NOT EDIT. Copied from the "bolt" package. +func (p *page) branchPageElement(index uint16) *branchPageElement { + return &((*[0x7FFFFFF]branchPageElement)(unsafe.Pointer(&p.ptr)))[index] +} + +// DO NOT EDIT. Copied from the "bolt" package. +type branchPageElement struct { + pos uint32 + ksize uint32 + pgid pgid +} + +// DO NOT EDIT. Copied from the "bolt" package. +func (n *branchPageElement) key() []byte { + buf := (*[maxAllocSize]byte)(unsafe.Pointer(n)) + return buf[n.pos : n.pos+n.ksize] +} + +// DO NOT EDIT. Copied from the "bolt" package. +type leafPageElement struct { + flags uint32 + pos uint32 + ksize uint32 + vsize uint32 +} + +// DO NOT EDIT. Copied from the "bolt" package. +func (n *leafPageElement) key() []byte { + buf := (*[maxAllocSize]byte)(unsafe.Pointer(n)) + return buf[n.pos : n.pos+n.ksize] +} + +// DO NOT EDIT. Copied from the "bolt" package. +func (n *leafPageElement) value() []byte { + buf := (*[maxAllocSize]byte)(unsafe.Pointer(n)) + return buf[n.pos+n.ksize : n.pos+n.ksize+n.vsize] +}