From 1879d88c4366fcc64829a95423cdb1d0024d3472 Mon Sep 17 00:00:00 2001 From: Kevin Gillette Date: Fri, 25 Apr 2014 15:38:42 -0600 Subject: [PATCH 1/4] Printf's %s and %q do the right thing with []byte; removed string conversion. --- bucket_test.go | 6 +++--- db_test.go | 8 ++++---- tx_test.go | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bucket_test.go b/bucket_test.go index d0133f8..c506b72 100644 --- a/bucket_test.go +++ b/bucket_test.go @@ -745,7 +745,7 @@ func ExampleBucket_Put() { // Read value back in a different read-only transaction. db.Update(func(tx *Tx) error { value := tx.Bucket([]byte("widgets")).Get([]byte("foo")) - fmt.Printf("The value of 'foo' is: %s\n", string(value)) + fmt.Printf("The value of 'foo' is: %s\n", value) return nil }) @@ -770,7 +770,7 @@ func ExampleBucket_Delete() { // Retrieve the key back from the database and verify it. value := b.Get([]byte("foo")) - fmt.Printf("The value of 'foo' was: %s\n", string(value)) + fmt.Printf("The value of 'foo' was: %s\n", value) return nil }) @@ -809,7 +809,7 @@ func ExampleBucket_ForEach() { // Iterate over items in sorted key order. b.ForEach(func(k, v []byte) error { - fmt.Printf("A %s is %s.\n", string(k), string(v)) + fmt.Printf("A %s is %s.\n", k, v) return nil }) return nil diff --git a/db_test.go b/db_test.go index 57c356e..d0933e8 100644 --- a/db_test.go +++ b/db_test.go @@ -375,7 +375,7 @@ func ExampleDB_Update() { if err == nil { db.View(func(tx *Tx) error { value := tx.Bucket([]byte("widgets")).Get([]byte("foo")) - fmt.Printf("The value of 'foo' is: %s\n", string(value)) + fmt.Printf("The value of 'foo' is: %s\n", value) return nil }) } @@ -402,7 +402,7 @@ func ExampleDB_View() { // Access data from within a read-only transactional block. db.View(func(tx *Tx) error { v := tx.Bucket([]byte("people")).Get([]byte("john")) - fmt.Printf("John's last name is %s.\n", string(v)) + fmt.Printf("John's last name is %s.\n", v) return nil }) @@ -434,7 +434,7 @@ func ExampleDB_Begin_ReadOnly() { tx, _ = db.Begin(false) c := tx.Bucket([]byte("widgets")).Cursor() for k, v := c.First(); k != nil; k, v = c.Next() { - fmt.Printf("%s likes %s\n", string(k), string(v)) + fmt.Printf("%s likes %s\n", k, v) } tx.Rollback() @@ -469,7 +469,7 @@ func ExampleDB_CopyFile() { // Ensure that the key exists in the copy. db2.View(func(tx *Tx) error { value := tx.Bucket([]byte("widgets")).Get([]byte("foo")) - fmt.Printf("The value for 'foo' in the clone is: %s\n", string(value)) + fmt.Printf("The value for 'foo' in the clone is: %s\n", value) return nil }) diff --git a/tx_test.go b/tx_test.go index f630e97..e3296ef 100644 --- a/tx_test.go +++ b/tx_test.go @@ -289,7 +289,7 @@ func ExampleTx_Rollback() { // Ensure that our original value is still set. db.View(func(tx *Tx) error { value := tx.Bucket([]byte("widgets")).Get([]byte("foo")) - fmt.Printf("The value for 'foo' is still: %s\n", string(value)) + fmt.Printf("The value for 'foo' is still: %s\n", value) return nil }) From e3ed19364616379fdf60432a025b0914cd193a73 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Tue, 29 Apr 2014 06:54:01 -0600 Subject: [PATCH 2/4] Copy key on Put() and CreateBucket(). This commit makes a copy of the key byte slice before inserting into the database. This fixes the issue where users may reuse byte buffers which can corrupt the database. Fixes #143. --- bucket.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/bucket.go b/bucket.go index 571cae4..9737128 100644 --- a/bucket.go +++ b/bucket.go @@ -154,6 +154,7 @@ func (b *Bucket) CreateBucket(key []byte) (*Bucket, error) { bucket.root = p.id // Insert into node. + key = cloneBytes(key) c.node().put(key, key, value, 0, bucketLeafFlag) return b.Bucket(key), nil @@ -262,6 +263,7 @@ func (b *Bucket) Put(key []byte, value []byte) error { } // Insert into node. + key = cloneBytes(key) c.node().put(key, key, value, 0, 0) return nil @@ -533,3 +535,10 @@ type BucketStats struct { LeafAlloc int // bytes allocated for physical leaf pages LeafInuse int // bytes actually used for leaf data } + +// cloneBytes returns a copy of a given slice. +func cloneBytes(v []byte) []byte { + var clone = make([]byte, len(v)) + copy(clone, v) + return clone +} From e3957cd0de7d86b772ec20e93700c3f6e44d378e Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Tue, 29 Apr 2014 07:25:14 -0600 Subject: [PATCH 3/4] Add Tx.Cursor(). This commit adds the Cursor() function to Tx. This allows iteration on the root bucket in a similar way to iteration on other buckets. Fixes #141. --- tx.go | 8 ++++++++ tx_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/tx.go b/tx.go index 5844ffe..612f493 100644 --- a/tx.go +++ b/tx.go @@ -75,6 +75,14 @@ func (tx *Tx) Writable() bool { return tx.writable } +// Cursor creates a cursor associated with the root bucket. +// All items in the cursor will return a nil value because all root bucket keys point to buckets. +// The cursor is only valid as long as the transaction is open. +// Do not use a cursor after the transaction is closed. +func (tx *Tx) Cursor() *Cursor { + return tx.root.Cursor() +} + // Stats retrieves a copy of the current transaction statistics. func (tx *Tx) Stats() TxStats { return tx.stats diff --git a/tx_test.go b/tx_test.go index e3296ef..61fa5d9 100644 --- a/tx_test.go +++ b/tx_test.go @@ -36,6 +36,31 @@ func TestTx_Commit_ReadOnly(t *testing.T) { }) } +// Ensure that a transaction can retrieve a cursor on the root bucket. +func TestTx_Cursor(t *testing.T) { + withOpenDB(func(db *DB, path string) { + db.Update(func(tx *Tx) error { + tx.CreateBucket([]byte("widgets")) + tx.CreateBucket([]byte("woojits")) + c := tx.Cursor() + + k, v := c.First() + assert.Equal(t, "widgets", string(k)) + assert.Nil(t, v) + + k, v = c.Next() + assert.Equal(t, "woojits", string(k)) + assert.Nil(t, v) + + k, v = c.Next() + assert.Nil(t, k) + assert.Nil(t, v) + + return nil + }) + }) +} + // Ensure that creating a bucket with a read-only transaction returns an error. func TestTx_CreateBucket_ReadOnly(t *testing.T) { withOpenDB(func(db *DB, path string) { From cabb44e01f70840e5a7057a1d778f5da968ac77e Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Tue, 29 Apr 2014 12:27:38 -0600 Subject: [PATCH 4/4] Add --batch-size to 'bolt bench'. This commit adds a --batch-size CLI argument to the 'bolt bench' tool. This argument will insert into Bolt in smaller batches which is a more typical use case. /cc @snormore --- cmd/bolt/bench.go | 44 ++++++++++++++++++++++++++++++++------------ cmd/bolt/main.go | 2 ++ 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/cmd/bolt/bench.go b/cmd/bolt/bench.go index 72144b8..193687b 100644 --- a/cmd/bolt/bench.go +++ b/cmd/bolt/bench.go @@ -88,22 +88,41 @@ func benchWrite(db *bolt.DB, options *BenchOptions, results *BenchResults) error } func benchWriteSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - results.WriteOps = options.Iterations + // Default batch size to iteration count, if not specified. + var batchSize, iterations = options.BatchSize, options.Iterations + if batchSize == 0 { + batchSize = iterations + } - return db.Update(func(tx *bolt.Tx) error { - b, _ := tx.CreateBucketIfNotExists(benchBucketName) + // Insert in batches. + var count int + for i := 0; i < (iterations/batchSize)+1; i++ { + err := db.Update(func(tx *bolt.Tx) error { + b, _ := tx.CreateBucketIfNotExists(benchBucketName) - for i := 0; i < options.Iterations; i++ { - var key = make([]byte, options.KeySize) - var value = make([]byte, options.ValueSize) - binary.BigEndian.PutUint32(key, uint32(i)) - if err := b.Put(key, value); err != nil { - return err + for j := 0; j < batchSize && count < iterations; j++ { + var key = make([]byte, options.KeySize) + var value = make([]byte, options.ValueSize) + binary.BigEndian.PutUint32(key, uint32(count)) + + if err := b.Put(key, value); err != nil { + return err + } + + count++ } - } - return nil - }) + return nil + }) + if err != nil { + return err + } + } + + // Update the write op count. + results.WriteOps = count + + return nil } // Reads from the database. @@ -213,6 +232,7 @@ type BenchOptions struct { WriteMode string ReadMode string Iterations int + BatchSize int KeySize int ValueSize int CPUProfile string diff --git a/cmd/bolt/main.go b/cmd/bolt/main.go index 719bf00..54e1796 100644 --- a/cmd/bolt/main.go +++ b/cmd/bolt/main.go @@ -98,6 +98,7 @@ func NewApp() *cli.App { &cli.StringFlag{Name: "write-mode", Value: "seq", Usage: "Write mode"}, &cli.StringFlag{Name: "read-mode", Value: "seq", Usage: "Read mode"}, &cli.IntFlag{Name: "count", Value: 1000, Usage: "Item count"}, + &cli.IntFlag{Name: "batch-size", Usage: "Insert batch size"}, &cli.IntFlag{Name: "key-size", Value: 8, Usage: "Key size"}, &cli.IntFlag{Name: "value-size", Value: 32, Usage: "Value size"}, &cli.StringFlag{Name: "cpuprofile", Usage: "CPU profile output path"}, @@ -110,6 +111,7 @@ func NewApp() *cli.App { WriteMode: c.String("write-mode"), ReadMode: c.String("read-mode"), Iterations: c.Int("count"), + BatchSize: c.Int("batch-size"), KeySize: c.Int("key-size"), ValueSize: c.Int("value-size"), CPUProfile: c.String("cpuprofile"),