From a544249dd8fb2ff76215a2563c2b8f3ab2ccffdb Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Sat, 22 Feb 2014 23:08:30 -0700 Subject: [PATCH] Refactor Bucket. --- bucket.go | 111 ++++++++++++++- bucket_test.go | 323 +++++++++++++++++++++++++++++++++++++++++- db.go | 45 +++++- db_test.go | 107 ++------------ error.go | 4 + example_test.go | 17 ++- functional_test.go | 5 +- rwtransaction.go | 89 +++--------- rwtransaction_test.go | 212 +++------------------------ transaction.go | 53 +------ transaction_test.go | 36 ++--- 11 files changed, 549 insertions(+), 453 deletions(-) diff --git a/bucket.go b/bucket.go index 6653389..d62d798 100644 --- a/bucket.go +++ b/bucket.go @@ -1,13 +1,15 @@ package bolt +import ( + "bytes" +) + // Bucket represents a collection of key/value pairs inside the database. -// All keys inside the bucket are unique. The Bucket type is not typically used -// directly. Instead the bucket name is typically passed into the Get(), Put(), -// or Delete() functions. type Bucket struct { *bucket - name string - transaction *Transaction + name string + transaction *Transaction + rwtransaction *RWTransaction } // bucket represents the on-file representation of a bucket. @@ -21,8 +23,15 @@ func (b *Bucket) Name() string { return b.name } -// cursor creates a new cursor for this bucket. -func (b *Bucket) cursor() *Cursor { +// Writable returns whether the bucket is writable. +func (b *Bucket) Writable() bool { + return (b.rwtransaction != nil) +} + +// Cursor creates a cursor associated with the bucket. +// The cursor is only valid as long as the Transaction is open. +// Do not use a cursor after the transaction is closed. +func (b *Bucket) Cursor() *Cursor { return &Cursor{ transaction: b.transaction, root: b.root, @@ -30,6 +39,94 @@ func (b *Bucket) cursor() *Cursor { } } +// Get retrieves the value for a key in the bucket. +// Returns a nil value if the key does not exist. +func (b *Bucket) Get(key []byte) []byte { + c := b.Cursor() + k, v := c.Seek(key) + + // If our target node isn't the same key as what's passed in then return nil. + if !bytes.Equal(key, k) { + return nil + } + return v +} + +// Put sets the value for a key in the bucket. +// If the key exist then its previous value will be overwritten. +// Returns an error if the bucket was created from a read-only transaction, if the key is blank, if the key is too large, or if the value is too large. +func (b *Bucket) Put(key []byte, value []byte) error { + if !b.Writable() { + return ErrBucketNotWritable + } + + // Validate the key and data size. + if len(key) == 0 { + return ErrKeyRequired + } else if len(key) > MaxKeySize { + return ErrKeyTooLarge + } else if len(value) > MaxValueSize { + return ErrValueTooLarge + } + + // Move cursor to correct position. + c := b.Cursor() + c.Seek(key) + + // Insert the key/value. + c.node(b.rwtransaction).put(key, key, value, 0) + + return nil +} + +// Delete removes a key from the bucket. +// If the key does not exist then nothing is done and a nil error is returned. +// Returns an error if the bucket was created from a read-only transaction. +func (b *Bucket) Delete(key []byte) error { + if !b.Writable() { + return ErrBucketNotWritable + } + + // Move cursor to correct position. + c := b.Cursor() + c.Seek(key) + + // Delete the node if we have a matching key. + c.node(b.rwtransaction).del(key) + + return nil +} + +// NextSequence returns an autoincrementing integer for the bucket. +func (b *Bucket) NextSequence() (int, error) { + if !b.Writable() { + return 0, ErrBucketNotWritable + } + + // Make sure next sequence number will not be larger than the maximum + // integer size of the system. + if b.bucket.sequence == uint64(maxInt) { + return 0, ErrSequenceOverflow + } + + // Increment and return the sequence. + b.bucket.sequence++ + + return int(b.bucket.sequence), nil +} + +// ForEach executes a function for each key/value pair in a bucket. +// An error is returned if the bucket cannot be found. +func (b *Bucket) ForEach(fn func(k, v []byte) error) error { + c := b.Cursor() + for k, v := c.First(); k != nil; k, v = c.Next() { + if err := fn(k, v); err != nil { + return err + } + } + return nil +} + // Stat returns stats on a bucket. func (b *Bucket) Stat() *BucketStat { s := &BucketStat{} diff --git a/bucket_test.go b/bucket_test.go index 5e33189..5a2c0ae 100644 --- a/bucket_test.go +++ b/bucket_test.go @@ -1,33 +1,229 @@ package bolt import ( + "bytes" + "fmt" + "os" "strconv" "strings" "testing" + "testing/quick" "github.com/stretchr/testify/assert" ) +// Ensure that a bucket that gets a non-existent key returns nil. +func TestBucketGetNonExistent(t *testing.T) { + withOpenDB(func(db *DB, path string) { + db.CreateBucket("widgets") + value, err := db.Get("widgets", []byte("foo")) + if assert.NoError(t, err) { + assert.Nil(t, value) + } + }) +} + +// Ensure that a bucket can write a key/value. +func TestBucketPut(t *testing.T) { + withOpenDB(func(db *DB, path string) { + db.CreateBucket("widgets") + err := db.Put("widgets", []byte("foo"), []byte("bar")) + assert.NoError(t, err) + value, err := db.Get("widgets", []byte("foo")) + if assert.NoError(t, err) { + assert.Equal(t, value, []byte("bar")) + } + }) +} + +// Ensure that setting a value on a read-only bucket returns an error. +func TestBucketPutReadOnly(t *testing.T) { + withOpenDB(func(db *DB, path string) { + db.CreateBucket("widgets") + db.With(func(txn *Transaction) error { + b := txn.Bucket("widgets") + err := b.Put([]byte("foo"), []byte("bar")) + assert.Equal(t, err, ErrBucketNotWritable) + return nil + }) + }) +} + +// Ensure that a bucket can delete an existing key. +func TestBucketDelete(t *testing.T) { + withOpenDB(func(db *DB, path string) { + db.CreateBucket("widgets") + db.Put("widgets", []byte("foo"), []byte("bar")) + err := db.Delete("widgets", []byte("foo")) + assert.NoError(t, err) + value, err := db.Get("widgets", []byte("foo")) + if assert.NoError(t, err) { + assert.Nil(t, value) + } + }) +} + +// Ensure that deleting a key on a read-only bucket returns an error. +func TestBucketDeleteReadOnly(t *testing.T) { + withOpenDB(func(db *DB, path string) { + db.CreateBucket("widgets") + db.With(func(txn *Transaction) error { + b := txn.Bucket("widgets") + err := b.Delete([]byte("foo")) + assert.Equal(t, err, ErrBucketNotWritable) + return nil + }) + }) +} + +// Ensure that a bucket can return an autoincrementing sequence. +func TestBucketNextSequence(t *testing.T) { + withOpenDB(func(db *DB, path string) { + db.CreateBucket("widgets") + db.CreateBucket("woojits") + + // Make sure sequence increments. + seq, err := db.NextSequence("widgets") + assert.NoError(t, err) + assert.Equal(t, seq, 1) + seq, err = db.NextSequence("widgets") + assert.NoError(t, err) + assert.Equal(t, seq, 2) + + // Buckets should be separate. + seq, err = db.NextSequence("woojits") + assert.NoError(t, err) + assert.Equal(t, seq, 1) + + // Missing buckets return an error. + seq, err = db.NextSequence("no_such_bucket") + assert.Equal(t, err, ErrBucketNotFound) + assert.Equal(t, seq, 0) + }) +} + +// Ensure that retrieving the next sequence on a read-only bucket returns an error. +func TestBucketNextSequenceReadOnly(t *testing.T) { + withOpenDB(func(db *DB, path string) { + db.CreateBucket("widgets") + db.With(func(txn *Transaction) error { + b := txn.Bucket("widgets") + i, err := b.NextSequence() + assert.Equal(t, i, 0) + assert.Equal(t, err, ErrBucketNotWritable) + return nil + }) + }) +} + +// Ensure that incrementing past the maximum sequence number will return an error. +func TestBucketNextSequenceOverflow(t *testing.T) { + withOpenDB(func(db *DB, path string) { + db.CreateBucket("widgets") + db.Do(func(txn *RWTransaction) error { + b := txn.Bucket("widgets") + b.bucket.sequence = uint64(maxInt) + seq, err := b.NextSequence() + assert.Equal(t, err, ErrSequenceOverflow) + assert.Equal(t, seq, 0) + return nil + }) + }) +} + +// Ensure a database can loop over all key/value pairs in a bucket. +func TestBucketForEach(t *testing.T) { + withOpenDB(func(db *DB, path string) { + db.CreateBucket("widgets") + db.Put("widgets", []byte("foo"), []byte("0000")) + db.Put("widgets", []byte("baz"), []byte("0001")) + db.Put("widgets", []byte("bar"), []byte("0002")) + + var index int + err := db.ForEach("widgets", func(k, v []byte) error { + switch index { + case 0: + assert.Equal(t, k, []byte("bar")) + assert.Equal(t, v, []byte("0002")) + case 1: + assert.Equal(t, k, []byte("baz")) + assert.Equal(t, v, []byte("0001")) + case 2: + assert.Equal(t, k, []byte("foo")) + assert.Equal(t, v, []byte("0000")) + } + index++ + return nil + }) + assert.NoError(t, err) + assert.Equal(t, index, 3) + }) +} + +// Ensure a database can stop iteration early. +func TestBucketForEachShortCircuit(t *testing.T) { + withOpenDB(func(db *DB, path string) { + db.CreateBucket("widgets") + db.Put("widgets", []byte("bar"), []byte("0000")) + db.Put("widgets", []byte("baz"), []byte("0000")) + db.Put("widgets", []byte("foo"), []byte("0000")) + + var index int + err := db.ForEach("widgets", func(k, v []byte) error { + index++ + if bytes.Equal(k, []byte("baz")) { + return &Error{"marker", nil} + } + return nil + }) + assert.Equal(t, err, &Error{"marker", nil}) + assert.Equal(t, index, 2) + }) +} + +// Ensure that an error is returned when inserting with an empty key. +func TestBucketPutEmptyKey(t *testing.T) { + withOpenDB(func(db *DB, path string) { + db.CreateBucket("widgets") + err := db.Put("widgets", []byte(""), []byte("bar")) + assert.Equal(t, err, ErrKeyRequired) + err = db.Put("widgets", nil, []byte("bar")) + assert.Equal(t, err, ErrKeyRequired) + }) +} + +// Ensure that an error is returned when inserting with a key that's too large. +func TestBucketPutKeyTooLarge(t *testing.T) { + withOpenDB(func(db *DB, path string) { + db.CreateBucket("widgets") + err := db.Put("widgets", make([]byte, 32769), []byte("bar")) + assert.Equal(t, err, ErrKeyTooLarge) + }) +} + // Ensure a bucket can calculate stats. func TestBucketStat(t *testing.T) { withOpenDB(func(db *DB, path string) { db.Do(func(txn *RWTransaction) error { // Add bucket with lots of keys. txn.CreateBucket("widgets") + b := txn.Bucket("widgets") for i := 0; i < 100000; i++ { - txn.Put("widgets", []byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) + b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) } // Add bucket with fewer keys but one big value. txn.CreateBucket("woojits") + b = txn.Bucket("woojits") for i := 0; i < 500; i++ { - txn.Put("woojits", []byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) + b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) } - txn.Put("woojits", []byte("really-big-value"), []byte(strings.Repeat("*", 10000))) + b.Put([]byte("really-big-value"), []byte(strings.Repeat("*", 10000))) // Add a bucket that fits on a single root leaf. txn.CreateBucket("whozawhats") - txn.Put("whozawhats", []byte("foo"), []byte("bar")) + b = txn.Bucket("whozawhats") + b.Put([]byte("foo"), []byte("bar")) return nil }) @@ -60,3 +256,122 @@ func TestBucketStat(t *testing.T) { }) }) } + +// Ensure that a bucket can write random keys and values across multiple txns. +func TestBucketPutSingle(t *testing.T) { + index := 0 + f := func(items testdata) bool { + withOpenDB(func(db *DB, path string) { + m := make(map[string][]byte) + + db.CreateBucket("widgets") + for _, item := range items { + if err := db.Put("widgets", item.Key, item.Value); err != nil { + panic("put error: " + err.Error()) + } + m[string(item.Key)] = item.Value + + // Verify all key/values so far. + i := 0 + for k, v := range m { + value, err := db.Get("widgets", []byte(k)) + if err != nil { + panic("get error: " + err.Error()) + } + if !bytes.Equal(value, v) { + db.CopyFile("/tmp/bolt.put.single.db", 0666) + t.Fatalf("value mismatch [run %d] (%d of %d):\nkey: %x\ngot: %x\nexp: %x", index, i, len(m), []byte(k), value, v) + } + i++ + } + } + + fmt.Fprint(os.Stderr, ".") + }) + index++ + return true + } + if err := quick.Check(f, qconfig()); err != nil { + t.Error(err) + } + fmt.Fprint(os.Stderr, "\n") +} + +// Ensure that a transaction can insert multiple key/value pairs at once. +func TestBucketPutMultiple(t *testing.T) { + f := func(items testdata) bool { + withOpenDB(func(db *DB, path string) { + // Bulk insert all values. + db.CreateBucket("widgets") + rwtxn, _ := db.RWTransaction() + b := rwtxn.Bucket("widgets") + for _, item := range items { + assert.NoError(t, b.Put(item.Key, item.Value)) + } + assert.NoError(t, rwtxn.Commit()) + + // Verify all items exist. + txn, _ := db.Transaction() + b = txn.Bucket("widgets") + for _, item := range items { + value := b.Get(item.Key) + if !assert.Equal(t, item.Value, value) { + db.CopyFile("/tmp/bolt.put.multiple.db", 0666) + t.FailNow() + } + } + txn.Close() + }) + fmt.Fprint(os.Stderr, ".") + return true + } + if err := quick.Check(f, qconfig()); err != nil { + t.Error(err) + } + fmt.Fprint(os.Stderr, "\n") +} + +// Ensure that a transaction can delete all key/value pairs and return to a single leaf page. +func TestBucketDeleteQuick(t *testing.T) { + f := func(items testdata) bool { + withOpenDB(func(db *DB, path string) { + // Bulk insert all values. + db.CreateBucket("widgets") + rwtxn, _ := db.RWTransaction() + b := rwtxn.Bucket("widgets") + for _, item := range items { + assert.NoError(t, b.Put(item.Key, item.Value)) + } + assert.NoError(t, rwtxn.Commit()) + + // Remove items one at a time and check consistency. + for i, item := range items { + assert.NoError(t, db.Delete("widgets", item.Key)) + + // Anything before our deletion index should be nil. + txn, _ := db.Transaction() + b := txn.Bucket("widgets") + for j, exp := range items { + if j > i { + value := b.Get(exp.Key) + if !assert.Equal(t, exp.Value, value) { + t.FailNow() + } + } else { + value := b.Get(exp.Key) + if !assert.Nil(t, value) { + t.FailNow() + } + } + } + txn.Close() + } + }) + fmt.Fprint(os.Stderr, ".") + return true + } + if err := quick.Check(f, qconfig()); err != nil { + t.Error(err) + } + fmt.Fprint(os.Stderr, "\n") +} diff --git a/db.go b/db.go index c9b0e92..5cf1c6f 100644 --- a/db.go +++ b/db.go @@ -376,7 +376,11 @@ func (db *DB) With(fn func(*Transaction) error) error { // An error is returned if the bucket cannot be found. func (db *DB) ForEach(name string, fn func(k, v []byte) error) error { return db.With(func(t *Transaction) error { - return t.ForEach(name, fn) + b := t.Bucket(name) + if b == nil { + return ErrBucketNotFound + } + return b.ForEach(fn) }) } @@ -431,8 +435,13 @@ func (db *DB) DeleteBucket(name string) error { func (db *DB) NextSequence(name string) (int, error) { var seq int err := db.Do(func(t *RWTransaction) error { + b := t.Bucket(name) + if b == nil { + return ErrBucketNotFound + } + var err error - seq, err = t.NextSequence(name) + seq, err = b.NextSequence() return err }) if err != nil { @@ -449,14 +458,36 @@ func (db *DB) Get(name string, key []byte) ([]byte, error) { return nil, err } defer t.Close() - return t.Get(name, key) + + // Open bucket and retrieve value for key. + b := t.Bucket(name) + if b == nil { + return nil, ErrBucketNotFound + } + value, err := b.Get(key), nil + if err != nil { + return nil, err + } else if value == nil { + return nil, nil + } + + // Copy the value out since the transaction will be closed after this + // function ends. The data can get reclaimed between now and when the + // value is used. + tmp := make([]byte, len(value)) + copy(tmp, value) + return tmp, nil } // Put sets the value for a key in a bucket. // Returns an error if the bucket is not found, if key is blank, if the key is too large, or if the value is too large. func (db *DB) Put(name string, key []byte, value []byte) error { return db.Do(func(t *RWTransaction) error { - return t.Put(name, key, value) + b := t.Bucket(name) + if b == nil { + return ErrBucketNotFound + } + return b.Put(key, value) }) } @@ -464,7 +495,11 @@ func (db *DB) Put(name string, key []byte, value []byte) error { // Returns an error if the bucket cannot be found. func (db *DB) Delete(name string, key []byte) error { return db.Do(func(t *RWTransaction) error { - return t.Delete(name, key) + b := t.Bucket(name) + if b == nil { + return ErrBucketNotFound + } + return b.Delete(key) }) } diff --git a/db_test.go b/db_test.go index 3d0b987..19e10f1 100644 --- a/db_test.go +++ b/db_test.go @@ -1,7 +1,6 @@ package bolt import ( - "bytes" "io" "io/ioutil" "os" @@ -142,44 +141,6 @@ func TestDBTransactionErrDatabaseNotOpen(t *testing.T) { }) } -// Ensure that a bucket that gets a non-existent key returns nil. -func TestDBGetNonExistent(t *testing.T) { - withOpenDB(func(db *DB, path string) { - db.CreateBucket("widgets") - value, err := db.Get("widgets", []byte("foo")) - if assert.NoError(t, err) { - assert.Nil(t, value) - } - }) -} - -// Ensure that a bucket can write a key/value. -func TestDBPut(t *testing.T) { - withOpenDB(func(db *DB, path string) { - db.CreateBucket("widgets") - err := db.Put("widgets", []byte("foo"), []byte("bar")) - assert.NoError(t, err) - value, err := db.Get("widgets", []byte("foo")) - if assert.NoError(t, err) { - assert.Equal(t, value, []byte("bar")) - } - }) -} - -// Ensure that a bucket can delete an existing key. -func TestDBDelete(t *testing.T) { - withOpenDB(func(db *DB, path string) { - db.CreateBucket("widgets") - db.Put("widgets", []byte("foo"), []byte("bar")) - err := db.Delete("widgets", []byte("foo")) - assert.NoError(t, err) - value, err := db.Get("widgets", []byte("foo")) - if assert.NoError(t, err) { - assert.Nil(t, value) - } - }) -} - // Ensure that a delete on a missing bucket returns an error. func TestDBDeleteFromMissingBucket(t *testing.T) { withOpenDB(func(db *DB, path string) { @@ -193,9 +154,10 @@ func TestDBTransactionBlock(t *testing.T) { withOpenDB(func(db *DB, path string) { err := db.Do(func(txn *RWTransaction) error { txn.CreateBucket("widgets") - txn.Put("widgets", []byte("foo"), []byte("bar")) - txn.Put("widgets", []byte("baz"), []byte("bat")) - txn.Delete("widgets", []byte("foo")) + b := txn.Bucket("widgets") + b.Put([]byte("foo"), []byte("bar")) + b.Put([]byte("baz"), []byte("bat")) + b.Delete([]byte("foo")) return nil }) assert.NoError(t, err) @@ -217,56 +179,6 @@ func TestDBTransactionBlockWhileClosed(t *testing.T) { }) } -// Ensure a database can loop over all key/value pairs in a bucket. -func TestDBForEach(t *testing.T) { - withOpenDB(func(db *DB, path string) { - db.CreateBucket("widgets") - db.Put("widgets", []byte("foo"), []byte("0000")) - db.Put("widgets", []byte("baz"), []byte("0001")) - db.Put("widgets", []byte("bar"), []byte("0002")) - - var index int - err := db.ForEach("widgets", func(k, v []byte) error { - switch index { - case 0: - assert.Equal(t, k, []byte("bar")) - assert.Equal(t, v, []byte("0002")) - case 1: - assert.Equal(t, k, []byte("baz")) - assert.Equal(t, v, []byte("0001")) - case 2: - assert.Equal(t, k, []byte("foo")) - assert.Equal(t, v, []byte("0000")) - } - index++ - return nil - }) - assert.NoError(t, err) - assert.Equal(t, index, 3) - }) -} - -// Ensure a database can stop iteration early. -func TestDBForEachShortCircuit(t *testing.T) { - withOpenDB(func(db *DB, path string) { - db.CreateBucket("widgets") - db.Put("widgets", []byte("bar"), []byte("0000")) - db.Put("widgets", []byte("baz"), []byte("0000")) - db.Put("widgets", []byte("foo"), []byte("0000")) - - var index int - err := db.ForEach("widgets", func(k, v []byte) error { - index++ - if bytes.Equal(k, []byte("baz")) { - return &Error{"marker", nil} - } - return nil - }) - assert.Equal(t, err, &Error{"marker", nil}) - assert.Equal(t, index, 2) - }) -} - // Ensure a database returns an error when trying to attempt a for each on a missing bucket. func TestDBForEachBucketNotFound(t *testing.T) { withOpenDB(func(db *DB, path string) { @@ -310,6 +222,14 @@ func TestDBGetWhileClosed(t *testing.T) { }) } +// Ensure that an error is returned when inserting into a bucket that doesn't exist. +func TestDBPutBucketNotFound(t *testing.T) { + withOpenDB(func(db *DB, path string) { + err := db.Put("widgets", []byte("foo"), []byte("bar")) + assert.Equal(t, err, ErrBucketNotFound) + }) +} + // Ensure that the database can be copied to a file path. func TestDBCopyFile(t *testing.T) { withOpenDB(func(db *DB, path string) { @@ -335,8 +255,9 @@ func TestDBStat(t *testing.T) { withOpenDB(func(db *DB, path string) { db.Do(func(txn *RWTransaction) error { txn.CreateBucket("widgets") + b := txn.Bucket("widgets") for i := 0; i < 10000; i++ { - txn.Put("widgets", []byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) + b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) } return nil }) diff --git a/error.go b/error.go index 7238203..7e879a1 100644 --- a/error.go +++ b/error.go @@ -30,6 +30,10 @@ var ( // that is longer than MaxBucketNameSize. ErrBucketNameTooLarge = &Error{"bucket name too large", nil} + // ErrBucketNotWritable is returned when changing data on a bucket + // reference that was created from a read-only transaction. + ErrBucketNotWritable = &Error{"bucket not writable", nil} + // ErrKeyRequired is returned when inserting a zero-length key. ErrKeyRequired = &Error{"key required", nil} diff --git a/example_test.go b/example_test.go index 8747f94..b4fac38 100644 --- a/example_test.go +++ b/example_test.go @@ -71,7 +71,8 @@ func ExampleDB_Do() { if err := t.CreateBucket("widgets"); err != nil { return err } - if err := t.Put("widgets", []byte("foo"), []byte("bar")); err != nil { + b := t.Bucket("widgets") + if err := b.Put([]byte("foo"), []byte("bar")); err != nil { return err } return nil @@ -100,7 +101,7 @@ func ExampleDB_With() { // Access data from within a read-only transactional block. db.With(func(t *Transaction) error { - v, _ := t.Get("people", []byte("john")) + v := t.Bucket("people").Get([]byte("john")) fmt.Printf("John's last name is %s.\n", string(v)) return nil }) @@ -144,14 +145,15 @@ func ExampleRWTransaction() { // Create several keys in a transaction. rwtxn, _ := db.RWTransaction() - rwtxn.Put("widgets", []byte("john"), []byte("blue")) - rwtxn.Put("widgets", []byte("abby"), []byte("red")) - rwtxn.Put("widgets", []byte("zephyr"), []byte("purple")) + b := rwtxn.Bucket("widgets") + b.Put([]byte("john"), []byte("blue")) + b.Put([]byte("abby"), []byte("red")) + b.Put([]byte("zephyr"), []byte("purple")) rwtxn.Commit() // Iterate over the values in sorted key order. txn, _ := db.Transaction() - c, _ := txn.Cursor("widgets") + c := txn.Bucket("widgets").Cursor() for k, v := c.First(); k != nil; k, v = c.Next() { fmt.Printf("%s likes %s\n", string(k), string(v)) } @@ -177,7 +179,8 @@ func ExampleRWTransaction_rollback() { // Update the key but rollback the transaction so it never saves. rwtxn, _ := db.RWTransaction() - rwtxn.Put("widgets", []byte("foo"), []byte("baz")) + b := rwtxn.Bucket("widgets") + b.Put([]byte("foo"), []byte("baz")) rwtxn.Rollback() // Ensure that our original value is still set. diff --git a/functional_test.go b/functional_test.go index f7c3c3a..61f09bc 100644 --- a/functional_test.go +++ b/functional_test.go @@ -56,7 +56,7 @@ func TestParallelTransactions(t *testing.T) { // Verify all data is in for local data list. for _, item := range local { - value, err := txn.Get("widgets", item.Key) + value := txn.Bucket("widgets").Get(item.Key) if !assert.NoError(t, err) || !assert.Equal(t, value, item.Value) { txn.Close() wg.Done() @@ -89,8 +89,9 @@ func TestParallelTransactions(t *testing.T) { } // Insert whole batch. + b := txn.Bucket("widgets") for _, item := range batchItems { - err := txn.Put("widgets", item.Key, item.Value) + err := b.Put(item.Key, item.Value) if !assert.NoError(t, err) { t.FailNow() } diff --git a/rwtransaction.go b/rwtransaction.go index e22c766..b88e2c8 100644 --- a/rwtransaction.go +++ b/rwtransaction.go @@ -24,6 +24,28 @@ func (t *RWTransaction) init(db *DB) { t.meta.txnid += txnid(1) } +// Bucket retrieves a writable bucket by name. +// Returns nil if the bucket does not exist. +func (t *RWTransaction) Bucket(name string) *Bucket { + b := t.Transaction.Bucket(name) + if b == nil { + return nil + } + + b.rwtransaction = t + return b +} + +// Buckets retrieves a list of all buckets. +// All returned buckets are writable. +func (t *RWTransaction) Buckets() []*Bucket { + buckets := t.Transaction.Buckets() + for _, b := range buckets { + b.rwtransaction = t + } + return buckets +} + // CreateBucket creates a new bucket. // Returns an error if the bucket already exists, if the bucket name is blank, or if the bucket name is too long. func (t *RWTransaction) CreateBucket(name string) error { @@ -74,73 +96,6 @@ func (t *RWTransaction) DeleteBucket(name string) error { return nil } -// NextSequence returns an autoincrementing integer for the bucket. -func (t *RWTransaction) NextSequence(name string) (int, error) { - // Check if bucket already exists. - b := t.Bucket(name) - if b == nil { - return 0, ErrBucketNotFound - } - - // Make sure next sequence number will not be larger than the maximum - // integer size of the system. - if b.bucket.sequence == uint64(maxInt) { - return 0, ErrSequenceOverflow - } - - // Increment and return the sequence. - b.bucket.sequence++ - - return int(b.bucket.sequence), nil -} - -// Put sets the value for a key inside of the named bucket. -// If the key exist then its previous value will be overwritten. -// Returns an error if the bucket is not found, if the key is blank, if the key is too large, or if the value is too large. -func (t *RWTransaction) Put(name string, key []byte, value []byte) error { - b := t.Bucket(name) - if b == nil { - return ErrBucketNotFound - } - - // Validate the key and data size. - if len(key) == 0 { - return ErrKeyRequired - } else if len(key) > MaxKeySize { - return ErrKeyTooLarge - } else if len(value) > MaxValueSize { - return ErrValueTooLarge - } - - // Move cursor to correct position. - c := b.cursor() - c.Seek(key) - - // Insert the key/value. - c.node(t).put(key, key, value, 0) - - return nil -} - -// Delete removes a key from the named bucket. -// If the key does not exist then nothing is done and a nil error is returned. -// Returns an error if the bucket cannot be found. -func (t *RWTransaction) Delete(name string, key []byte) error { - b := t.Bucket(name) - if b == nil { - return ErrBucketNotFound - } - - // Move cursor to correct position. - c := b.cursor() - c.Seek(key) - - // Delete the node if we have a matching key. - c.node(t).del(key) - - return nil -} - // Commit writes all changes to disk and updates the meta page. // Returns an error if a disk write error occurs. func (t *RWTransaction) Commit() error { diff --git a/rwtransaction_test.go b/rwtransaction_test.go index 18b6ae9..83b2d1f 100644 --- a/rwtransaction_test.go +++ b/rwtransaction_test.go @@ -1,12 +1,8 @@ package bolt import ( - "bytes" - "fmt" - "os" "strings" "testing" - "testing/quick" "github.com/stretchr/testify/assert" ) @@ -30,6 +26,27 @@ func TestRWTransactionOpenWithClosedDB(t *testing.T) { }) } +// Ensure that retrieving all buckets returns writable buckets. +func TestRWTransactionBuckets(t *testing.T) { + withOpenDB(func(db *DB, path string) { + db.CreateBucket("widgets") + db.CreateBucket("woojits") + db.Do(func(txn *RWTransaction) error { + buckets := txn.Buckets() + assert.Equal(t, len(buckets), 2) + assert.Equal(t, buckets[0].Name(), "widgets") + assert.Equal(t, buckets[1].Name(), "woojits") + buckets[0].Put([]byte("foo"), []byte("0000")) + buckets[1].Put([]byte("bar"), []byte("0001")) + return nil + }) + v, _ := db.Get("widgets", []byte("foo")) + assert.Equal(t, v, []byte("0000")) + v, _ = db.Get("woojits", []byte("bar")) + assert.Equal(t, v, []byte("0001")) + }) +} + // Ensure that a bucket can be created and retrieved. func TestRWTransactionCreateBucket(t *testing.T) { withOpenDB(func(db *DB, path string) { @@ -110,75 +127,6 @@ func TestRWTransactionDeleteBucket(t *testing.T) { }) } -// Ensure that a bucket can return an autoincrementing sequence. -func TestRWTransactionNextSequence(t *testing.T) { - withOpenDB(func(db *DB, path string) { - db.CreateBucket("widgets") - db.CreateBucket("woojits") - - // Make sure sequence increments. - seq, err := db.NextSequence("widgets") - assert.NoError(t, err) - assert.Equal(t, seq, 1) - seq, err = db.NextSequence("widgets") - assert.NoError(t, err) - assert.Equal(t, seq, 2) - - // Buckets should be separate. - seq, err = db.NextSequence("woojits") - assert.NoError(t, err) - assert.Equal(t, seq, 1) - - // Missing buckets return an error. - seq, err = db.NextSequence("no_such_bucket") - assert.Equal(t, err, ErrBucketNotFound) - assert.Equal(t, seq, 0) - }) -} - -// Ensure that incrementing past the maximum sequence number will return an error. -func TestRWTransactionNextSequenceOverflow(t *testing.T) { - withOpenDB(func(db *DB, path string) { - db.CreateBucket("widgets") - db.Do(func(txn *RWTransaction) error { - b := txn.Bucket("widgets") - b.bucket.sequence = uint64(maxInt) - seq, err := txn.NextSequence("widgets") - assert.Equal(t, err, ErrSequenceOverflow) - assert.Equal(t, seq, 0) - return nil - }) - }) -} - -// Ensure that an error is returned when inserting into a bucket that doesn't exist. -func TestRWTransactionPutBucketNotFound(t *testing.T) { - withOpenDB(func(db *DB, path string) { - err := db.Put("widgets", []byte("foo"), []byte("bar")) - assert.Equal(t, err, ErrBucketNotFound) - }) -} - -// Ensure that an error is returned when inserting with an empty key. -func TestRWTransactionPutEmptyKey(t *testing.T) { - withOpenDB(func(db *DB, path string) { - db.CreateBucket("widgets") - err := db.Put("widgets", []byte(""), []byte("bar")) - assert.Equal(t, err, ErrKeyRequired) - err = db.Put("widgets", nil, []byte("bar")) - assert.Equal(t, err, ErrKeyRequired) - }) -} - -// Ensure that an error is returned when inserting with a key that's too large. -func TestRWTransactionPutKeyTooLarge(t *testing.T) { - withOpenDB(func(db *DB, path string) { - db.CreateBucket("widgets") - err := db.Put("widgets", make([]byte, 32769), []byte("bar")) - assert.Equal(t, err, ErrKeyTooLarge) - }) -} - // Ensure that an error is returned when deleting from a bucket that doesn't exist. func TestRWTransactionDeleteBucketNotFound(t *testing.T) { withOpenDB(func(db *DB, path string) { @@ -186,121 +134,3 @@ func TestRWTransactionDeleteBucketNotFound(t *testing.T) { assert.Equal(t, err, ErrBucketNotFound) }) } - -// Ensure that a bucket can write random keys and values across multiple txns. -func TestRWTransactionPutSingle(t *testing.T) { - index := 0 - f := func(items testdata) bool { - withOpenDB(func(db *DB, path string) { - m := make(map[string][]byte) - - db.CreateBucket("widgets") - for _, item := range items { - if err := db.Put("widgets", item.Key, item.Value); err != nil { - panic("put error: " + err.Error()) - } - m[string(item.Key)] = item.Value - - // Verify all key/values so far. - i := 0 - for k, v := range m { - value, err := db.Get("widgets", []byte(k)) - if err != nil { - panic("get error: " + err.Error()) - } - if !bytes.Equal(value, v) { - db.CopyFile("/tmp/bolt.put.single.db", 0666) - t.Fatalf("value mismatch [run %d] (%d of %d):\nkey: %x\ngot: %x\nexp: %x", index, i, len(m), []byte(k), value, v) - } - i++ - } - } - - fmt.Fprint(os.Stderr, ".") - }) - index++ - return true - } - if err := quick.Check(f, qconfig()); err != nil { - t.Error(err) - } - fmt.Fprint(os.Stderr, "\n") -} - -// Ensure that a transaction can insert multiple key/value pairs at once. -func TestRWTransactionPutMultiple(t *testing.T) { - f := func(items testdata) bool { - withOpenDB(func(db *DB, path string) { - // Bulk insert all values. - db.CreateBucket("widgets") - rwtxn, _ := db.RWTransaction() - for _, item := range items { - assert.NoError(t, rwtxn.Put("widgets", item.Key, item.Value)) - } - assert.NoError(t, rwtxn.Commit()) - - // Verify all items exist. - txn, _ := db.Transaction() - for _, item := range items { - value, err := txn.Get("widgets", item.Key) - assert.NoError(t, err) - if !assert.Equal(t, item.Value, value) { - db.CopyFile("/tmp/bolt.put.multiple.db", 0666) - t.FailNow() - } - } - txn.Close() - }) - fmt.Fprint(os.Stderr, ".") - return true - } - if err := quick.Check(f, qconfig()); err != nil { - t.Error(err) - } - fmt.Fprint(os.Stderr, "\n") -} - -// Ensure that a transaction can delete all key/value pairs and return to a single leaf page. -func TestRWTransactionDelete(t *testing.T) { - f := func(items testdata) bool { - withOpenDB(func(db *DB, path string) { - // Bulk insert all values. - db.CreateBucket("widgets") - rwtxn, _ := db.RWTransaction() - for _, item := range items { - assert.NoError(t, rwtxn.Put("widgets", item.Key, item.Value)) - } - assert.NoError(t, rwtxn.Commit()) - - // Remove items one at a time and check consistency. - for i, item := range items { - assert.NoError(t, db.Delete("widgets", item.Key)) - - // Anything before our deletion index should be nil. - txn, _ := db.Transaction() - for j, exp := range items { - if j > i { - value, err := txn.Get("widgets", exp.Key) - assert.NoError(t, err) - if !assert.Equal(t, exp.Value, value) { - t.FailNow() - } - } else { - value, err := txn.Get("widgets", exp.Key) - assert.NoError(t, err) - if !assert.Nil(t, value) { - t.FailNow() - } - } - } - txn.Close() - } - }) - fmt.Fprint(os.Stderr, ".") - return true - } - if err := quick.Check(f, qconfig()); err != nil { - t.Error(err) - } - fmt.Fprint(os.Stderr, "\n") -} diff --git a/transaction.go b/transaction.go index 6e9ca8f..f62db8c 100644 --- a/transaction.go +++ b/transaction.go @@ -1,9 +1,5 @@ package bolt -import ( - "bytes" -) - // Transaction represents a read-only transaction on the database. // It can be used for retrieving values for keys as well as creating cursors for // iterating over the data. @@ -50,7 +46,7 @@ func (t *Transaction) DB() *DB { return t.db } -// Bucket retrieves a bucket by name. +// Bucket retrieves a read-only bucket by name. // Returns nil if the bucket does not exist. func (t *Transaction) Bucket(name string) *Bucket { b := t.buckets.get(name) @@ -66,6 +62,7 @@ func (t *Transaction) Bucket(name string) *Bucket { } // Buckets retrieves a list of all buckets. +// All returned buckets are read-only. func (t *Transaction) Buckets() []*Bucket { buckets := make([]*Bucket, 0, len(t.buckets.items)) for name, b := range t.buckets.items { @@ -75,52 +72,6 @@ func (t *Transaction) Buckets() []*Bucket { return buckets } -// Cursor creates a cursor associated with a given bucket. -// The cursor is only valid as long as the Transaction is open. -// Do not use a cursor after the transaction is closed. -func (t *Transaction) Cursor(name string) (*Cursor, error) { - b := t.Bucket(name) - if b == nil { - return nil, ErrBucketNotFound - } - return b.cursor(), nil -} - -// Get retrieves the value for a key in a named bucket. -// Returns a nil value if the key does not exist. -// Returns an error if the bucket does not exist. -func (t *Transaction) Get(name string, key []byte) (value []byte, err error) { - c, err := t.Cursor(name) - if err != nil { - return nil, err - } - k, v := c.Seek(key) - // If our target node isn't the same key as what's passed in then return nil. - if !bytes.Equal(key, k) { - return nil, nil - } - return v, nil -} - -// ForEach executes a function for each key/value pair in a bucket. -// An error is returned if the bucket cannot be found. -func (t *Transaction) ForEach(name string, fn func(k, v []byte) error) error { - // Open a cursor on the bucket. - c, err := t.Cursor(name) - if err != nil { - return err - } - - // Iterate over each key/value pair in the bucket. - for k, v := c.First(); k != nil; k, v = c.Next() { - if err := fn(k, v); err != nil { - return err - } - } - - return nil -} - // page returns a reference to the page with a given id. // If page has been written to then a temporary bufferred page is returned. func (t *Transaction) page(id pgid) *page { diff --git a/transaction_test.go b/transaction_test.go index 4a7170c..8cf9dfa 100644 --- a/transaction_test.go +++ b/transaction_test.go @@ -53,8 +53,7 @@ func TestTransactionCursorEmptyBucket(t *testing.T) { withOpenDB(func(db *DB, path string) { db.CreateBucket("widgets") txn, _ := db.Transaction() - c, err := txn.Cursor("widgets") - assert.NoError(t, err) + c := txn.Bucket("widgets").Cursor() k, v := c.First() assert.Nil(t, k) assert.Nil(t, v) @@ -62,18 +61,6 @@ func TestTransactionCursorEmptyBucket(t *testing.T) { }) } -// Ensure that a Transaction returns a nil when a bucket doesn't exist. -func TestTransactionCursorMissingBucket(t *testing.T) { - withOpenDB(func(db *DB, path string) { - db.CreateBucket("widgets") - txn, _ := db.Transaction() - c, err := txn.Cursor("woojits") - assert.Nil(t, c) - assert.Equal(t, err, ErrBucketNotFound) - txn.Close() - }) -} - // Ensure that a Transaction cursor can iterate over a single root with a couple elements. func TestTransactionCursorLeafRoot(t *testing.T) { withOpenDB(func(db *DB, path string) { @@ -82,8 +69,7 @@ func TestTransactionCursorLeafRoot(t *testing.T) { db.Put("widgets", []byte("foo"), []byte{0}) db.Put("widgets", []byte("bar"), []byte{1}) txn, _ := db.Transaction() - c, err := txn.Cursor("widgets") - assert.NoError(t, err) + c := txn.Bucket("widgets").Cursor() k, v := c.First() assert.Equal(t, string(k), "bar") @@ -117,8 +103,7 @@ func TestTransactionCursorLeafRootReverse(t *testing.T) { db.Put("widgets", []byte("foo"), []byte{0}) db.Put("widgets", []byte("bar"), []byte{1}) txn, _ := db.Transaction() - c, err := txn.Cursor("widgets") - assert.NoError(t, err) + c := txn.Bucket("widgets").Cursor() k, v := c.Last() assert.Equal(t, string(k), "foo") @@ -152,8 +137,7 @@ func TestTransactionCursorRestart(t *testing.T) { db.Put("widgets", []byte("foo"), []byte{}) txn, _ := db.Transaction() - c, err := txn.Cursor("widgets") - assert.NoError(t, err) + c := txn.Bucket("widgets").Cursor() k, _ := c.First() assert.Equal(t, string(k), "bar") @@ -178,8 +162,9 @@ func TestTransactionCursorIterate(t *testing.T) { // Bulk insert all values. db.CreateBucket("widgets") rwtxn, _ := db.RWTransaction() + b := rwtxn.Bucket("widgets") for _, item := range items { - assert.NoError(t, rwtxn.Put("widgets", item.Key, item.Value)) + assert.NoError(t, b.Put(item.Key, item.Value)) } assert.NoError(t, rwtxn.Commit()) @@ -189,8 +174,7 @@ func TestTransactionCursorIterate(t *testing.T) { // Iterate over all items and check consistency. var index = 0 txn, _ := db.Transaction() - c, err := txn.Cursor("widgets") - assert.NoError(t, err) + c := txn.Bucket("widgets").Cursor() for k, v := c.First(); k != nil && index < len(items); k, v = c.Next() { assert.Equal(t, k, items[index].Key) assert.Equal(t, v, items[index].Value) @@ -215,8 +199,9 @@ func TestTransactionCursorIterateReverse(t *testing.T) { // Bulk insert all values. db.CreateBucket("widgets") rwtxn, _ := db.RWTransaction() + b := rwtxn.Bucket("widgets") for _, item := range items { - assert.NoError(t, rwtxn.Put("widgets", item.Key, item.Value)) + assert.NoError(t, b.Put(item.Key, item.Value)) } assert.NoError(t, rwtxn.Commit()) @@ -226,8 +211,7 @@ func TestTransactionCursorIterateReverse(t *testing.T) { // Iterate over all items and check consistency. var index = 0 txn, _ := db.Transaction() - c, err := txn.Cursor("widgets") - assert.NoError(t, err) + c := txn.Bucket("widgets").Cursor() for k, v := c.Last(); k != nil && index < len(items); k, v = c.Prev() { assert.Equal(t, k, items[index].Key) assert.Equal(t, v, items[index].Value)