From c0ae4881ab0f827f7964b47cff59d60d8c85f499 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Wed, 16 Apr 2014 15:05:30 -0400 Subject: [PATCH] Add Cursor.Next() to C cursor. --- c/cursor.go | 245 +++++++++++++++++++++++++++++------------------ c/cursor_test.go | 168 +++++++++++++++----------------- c/doc.go | 4 + c/quick_test.go | 77 --------------- cursor_test.go | 10 +- 5 files changed, 235 insertions(+), 269 deletions(-) create mode 100644 c/doc.go delete mode 100644 c/quick_test.go diff --git a/c/cursor.go b/c/cursor.go index 73c5269..000dd9d 100644 --- a/c/cursor.go +++ b/c/cursor.go @@ -6,137 +6,190 @@ package c #include #include +//------------------------------------------------------------------------------ +// Constants +//------------------------------------------------------------------------------ + +// This represents the maximum number of levels that a cursor can traverse. #define MAX_DEPTH 100 -#define BRANCH_PAGE 1 + +// These flags mark the type of page and are set in the page.flags. +#define PAGE_BRANCH 0x01 +#define PAGE_LEAF 0x02 +#define PAGE_META 0x04 +#define PAGE_FREELIST 0x10 + + +//------------------------------------------------------------------------------ +// Typedefs +//------------------------------------------------------------------------------ // These types MUST have the same layout as their corresponding Go types typedef int64_t pgid; +// Page represents a header struct of a block in the mmap. typedef struct page { - pgid id; - uint16_t flags; - uint16_t count; - uint32_t overflow; + pgid id; + uint16_t flags; + uint16_t count; + uint32_t overflow; } page; -typedef struct branch_elem { - uint32_t pos; - uint32_t ksize; - pgid page; -} branch_elem; +// The branch element represents an a item in a branch page +// that points to a child page. +typedef struct branch_element { + uint32_t pos; + uint32_t ksize; + pgid page; +} branch_element; -typedef struct leaf_elem { - uint32_t flags; - uint32_t pos; - uint32_t ksize; - uint32_t vsize; -} leaf_elem; - -// private types +// The leaf element represents an a item in a leaf page +// that points to a key/value pair. +typedef struct leaf_element { + uint32_t flags; + uint32_t pos; + uint32_t ksize; + uint32_t vsize; +} leaf_element; +// elem_ref represents a pointer to an element inside of a page. +// It is used by the cursor stack to track the position at each level. typedef struct elem_ref { - page *page; - uint16_t index; + page *page; + uint16_t index; } elem_ref; -// public types - +// bolt_val represents a pointer to a fixed-length series of bytes. +// It is used to represent keys and values returned by the cursor. typedef struct bolt_val { uint32_t size; void *data; } bolt_val; +// bolt_cursor represents a cursor attached to a bucket. typedef struct bolt_cursor { - void *data; - pgid root; - size_t pgsz; - unsigned int stackp; - elem_ref stack[MAX_DEPTH]; + void *data; + pgid root; + size_t pgsz; + int top; + elem_ref stack[MAX_DEPTH]; } bolt_cursor; -// int bolt_cursor_seek(bolt_cursor *c, bolt_val *key, bolt_val *actual_key, bolt_val *value) +//------------------------------------------------------------------------------ +// Forward Declarations +//------------------------------------------------------------------------------ -// private functions +page *cursor_page(bolt_cursor *c, pgid id); -// Returns a page pointer from a page identifier. -page *get_page(bolt_cursor *c, pgid id) { - return (page *)(c->data + (c->pgsz * id)); -} +void cursor_first_leaf(bolt_cursor *c); -// Returns the leaf element at a given index on a given page. -branch_elem *branch_page_element(page *p, uint16_t index) { - branch_elem *elements = (branch_elem*)((void*)(p) + sizeof(page)); - return &elements[index]; -} +void cursor_key_value(bolt_cursor *c, bolt_val *key, bolt_val *value, uint32_t *flags); -// Returns the leaf element at a given index on a given page. -leaf_elem *leaf_page_element(page *p, uint16_t index) { - leaf_elem *elements = (leaf_elem*)((void*)(p) + sizeof(page)); - return &elements[index]; -} -// Sets the key and value for a leaf element to a bolt value. -void key_value(leaf_elem *leaf, bolt_val *key, bolt_val *value) { - key->size = leaf->ksize; - key->data = ((void*)leaf) + leaf->pos; - value->size = leaf->vsize; - value->data = key->data + key->size; -} - -// Traverses from the current stack position down to the first leaf element. -int bolt_cursor_first_leaf(bolt_cursor *c, bolt_val *key, bolt_val *value) { - elem_ref *ref = &(c->stack[c->stackp]); - branch_elem *branch; - while (ref->page->flags & BRANCH_PAGE) { - branch = branch_page_element(ref->page,ref->index); - c->stackp++; - ref = &c->stack[c->stackp]; - ref->index = 0; - ref->page = get_page(c, branch->page); - }; - key_value(leaf_page_element(ref->page,ref->index), key, value); - return 0; -} - -// public functions +//------------------------------------------------------------------------------ +// Public Functions +//------------------------------------------------------------------------------ +// Initializes a cursor. void bolt_cursor_init(bolt_cursor *c, void *data, size_t pgsz, pgid root) { c->data = data; c->root = root; c->pgsz = pgsz; } -int bolt_cursor_first(bolt_cursor *c, bolt_val *key, bolt_val *value) { - leaf_elem *leaf; - elem_ref *ref; - +// Positions the cursor to the first leaf element and returns the key/value pair. +void bolt_cursor_first(bolt_cursor *c, bolt_val *key, bolt_val *value, uint32_t *flags) { // reset stack to initial state - c->stackp = 0; - ref = &(c->stack[c->stackp]); - ref->page = get_page(c, c->root); + c->top = 0; + elem_ref *ref = &(c->stack[c->top]); + ref->page = cursor_page(c, c->root); ref->index = 0; - // get current leaf element - return bolt_cursor_first_leaf(c, key, value); + // Find first leaf and return key/value. + cursor_first_leaf(c); + cursor_key_value(c, key, value, flags); } -int bolt_cursor_next(bolt_cursor *c, bolt_val *key, bolt_val *value) { - elem_ref *ref = &c->stack[c->stackp]; +// Positions the cursor to the next leaf element and returns the key/value pair. +void bolt_cursor_next(bolt_cursor *c, bolt_val *key, bolt_val *value, uint32_t *flags) { + // Attempt to move over one element until we're successful. + // Move up the stack as we hit the end of each page in our stack. + for (int i = c->top; i >= 0; i--) { + elem_ref *elem = &c->stack[i]; + if (elem->index < elem->page->count - 1) { + elem->index++; + break; + } + c->top--; + } - // increment element index - ref->index++; - // if we're past last element pop the stack and repeat - while (ref->index >= ref->page->count ) { - c->stackp--; - ref = &c->stack[c->stackp]; - ref->index++; + // If we are at the top of the stack then return a blank key/value pair. + if (c->top == -1) { + key->size = value->size = 0; + key->data = value->data = NULL; + *flags = 0; + return; + } + + // Find first leaf and return key/value. + cursor_first_leaf(c); + cursor_key_value(c, key, value, flags); +} + + +//------------------------------------------------------------------------------ +// Private Functions +//------------------------------------------------------------------------------ + +// Returns a page pointer from a page identifier. +page *cursor_page(bolt_cursor *c, pgid id) { + return (page *)(c->data + (c->pgsz * id)); +} + +// Returns the leaf element at a given index on a given page. +branch_element *branch_page_element(page *p, uint16_t index) { + branch_element *elements = (branch_element*)((void*)(p) + sizeof(page)); + return &elements[index]; +} + +// Returns the leaf element at a given index on a given page. +leaf_element *page_leaf_element(page *p, uint16_t index) { + leaf_element *elements = (leaf_element*)((void*)(p) + sizeof(page)); + return &elements[index]; +} + +// Returns the key/value pair for the current position of the cursor. +void cursor_key_value(bolt_cursor *c, bolt_val *key, bolt_val *value, uint32_t *flags) { + elem_ref *ref = &(c->stack[c->top]); + leaf_element *elem = page_leaf_element(ref->page,ref->index); + + // Assign key pointer. + key->size = elem->ksize; + key->data = ((void*)elem) + elem->pos; + + // Assign value pointer. + value->size = elem->vsize; + value->data = key->data + key->size; + + // Return the element flags. + *flags = elem->flags; +} + +// Traverses from the current stack position down to the first leaf element. +void cursor_first_leaf(bolt_cursor *c) { + elem_ref *ref = &(c->stack[c->top]); + branch_element *branch; + while (ref->page->flags & PAGE_BRANCH) { + branch = branch_page_element(ref->page,ref->index); + c->top++; + ref = &c->stack[c->top]; + ref->index = 0; + ref->page = cursor_page(c, branch->page); }; - - // get current leaf element - return bolt_cursor_first_leaf(c, key, value); } + */ import "C" @@ -162,19 +215,21 @@ func NewCursor(b *bolt.Bucket) *Cursor { return c } -// first moves the cursor to the first element and returns the key and value. +// Next moves the cursor to the first element and returns the key and value. // Returns a nil key if there are no elements. -func first(c *Cursor) (key, value []byte) { +func (c *Cursor) First() (key, value []byte) { var k, v C.bolt_val - C.bolt_cursor_first(c.C, &k, &v) + var flags C.uint32_t + C.bolt_cursor_first(c.C, &k, &v, &flags) return C.GoBytes(k.data, C.int(k.size)), C.GoBytes(v.data, C.int(v.size)) } -// next moves the cursor to the next element and returns the key and value. -// Returns a nil key if at the end of the bucket. -func next(c *Cursor) (key, value []byte) { +// Next moves the cursor to the next element and returns the key and value. +// Returns a nil key if there are no more key/value pairs. +func (c *Cursor) Next() (key, value []byte) { var k, v C.bolt_val - C.bolt_cursor_next(c.C, &k, &v) + var flags C.uint32_t + C.bolt_cursor_next(c.C, &k, &v, &flags) return C.GoBytes(k.data, C.int(k.size)), C.GoBytes(v.data, C.int(v.size)) } diff --git a/c/cursor_test.go b/c/cursor_test.go index 3ac621f..fd0fa1a 100644 --- a/c/cursor_test.go +++ b/c/cursor_test.go @@ -1,123 +1,107 @@ -package c +package c_test import ( + "fmt" "io/ioutil" "os" - // "sort" "testing" - // "testing/quick" "github.com/boltdb/bolt" + . "github.com/boltdb/bolt/c" "github.com/stretchr/testify/assert" ) -// Test when cursor hits the end -// Implement seek; binary search within the page (branch page and element page) - -// Ensure that a cursor can iterate over all elements in a bucket. -// func TestIterate(t *testing.T) { -// if testing.Short() { -// t.Skip("skipping test in short mode.") -// } - -// f := func(items testdata) bool { -// withOpenDB(func(db *bolt.DB, path string) { -// // Bulk insert all values. -// tx, _ := db.Begin(true) -// tx.CreateBucket("widgets") -// b := tx.Bucket("widgets") -// for _, item := range items { -// assert.NoError(t, b.Put(item.Key, item.Value)) -// } -// assert.NoError(t, tx.Commit()) - -// // Sort test data. -// sort.Sort(items) - -// // Iterate over all items and check consistency. -// var index = 0 -// tx, _ = db.Begin(false) -// c := NewCursor(tx.Bucket("widgets")) -// for key, value := first(c); key != nil && index < len(items); key, value = next(c) { -// assert.Equal(t, key, items[index].Key) -// assert.Equal(t, value, items[index].Value) -// index++ -// } -// assert.Equal(t, len(items), index) -// assert.Equal(t, len(items), index) -// tx.Rollback() -// }) -// return true -// } -// if err := quick.Check(f, qconfig()); err != nil { -// t.Error(err) -// } -// fmt.Fprint(os.Stderr, "\n") -// } - -func TestCursorFirst(t *testing.T) { - withOpenDB(func(db *bolt.DB, path string) { - - // Bulk insert all values. - tx, _ := db.Begin(true) - b, _ := tx.CreateBucket([]byte("widgets")) - assert.NoError(t, b.Put([]byte("foo"), []byte("barz"))) - assert.NoError(t, tx.Commit()) - - // Get first and check consistency - tx, _ = db.Begin(false) - c := NewCursor(tx.Bucket([]byte("widgets"))) - key, value := first(c) - assert.Equal(t, key, []byte("foo")) - assert.Equal(t, value, []byte("barz")) - - tx.Rollback() +// Ensure that the C cursor can +func TestCursor_First(t *testing.T) { + withDB(func(db *bolt.DB) { + db.Update(func(tx *bolt.Tx) error { + b, _ := tx.CreateBucket([]byte("widgets")) + return b.Put([]byte("foo"), []byte("barz")) + }) + db.View(func(tx *bolt.Tx) error { + c := NewCursor(tx.Bucket([]byte("widgets"))) + key, value := c.First() + assert.Equal(t, []byte("foo"), key) + assert.Equal(t, []byte("barz"), value) + return nil + }) }) } -// withTempPath executes a function with a database reference. -func withTempPath(fn func(string)) { - f, _ := ioutil.TempFile("", "bolt-") - path := f.Name() +// Ensure that a C cursor can iterate over a single root with a couple elements. +func TestCursor_Iterate_Leaf(t *testing.T) { + withDB(func(db *bolt.DB) { + db.Update(func(tx *bolt.Tx) error { + tx.CreateBucket([]byte("widgets")) + tx.Bucket([]byte("widgets")).Put([]byte("baz"), []byte{}) + tx.Bucket([]byte("widgets")).Put([]byte("foo"), []byte{0}) + tx.Bucket([]byte("widgets")).Put([]byte("bar"), []byte{1}) + return nil + }) + db.View(func(tx *bolt.Tx) error { + c := NewCursor(tx.Bucket([]byte("widgets"))) + + k, v := c.First() + assert.Equal(t, string(k), "bar") + assert.Equal(t, []byte{1}, v) + + k, v = c.Next() + assert.Equal(t, string(k), "baz") + assert.Equal(t, []byte{}, v) + + k, v = c.Next() + assert.Equal(t, string(k), "foo") + assert.Equal(t, []byte{0}, v) + + k, v = c.Next() + assert.Equal(t, []byte{}, k) + assert.Equal(t, []byte{}, v) + + k, v = c.Next() + assert.Equal(t, []byte{}, k) + assert.Equal(t, []byte{}, v) + return nil + }) + }) +} + +// tempfile returns a temporary path. +func tempfile() string { + f, _ := ioutil.TempFile("", "bolt-c-") f.Close() - os.Remove(path) - defer os.RemoveAll(path) - - fn(path) + os.Remove(f.Name()) + return f.Name() } -// withOpenDB executes a function with an already opened database. -func withOpenDB(fn func(*bolt.DB, string)) { - withTempPath(func(path string) { - db, err := bolt.Open(path, 0666) - if err != nil { - panic("cannot open db: " + err.Error()) - } - defer db.Close() - fn(db, path) +// withDB executes a function with an already opened database. +func withDB(fn func(*bolt.DB)) { + path := tempfile() + db, err := bolt.Open(path, 0666) + if err != nil { + panic("cannot open db: " + err.Error()) + } + defer os.Remove(path) + defer db.Close() - // Log statistics. - // if *statsFlag { - // logStats(db) - // } + fn(db) - // Check database consistency after every test. - mustCheck(db) - }) + // Check database consistency after every test. + mustCheck(db) } // mustCheck runs a consistency check on the database and panics if any errors are found. func mustCheck(db *bolt.DB) { if err := db.Check(); err != nil { // Copy db off first. - db.CopyFile("/tmp/check.db", 0600) + var path = tempfile() + db.CopyFile(path, 0600) if errors, ok := err.(bolt.ErrorList); ok { for _, err := range errors { - warn(err) + fmt.Println(err) } } - warn(err) - panic("check failure: see /tmp/check.db") + fmt.Println(err) + panic("check failure: " + path) } } diff --git a/c/doc.go b/c/doc.go new file mode 100644 index 0000000..cbb98fe --- /dev/null +++ b/c/doc.go @@ -0,0 +1,4 @@ +/* +Package c provides a C interface to Bolt. +*/ +package c diff --git a/c/quick_test.go b/c/quick_test.go deleted file mode 100644 index 7d87249..0000000 --- a/c/quick_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package c - -import ( - "bytes" - "flag" - "math/rand" - "reflect" - "testing/quick" - "time" -) - -// testing/quick defaults to 5 iterations and a random seed. -// You can override these settings from the command line: -// -// -quick.count The number of iterations to perform. -// -quick.seed The seed to use for randomizing. -// -quick.maxitems The maximum number of items to insert into a DB. -// -quick.maxksize The maximum size of a key. -// -quick.maxvsize The maximum size of a value. -// - -var qcount, qseed, qmaxitems, qmaxksize, qmaxvsize int - -func init() { - flag.IntVar(&qcount, "quick.count", 5, "") - flag.IntVar(&qseed, "quick.seed", int(time.Now().UnixNano())%100000, "") - flag.IntVar(&qmaxitems, "quick.maxitems", 1000, "") - flag.IntVar(&qmaxksize, "quick.maxksize", 1024, "") - flag.IntVar(&qmaxvsize, "quick.maxvsize", 1024, "") - flag.Parse() - warn("seed:", qseed) - warnf("quick settings: count=%v, items=%v, ksize=%v, vsize=%v", qcount, qmaxitems, qmaxksize, qmaxvsize) -} - -func qconfig() *quick.Config { - return &quick.Config{ - MaxCount: qcount, - Rand: rand.New(rand.NewSource(int64(qseed))), - } -} - -type testdata []testdataitem - -func (t testdata) Len() int { return len(t) } -func (t testdata) Swap(i, j int) { t[i], t[j] = t[j], t[i] } -func (t testdata) Less(i, j int) bool { return bytes.Compare(t[i].Key, t[j].Key) == -1 } - -func (t testdata) Generate(rand *rand.Rand, size int) reflect.Value { - n := rand.Intn(qmaxitems-1) + 1 - items := make(testdata, n) - for i := 0; i < n; i++ { - item := &items[i] - item.Key = randByteSlice(rand, 1, qmaxksize) - item.Value = randByteSlice(rand, 0, qmaxvsize) - } - return reflect.ValueOf(items) -} - -type revtestdata []testdataitem - -func (t revtestdata) Len() int { return len(t) } -func (t revtestdata) Swap(i, j int) { t[i], t[j] = t[j], t[i] } -func (t revtestdata) Less(i, j int) bool { return bytes.Compare(t[i].Key, t[j].Key) == 1 } - -type testdataitem struct { - Key []byte - Value []byte -} - -func randByteSlice(rand *rand.Rand, minSize, maxSize int) []byte { - n := rand.Intn(maxSize-minSize) + minSize - b := make([]byte, n) - for i := 0; i < n; i++ { - b[i] = byte(rand.Intn(255)) - } - return b -} diff --git a/cursor_test.go b/cursor_test.go index 00fc561..fe406bf 100644 --- a/cursor_test.go +++ b/cursor_test.go @@ -89,7 +89,7 @@ func TestCursor_EmptyBucketReverse(t *testing.T) { } // Ensure that a Tx cursor can iterate over a single root with a couple elements. -func TestCursor_LeafRoot(t *testing.T) { +func TestCursor_Iterate_Leaf(t *testing.T) { withOpenDB(func(db *DB, path string) { db.Update(func(tx *Tx) error { tx.CreateBucket([]byte("widgets")) @@ -192,7 +192,7 @@ func TestCursor_Restart(t *testing.T) { } // Ensure that a Tx can iterate over all elements in a bucket. -func TestCursor_Iterate(t *testing.T) { +func TestCursor_QuickCheck(t *testing.T) { f := func(items testdata) bool { withOpenDB(func(db *DB, path string) { // Bulk insert all values. @@ -227,7 +227,7 @@ func TestCursor_Iterate(t *testing.T) { } // Ensure that a transaction can iterate over all elements in a bucket in reverse. -func TestCursor_Iterate_Reverse(t *testing.T) { +func TestCursor_QuickCheck_Reverse(t *testing.T) { f := func(items testdata) bool { withOpenDB(func(db *DB, path string) { // Bulk insert all values. @@ -262,7 +262,7 @@ func TestCursor_Iterate_Reverse(t *testing.T) { } // Ensure that a Tx cursor can iterate over subbuckets. -func TestCursor_Iterate_BucketsOnly(t *testing.T) { +func TestCursor_QuickCheck_BucketsOnly(t *testing.T) { withOpenDB(func(db *DB, path string) { db.Update(func(tx *Tx) error { b, err := tx.CreateBucket([]byte("widgets")) @@ -289,7 +289,7 @@ func TestCursor_Iterate_BucketsOnly(t *testing.T) { } // Ensure that a Tx cursor can reverse iterate over subbuckets. -func TestCursor_Iterate_BucketsOnly_Reverse(t *testing.T) { +func TestCursor_QuickCheck_BucketsOnly_Reverse(t *testing.T) { withOpenDB(func(db *DB, path string) { db.Update(func(tx *Tx) error { b, err := tx.CreateBucket([]byte("widgets"))