From fd4263d944f1e0bfef22c3f7a042c80d1a54972d Mon Sep 17 00:00:00 2001 From: Martin Kobetic Date: Fri, 11 Apr 2014 18:02:39 -0400 Subject: [PATCH 01/14] first draft --- c/cursor.go | 138 +++++++++++++++++++++++++++++++++++++++++++++++ c/cursor_test.go | 55 +++++++++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 c/cursor.go create mode 100644 c/cursor_test.go diff --git a/c/cursor.go b/c/cursor.go new file mode 100644 index 0000000..5874b20 --- /dev/null +++ b/c/cursor.go @@ -0,0 +1,138 @@ +package c + +/* + +#define MAX_DEPTH 100 +#define BRANCH_PAGE 1 + +// These types MUST have the same layout as their corresponding Go types + +typedef unsigned long long pageid; +typedef unsigned short elemid; + +typedef struct page { + pgid id; + unsigned short flags; + elemid count; + unsigned long overflow; +} page; + +typedef struct branch_elem { + unsigned long pos; + unsigned long ksize; + pgid pgid; +} branch_elem; + +typedef struct leaf_elem { + unsigned long flags; + unsigned long pos; + unsigned long ksize; + unsigned long vsize; +} leaf_elem; + +// private types + +typedef struct elem_ref { + void *element; + page *page; + elemid index; +} elem_ref; + +// public types + +typedef struct bolt_val { + uint32_t size; + void *data; +} bolt_val; + +typedef struct bolt_cursor { + void *data; + pgid root; + size_t pgsz; + elemid[MAX_DEPTH] stack; + unsigned int stackp; +} bolt_cursor; + +void bolt_cursor_init(bolt_cursor* c, void *data, size_t pgsz, pgid root) { + c->data = data; + c->pgid = pgid; + c->pgsz = pgsz; +} + +// public functions + +int bolt_cursor_first(bolt_cursor* c, bolt_val *key, bolt_val *value) { + leaf_elem* leaf; + elem_ref* ref; + c->stackp = 0; + ref = &c->stack[c->stackp] + ref->page = page(c, c->pgid); + ref->index = 0; + return next_element(c, key, value); +} + +int bolt_cursor_next(bolt_cursor* c, bolt_val *key, bolt_val *value) { + elem_ref* ref= &c->stack[c->stackp]; + ref->index++; + while (ref->index >= ref->page->count ) { + c->stackp--; + ref = &c->stack[c->stackp]; + ref->index++; + } + if(ref->page | BRANCH_PAGE) + ref->element += sizeof(branch_elem); + else + ref->element += sizeof(leaf_elem); + return next_element(c, key, value); +} + +// int bolt_cursor_seek(bolt_cursor* c, bolt_val key, bolt_val *actual_key, bolt_val *value) + + +// private functions + +page* page(bolt_cursor* c, pgid id) { + return (page*)(c->data + (c->pgsz * id)); +} + +branch_elem* branch_page_element(*p, elemid index) { + return p + sizeof(page) + index * sizeof(branch_elem); +} + +leaf_elem* leaf_page_element(page* p, elemid index) { + return p + sizeof(page) + index * sizeof(leaf_elem); +} + +set_key_value(leaf_elem* leaf, bolt_val* key, bolt_val *value) { + key.size = leaf->ksize; + key.data = leaf + leaf->pos; + value.size = leaf->vsize; + value.data = key.data + key.size; +} + +int next_element(bolt_cursor* c, bolt_val *key, bolt_val *value) { + elem_ref* ref = &c->stack[c->stackp]; + branch_elem* branch; + for(ref->page->flags | BRANCH_PAGE) { + branch = branch_page_element(ref->page,ref->index); + c->stackp++; + ref = &c->stack[c->stackp]; + ref->index = 0; + ref->element = branch; + ref->page = page(c, branch->pgid); + }; + set_key_value(leaf_page_element(ref->page,ref->index), key, value); + return 0 +} + +*/ +import "C" +import "github.com/boltdb/bolt" + +func NewCursor(b *bolt.Bucket) *C.bolt_cursor { + data := (*C.void)(&b.tx.db.data[0])] + pgsz := (C.size_t)(b.tx.db.pageSize) + cursor := new(C.bolt_cursor) + C.bolt_cursor_init(cursor, data, pgsz, (C.pgid)(b.root)) + return cursor +} diff --git a/c/cursor_test.go b/c/cursor_test.go new file mode 100644 index 0000000..7c45346 --- /dev/null +++ b/c/cursor_test.go @@ -0,0 +1,55 @@ +package c + +import "C" +import ( + "github.com/boltdb/bolt" + "github.com/stretchr/testify/assert" + "testing" + "testing/quick" +) + +// Ensure that a cursor can iterate over all elements in a bucket. +func TestIterate(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + + f := func(items testdata) bool { + withOpenDB(func(db *DB, path string) { + // Bulk insert all values. + tx, _ := db.Begin(true) + tx.CreateBucket("widgets") + b := tx.Bucket("widgets") + for _, item := range items { + assert.NoError(t, b.Put(item.Key, item.Value)) + } + assert.NoError(t, tx.Commit()) + + // Sort test data. + sort.Sort(items) + + // Iterate over all items and check consistency. + var index = 0 + tx, _ = db.Begin(false) + var k, v C.bolt_val + c := NewCursor(tx.Bucket("widgets")) + C.bolt_cursor_first(c, &k, &v) + key := C.GoBytes(k.data, k.size) + for key != nil && index < len(items) { + assert.Equal(t, key, items[index].Key) + assert.Equal(t, C.GoBytes(v.data, v.size), items[index].Value) + index++ + C.bolt_cursor_next(c, &k, &v) + key := C.GoBytes(k.data, k.size) + } + assert.Equal(t, len(items), index) + assert.Equal(t, len(items), index) + tx.Rollback() + }) + return true + } + if err := quick.Check(f, qconfig()); err != nil { + t.Error(err) + } + fmt.Fprint(os.Stderr, "\n") +} From 8a24e16dfd8f24a970af68ed2f4757887ee025c6 Mon Sep 17 00:00:00 2001 From: Martin Kobetic Date: Sun, 13 Apr 2014 10:58:02 -0400 Subject: [PATCH 02/14] minor cleanup --- c/cursor.go | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/c/cursor.go b/c/cursor.go index 5874b20..436bece 100644 --- a/c/cursor.go +++ b/c/cursor.go @@ -53,49 +53,58 @@ typedef struct bolt_cursor { unsigned int stackp; } bolt_cursor; +// public functions + void bolt_cursor_init(bolt_cursor* c, void *data, size_t pgsz, pgid root) { c->data = data; c->pgid = pgid; c->pgsz = pgsz; } -// public functions - int bolt_cursor_first(bolt_cursor* c, bolt_val *key, bolt_val *value) { leaf_elem* leaf; elem_ref* ref; + + // reset stack to initial state c->stackp = 0; ref = &c->stack[c->stackp] ref->page = page(c, c->pgid); ref->index = 0; - return next_element(c, key, value); + + // get current leaf element + return leaf_element(c, key, value); } int bolt_cursor_next(bolt_cursor* c, bolt_val *key, bolt_val *value) { elem_ref* ref= &c->stack[c->stackp]; + + // increment element index ref->index++; + // if we're past last element pop the stack and repeat while (ref->index >= ref->page->count ) { c->stackp--; ref = &c->stack[c->stackp]; ref->index++; } + // increment element pointer if(ref->page | BRANCH_PAGE) ref->element += sizeof(branch_elem); else ref->element += sizeof(leaf_elem); - return next_element(c, key, value); + + // get current leaf element + return leaf_element(c, key, value); } // int bolt_cursor_seek(bolt_cursor* c, bolt_val key, bolt_val *actual_key, bolt_val *value) - // private functions page* page(bolt_cursor* c, pgid id) { return (page*)(c->data + (c->pgsz * id)); } -branch_elem* branch_page_element(*p, elemid index) { +branch_elem* branch_page_element(page* p, elemid index) { return p + sizeof(page) + index * sizeof(branch_elem); } @@ -103,14 +112,10 @@ leaf_elem* leaf_page_element(page* p, elemid index) { return p + sizeof(page) + index * sizeof(leaf_elem); } -set_key_value(leaf_elem* leaf, bolt_val* key, bolt_val *value) { - key.size = leaf->ksize; - key.data = leaf + leaf->pos; - value.size = leaf->vsize; - value.data = key.data + key.size; -} - -int next_element(bolt_cursor* c, bolt_val *key, bolt_val *value) { +// return current leaf element +// if stack points at a branch page descend down to the first elemenet +// of the first leaf page +int leaf_element(bolt_cursor* c, bolt_val *key, bolt_val *value) { elem_ref* ref = &c->stack[c->stackp]; branch_elem* branch; for(ref->page->flags | BRANCH_PAGE) { @@ -125,6 +130,13 @@ int next_element(bolt_cursor* c, bolt_val *key, bolt_val *value) { return 0 } +set_key_value(leaf_elem* leaf, bolt_val* key, bolt_val *value) { + key.size = leaf->ksize; + key.data = leaf + leaf->pos; + value.size = leaf->vsize; + value.data = key.data + key.size; +} + */ import "C" import "github.com/boltdb/bolt" From 3ae69d73fbfbe132b6bf305ff58beb12640b874e Mon Sep 17 00:00:00 2001 From: Martin Kobetic Date: Sun, 13 Apr 2014 15:05:28 +0000 Subject: [PATCH 03/14] cleanup --- c/cursor.go | 14 +++++++------- c/cursor_test.go | 1 - 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/c/cursor.go b/c/cursor.go index 436bece..0af66af 100644 --- a/c/cursor.go +++ b/c/cursor.go @@ -70,14 +70,14 @@ int bolt_cursor_first(bolt_cursor* c, bolt_val *key, bolt_val *value) { ref = &c->stack[c->stackp] ref->page = page(c, c->pgid); ref->index = 0; - + // get current leaf element return leaf_element(c, key, value); } int bolt_cursor_next(bolt_cursor* c, bolt_val *key, bolt_val *value) { elem_ref* ref= &c->stack[c->stackp]; - + // increment element index ref->index++; // if we're past last element pop the stack and repeat @@ -91,9 +91,9 @@ int bolt_cursor_next(bolt_cursor* c, bolt_val *key, bolt_val *value) { ref->element += sizeof(branch_elem); else ref->element += sizeof(leaf_elem); - + // get current leaf element - return leaf_element(c, key, value); + return leaf_element(c, key, value); } // int bolt_cursor_seek(bolt_cursor* c, bolt_val key, bolt_val *actual_key, bolt_val *value) @@ -134,15 +134,15 @@ set_key_value(leaf_elem* leaf, bolt_val* key, bolt_val *value) { key.size = leaf->ksize; key.data = leaf + leaf->pos; value.size = leaf->vsize; - value.data = key.data + key.size; + value.data = key.data + key.size; } */ import "C" import "github.com/boltdb/bolt" -func NewCursor(b *bolt.Bucket) *C.bolt_cursor { - data := (*C.void)(&b.tx.db.data[0])] +func NewCursor(b *bolt.Bucket) *C.bolt_cursor { + data := (*C.void)(&b.tx.db.data[0]) pgsz := (C.size_t)(b.tx.db.pageSize) cursor := new(C.bolt_cursor) C.bolt_cursor_init(cursor, data, pgsz, (C.pgid)(b.root)) diff --git a/c/cursor_test.go b/c/cursor_test.go index 7c45346..12710d9 100644 --- a/c/cursor_test.go +++ b/c/cursor_test.go @@ -1,6 +1,5 @@ package c -import "C" import ( "github.com/boltdb/bolt" "github.com/stretchr/testify/assert" From 4e01c9fd81eec87e54ed79a6b7c04f0d2e39ea50 Mon Sep 17 00:00:00 2001 From: Martin Kobetic Date: Mon, 14 Apr 2014 19:21:15 +0000 Subject: [PATCH 04/14] fix up the C bits to compile --- c/cursor.go | 134 +++++++++++++++++++++++++---------------------- c/cursor_test.go | 9 +--- 2 files changed, 74 insertions(+), 69 deletions(-) diff --git a/c/cursor.go b/c/cursor.go index 0af66af..84eae03 100644 --- a/c/cursor.go +++ b/c/cursor.go @@ -1,13 +1,15 @@ package c /* +#include +#include #define MAX_DEPTH 100 #define BRANCH_PAGE 1 // These types MUST have the same layout as their corresponding Go types -typedef unsigned long long pageid; +typedef unsigned long long pgid; typedef unsigned short elemid; typedef struct page { @@ -20,7 +22,7 @@ typedef struct page { typedef struct branch_elem { unsigned long pos; unsigned long ksize; - pgid pgid; + pgid page; } branch_elem; typedef struct leaf_elem { @@ -33,7 +35,6 @@ typedef struct leaf_elem { // private types typedef struct elem_ref { - void *element; page *page; elemid index; } elem_ref; @@ -49,34 +50,75 @@ typedef struct bolt_cursor { void *data; pgid root; size_t pgsz; - elemid[MAX_DEPTH] stack; unsigned int stackp; + elem_ref stack[MAX_DEPTH]; } bolt_cursor; + +// int bolt_cursor_seek(bolt_cursor *c, bolt_val *key, bolt_val *actual_key, bolt_val *value) + +// private functions + +page *get_page(bolt_cursor *c, pgid id) { + return (page *)(c->data + (c->pgsz * id)); +} + +branch_elem *branch_page_element(page *p, elemid index) { + return (branch_elem*)(p + sizeof(page) + index * sizeof(branch_elem)); +} + +leaf_elem *leaf_page_element(page *p, elemid index) { + return (leaf_elem*)(p + sizeof(page) + index * sizeof(leaf_elem)); +} + +// return current leaf element +// if stack points at a branch page descend down to the first elemenet +// of the first leaf page +int leaf_element(bolt_cursor *c, bolt_val *key, bolt_val *value) { + elem_ref *ref = &(c->stack[c->stackp]); + branch_elem *branch; + while (ref->page->flags | BRANCH_PAGE) { + branch = branch_page_element(ref->page,ref->index); + c->stackp++; + ref = &c->stack[c->stackp]; + ref->index = 0; + ref->page = get_page(c, branch->page); + }; + set_key_value(leaf_page_element(ref->page,ref->index), key, value); + return 0; +} + +set_key_value(leaf_elem *leaf, bolt_val *key, bolt_val *value) { + key->size = leaf->ksize; + key->data = leaf + leaf->pos; + value->size = leaf->vsize; + value->data = key->data + key->size; +} + // public functions -void bolt_cursor_init(bolt_cursor* c, void *data, size_t pgsz, pgid root) { +void bolt_cursor_init(bolt_cursor *c, void *data, size_t pgsz, pgid root) { c->data = data; - c->pgid = pgid; + c->root = root; c->pgsz = pgsz; } -int bolt_cursor_first(bolt_cursor* c, bolt_val *key, bolt_val *value) { - leaf_elem* leaf; - elem_ref* ref; +int bolt_cursor_first(bolt_cursor *c, bolt_val *key, bolt_val *value) { + leaf_elem *leaf; + elem_ref *ref; // reset stack to initial state c->stackp = 0; - ref = &c->stack[c->stackp] - ref->page = page(c, c->pgid); + ref = &(c->stack[c->stackp]); + ref->page = get_page(c, c->root); ref->index = 0; // get current leaf element return leaf_element(c, key, value); } -int bolt_cursor_next(bolt_cursor* c, bolt_val *key, bolt_val *value) { - elem_ref* ref= &c->stack[c->stackp]; +int bolt_cursor_next(bolt_cursor *c, bolt_val *key, bolt_val *value) { + elem_ref *ref = &c->stack[c->stackp]; // increment element index ref->index++; @@ -85,66 +127,34 @@ int bolt_cursor_next(bolt_cursor* c, bolt_val *key, bolt_val *value) { c->stackp--; ref = &c->stack[c->stackp]; ref->index++; - } - // increment element pointer - if(ref->page | BRANCH_PAGE) - ref->element += sizeof(branch_elem); - else - ref->element += sizeof(leaf_elem); + }; // get current leaf element return leaf_element(c, key, value); } - -// int bolt_cursor_seek(bolt_cursor* c, bolt_val key, bolt_val *actual_key, bolt_val *value) - -// private functions - -page* page(bolt_cursor* c, pgid id) { - return (page*)(c->data + (c->pgsz * id)); -} - -branch_elem* branch_page_element(page* p, elemid index) { - return p + sizeof(page) + index * sizeof(branch_elem); -} - -leaf_elem* leaf_page_element(page* p, elemid index) { - return p + sizeof(page) + index * sizeof(leaf_elem); -} - -// return current leaf element -// if stack points at a branch page descend down to the first elemenet -// of the first leaf page -int leaf_element(bolt_cursor* c, bolt_val *key, bolt_val *value) { - elem_ref* ref = &c->stack[c->stackp]; - branch_elem* branch; - for(ref->page->flags | BRANCH_PAGE) { - branch = branch_page_element(ref->page,ref->index); - c->stackp++; - ref = &c->stack[c->stackp]; - ref->index = 0; - ref->element = branch; - ref->page = page(c, branch->pgid); - }; - set_key_value(leaf_page_element(ref->page,ref->index), key, value); - return 0 -} - -set_key_value(leaf_elem* leaf, bolt_val* key, bolt_val *value) { - key.size = leaf->ksize; - key.data = leaf + leaf->pos; - value.size = leaf->vsize; - value.data = key.data + key.size; -} - */ import "C" + import "github.com/boltdb/bolt" -func NewCursor(b *bolt.Bucket) *C.bolt_cursor { +type bolt_cursor *C.bolt_cursor + +func NewCursor(b *bolt.Bucket) bolt_cursor { data := (*C.void)(&b.tx.db.data[0]) pgsz := (C.size_t)(b.tx.db.pageSize) cursor := new(C.bolt_cursor) C.bolt_cursor_init(cursor, data, pgsz, (C.pgid)(b.root)) return cursor } + +func (c bolt_cursor) first() (key, value []byte) { + var k, v C.bolt_val + C.bolt_cursor_first(c, &k, &v) + return C.GoBytes(k.data, k.size), C.GoBytes(v.data, v.size) +} + +func (c bolt_cursor) next() (key, value []byte) { + var k, v C.bolt_val + C.bolt_cursor_next(c, &k, &v) + return C.GoBytes(k.data, k.size), C.GoBytes(v.data, v.size) +} diff --git a/c/cursor_test.go b/c/cursor_test.go index 12710d9..ff6d547 100644 --- a/c/cursor_test.go +++ b/c/cursor_test.go @@ -30,16 +30,11 @@ func TestIterate(t *testing.T) { // Iterate over all items and check consistency. var index = 0 tx, _ = db.Begin(false) - var k, v C.bolt_val c := NewCursor(tx.Bucket("widgets")) - C.bolt_cursor_first(c, &k, &v) - key := C.GoBytes(k.data, k.size) - for key != nil && index < len(items) { + for key, value := c.first(); key != nil && index < len(items); key, value = c.next() { assert.Equal(t, key, items[index].Key) - assert.Equal(t, C.GoBytes(v.data, v.size), items[index].Value) + assert.Equal(t, value, items[index].Value) index++ - C.bolt_cursor_next(c, &k, &v) - key := C.GoBytes(k.data, k.size) } assert.Equal(t, len(items), index) assert.Equal(t, len(items), index) From b178373351a2d9711c8614255843038c5bed5872 Mon Sep 17 00:00:00 2001 From: Steven Normore Date: Tue, 15 Apr 2014 17:56:53 +0000 Subject: [PATCH 05/14] build c/cursor and running tests --- bucket.go | 10 +++++++ c/cursor.go | 19 +++++++----- c/cursor_test.go | 71 +++++++++++++++++++++++++++++++++++++++++--- c/quick_test.go | 77 ++++++++++++++++++++++++++++++++++++++++++++++++ db.go | 6 ++++ 5 files changed, 171 insertions(+), 12 deletions(-) create mode 100644 c/quick_test.go diff --git a/bucket.go b/bucket.go index 2ce2350..77d31c9 100644 --- a/bucket.go +++ b/bucket.go @@ -63,6 +63,16 @@ func newBucket(tx *Tx) Bucket { return b } +// Tx returns the tx of the bucket. +func (b *Bucket) Tx() *Tx { + return b.tx +} + +// Root returns the root of the bucket. +func (b *Bucket) Root() pgid { + return b.root +} + // Writable returns whether the bucket is writable. func (b *Bucket) Writable() bool { return b.tx.writable diff --git a/c/cursor.go b/c/cursor.go index 84eae03..0aebd47 100644 --- a/c/cursor.go +++ b/c/cursor.go @@ -135,26 +135,29 @@ int bolt_cursor_next(bolt_cursor *c, bolt_val *key, bolt_val *value) { */ import "C" -import "github.com/boltdb/bolt" +import ( + "unsafe" + + "github.com/boltdb/bolt" +) type bolt_cursor *C.bolt_cursor func NewCursor(b *bolt.Bucket) bolt_cursor { - data := (*C.void)(&b.tx.db.data[0]) - pgsz := (C.size_t)(b.tx.db.pageSize) + data, pgsz := b.Tx().DB().RawData() cursor := new(C.bolt_cursor) - C.bolt_cursor_init(cursor, data, pgsz, (C.pgid)(b.root)) + C.bolt_cursor_init(cursor, unsafe.Pointer(&data[0]), (C.size_t)(pgsz), (C.pgid)(b.Root())) return cursor } -func (c bolt_cursor) first() (key, value []byte) { +func first(c bolt_cursor) (key, value []byte) { var k, v C.bolt_val C.bolt_cursor_first(c, &k, &v) - return C.GoBytes(k.data, k.size), C.GoBytes(v.data, v.size) + return C.GoBytes(k.data, C.int(k.size)), C.GoBytes(v.data, C.int(v.size)) } -func (c bolt_cursor) next() (key, value []byte) { +func next(c bolt_cursor) (key, value []byte) { var k, v C.bolt_val C.bolt_cursor_next(c, &k, &v) - return C.GoBytes(k.data, k.size), C.GoBytes(v.data, v.size) + return C.GoBytes(k.data, C.int(k.size)), C.GoBytes(v.data, C.int(v.size)) } diff --git a/c/cursor_test.go b/c/cursor_test.go index ff6d547..3208757 100644 --- a/c/cursor_test.go +++ b/c/cursor_test.go @@ -1,12 +1,20 @@ package c import ( - "github.com/boltdb/bolt" - "github.com/stretchr/testify/assert" + "fmt" + "io/ioutil" + "os" + "sort" "testing" "testing/quick" + + "github.com/boltdb/bolt" + "github.com/stretchr/testify/assert" ) +// Test when cursor hits the end +// Implement seek; binary search within the page (branch page and element page) + // Ensure that a cursor can iterate over all elements in a bucket. func TestIterate(t *testing.T) { if testing.Short() { @@ -14,7 +22,7 @@ func TestIterate(t *testing.T) { } f := func(items testdata) bool { - withOpenDB(func(db *DB, path string) { + withOpenDB(func(db *bolt.DB, path string) { // Bulk insert all values. tx, _ := db.Begin(true) tx.CreateBucket("widgets") @@ -31,7 +39,7 @@ func TestIterate(t *testing.T) { var index = 0 tx, _ = db.Begin(false) c := NewCursor(tx.Bucket("widgets")) - for key, value := c.first(); key != nil && index < len(items); key, value = c.next() { + for key, value := first(c); key != nil && index < len(items); key, value = next(c) { assert.Equal(t, key, items[index].Key) assert.Equal(t, value, items[index].Value) index++ @@ -47,3 +55,58 @@ func TestIterate(t *testing.T) { } fmt.Fprint(os.Stderr, "\n") } + +// withTempPath executes a function with a database reference. +func withTempPath(fn func(string)) { + f, _ := ioutil.TempFile("", "bolt-") + path := f.Name() + f.Close() + os.Remove(path) + defer os.RemoveAll(path) + + fn(path) +} + +// withOpenDB executes a function with an already opened database. +func withOpenDB(fn func(*bolt.DB, string)) { + withTempPath(func(path string) { + db, err := bolt.Open(path, 0666) + if err != nil { + panic("cannot open db: " + err.Error()) + } + defer db.Close() + fn(db, path) + + // Log statistics. + // if *statsFlag { + // logStats(db) + // } + + // Check database consistency after every test. + mustCheck(db) + }) +} + +// mustCheck runs a consistency check on the database and panics if any errors are found. +func mustCheck(db *bolt.DB) { + if err := db.Check(); err != nil { + // Copy db off first. + db.CopyFile("/tmp/check.db", 0600) + + if errors, ok := err.(bolt.ErrorList); ok { + for _, err := range errors { + warn(err) + } + } + warn(err) + panic("check failure: see /tmp/check.db") + } +} + +func warn(v ...interface{}) { + fmt.Fprintln(os.Stderr, v...) +} + +func warnf(msg string, v ...interface{}) { + fmt.Fprintf(os.Stderr, msg+"\n", v...) +} diff --git a/c/quick_test.go b/c/quick_test.go new file mode 100644 index 0000000..7d87249 --- /dev/null +++ b/c/quick_test.go @@ -0,0 +1,77 @@ +package c + +import ( + "bytes" + "flag" + "math/rand" + "reflect" + "testing/quick" + "time" +) + +// testing/quick defaults to 5 iterations and a random seed. +// You can override these settings from the command line: +// +// -quick.count The number of iterations to perform. +// -quick.seed The seed to use for randomizing. +// -quick.maxitems The maximum number of items to insert into a DB. +// -quick.maxksize The maximum size of a key. +// -quick.maxvsize The maximum size of a value. +// + +var qcount, qseed, qmaxitems, qmaxksize, qmaxvsize int + +func init() { + flag.IntVar(&qcount, "quick.count", 5, "") + flag.IntVar(&qseed, "quick.seed", int(time.Now().UnixNano())%100000, "") + flag.IntVar(&qmaxitems, "quick.maxitems", 1000, "") + flag.IntVar(&qmaxksize, "quick.maxksize", 1024, "") + flag.IntVar(&qmaxvsize, "quick.maxvsize", 1024, "") + flag.Parse() + warn("seed:", qseed) + warnf("quick settings: count=%v, items=%v, ksize=%v, vsize=%v", qcount, qmaxitems, qmaxksize, qmaxvsize) +} + +func qconfig() *quick.Config { + return &quick.Config{ + MaxCount: qcount, + Rand: rand.New(rand.NewSource(int64(qseed))), + } +} + +type testdata []testdataitem + +func (t testdata) Len() int { return len(t) } +func (t testdata) Swap(i, j int) { t[i], t[j] = t[j], t[i] } +func (t testdata) Less(i, j int) bool { return bytes.Compare(t[i].Key, t[j].Key) == -1 } + +func (t testdata) Generate(rand *rand.Rand, size int) reflect.Value { + n := rand.Intn(qmaxitems-1) + 1 + items := make(testdata, n) + for i := 0; i < n; i++ { + item := &items[i] + item.Key = randByteSlice(rand, 1, qmaxksize) + item.Value = randByteSlice(rand, 0, qmaxvsize) + } + return reflect.ValueOf(items) +} + +type revtestdata []testdataitem + +func (t revtestdata) Len() int { return len(t) } +func (t revtestdata) Swap(i, j int) { t[i], t[j] = t[j], t[i] } +func (t revtestdata) Less(i, j int) bool { return bytes.Compare(t[i].Key, t[j].Key) == 1 } + +type testdataitem struct { + Key []byte + Value []byte +} + +func randByteSlice(rand *rand.Rand, minSize, maxSize int) []byte { + n := rand.Intn(maxSize-minSize) + minSize + b := make([]byte, n) + for i := 0; i < n; i++ { + b[i] = byte(rand.Intn(255)) + } + return b +} diff --git a/db.go b/db.go index c9611f9..54f726e 100644 --- a/db.go +++ b/db.go @@ -578,6 +578,12 @@ func (db *DB) checkBucket(b *Bucket, reachable map[pgid]*page, errors *ErrorList }) } +// This is for internal access to the raw data bytes from the C cursor, use +// carefully, or not at all. +func (db *DB) RawData() ([]byte, int) { + return db.data, db.pageSize +} + // page retrieves a page reference from the mmap based on the current page size. func (db *DB) page(id pgid) *page { pos := id * pgid(db.pageSize) From 32937280c36f9af3755472c4e8c825ec3a9d7539 Mon Sep 17 00:00:00 2001 From: Steven Normore Date: Wed, 16 Apr 2014 15:00:26 +0000 Subject: [PATCH 06/14] wip --- c/cursor.go | 58 ++++++++++++++++++++---------- c/cursor_test.go | 94 +++++++++++++++++++++++++++++------------------- db.go | 9 +++-- 3 files changed, 103 insertions(+), 58 deletions(-) diff --git a/c/cursor.go b/c/cursor.go index 0aebd47..17ec43b 100644 --- a/c/cursor.go +++ b/c/cursor.go @@ -3,40 +3,41 @@ package c /* #include #include +#include +#include #define MAX_DEPTH 100 #define BRANCH_PAGE 1 // These types MUST have the same layout as their corresponding Go types -typedef unsigned long long pgid; -typedef unsigned short elemid; +typedef int64_t pgid; typedef struct page { - pgid id; - unsigned short flags; - elemid count; - unsigned long overflow; + pgid id; + uint16_t flags; + uint16_t count; + uint32_t overflow; } page; typedef struct branch_elem { - unsigned long pos; - unsigned long ksize; - pgid page; + uint32_t pos; + uint32_t ksize; + pgid page; } branch_elem; typedef struct leaf_elem { - unsigned long flags; - unsigned long pos; - unsigned long ksize; - unsigned long vsize; + uint32_t flags; + uint32_t pos; + uint32_t ksize; + uint32_t vsize; } leaf_elem; // private types typedef struct elem_ref { page *page; - elemid index; + uint16_t index; } elem_ref; // public types @@ -60,14 +61,17 @@ typedef struct bolt_cursor { // private functions page *get_page(bolt_cursor *c, pgid id) { + printf("get_page: c->data=%d, c->pgsz=%d, pgid=%d\n\n", c->data, c->pgsz, id); return (page *)(c->data + (c->pgsz * id)); } -branch_elem *branch_page_element(page *p, elemid index) { +branch_elem *branch_page_element(page *p, uint16_t index) { return (branch_elem*)(p + sizeof(page) + index * sizeof(branch_elem)); } -leaf_elem *leaf_page_element(page *p, elemid index) { +leaf_elem *leaf_page_element(page *p, uint16_t index) { + printf("leaf_page_element: page=%d, index=%d, sizeof(page)=%d, sizeof(leaf_elem)=%d\n\n", p, index, sizeof(page), sizeof(leaf_elem)); + printf("leaf_page_element: elem=%x\n", (leaf_elem*)(p + sizeof(page) + index * sizeof(leaf_elem))[0]); return (leaf_elem*)(p + sizeof(page) + index * sizeof(leaf_elem)); } @@ -75,16 +79,26 @@ leaf_elem *leaf_page_element(page *p, elemid index) { // if stack points at a branch page descend down to the first elemenet // of the first leaf page int leaf_element(bolt_cursor *c, bolt_val *key, bolt_val *value) { + printf("leaf_element:1:\n\n"); elem_ref *ref = &(c->stack[c->stackp]); + printf("leaf_element:2:, ref->page->flags=%d\n\n", ref->page->flags); branch_elem *branch; - while (ref->page->flags | BRANCH_PAGE) { + while (ref->page->flags & BRANCH_PAGE) { + printf("leaf_element:2.1, ref->page->flags=%d\n\n", ref->page->flags); branch = branch_page_element(ref->page,ref->index); + printf("leaf_element:2.2\n\n"); c->stackp++; + //printf("leaf_element:2.3, c->stack=%d, c->stackp=%d\n\n", c->stack, c->stackp); ref = &c->stack[c->stackp]; + //printf("leaf_element:2.4, ref=%d\n\n", ref); ref->index = 0; + printf("leaf_element:2.5\n\n"); ref->page = get_page(c, branch->page); + printf("leaf_element:2.6\n\n"); }; + printf("leaf_element:3, key=%s, value=%s\n\n", key, value); set_key_value(leaf_page_element(ref->page,ref->index), key, value); + printf("leaf_element:3, key=%s, value=%s\n\n", key, value); return 0; } @@ -93,6 +107,7 @@ set_key_value(leaf_elem *leaf, bolt_val *key, bolt_val *value) { key->data = leaf + leaf->pos; value->size = leaf->vsize; value->data = key->data + key->size; + printf("set_key_value: key=%s (%d), value=%s (%d)\n\n", key->data, key->size, value->data, value->size); } // public functions @@ -136,6 +151,7 @@ int bolt_cursor_next(bolt_cursor *c, bolt_val *key, bolt_val *value) { import "C" import ( + // "fmt" "unsafe" "github.com/boltdb/bolt" @@ -144,14 +160,18 @@ import ( type bolt_cursor *C.bolt_cursor func NewCursor(b *bolt.Bucket) bolt_cursor { - data, pgsz := b.Tx().DB().RawData() + info := b.Tx().DB().Info() + root := b.Root() cursor := new(C.bolt_cursor) - C.bolt_cursor_init(cursor, unsafe.Pointer(&data[0]), (C.size_t)(pgsz), (C.pgid)(b.Root())) + C.bolt_cursor_init(cursor, unsafe.Pointer(&info.Data[0]), (C.size_t)(info.PageSize), (C.pgid)(root)) return cursor } func first(c bolt_cursor) (key, value []byte) { var k, v C.bolt_val + // fmt.Println("cursor =", c) + // fmt.Println("key =", k) + // fmt.Println("value =", v) C.bolt_cursor_first(c, &k, &v) return C.GoBytes(k.data, C.int(k.size)), C.GoBytes(v.data, C.int(v.size)) } diff --git a/c/cursor_test.go b/c/cursor_test.go index 3208757..f938881 100644 --- a/c/cursor_test.go +++ b/c/cursor_test.go @@ -4,9 +4,9 @@ import ( "fmt" "io/ioutil" "os" - "sort" + // "sort" "testing" - "testing/quick" + // "testing/quick" "github.com/boltdb/bolt" "github.com/stretchr/testify/assert" @@ -16,44 +16,64 @@ import ( // Implement seek; binary search within the page (branch page and element page) // Ensure that a cursor can iterate over all elements in a bucket. -func TestIterate(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - } +// func TestIterate(t *testing.T) { +// if testing.Short() { +// t.Skip("skipping test in short mode.") +// } - f := func(items testdata) bool { - withOpenDB(func(db *bolt.DB, path string) { - // Bulk insert all values. - tx, _ := db.Begin(true) - tx.CreateBucket("widgets") - b := tx.Bucket("widgets") - for _, item := range items { - assert.NoError(t, b.Put(item.Key, item.Value)) - } - assert.NoError(t, tx.Commit()) +// f := func(items testdata) bool { +// withOpenDB(func(db *bolt.DB, path string) { +// // Bulk insert all values. +// tx, _ := db.Begin(true) +// tx.CreateBucket("widgets") +// b := tx.Bucket("widgets") +// for _, item := range items { +// assert.NoError(t, b.Put(item.Key, item.Value)) +// } +// assert.NoError(t, tx.Commit()) - // Sort test data. - sort.Sort(items) +// // Sort test data. +// sort.Sort(items) - // Iterate over all items and check consistency. - var index = 0 - tx, _ = db.Begin(false) - c := NewCursor(tx.Bucket("widgets")) - for key, value := first(c); key != nil && index < len(items); key, value = next(c) { - assert.Equal(t, key, items[index].Key) - assert.Equal(t, value, items[index].Value) - index++ - } - assert.Equal(t, len(items), index) - assert.Equal(t, len(items), index) - tx.Rollback() - }) - return true - } - if err := quick.Check(f, qconfig()); err != nil { - t.Error(err) - } - fmt.Fprint(os.Stderr, "\n") +// // Iterate over all items and check consistency. +// var index = 0 +// tx, _ = db.Begin(false) +// c := NewCursor(tx.Bucket("widgets")) +// for key, value := first(c); key != nil && index < len(items); key, value = next(c) { +// assert.Equal(t, key, items[index].Key) +// assert.Equal(t, value, items[index].Value) +// index++ +// } +// assert.Equal(t, len(items), index) +// assert.Equal(t, len(items), index) +// tx.Rollback() +// }) +// return true +// } +// if err := quick.Check(f, qconfig()); err != nil { +// t.Error(err) +// } +// fmt.Fprint(os.Stderr, "\n") +// } + +func TestCursorFirst(t *testing.T) { + withOpenDB(func(db *bolt.DB, path string) { + + // Bulk insert all values. + tx, _ := db.Begin(true) + b, _ := tx.CreateBucket([]byte("widgets")) + assert.NoError(t, b.Put([]byte("foo"), []byte("bar"))) + assert.NoError(t, tx.Commit()) + + // Get first and check consistency + tx, _ = db.Begin(false) + c := NewCursor(tx.Bucket([]byte("widgets"))) + key, value := first(c) + assert.Equal(t, key, []byte("foo")) + assert.Equal(t, value, []byte("bar")) + + tx.Rollback() + }) } // withTempPath executes a function with a database reference. diff --git a/db.go b/db.go index 54f726e..7e8dd7f 100644 --- a/db.go +++ b/db.go @@ -580,8 +580,8 @@ func (db *DB) checkBucket(b *Bucket, reachable map[pgid]*page, errors *ErrorList // This is for internal access to the raw data bytes from the C cursor, use // carefully, or not at all. -func (db *DB) RawData() ([]byte, int) { - return db.data, db.pageSize +func (db *DB) Info() *Info { + return &Info{db.data, db.pageSize} } // page retrieves a page reference from the mmap based on the current page size. @@ -647,3 +647,8 @@ func (s *Stats) Sub(other *Stats) Stats { func (s *Stats) add(other *Stats) { s.TxStats.add(&other.TxStats) } + +type Info struct { + Data []byte + PageSize int +} From a8cb83c00826f8acbad841e30cbdeba75eaae21f Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Wed, 16 Apr 2014 11:30:03 -0400 Subject: [PATCH 07/14] Fix pointer arithematic. --- c/cursor.go | 104 ++++++++++++++++++++++++----------------------- c/cursor_test.go | 13 +----- 2 files changed, 56 insertions(+), 61 deletions(-) diff --git a/c/cursor.go b/c/cursor.go index 17ec43b..73c5269 100644 --- a/c/cursor.go +++ b/c/cursor.go @@ -60,54 +60,44 @@ typedef struct bolt_cursor { // private functions +// Returns a page pointer from a page identifier. page *get_page(bolt_cursor *c, pgid id) { - printf("get_page: c->data=%d, c->pgsz=%d, pgid=%d\n\n", c->data, c->pgsz, id); return (page *)(c->data + (c->pgsz * id)); } +// Returns the leaf element at a given index on a given page. branch_elem *branch_page_element(page *p, uint16_t index) { - return (branch_elem*)(p + sizeof(page) + index * sizeof(branch_elem)); + branch_elem *elements = (branch_elem*)((void*)(p) + sizeof(page)); + return &elements[index]; } +// Returns the leaf element at a given index on a given page. leaf_elem *leaf_page_element(page *p, uint16_t index) { - printf("leaf_page_element: page=%d, index=%d, sizeof(page)=%d, sizeof(leaf_elem)=%d\n\n", p, index, sizeof(page), sizeof(leaf_elem)); - printf("leaf_page_element: elem=%x\n", (leaf_elem*)(p + sizeof(page) + index * sizeof(leaf_elem))[0]); - return (leaf_elem*)(p + sizeof(page) + index * sizeof(leaf_elem)); + leaf_elem *elements = (leaf_elem*)((void*)(p) + sizeof(page)); + return &elements[index]; } -// return current leaf element -// if stack points at a branch page descend down to the first elemenet -// of the first leaf page -int leaf_element(bolt_cursor *c, bolt_val *key, bolt_val *value) { - printf("leaf_element:1:\n\n"); - elem_ref *ref = &(c->stack[c->stackp]); - printf("leaf_element:2:, ref->page->flags=%d\n\n", ref->page->flags); - branch_elem *branch; - while (ref->page->flags & BRANCH_PAGE) { - printf("leaf_element:2.1, ref->page->flags=%d\n\n", ref->page->flags); - branch = branch_page_element(ref->page,ref->index); - printf("leaf_element:2.2\n\n"); - c->stackp++; - //printf("leaf_element:2.3, c->stack=%d, c->stackp=%d\n\n", c->stack, c->stackp); - ref = &c->stack[c->stackp]; - //printf("leaf_element:2.4, ref=%d\n\n", ref); - ref->index = 0; - printf("leaf_element:2.5\n\n"); - ref->page = get_page(c, branch->page); - printf("leaf_element:2.6\n\n"); - }; - printf("leaf_element:3, key=%s, value=%s\n\n", key, value); - set_key_value(leaf_page_element(ref->page,ref->index), key, value); - printf("leaf_element:3, key=%s, value=%s\n\n", key, value); - return 0; -} - -set_key_value(leaf_elem *leaf, bolt_val *key, bolt_val *value) { +// Sets the key and value for a leaf element to a bolt value. +void key_value(leaf_elem *leaf, bolt_val *key, bolt_val *value) { key->size = leaf->ksize; - key->data = leaf + leaf->pos; + key->data = ((void*)leaf) + leaf->pos; value->size = leaf->vsize; value->data = key->data + key->size; - printf("set_key_value: key=%s (%d), value=%s (%d)\n\n", key->data, key->size, value->data, value->size); +} + +// Traverses from the current stack position down to the first leaf element. +int bolt_cursor_first_leaf(bolt_cursor *c, bolt_val *key, bolt_val *value) { + elem_ref *ref = &(c->stack[c->stackp]); + branch_elem *branch; + while (ref->page->flags & BRANCH_PAGE) { + branch = branch_page_element(ref->page,ref->index); + c->stackp++; + ref = &c->stack[c->stackp]; + ref->index = 0; + ref->page = get_page(c, branch->page); + }; + key_value(leaf_page_element(ref->page,ref->index), key, value); + return 0; } // public functions @@ -129,7 +119,7 @@ int bolt_cursor_first(bolt_cursor *c, bolt_val *key, bolt_val *value) { ref->index = 0; // get current leaf element - return leaf_element(c, key, value); + return bolt_cursor_first_leaf(c, key, value); } int bolt_cursor_next(bolt_cursor *c, bolt_val *key, bolt_val *value) { @@ -145,39 +135,53 @@ int bolt_cursor_next(bolt_cursor *c, bolt_val *key, bolt_val *value) { }; // get current leaf element - return leaf_element(c, key, value); + return bolt_cursor_first_leaf(c, key, value); } */ import "C" import ( - // "fmt" + "fmt" + "os" "unsafe" "github.com/boltdb/bolt" ) -type bolt_cursor *C.bolt_cursor +// Cursor represents a wrapper around a Bolt C cursor. +type Cursor struct { + C *C.bolt_cursor +} -func NewCursor(b *bolt.Bucket) bolt_cursor { +// NewCursor creates a C cursor from a Bucket. +func NewCursor(b *bolt.Bucket) *Cursor { info := b.Tx().DB().Info() root := b.Root() - cursor := new(C.bolt_cursor) - C.bolt_cursor_init(cursor, unsafe.Pointer(&info.Data[0]), (C.size_t)(info.PageSize), (C.pgid)(root)) - return cursor + c := &Cursor{C: new(C.bolt_cursor)} + C.bolt_cursor_init(c.C, unsafe.Pointer(&info.Data[0]), C.size_t(info.PageSize), C.pgid(root)) + return c } -func first(c bolt_cursor) (key, value []byte) { +// first moves the cursor to the first element and returns the key and value. +// Returns a nil key if there are no elements. +func first(c *Cursor) (key, value []byte) { var k, v C.bolt_val - // fmt.Println("cursor =", c) - // fmt.Println("key =", k) - // fmt.Println("value =", v) - C.bolt_cursor_first(c, &k, &v) + C.bolt_cursor_first(c.C, &k, &v) return C.GoBytes(k.data, C.int(k.size)), C.GoBytes(v.data, C.int(v.size)) } -func next(c bolt_cursor) (key, value []byte) { +// next moves the cursor to the next element and returns the key and value. +// Returns a nil key if at the end of the bucket. +func next(c *Cursor) (key, value []byte) { var k, v C.bolt_val - C.bolt_cursor_next(c, &k, &v) + C.bolt_cursor_next(c.C, &k, &v) return C.GoBytes(k.data, C.int(k.size)), C.GoBytes(v.data, C.int(v.size)) } + +func warn(v ...interface{}) { + fmt.Fprintln(os.Stderr, v...) +} + +func warnf(msg string, v ...interface{}) { + fmt.Fprintf(os.Stderr, msg+"\n", v...) +} diff --git a/c/cursor_test.go b/c/cursor_test.go index f938881..3ac621f 100644 --- a/c/cursor_test.go +++ b/c/cursor_test.go @@ -1,7 +1,6 @@ package c import ( - "fmt" "io/ioutil" "os" // "sort" @@ -62,7 +61,7 @@ func TestCursorFirst(t *testing.T) { // Bulk insert all values. tx, _ := db.Begin(true) b, _ := tx.CreateBucket([]byte("widgets")) - assert.NoError(t, b.Put([]byte("foo"), []byte("bar"))) + assert.NoError(t, b.Put([]byte("foo"), []byte("barz"))) assert.NoError(t, tx.Commit()) // Get first and check consistency @@ -70,7 +69,7 @@ func TestCursorFirst(t *testing.T) { c := NewCursor(tx.Bucket([]byte("widgets"))) key, value := first(c) assert.Equal(t, key, []byte("foo")) - assert.Equal(t, value, []byte("bar")) + assert.Equal(t, value, []byte("barz")) tx.Rollback() }) @@ -122,11 +121,3 @@ func mustCheck(db *bolt.DB) { panic("check failure: see /tmp/check.db") } } - -func warn(v ...interface{}) { - fmt.Fprintln(os.Stderr, v...) -} - -func warnf(msg string, v ...interface{}) { - fmt.Fprintf(os.Stderr, msg+"\n", v...) -} From fe1f861f745669823fc0eb0d9eb7ecfd156e532f Mon Sep 17 00:00:00 2001 From: Steven Normore Date: Wed, 16 Apr 2014 15:38:23 +0000 Subject: [PATCH 08/14] adds cursor iterate test --- c/cursor_test.go | 102 +++++++++++++++++++++++++---------------------- 1 file changed, 54 insertions(+), 48 deletions(-) diff --git a/c/cursor_test.go b/c/cursor_test.go index 3ac621f..8d40d8d 100644 --- a/c/cursor_test.go +++ b/c/cursor_test.go @@ -1,11 +1,12 @@ package c import ( + "fmt" "io/ioutil" "os" - // "sort" + "sort" "testing" - // "testing/quick" + "testing/quick" "github.com/boltdb/bolt" "github.com/stretchr/testify/assert" @@ -14,67 +15,72 @@ import ( // Test when cursor hits the end // Implement seek; binary search within the page (branch page and element page) -// Ensure that a cursor can iterate over all elements in a bucket. -// func TestIterate(t *testing.T) { -// if testing.Short() { -// t.Skip("skipping test in short mode.") -// } - -// f := func(items testdata) bool { -// withOpenDB(func(db *bolt.DB, path string) { -// // Bulk insert all values. -// tx, _ := db.Begin(true) -// tx.CreateBucket("widgets") -// b := tx.Bucket("widgets") -// for _, item := range items { -// assert.NoError(t, b.Put(item.Key, item.Value)) -// } -// assert.NoError(t, tx.Commit()) - -// // Sort test data. -// sort.Sort(items) - -// // Iterate over all items and check consistency. -// var index = 0 -// tx, _ = db.Begin(false) -// c := NewCursor(tx.Bucket("widgets")) -// for key, value := first(c); key != nil && index < len(items); key, value = next(c) { -// assert.Equal(t, key, items[index].Key) -// assert.Equal(t, value, items[index].Value) -// index++ -// } -// assert.Equal(t, len(items), index) -// assert.Equal(t, len(items), index) -// tx.Rollback() -// }) -// return true -// } -// if err := quick.Check(f, qconfig()); err != nil { -// t.Error(err) -// } -// fmt.Fprint(os.Stderr, "\n") -// } - +// Ensure that a cursor can get the first element of a bucket. func TestCursorFirst(t *testing.T) { withOpenDB(func(db *bolt.DB, path string) { // Bulk insert all values. tx, _ := db.Begin(true) - b, _ := tx.CreateBucket([]byte("widgets")) - assert.NoError(t, b.Put([]byte("foo"), []byte("barz"))) + b, _ := tx.CreateBucket(toBytes("widgets")) + assert.NoError(t, b.Put(toBytes("foo"), toBytes("barz"))) assert.NoError(t, tx.Commit()) // Get first and check consistency tx, _ = db.Begin(false) - c := NewCursor(tx.Bucket([]byte("widgets"))) + c := NewCursor(tx.Bucket(toBytes("widgets"))) key, value := first(c) - assert.Equal(t, key, []byte("foo")) - assert.Equal(t, value, []byte("barz")) + assert.Equal(t, key, toBytes("foo")) + assert.Equal(t, value, toBytes("barz")) tx.Rollback() }) } +// Ensure that a cursor can iterate over all elements in a bucket. +func TestIterate(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + + f := func(items testdata) bool { + withOpenDB(func(db *bolt.DB, path string) { + // Bulk insert all values. + tx, _ := db.Begin(true) + b, _ := tx.CreateBucket(toBytes("widgets")) + for _, item := range items { + assert.NoError(t, b.Put(item.Key, item.Value)) + } + assert.NoError(t, tx.Commit()) + + // Sort test data. + sort.Sort(items) + + // Iterate over all items and check consistency. + var index = 0 + tx, _ = db.Begin(false) + c := NewCursor(tx.Bucket(toBytes("widgets"))) + for key, value := first(c); key != nil && index < len(items); key, value = next(c) { + assert.Equal(t, key, items[index].Key) + assert.Equal(t, value, items[index].Value) + index++ + } + assert.Equal(t, len(items), index) + assert.Equal(t, len(items), index) + tx.Rollback() + }) + return true + } + if err := quick.Check(f, qconfig()); err != nil { + t.Error(err) + } + fmt.Fprint(os.Stderr, "\n") +} + +// toBytes converts a string to an array of bytes. +func toBytes(s string) []byte { + return []byte(s) +} + // withTempPath executes a function with a database reference. func withTempPath(fn func(string)) { f, _ := ioutil.TempFile("", "bolt-") From c0ae4881ab0f827f7964b47cff59d60d8c85f499 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Wed, 16 Apr 2014 15:05:30 -0400 Subject: [PATCH 09/14] Add Cursor.Next() to C cursor. --- c/cursor.go | 245 +++++++++++++++++++++++++++++------------------ c/cursor_test.go | 168 +++++++++++++++----------------- c/doc.go | 4 + c/quick_test.go | 77 --------------- cursor_test.go | 10 +- 5 files changed, 235 insertions(+), 269 deletions(-) create mode 100644 c/doc.go delete mode 100644 c/quick_test.go diff --git a/c/cursor.go b/c/cursor.go index 73c5269..000dd9d 100644 --- a/c/cursor.go +++ b/c/cursor.go @@ -6,137 +6,190 @@ package c #include #include +//------------------------------------------------------------------------------ +// Constants +//------------------------------------------------------------------------------ + +// This represents the maximum number of levels that a cursor can traverse. #define MAX_DEPTH 100 -#define BRANCH_PAGE 1 + +// These flags mark the type of page and are set in the page.flags. +#define PAGE_BRANCH 0x01 +#define PAGE_LEAF 0x02 +#define PAGE_META 0x04 +#define PAGE_FREELIST 0x10 + + +//------------------------------------------------------------------------------ +// Typedefs +//------------------------------------------------------------------------------ // These types MUST have the same layout as their corresponding Go types typedef int64_t pgid; +// Page represents a header struct of a block in the mmap. typedef struct page { - pgid id; - uint16_t flags; - uint16_t count; - uint32_t overflow; + pgid id; + uint16_t flags; + uint16_t count; + uint32_t overflow; } page; -typedef struct branch_elem { - uint32_t pos; - uint32_t ksize; - pgid page; -} branch_elem; +// The branch element represents an a item in a branch page +// that points to a child page. +typedef struct branch_element { + uint32_t pos; + uint32_t ksize; + pgid page; +} branch_element; -typedef struct leaf_elem { - uint32_t flags; - uint32_t pos; - uint32_t ksize; - uint32_t vsize; -} leaf_elem; - -// private types +// The leaf element represents an a item in a leaf page +// that points to a key/value pair. +typedef struct leaf_element { + uint32_t flags; + uint32_t pos; + uint32_t ksize; + uint32_t vsize; +} leaf_element; +// elem_ref represents a pointer to an element inside of a page. +// It is used by the cursor stack to track the position at each level. typedef struct elem_ref { - page *page; - uint16_t index; + page *page; + uint16_t index; } elem_ref; -// public types - +// bolt_val represents a pointer to a fixed-length series of bytes. +// It is used to represent keys and values returned by the cursor. typedef struct bolt_val { uint32_t size; void *data; } bolt_val; +// bolt_cursor represents a cursor attached to a bucket. typedef struct bolt_cursor { - void *data; - pgid root; - size_t pgsz; - unsigned int stackp; - elem_ref stack[MAX_DEPTH]; + void *data; + pgid root; + size_t pgsz; + int top; + elem_ref stack[MAX_DEPTH]; } bolt_cursor; -// int bolt_cursor_seek(bolt_cursor *c, bolt_val *key, bolt_val *actual_key, bolt_val *value) +//------------------------------------------------------------------------------ +// Forward Declarations +//------------------------------------------------------------------------------ -// private functions +page *cursor_page(bolt_cursor *c, pgid id); -// Returns a page pointer from a page identifier. -page *get_page(bolt_cursor *c, pgid id) { - return (page *)(c->data + (c->pgsz * id)); -} +void cursor_first_leaf(bolt_cursor *c); -// Returns the leaf element at a given index on a given page. -branch_elem *branch_page_element(page *p, uint16_t index) { - branch_elem *elements = (branch_elem*)((void*)(p) + sizeof(page)); - return &elements[index]; -} +void cursor_key_value(bolt_cursor *c, bolt_val *key, bolt_val *value, uint32_t *flags); -// Returns the leaf element at a given index on a given page. -leaf_elem *leaf_page_element(page *p, uint16_t index) { - leaf_elem *elements = (leaf_elem*)((void*)(p) + sizeof(page)); - return &elements[index]; -} -// Sets the key and value for a leaf element to a bolt value. -void key_value(leaf_elem *leaf, bolt_val *key, bolt_val *value) { - key->size = leaf->ksize; - key->data = ((void*)leaf) + leaf->pos; - value->size = leaf->vsize; - value->data = key->data + key->size; -} - -// Traverses from the current stack position down to the first leaf element. -int bolt_cursor_first_leaf(bolt_cursor *c, bolt_val *key, bolt_val *value) { - elem_ref *ref = &(c->stack[c->stackp]); - branch_elem *branch; - while (ref->page->flags & BRANCH_PAGE) { - branch = branch_page_element(ref->page,ref->index); - c->stackp++; - ref = &c->stack[c->stackp]; - ref->index = 0; - ref->page = get_page(c, branch->page); - }; - key_value(leaf_page_element(ref->page,ref->index), key, value); - return 0; -} - -// public functions +//------------------------------------------------------------------------------ +// Public Functions +//------------------------------------------------------------------------------ +// Initializes a cursor. void bolt_cursor_init(bolt_cursor *c, void *data, size_t pgsz, pgid root) { c->data = data; c->root = root; c->pgsz = pgsz; } -int bolt_cursor_first(bolt_cursor *c, bolt_val *key, bolt_val *value) { - leaf_elem *leaf; - elem_ref *ref; - +// Positions the cursor to the first leaf element and returns the key/value pair. +void bolt_cursor_first(bolt_cursor *c, bolt_val *key, bolt_val *value, uint32_t *flags) { // reset stack to initial state - c->stackp = 0; - ref = &(c->stack[c->stackp]); - ref->page = get_page(c, c->root); + c->top = 0; + elem_ref *ref = &(c->stack[c->top]); + ref->page = cursor_page(c, c->root); ref->index = 0; - // get current leaf element - return bolt_cursor_first_leaf(c, key, value); + // Find first leaf and return key/value. + cursor_first_leaf(c); + cursor_key_value(c, key, value, flags); } -int bolt_cursor_next(bolt_cursor *c, bolt_val *key, bolt_val *value) { - elem_ref *ref = &c->stack[c->stackp]; +// Positions the cursor to the next leaf element and returns the key/value pair. +void bolt_cursor_next(bolt_cursor *c, bolt_val *key, bolt_val *value, uint32_t *flags) { + // Attempt to move over one element until we're successful. + // Move up the stack as we hit the end of each page in our stack. + for (int i = c->top; i >= 0; i--) { + elem_ref *elem = &c->stack[i]; + if (elem->index < elem->page->count - 1) { + elem->index++; + break; + } + c->top--; + } - // increment element index - ref->index++; - // if we're past last element pop the stack and repeat - while (ref->index >= ref->page->count ) { - c->stackp--; - ref = &c->stack[c->stackp]; - ref->index++; + // If we are at the top of the stack then return a blank key/value pair. + if (c->top == -1) { + key->size = value->size = 0; + key->data = value->data = NULL; + *flags = 0; + return; + } + + // Find first leaf and return key/value. + cursor_first_leaf(c); + cursor_key_value(c, key, value, flags); +} + + +//------------------------------------------------------------------------------ +// Private Functions +//------------------------------------------------------------------------------ + +// Returns a page pointer from a page identifier. +page *cursor_page(bolt_cursor *c, pgid id) { + return (page *)(c->data + (c->pgsz * id)); +} + +// Returns the leaf element at a given index on a given page. +branch_element *branch_page_element(page *p, uint16_t index) { + branch_element *elements = (branch_element*)((void*)(p) + sizeof(page)); + return &elements[index]; +} + +// Returns the leaf element at a given index on a given page. +leaf_element *page_leaf_element(page *p, uint16_t index) { + leaf_element *elements = (leaf_element*)((void*)(p) + sizeof(page)); + return &elements[index]; +} + +// Returns the key/value pair for the current position of the cursor. +void cursor_key_value(bolt_cursor *c, bolt_val *key, bolt_val *value, uint32_t *flags) { + elem_ref *ref = &(c->stack[c->top]); + leaf_element *elem = page_leaf_element(ref->page,ref->index); + + // Assign key pointer. + key->size = elem->ksize; + key->data = ((void*)elem) + elem->pos; + + // Assign value pointer. + value->size = elem->vsize; + value->data = key->data + key->size; + + // Return the element flags. + *flags = elem->flags; +} + +// Traverses from the current stack position down to the first leaf element. +void cursor_first_leaf(bolt_cursor *c) { + elem_ref *ref = &(c->stack[c->top]); + branch_element *branch; + while (ref->page->flags & PAGE_BRANCH) { + branch = branch_page_element(ref->page,ref->index); + c->top++; + ref = &c->stack[c->top]; + ref->index = 0; + ref->page = cursor_page(c, branch->page); }; - - // get current leaf element - return bolt_cursor_first_leaf(c, key, value); } + */ import "C" @@ -162,19 +215,21 @@ func NewCursor(b *bolt.Bucket) *Cursor { return c } -// first moves the cursor to the first element and returns the key and value. +// Next moves the cursor to the first element and returns the key and value. // Returns a nil key if there are no elements. -func first(c *Cursor) (key, value []byte) { +func (c *Cursor) First() (key, value []byte) { var k, v C.bolt_val - C.bolt_cursor_first(c.C, &k, &v) + var flags C.uint32_t + C.bolt_cursor_first(c.C, &k, &v, &flags) return C.GoBytes(k.data, C.int(k.size)), C.GoBytes(v.data, C.int(v.size)) } -// next moves the cursor to the next element and returns the key and value. -// Returns a nil key if at the end of the bucket. -func next(c *Cursor) (key, value []byte) { +// Next moves the cursor to the next element and returns the key and value. +// Returns a nil key if there are no more key/value pairs. +func (c *Cursor) Next() (key, value []byte) { var k, v C.bolt_val - C.bolt_cursor_next(c.C, &k, &v) + var flags C.uint32_t + C.bolt_cursor_next(c.C, &k, &v, &flags) return C.GoBytes(k.data, C.int(k.size)), C.GoBytes(v.data, C.int(v.size)) } diff --git a/c/cursor_test.go b/c/cursor_test.go index 3ac621f..fd0fa1a 100644 --- a/c/cursor_test.go +++ b/c/cursor_test.go @@ -1,123 +1,107 @@ -package c +package c_test import ( + "fmt" "io/ioutil" "os" - // "sort" "testing" - // "testing/quick" "github.com/boltdb/bolt" + . "github.com/boltdb/bolt/c" "github.com/stretchr/testify/assert" ) -// Test when cursor hits the end -// Implement seek; binary search within the page (branch page and element page) - -// Ensure that a cursor can iterate over all elements in a bucket. -// func TestIterate(t *testing.T) { -// if testing.Short() { -// t.Skip("skipping test in short mode.") -// } - -// f := func(items testdata) bool { -// withOpenDB(func(db *bolt.DB, path string) { -// // Bulk insert all values. -// tx, _ := db.Begin(true) -// tx.CreateBucket("widgets") -// b := tx.Bucket("widgets") -// for _, item := range items { -// assert.NoError(t, b.Put(item.Key, item.Value)) -// } -// assert.NoError(t, tx.Commit()) - -// // Sort test data. -// sort.Sort(items) - -// // Iterate over all items and check consistency. -// var index = 0 -// tx, _ = db.Begin(false) -// c := NewCursor(tx.Bucket("widgets")) -// for key, value := first(c); key != nil && index < len(items); key, value = next(c) { -// assert.Equal(t, key, items[index].Key) -// assert.Equal(t, value, items[index].Value) -// index++ -// } -// assert.Equal(t, len(items), index) -// assert.Equal(t, len(items), index) -// tx.Rollback() -// }) -// return true -// } -// if err := quick.Check(f, qconfig()); err != nil { -// t.Error(err) -// } -// fmt.Fprint(os.Stderr, "\n") -// } - -func TestCursorFirst(t *testing.T) { - withOpenDB(func(db *bolt.DB, path string) { - - // Bulk insert all values. - tx, _ := db.Begin(true) - b, _ := tx.CreateBucket([]byte("widgets")) - assert.NoError(t, b.Put([]byte("foo"), []byte("barz"))) - assert.NoError(t, tx.Commit()) - - // Get first and check consistency - tx, _ = db.Begin(false) - c := NewCursor(tx.Bucket([]byte("widgets"))) - key, value := first(c) - assert.Equal(t, key, []byte("foo")) - assert.Equal(t, value, []byte("barz")) - - tx.Rollback() +// Ensure that the C cursor can +func TestCursor_First(t *testing.T) { + withDB(func(db *bolt.DB) { + db.Update(func(tx *bolt.Tx) error { + b, _ := tx.CreateBucket([]byte("widgets")) + return b.Put([]byte("foo"), []byte("barz")) + }) + db.View(func(tx *bolt.Tx) error { + c := NewCursor(tx.Bucket([]byte("widgets"))) + key, value := c.First() + assert.Equal(t, []byte("foo"), key) + assert.Equal(t, []byte("barz"), value) + return nil + }) }) } -// withTempPath executes a function with a database reference. -func withTempPath(fn func(string)) { - f, _ := ioutil.TempFile("", "bolt-") - path := f.Name() +// Ensure that a C cursor can iterate over a single root with a couple elements. +func TestCursor_Iterate_Leaf(t *testing.T) { + withDB(func(db *bolt.DB) { + db.Update(func(tx *bolt.Tx) error { + tx.CreateBucket([]byte("widgets")) + tx.Bucket([]byte("widgets")).Put([]byte("baz"), []byte{}) + tx.Bucket([]byte("widgets")).Put([]byte("foo"), []byte{0}) + tx.Bucket([]byte("widgets")).Put([]byte("bar"), []byte{1}) + return nil + }) + db.View(func(tx *bolt.Tx) error { + c := NewCursor(tx.Bucket([]byte("widgets"))) + + k, v := c.First() + assert.Equal(t, string(k), "bar") + assert.Equal(t, []byte{1}, v) + + k, v = c.Next() + assert.Equal(t, string(k), "baz") + assert.Equal(t, []byte{}, v) + + k, v = c.Next() + assert.Equal(t, string(k), "foo") + assert.Equal(t, []byte{0}, v) + + k, v = c.Next() + assert.Equal(t, []byte{}, k) + assert.Equal(t, []byte{}, v) + + k, v = c.Next() + assert.Equal(t, []byte{}, k) + assert.Equal(t, []byte{}, v) + return nil + }) + }) +} + +// tempfile returns a temporary path. +func tempfile() string { + f, _ := ioutil.TempFile("", "bolt-c-") f.Close() - os.Remove(path) - defer os.RemoveAll(path) - - fn(path) + os.Remove(f.Name()) + return f.Name() } -// withOpenDB executes a function with an already opened database. -func withOpenDB(fn func(*bolt.DB, string)) { - withTempPath(func(path string) { - db, err := bolt.Open(path, 0666) - if err != nil { - panic("cannot open db: " + err.Error()) - } - defer db.Close() - fn(db, path) +// withDB executes a function with an already opened database. +func withDB(fn func(*bolt.DB)) { + path := tempfile() + db, err := bolt.Open(path, 0666) + if err != nil { + panic("cannot open db: " + err.Error()) + } + defer os.Remove(path) + defer db.Close() - // Log statistics. - // if *statsFlag { - // logStats(db) - // } + fn(db) - // Check database consistency after every test. - mustCheck(db) - }) + // Check database consistency after every test. + mustCheck(db) } // mustCheck runs a consistency check on the database and panics if any errors are found. func mustCheck(db *bolt.DB) { if err := db.Check(); err != nil { // Copy db off first. - db.CopyFile("/tmp/check.db", 0600) + var path = tempfile() + db.CopyFile(path, 0600) if errors, ok := err.(bolt.ErrorList); ok { for _, err := range errors { - warn(err) + fmt.Println(err) } } - warn(err) - panic("check failure: see /tmp/check.db") + fmt.Println(err) + panic("check failure: " + path) } } diff --git a/c/doc.go b/c/doc.go new file mode 100644 index 0000000..cbb98fe --- /dev/null +++ b/c/doc.go @@ -0,0 +1,4 @@ +/* +Package c provides a C interface to Bolt. +*/ +package c diff --git a/c/quick_test.go b/c/quick_test.go deleted file mode 100644 index 7d87249..0000000 --- a/c/quick_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package c - -import ( - "bytes" - "flag" - "math/rand" - "reflect" - "testing/quick" - "time" -) - -// testing/quick defaults to 5 iterations and a random seed. -// You can override these settings from the command line: -// -// -quick.count The number of iterations to perform. -// -quick.seed The seed to use for randomizing. -// -quick.maxitems The maximum number of items to insert into a DB. -// -quick.maxksize The maximum size of a key. -// -quick.maxvsize The maximum size of a value. -// - -var qcount, qseed, qmaxitems, qmaxksize, qmaxvsize int - -func init() { - flag.IntVar(&qcount, "quick.count", 5, "") - flag.IntVar(&qseed, "quick.seed", int(time.Now().UnixNano())%100000, "") - flag.IntVar(&qmaxitems, "quick.maxitems", 1000, "") - flag.IntVar(&qmaxksize, "quick.maxksize", 1024, "") - flag.IntVar(&qmaxvsize, "quick.maxvsize", 1024, "") - flag.Parse() - warn("seed:", qseed) - warnf("quick settings: count=%v, items=%v, ksize=%v, vsize=%v", qcount, qmaxitems, qmaxksize, qmaxvsize) -} - -func qconfig() *quick.Config { - return &quick.Config{ - MaxCount: qcount, - Rand: rand.New(rand.NewSource(int64(qseed))), - } -} - -type testdata []testdataitem - -func (t testdata) Len() int { return len(t) } -func (t testdata) Swap(i, j int) { t[i], t[j] = t[j], t[i] } -func (t testdata) Less(i, j int) bool { return bytes.Compare(t[i].Key, t[j].Key) == -1 } - -func (t testdata) Generate(rand *rand.Rand, size int) reflect.Value { - n := rand.Intn(qmaxitems-1) + 1 - items := make(testdata, n) - for i := 0; i < n; i++ { - item := &items[i] - item.Key = randByteSlice(rand, 1, qmaxksize) - item.Value = randByteSlice(rand, 0, qmaxvsize) - } - return reflect.ValueOf(items) -} - -type revtestdata []testdataitem - -func (t revtestdata) Len() int { return len(t) } -func (t revtestdata) Swap(i, j int) { t[i], t[j] = t[j], t[i] } -func (t revtestdata) Less(i, j int) bool { return bytes.Compare(t[i].Key, t[j].Key) == 1 } - -type testdataitem struct { - Key []byte - Value []byte -} - -func randByteSlice(rand *rand.Rand, minSize, maxSize int) []byte { - n := rand.Intn(maxSize-minSize) + minSize - b := make([]byte, n) - for i := 0; i < n; i++ { - b[i] = byte(rand.Intn(255)) - } - return b -} diff --git a/cursor_test.go b/cursor_test.go index 00fc561..fe406bf 100644 --- a/cursor_test.go +++ b/cursor_test.go @@ -89,7 +89,7 @@ func TestCursor_EmptyBucketReverse(t *testing.T) { } // Ensure that a Tx cursor can iterate over a single root with a couple elements. -func TestCursor_LeafRoot(t *testing.T) { +func TestCursor_Iterate_Leaf(t *testing.T) { withOpenDB(func(db *DB, path string) { db.Update(func(tx *Tx) error { tx.CreateBucket([]byte("widgets")) @@ -192,7 +192,7 @@ func TestCursor_Restart(t *testing.T) { } // Ensure that a Tx can iterate over all elements in a bucket. -func TestCursor_Iterate(t *testing.T) { +func TestCursor_QuickCheck(t *testing.T) { f := func(items testdata) bool { withOpenDB(func(db *DB, path string) { // Bulk insert all values. @@ -227,7 +227,7 @@ func TestCursor_Iterate(t *testing.T) { } // Ensure that a transaction can iterate over all elements in a bucket in reverse. -func TestCursor_Iterate_Reverse(t *testing.T) { +func TestCursor_QuickCheck_Reverse(t *testing.T) { f := func(items testdata) bool { withOpenDB(func(db *DB, path string) { // Bulk insert all values. @@ -262,7 +262,7 @@ func TestCursor_Iterate_Reverse(t *testing.T) { } // Ensure that a Tx cursor can iterate over subbuckets. -func TestCursor_Iterate_BucketsOnly(t *testing.T) { +func TestCursor_QuickCheck_BucketsOnly(t *testing.T) { withOpenDB(func(db *DB, path string) { db.Update(func(tx *Tx) error { b, err := tx.CreateBucket([]byte("widgets")) @@ -289,7 +289,7 @@ func TestCursor_Iterate_BucketsOnly(t *testing.T) { } // Ensure that a Tx cursor can reverse iterate over subbuckets. -func TestCursor_Iterate_BucketsOnly_Reverse(t *testing.T) { +func TestCursor_QuickCheck_BucketsOnly_Reverse(t *testing.T) { withOpenDB(func(db *DB, path string) { db.Update(func(tx *Tx) error { b, err := tx.CreateBucket([]byte("widgets")) From a233966d0c349302d7a800e71816702312144624 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Thu, 17 Apr 2014 08:25:26 -0400 Subject: [PATCH 10/14] Clean up --- c/cursor.go | 109 +++++++++++++++++++++++++++++++++++++++++++++-- c/cursor_test.go | 75 ++++++++++++++++++++++++++++++++ 2 files changed, 180 insertions(+), 4 deletions(-) diff --git a/c/cursor.go b/c/cursor.go index 000dd9d..92e93eb 100644 --- a/c/cursor.go +++ b/c/cursor.go @@ -4,6 +4,7 @@ package c #include #include #include +#include #include //------------------------------------------------------------------------------ @@ -41,7 +42,7 @@ typedef struct page { typedef struct branch_element { uint32_t pos; uint32_t ksize; - pgid page; + pgid pgid; } branch_element; // The leaf element represents an a item in a leaf page @@ -87,6 +88,11 @@ void cursor_first_leaf(bolt_cursor *c); void cursor_key_value(bolt_cursor *c, bolt_val *key, bolt_val *value, uint32_t *flags); +void cursor_search(bolt_cursor *c, bolt_val key, pgid id); + +void cursor_search_branch(bolt_cursor *c, bolt_val key); + +void cursor_search_leaf(bolt_cursor *c, bolt_val key); //------------------------------------------------------------------------------ // Public Functions @@ -138,6 +144,27 @@ void bolt_cursor_next(bolt_cursor *c, bolt_val *key, bolt_val *value, uint32_t * cursor_key_value(c, key, value, flags); } +// Positions the cursor first leaf element starting from a given key. +// If there is a matching key then the cursor will be place on that key. +// If there not a match then the cursor will be placed on the next key, if available. +void bolt_cursor_seek(bolt_cursor *c, bolt_val seek, bolt_val *key, bolt_val *value, uint32_t *flags) { + // Start from root page/node and traverse to correct page. + c->top = -1; + cursor_search(c, seek, c->root); + elem_ref *ref = &c->stack[c->top]; + + // If the cursor is pointing to the end of page then return nil. + if (ref->index >= ref->page->count) { + key->size = value->size = 0; + key->data = value->data = NULL; + *flags = 0; + return; + } + + // Set the key/value for the current position. + cursor_key_value(c, key, value, flags); +} + //------------------------------------------------------------------------------ // Private Functions @@ -180,16 +207,76 @@ void cursor_key_value(bolt_cursor *c, bolt_val *key, bolt_val *value, uint32_t * // Traverses from the current stack position down to the first leaf element. void cursor_first_leaf(bolt_cursor *c) { elem_ref *ref = &(c->stack[c->top]); - branch_element *branch; while (ref->page->flags & PAGE_BRANCH) { - branch = branch_page_element(ref->page,ref->index); + branch_element *elem = branch_page_element(ref->page,ref->index); c->top++; ref = &c->stack[c->top]; ref->index = 0; - ref->page = cursor_page(c, branch->page); + ref->page = cursor_page(c, elem->pgid); }; } +// Recursively performs a binary search against a given page/node until it finds a given key. +void cursor_search(bolt_cursor *c, bolt_val key, pgid id) { + // Push page onto the cursor stack. + c->top++; + elem_ref *ref = &c->stack[c->top]; + ref->page = cursor_page(c, id); + ref->index = 0; + + // If we're on a leaf page/node then find the specific node. + if (ref->page->flags & PAGE_LEAF) { + cursor_search_leaf(c, key); + return; + } + + // Otherwise search the branch page. + cursor_search_branch(c, key); +} + +// Recursively search over a leaf page for a key. +void cursor_search_leaf(bolt_cursor *c, bolt_val key) { + elem_ref *ref = &c->stack[c->top]; + + // HACK: Simply loop over elements to find the right one. Replace with a binary search. + leaf_element *elems = (leaf_element*)((void*)(ref->page) + sizeof(page)); + for (int i=0; ipage->count; i++) { + leaf_element *elem = &elems[i]; + int rc = memcmp(key.data, ((void*)elem) + elem->pos, (elem->ksize < key.size ? elem->ksize : key.size)); + + // printf("? %.*s | %.*s\n", key.size, key.data, elem->ksize, ((void*)elem) + elem->pos); + // printf("rc=%d; key.size(%d) >= elem->ksize(%d)\n", rc, key.size, elem->ksize); + if (key.size == 0 || (rc == 0 && key.size >= elem->ksize) || rc < 0) { + ref->index = i; + return; + } + } + + // If nothing was matched then move the cursor to the end. + ref->index = ref->page->count; +} + +// Recursively search over a branch page for a key. +void cursor_search_branch(bolt_cursor *c, bolt_val key) { + elem_ref *ref = &c->stack[c->top]; + + // HACK: Simply loop over elements to find the right one. Replace with a binary search. + branch_element *elems = (branch_element*)((void*)(ref->page) + sizeof(page)); + for (int i=0; ipage->count; i++) { + branch_element *elem = &elems[i]; + int rc = memcmp(key.data, ((void*)elem) + elem->pos, (elem->ksize < key.size ? elem->ksize : key.size)); + + if (key.size == 0 || (rc == 0 && key.size >= elem->ksize) || rc < 0) { + ref->index = i; + cursor_search(c, key, elem->pgid); + return; + } + } + + // If nothing was matched then move the cursor to the end. + ref->index = ref->page->count; +} + */ import "C" @@ -233,6 +320,20 @@ func (c *Cursor) Next() (key, value []byte) { return C.GoBytes(k.data, C.int(k.size)), C.GoBytes(v.data, C.int(v.size)) } +// Seek moves the cursor to a given key and returns it. +// If the key does not exist then the next key is used. If no keys +// follow, an empty value is returned. +func (c *Cursor) Seek(seek []byte) (key, value []byte, flags int) { + var _flags C.uint32_t + var _seek, k, v C.bolt_val + if len(seek) > 0 { + _seek.size = C.uint32_t(len(seek)) + _seek.data = unsafe.Pointer(&seek[0]) + } + C.bolt_cursor_seek(c.C, _seek, &k, &v, &_flags) + return C.GoBytes(k.data, C.int(k.size)), C.GoBytes(v.data, C.int(v.size)), int(_flags) +} + func warn(v ...interface{}) { fmt.Fprintln(os.Stderr, v...) } diff --git a/c/cursor_test.go b/c/cursor_test.go index fd0fa1a..560d03c 100644 --- a/c/cursor_test.go +++ b/c/cursor_test.go @@ -28,6 +28,57 @@ func TestCursor_First(t *testing.T) { }) } +// Ensure that a C cursor can seek to the appropriate keys. +func TestCursor_Seek(t *testing.T) { + withDB(func(db *bolt.DB) { + db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucket([]byte("widgets")) + assert.NoError(t, err) + assert.NoError(t, b.Put([]byte("foo"), []byte("0001"))) + assert.NoError(t, b.Put([]byte("bar"), []byte("0002"))) + assert.NoError(t, b.Put([]byte("baz"), []byte("0003"))) + _, err = b.CreateBucket([]byte("bkt")) + assert.NoError(t, err) + return nil + }) + db.View(func(tx *bolt.Tx) error { + c := NewCursor(tx.Bucket([]byte("widgets"))) + + // Exact match should go to the key. + k, v, flags := c.Seek([]byte("bar")) + assert.Equal(t, "bar", string(k)) + assert.Equal(t, "0002", string(v)) + assert.Equal(t, 0, flags) + + // Inexact match should go to the next key. + k, v, flags = c.Seek([]byte("bas")) + assert.Equal(t, "baz", string(k)) + assert.Equal(t, "0003", string(v)) + assert.Equal(t, 0, flags) + + // Low key should go to the first key. + k, v, flags = c.Seek([]byte("")) + assert.Equal(t, "bar", string(k)) + assert.Equal(t, "0002", string(v)) + assert.Equal(t, 0, flags) + + // High key should return no key. + k, v, flags = c.Seek([]byte("zzz")) + assert.Equal(t, "", string(k)) + assert.Equal(t, "", string(v)) + assert.Equal(t, 0, flags) + + // Buckets should return their key but no value. + k, v, flags = c.Seek([]byte("bkt")) + assert.Equal(t, []byte("bkt"), k) + assert.True(t, len(v) > 0) + assert.Equal(t, 1, flags) // bucketLeafFlag + + return nil + }) + }) +} + // Ensure that a C cursor can iterate over a single root with a couple elements. func TestCursor_Iterate_Leaf(t *testing.T) { withDB(func(db *bolt.DB) { @@ -65,6 +116,30 @@ func TestCursor_Iterate_Leaf(t *testing.T) { }) } +// Ensure that a C cursor can iterate over a branches and leafs. +func TestCursor_Iterate_Large(t *testing.T) { + withDB(func(db *bolt.DB) { + db.Update(func(tx *bolt.Tx) error { + b, _ := tx.CreateBucket([]byte("widgets")) + for i := 0; i < 1000; i++ { + b.Put([]byte(fmt.Sprintf("%d", i)), []byte(fmt.Sprintf("%020d", i))) + } + return nil + }) + db.View(func(tx *bolt.Tx) error { + var index int + c := NewCursor(tx.Bucket([]byte("widgets"))) + for k, v := c.First(); len(k) > 0; k, v = c.Next() { + assert.Equal(t, fmt.Sprintf("%d", index), string(k)) + assert.Equal(t, fmt.Sprintf("%020d", index), string(v)) + index++ + } + assert.Equal(t, 1000, index) + return nil + }) + }) +} + // tempfile returns a temporary path. func tempfile() string { f, _ := ioutil.TempFile("", "bolt-c-") From 846cb9db039e600689ba2bd70be9138f2ced12de Mon Sep 17 00:00:00 2001 From: Martin Kobetic Date: Thu, 17 Apr 2014 14:02:31 +0000 Subject: [PATCH 11/14] tests pass --- c/cursor.go | 12 ++++++++---- c/cursor_test.go | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/c/cursor.go b/c/cursor.go index 92e93eb..b98211e 100644 --- a/c/cursor.go +++ b/c/cursor.go @@ -120,9 +120,11 @@ void bolt_cursor_first(bolt_cursor *c, bolt_val *key, bolt_val *value, uint32_t // Positions the cursor to the next leaf element and returns the key/value pair. void bolt_cursor_next(bolt_cursor *c, bolt_val *key, bolt_val *value, uint32_t *flags) { + int i; + // Attempt to move over one element until we're successful. // Move up the stack as we hit the end of each page in our stack. - for (int i = c->top; i >= 0; i--) { + for (i = c->top; i >= 0; i--) { elem_ref *elem = &c->stack[i]; if (elem->index < elem->page->count - 1) { elem->index++; @@ -175,7 +177,7 @@ page *cursor_page(bolt_cursor *c, pgid id) { return (page *)(c->data + (c->pgsz * id)); } -// Returns the leaf element at a given index on a given page. +// Returns the branch element at a given index on a given page. branch_element *branch_page_element(page *p, uint16_t index) { branch_element *elements = (branch_element*)((void*)(p) + sizeof(page)); return &elements[index]; @@ -237,10 +239,11 @@ void cursor_search(bolt_cursor *c, bolt_val key, pgid id) { // Recursively search over a leaf page for a key. void cursor_search_leaf(bolt_cursor *c, bolt_val key) { elem_ref *ref = &c->stack[c->top]; + int i; // HACK: Simply loop over elements to find the right one. Replace with a binary search. leaf_element *elems = (leaf_element*)((void*)(ref->page) + sizeof(page)); - for (int i=0; ipage->count; i++) { + for (i=0; ipage->count; i++) { leaf_element *elem = &elems[i]; int rc = memcmp(key.data, ((void*)elem) + elem->pos, (elem->ksize < key.size ? elem->ksize : key.size)); @@ -259,10 +262,11 @@ void cursor_search_leaf(bolt_cursor *c, bolt_val key) { // Recursively search over a branch page for a key. void cursor_search_branch(bolt_cursor *c, bolt_val key) { elem_ref *ref = &c->stack[c->top]; + int i; // HACK: Simply loop over elements to find the right one. Replace with a binary search. branch_element *elems = (branch_element*)((void*)(ref->page) + sizeof(page)); - for (int i=0; ipage->count; i++) { + for (i=0; ipage->count; i++) { branch_element *elem = &elems[i]; int rc = memcmp(key.data, ((void*)elem) + elem->pos, (elem->ksize < key.size ? elem->ksize : key.size)); diff --git a/c/cursor_test.go b/c/cursor_test.go index 560d03c..5be85de 100644 --- a/c/cursor_test.go +++ b/c/cursor_test.go @@ -122,7 +122,7 @@ func TestCursor_Iterate_Large(t *testing.T) { db.Update(func(tx *bolt.Tx) error { b, _ := tx.CreateBucket([]byte("widgets")) for i := 0; i < 1000; i++ { - b.Put([]byte(fmt.Sprintf("%d", i)), []byte(fmt.Sprintf("%020d", i))) + b.Put([]byte(fmt.Sprintf("%05d", i)), []byte(fmt.Sprintf("%020d", i))) } return nil }) @@ -130,7 +130,7 @@ func TestCursor_Iterate_Large(t *testing.T) { var index int c := NewCursor(tx.Bucket([]byte("widgets"))) for k, v := c.First(); len(k) > 0; k, v = c.Next() { - assert.Equal(t, fmt.Sprintf("%d", index), string(k)) + assert.Equal(t, fmt.Sprintf("%05d", index), string(k)) assert.Equal(t, fmt.Sprintf("%020d", index), string(v)) index++ } From 194a0967b44932e62d10955d8f7643bd072b1a7e Mon Sep 17 00:00:00 2001 From: Martin Kobetic Date: Thu, 17 Apr 2014 21:25:03 +0000 Subject: [PATCH 12/14] hide the cursor stack details behind function calls (pop/push/current) --- c/cursor.go | 102 ++++++++++++++++++++++++++++------------------- c/cursor_test.go | 40 ++++++++++++++++++- 2 files changed, 101 insertions(+), 41 deletions(-) diff --git a/c/cursor.go b/c/cursor.go index b98211e..e3495b3 100644 --- a/c/cursor.go +++ b/c/cursor.go @@ -82,7 +82,11 @@ typedef struct bolt_cursor { // Forward Declarations //------------------------------------------------------------------------------ -page *cursor_page(bolt_cursor *c, pgid id); +elem_ref *cursor_push(bolt_cursor *c, pgid id); + +elem_ref *cursor_current(bolt_cursor *c); + +elem_ref *cursor_pop(bolt_cursor *c); void cursor_first_leaf(bolt_cursor *c); @@ -103,15 +107,13 @@ void bolt_cursor_init(bolt_cursor *c, void *data, size_t pgsz, pgid root) { c->data = data; c->root = root; c->pgsz = pgsz; + c->top = -1; } // Positions the cursor to the first leaf element and returns the key/value pair. void bolt_cursor_first(bolt_cursor *c, bolt_val *key, bolt_val *value, uint32_t *flags) { // reset stack to initial state - c->top = 0; - elem_ref *ref = &(c->stack[c->top]); - ref->page = cursor_page(c, c->root); - ref->index = 0; + elem_ref *ref = cursor_push(c, c->root); // Find first leaf and return key/value. cursor_first_leaf(c); @@ -121,25 +123,23 @@ void bolt_cursor_first(bolt_cursor *c, bolt_val *key, bolt_val *value, uint32_t // Positions the cursor to the next leaf element and returns the key/value pair. void bolt_cursor_next(bolt_cursor *c, bolt_val *key, bolt_val *value, uint32_t *flags) { int i; + elem_ref *ref; // Attempt to move over one element until we're successful. // Move up the stack as we hit the end of each page in our stack. - for (i = c->top; i >= 0; i--) { - elem_ref *elem = &c->stack[i]; - if (elem->index < elem->page->count - 1) { - elem->index++; - break; - } - c->top--; - } + for (ref = cursor_current(c); ref != NULL; ref = cursor_current(c)) { + ref->index++; + if (ref->index < ref->page->count) break; + cursor_pop(c); + }; // If we are at the top of the stack then return a blank key/value pair. - if (c->top == -1) { + if (ref == NULL) { key->size = value->size = 0; key->data = value->data = NULL; *flags = 0; return; - } + }; // Find first leaf and return key/value. cursor_first_leaf(c); @@ -151,17 +151,17 @@ void bolt_cursor_next(bolt_cursor *c, bolt_val *key, bolt_val *value, uint32_t * // If there not a match then the cursor will be placed on the next key, if available. void bolt_cursor_seek(bolt_cursor *c, bolt_val seek, bolt_val *key, bolt_val *value, uint32_t *flags) { // Start from root page/node and traverse to correct page. - c->top = -1; + cursor_push(c, c->root); cursor_search(c, seek, c->root); - elem_ref *ref = &c->stack[c->top]; + elem_ref *ref = cursor_current(c); // If the cursor is pointing to the end of page then return nil. - if (ref->index >= ref->page->count) { + if (ref == NULL) { key->size = value->size = 0; key->data = value->data = NULL; *flags = 0; return; - } + }; // Set the key/value for the current position. cursor_key_value(c, key, value, flags); @@ -172,13 +172,36 @@ void bolt_cursor_seek(bolt_cursor *c, bolt_val seek, bolt_val *key, bolt_val *va // Private Functions //------------------------------------------------------------------------------ -// Returns a page pointer from a page identifier. -page *cursor_page(bolt_cursor *c, pgid id) { - return (page *)(c->data + (c->pgsz * id)); +// Push ref to the first element of the page onto the cursor stack +// If the page is the root page reset the stack to initial state +elem_ref *cursor_push(bolt_cursor *c, pgid id) { + elem_ref *ref; + if (id == c->root) + c->top = 0; + else + c->top++; + ref = &(c->stack[c->top]); + ref->page = (page *)(c->data + (c->pgsz * id)); + ref->index = 0; + return ref; +} + +// Return current element ref from the cursor stack +// If stack is empty return null +elem_ref *cursor_current(bolt_cursor *c) { + if (c->top < 0) return NULL; + return &c->stack[c->top]; +} + +// Pop current element ref off the cursor stack +elem_ref *cursor_pop(bolt_cursor *c) { + elem_ref *ref = cursor_current(c); + if (ref != NULL) c->top--; + return ref; } // Returns the branch element at a given index on a given page. -branch_element *branch_page_element(page *p, uint16_t index) { +branch_element *page_branch_element(page *p, uint16_t index) { branch_element *elements = (branch_element*)((void*)(p) + sizeof(page)); return &elements[index]; } @@ -191,7 +214,7 @@ leaf_element *page_leaf_element(page *p, uint16_t index) { // Returns the key/value pair for the current position of the cursor. void cursor_key_value(bolt_cursor *c, bolt_val *key, bolt_val *value, uint32_t *flags) { - elem_ref *ref = &(c->stack[c->top]); + elem_ref *ref = cursor_current(c); leaf_element *elem = page_leaf_element(ref->page,ref->index); // Assign key pointer. @@ -208,23 +231,17 @@ void cursor_key_value(bolt_cursor *c, bolt_val *key, bolt_val *value, uint32_t * // Traverses from the current stack position down to the first leaf element. void cursor_first_leaf(bolt_cursor *c) { - elem_ref *ref = &(c->stack[c->top]); + elem_ref *ref = cursor_current(c); while (ref->page->flags & PAGE_BRANCH) { - branch_element *elem = branch_page_element(ref->page,ref->index); - c->top++; - ref = &c->stack[c->top]; - ref->index = 0; - ref->page = cursor_page(c, elem->pgid); + branch_element *elem = page_branch_element(ref->page,ref->index); + ref = cursor_push(c, elem->pgid); }; } // Recursively performs a binary search against a given page/node until it finds a given key. void cursor_search(bolt_cursor *c, bolt_val key, pgid id) { // Push page onto the cursor stack. - c->top++; - elem_ref *ref = &c->stack[c->top]; - ref->page = cursor_page(c, id); - ref->index = 0; + elem_ref *ref = cursor_push(c, id); // If we're on a leaf page/node then find the specific node. if (ref->page->flags & PAGE_LEAF) { @@ -238,7 +255,7 @@ void cursor_search(bolt_cursor *c, bolt_val key, pgid id) { // Recursively search over a leaf page for a key. void cursor_search_leaf(bolt_cursor *c, bolt_val key) { - elem_ref *ref = &c->stack[c->top]; + elem_ref *ref = cursor_current(c); int i; // HACK: Simply loop over elements to find the right one. Replace with a binary search. @@ -255,13 +272,13 @@ void cursor_search_leaf(bolt_cursor *c, bolt_val key) { } } - // If nothing was matched then move the cursor to the end. - ref->index = ref->page->count; + // If nothing was matched then pop the current page off the stack. + cursor_pop(c); } // Recursively search over a branch page for a key. void cursor_search_branch(bolt_cursor *c, bolt_val key) { - elem_ref *ref = &c->stack[c->top]; + elem_ref *ref = cursor_current(c); int i; // HACK: Simply loop over elements to find the right one. Replace with a binary search. @@ -277,8 +294,8 @@ void cursor_search_branch(bolt_cursor *c, bolt_val key) { } } - // If nothing was matched then move the cursor to the end. - ref->index = ref->page->count; + // If nothing was matched then pop the current page off the stack. + cursor_pop(c); } */ @@ -335,6 +352,11 @@ func (c *Cursor) Seek(seek []byte) (key, value []byte, flags int) { _seek.data = unsafe.Pointer(&seek[0]) } C.bolt_cursor_seek(c.C, _seek, &k, &v, &_flags) + //fmt.Printf("Key %v [%v]\n", k.data, k.size) + //fmt.Printf("Value %v [%v]\n", k.data, k.size) + if k.data == nil { + return nil, nil, 0 + } return C.GoBytes(k.data, C.int(k.size)), C.GoBytes(v.data, C.int(v.size)), int(_flags) } diff --git a/c/cursor_test.go b/c/cursor_test.go index 5be85de..21b6696 100644 --- a/c/cursor_test.go +++ b/c/cursor_test.go @@ -116,7 +116,7 @@ func TestCursor_Iterate_Leaf(t *testing.T) { }) } -// Ensure that a C cursor can iterate over a branches and leafs. +// Ensure that a C cursor can iterate over branches and leafs. func TestCursor_Iterate_Large(t *testing.T) { withDB(func(db *bolt.DB) { db.Update(func(tx *bolt.Tx) error { @@ -140,6 +140,44 @@ func TestCursor_Iterate_Large(t *testing.T) { }) } +// Ensure that a C cursor can seek over branches and leafs. +func TestCursor_Seek_Large(t *testing.T) { + withDB(func(db *bolt.DB) { + db.Update(func(tx *bolt.Tx) error { + b, _ := tx.CreateBucket([]byte("widgets")) + for i := 1; i < 1000; i++ { + b.Put([]byte(fmt.Sprintf("%05d", i*10)), []byte(fmt.Sprintf("%020d", i*10))) + } + return nil + }) + db.View(func(tx *bolt.Tx) error { + c := NewCursor(tx.Bucket([]byte("widgets"))) + + // Exact match should go to the key. + k, v, _ := c.Seek([]byte("05000")) + assert.Equal(t, "05000", string(k)) + assert.Equal(t, fmt.Sprintf("%020d", 5000), string(v)) + + // Inexact match should go to the next key. + k, v, _ = c.Seek([]byte("07495")) + assert.Equal(t, "07500", string(k)) + assert.Equal(t, fmt.Sprintf("%020d", 7500), string(v)) + + // Low key should go to the first key. + k, v, _ = c.Seek([]byte("00000")) + assert.Equal(t, "00010", string(k)) + assert.Equal(t, fmt.Sprintf("%020d", 10), string(v)) + + // High key should return no key. + k, v, _ = c.Seek([]byte("40000")) + assert.Equal(t, "", string(k)) + assert.Equal(t, "", string(v)) + + return nil + }) + }) +} + // tempfile returns a temporary path. func tempfile() string { f, _ := ioutil.TempFile("", "bolt-c-") From 9a3d0db1b2f46e79f391c1cd839c7fce6bfb8918 Mon Sep 17 00:00:00 2001 From: Martin Kobetic Date: Thu, 17 Apr 2014 22:01:16 +0000 Subject: [PATCH 13/14] trying to fix large seek --- c/cursor.go | 8 ++++++-- c/cursor_test.go | 24 ++++++++++++------------ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/c/cursor.go b/c/cursor.go index e3495b3..189d385 100644 --- a/c/cursor.go +++ b/c/cursor.go @@ -163,7 +163,8 @@ void bolt_cursor_seek(bolt_cursor *c, bolt_val seek, bolt_val *key, bolt_val *va return; }; - // Set the key/value for the current position. + // Find first leaf and return key/value. + cursor_first_leaf(c); cursor_key_value(c, key, value, flags); } @@ -243,6 +244,7 @@ void cursor_search(bolt_cursor *c, bolt_val key, pgid id) { // Push page onto the cursor stack. elem_ref *ref = cursor_push(c, id); + printf("search page id=%d depth=%d\n", (int)id, c->top); // If we're on a leaf page/node then find the specific node. if (ref->page->flags & PAGE_LEAF) { cursor_search_leaf(c, key); @@ -290,7 +292,9 @@ void cursor_search_branch(bolt_cursor *c, bolt_val key) { if (key.size == 0 || (rc == 0 && key.size >= elem->ksize) || rc < 0) { ref->index = i; cursor_search(c, key, elem->pgid); - return; + if (cursor_current(c) == ref) ref->index++; + if (ref->index < ref->page->count) return; + break; } } diff --git a/c/cursor_test.go b/c/cursor_test.go index 21b6696..ef0f4bc 100644 --- a/c/cursor_test.go +++ b/c/cursor_test.go @@ -146,30 +146,30 @@ func TestCursor_Seek_Large(t *testing.T) { db.Update(func(tx *bolt.Tx) error { b, _ := tx.CreateBucket([]byte("widgets")) for i := 1; i < 1000; i++ { - b.Put([]byte(fmt.Sprintf("%05d", i*10)), []byte(fmt.Sprintf("%020d", i*10))) + b.Put([]byte(fmt.Sprintf("%05d\000", i*10)), []byte(fmt.Sprintf("%020d", i*10))) } return nil }) db.View(func(tx *bolt.Tx) error { c := NewCursor(tx.Bucket([]byte("widgets"))) - // Exact match should go to the key. - k, v, _ := c.Seek([]byte("05000")) - assert.Equal(t, "05000", string(k)) + fmt.Println("// Exact match should go to the key.") + k, v, _ := c.Seek([]byte("05000\000")) + assert.Equal(t, "05000\000", string(k)) assert.Equal(t, fmt.Sprintf("%020d", 5000), string(v)) - // Inexact match should go to the next key. - k, v, _ = c.Seek([]byte("07495")) - assert.Equal(t, "07500", string(k)) + fmt.Println("// Inexact match should go to the next key.") + k, v, _ = c.Seek([]byte("07495\000")) + assert.Equal(t, "07500\000", string(k)) assert.Equal(t, fmt.Sprintf("%020d", 7500), string(v)) - // Low key should go to the first key. - k, v, _ = c.Seek([]byte("00000")) - assert.Equal(t, "00010", string(k)) + fmt.Println("// Low key should go to the first key.") + k, v, _ = c.Seek([]byte("00000\000")) + assert.Equal(t, "00010\000", string(k)) assert.Equal(t, fmt.Sprintf("%020d", 10), string(v)) - // High key should return no key. - k, v, _ = c.Seek([]byte("40000")) + fmt.Println("// High key should return no key.") + k, v, _ = c.Seek([]byte("40000\000")) assert.Equal(t, "", string(k)) assert.Equal(t, "", string(v)) From 1bead4401cfa87409d9e6aef5d6556c0aae7e3ff Mon Sep 17 00:00:00 2001 From: Martin Kobetic Date: Fri, 18 Apr 2014 00:14:56 +0000 Subject: [PATCH 14/14] all tests pass --- c/cursor.go | 28 ++++++++++++++++++---------- c/cursor_test.go | 8 ++++---- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/c/cursor.go b/c/cursor.go index 189d385..49fdcd8 100644 --- a/c/cursor.go +++ b/c/cursor.go @@ -152,7 +152,7 @@ void bolt_cursor_next(bolt_cursor *c, bolt_val *key, bolt_val *value, uint32_t * void bolt_cursor_seek(bolt_cursor *c, bolt_val seek, bolt_val *key, bolt_val *value, uint32_t *flags) { // Start from root page/node and traverse to correct page. cursor_push(c, c->root); - cursor_search(c, seek, c->root); + if (seek.size > 0) cursor_search(c, seek, c->root); elem_ref *ref = cursor_current(c); // If the cursor is pointing to the end of page then return nil. @@ -244,7 +244,6 @@ void cursor_search(bolt_cursor *c, bolt_val key, pgid id) { // Push page onto the cursor stack. elem_ref *ref = cursor_push(c, id); - printf("search page id=%d depth=%d\n", (int)id, c->top); // If we're on a leaf page/node then find the specific node. if (ref->page->flags & PAGE_LEAF) { cursor_search_leaf(c, key); @@ -268,13 +267,13 @@ void cursor_search_leaf(bolt_cursor *c, bolt_val key) { // printf("? %.*s | %.*s\n", key.size, key.data, elem->ksize, ((void*)elem) + elem->pos); // printf("rc=%d; key.size(%d) >= elem->ksize(%d)\n", rc, key.size, elem->ksize); - if (key.size == 0 || (rc == 0 && key.size >= elem->ksize) || rc < 0) { + if ((rc == 0 && key.size >= elem->ksize) || rc < 0) { ref->index = i; return; } } - // If nothing was matched then pop the current page off the stack. + // If nothing was greater than the key then pop the current page off the stack. cursor_pop(c); } @@ -289,16 +288,25 @@ void cursor_search_branch(bolt_cursor *c, bolt_val key) { branch_element *elem = &elems[i]; int rc = memcmp(key.data, ((void*)elem) + elem->pos, (elem->ksize < key.size ? elem->ksize : key.size)); - if (key.size == 0 || (rc == 0 && key.size >= elem->ksize) || rc < 0) { + if (rc == 0 && key.size == elem->ksize) { + // exact match, done ref->index = i; - cursor_search(c, key, elem->pgid); - if (cursor_current(c) == ref) ref->index++; - if (ref->index < ref->page->count) return; - break; + return; + } else if ((rc == 0 && key.size < elem->ksize) || rc < 0) { + // if key is less than anything in this subtree we are done + if (i == 0) return; + // otherwise search the previous subtree + cursor_search(c, key, elems[i-1].pgid); + // didn't find anything greater than key? + if (cursor_current(c) == ref) + ref->index = i; + else + ref->index = i-1; + return; } } - // If nothing was matched then pop the current page off the stack. + // If nothing was greater than the key then pop the current page off the stack. cursor_pop(c); } diff --git a/c/cursor_test.go b/c/cursor_test.go index ef0f4bc..9e7cb1b 100644 --- a/c/cursor_test.go +++ b/c/cursor_test.go @@ -153,22 +153,22 @@ func TestCursor_Seek_Large(t *testing.T) { db.View(func(tx *bolt.Tx) error { c := NewCursor(tx.Bucket([]byte("widgets"))) - fmt.Println("// Exact match should go to the key.") + // Exact match should go to the key. k, v, _ := c.Seek([]byte("05000\000")) assert.Equal(t, "05000\000", string(k)) assert.Equal(t, fmt.Sprintf("%020d", 5000), string(v)) - fmt.Println("// Inexact match should go to the next key.") + // Inexact match should go to the next key. k, v, _ = c.Seek([]byte("07495\000")) assert.Equal(t, "07500\000", string(k)) assert.Equal(t, fmt.Sprintf("%020d", 7500), string(v)) - fmt.Println("// Low key should go to the first key.") + // Low key should go to the first key. k, v, _ = c.Seek([]byte("00000\000")) assert.Equal(t, "00010\000", string(k)) assert.Equal(t, fmt.Sprintf("%020d", 10), string(v)) - fmt.Println("// High key should return no key.") + // High key should return no key. k, v, _ = c.Seek([]byte("40000\000")) assert.Equal(t, "", string(k)) assert.Equal(t, "", string(v))