Add Cursor.Next() to C cursor.

pull/34/head
Ben Johnson 2014-04-16 15:05:30 -04:00
parent a8cb83c008
commit c0ae4881ab
5 changed files with 235 additions and 269 deletions

View File

@ -6,137 +6,190 @@ package c
#include <stdio.h>
#include <inttypes.h>
//------------------------------------------------------------------------------
// Constants
//------------------------------------------------------------------------------
// This represents the maximum number of levels that a cursor can traverse.
#define MAX_DEPTH 100
#define BRANCH_PAGE 1
// These flags mark the type of page and are set in the page.flags.
#define PAGE_BRANCH 0x01
#define PAGE_LEAF 0x02
#define PAGE_META 0x04
#define PAGE_FREELIST 0x10
//------------------------------------------------------------------------------
// Typedefs
//------------------------------------------------------------------------------
// These types MUST have the same layout as their corresponding Go types
typedef int64_t pgid;
// Page represents a header struct of a block in the mmap.
typedef struct page {
pgid id;
uint16_t flags;
uint16_t count;
uint32_t overflow;
pgid id;
uint16_t flags;
uint16_t count;
uint32_t overflow;
} page;
typedef struct branch_elem {
uint32_t pos;
uint32_t ksize;
pgid page;
} branch_elem;
// The branch element represents an a item in a branch page
// that points to a child page.
typedef struct branch_element {
uint32_t pos;
uint32_t ksize;
pgid page;
} branch_element;
typedef struct leaf_elem {
uint32_t flags;
uint32_t pos;
uint32_t ksize;
uint32_t vsize;
} leaf_elem;
// private types
// The leaf element represents an a item in a leaf page
// that points to a key/value pair.
typedef struct leaf_element {
uint32_t flags;
uint32_t pos;
uint32_t ksize;
uint32_t vsize;
} leaf_element;
// elem_ref represents a pointer to an element inside of a page.
// It is used by the cursor stack to track the position at each level.
typedef struct elem_ref {
page *page;
uint16_t index;
page *page;
uint16_t index;
} elem_ref;
// public types
// bolt_val represents a pointer to a fixed-length series of bytes.
// It is used to represent keys and values returned by the cursor.
typedef struct bolt_val {
uint32_t size;
void *data;
} bolt_val;
// bolt_cursor represents a cursor attached to a bucket.
typedef struct bolt_cursor {
void *data;
pgid root;
size_t pgsz;
unsigned int stackp;
elem_ref stack[MAX_DEPTH];
void *data;
pgid root;
size_t pgsz;
int top;
elem_ref stack[MAX_DEPTH];
} bolt_cursor;
// int bolt_cursor_seek(bolt_cursor *c, bolt_val *key, bolt_val *actual_key, bolt_val *value)
//------------------------------------------------------------------------------
// Forward Declarations
//------------------------------------------------------------------------------
// private functions
page *cursor_page(bolt_cursor *c, pgid id);
// Returns a page pointer from a page identifier.
page *get_page(bolt_cursor *c, pgid id) {
return (page *)(c->data + (c->pgsz * id));
}
void cursor_first_leaf(bolt_cursor *c);
// Returns the leaf element at a given index on a given page.
branch_elem *branch_page_element(page *p, uint16_t index) {
branch_elem *elements = (branch_elem*)((void*)(p) + sizeof(page));
return &elements[index];
}
void cursor_key_value(bolt_cursor *c, bolt_val *key, bolt_val *value, uint32_t *flags);
// Returns the leaf element at a given index on a given page.
leaf_elem *leaf_page_element(page *p, uint16_t index) {
leaf_elem *elements = (leaf_elem*)((void*)(p) + sizeof(page));
return &elements[index];
}
// Sets the key and value for a leaf element to a bolt value.
void key_value(leaf_elem *leaf, bolt_val *key, bolt_val *value) {
key->size = leaf->ksize;
key->data = ((void*)leaf) + leaf->pos;
value->size = leaf->vsize;
value->data = key->data + key->size;
}
// Traverses from the current stack position down to the first leaf element.
int bolt_cursor_first_leaf(bolt_cursor *c, bolt_val *key, bolt_val *value) {
elem_ref *ref = &(c->stack[c->stackp]);
branch_elem *branch;
while (ref->page->flags & BRANCH_PAGE) {
branch = branch_page_element(ref->page,ref->index);
c->stackp++;
ref = &c->stack[c->stackp];
ref->index = 0;
ref->page = get_page(c, branch->page);
};
key_value(leaf_page_element(ref->page,ref->index), key, value);
return 0;
}
// public functions
//------------------------------------------------------------------------------
// Public Functions
//------------------------------------------------------------------------------
// Initializes a cursor.
void bolt_cursor_init(bolt_cursor *c, void *data, size_t pgsz, pgid root) {
c->data = data;
c->root = root;
c->pgsz = pgsz;
}
int bolt_cursor_first(bolt_cursor *c, bolt_val *key, bolt_val *value) {
leaf_elem *leaf;
elem_ref *ref;
// Positions the cursor to the first leaf element and returns the key/value pair.
void bolt_cursor_first(bolt_cursor *c, bolt_val *key, bolt_val *value, uint32_t *flags) {
// reset stack to initial state
c->stackp = 0;
ref = &(c->stack[c->stackp]);
ref->page = get_page(c, c->root);
c->top = 0;
elem_ref *ref = &(c->stack[c->top]);
ref->page = cursor_page(c, c->root);
ref->index = 0;
// get current leaf element
return bolt_cursor_first_leaf(c, key, value);
// Find first leaf and return key/value.
cursor_first_leaf(c);
cursor_key_value(c, key, value, flags);
}
int bolt_cursor_next(bolt_cursor *c, bolt_val *key, bolt_val *value) {
elem_ref *ref = &c->stack[c->stackp];
// Positions the cursor to the next leaf element and returns the key/value pair.
void bolt_cursor_next(bolt_cursor *c, bolt_val *key, bolt_val *value, uint32_t *flags) {
// Attempt to move over one element until we're successful.
// Move up the stack as we hit the end of each page in our stack.
for (int i = c->top; i >= 0; i--) {
elem_ref *elem = &c->stack[i];
if (elem->index < elem->page->count - 1) {
elem->index++;
break;
}
c->top--;
}
// increment element index
ref->index++;
// if we're past last element pop the stack and repeat
while (ref->index >= ref->page->count ) {
c->stackp--;
ref = &c->stack[c->stackp];
ref->index++;
// If we are at the top of the stack then return a blank key/value pair.
if (c->top == -1) {
key->size = value->size = 0;
key->data = value->data = NULL;
*flags = 0;
return;
}
// Find first leaf and return key/value.
cursor_first_leaf(c);
cursor_key_value(c, key, value, flags);
}
//------------------------------------------------------------------------------
// Private Functions
//------------------------------------------------------------------------------
// Returns a page pointer from a page identifier.
page *cursor_page(bolt_cursor *c, pgid id) {
return (page *)(c->data + (c->pgsz * id));
}
// Returns the leaf element at a given index on a given page.
branch_element *branch_page_element(page *p, uint16_t index) {
branch_element *elements = (branch_element*)((void*)(p) + sizeof(page));
return &elements[index];
}
// Returns the leaf element at a given index on a given page.
leaf_element *page_leaf_element(page *p, uint16_t index) {
leaf_element *elements = (leaf_element*)((void*)(p) + sizeof(page));
return &elements[index];
}
// Returns the key/value pair for the current position of the cursor.
void cursor_key_value(bolt_cursor *c, bolt_val *key, bolt_val *value, uint32_t *flags) {
elem_ref *ref = &(c->stack[c->top]);
leaf_element *elem = page_leaf_element(ref->page,ref->index);
// Assign key pointer.
key->size = elem->ksize;
key->data = ((void*)elem) + elem->pos;
// Assign value pointer.
value->size = elem->vsize;
value->data = key->data + key->size;
// Return the element flags.
*flags = elem->flags;
}
// Traverses from the current stack position down to the first leaf element.
void cursor_first_leaf(bolt_cursor *c) {
elem_ref *ref = &(c->stack[c->top]);
branch_element *branch;
while (ref->page->flags & PAGE_BRANCH) {
branch = branch_page_element(ref->page,ref->index);
c->top++;
ref = &c->stack[c->top];
ref->index = 0;
ref->page = cursor_page(c, branch->page);
};
// get current leaf element
return bolt_cursor_first_leaf(c, key, value);
}
*/
import "C"
@ -162,19 +215,21 @@ func NewCursor(b *bolt.Bucket) *Cursor {
return c
}
// first moves the cursor to the first element and returns the key and value.
// Next moves the cursor to the first element and returns the key and value.
// Returns a nil key if there are no elements.
func first(c *Cursor) (key, value []byte) {
func (c *Cursor) First() (key, value []byte) {
var k, v C.bolt_val
C.bolt_cursor_first(c.C, &k, &v)
var flags C.uint32_t
C.bolt_cursor_first(c.C, &k, &v, &flags)
return C.GoBytes(k.data, C.int(k.size)), C.GoBytes(v.data, C.int(v.size))
}
// next moves the cursor to the next element and returns the key and value.
// Returns a nil key if at the end of the bucket.
func next(c *Cursor) (key, value []byte) {
// Next moves the cursor to the next element and returns the key and value.
// Returns a nil key if there are no more key/value pairs.
func (c *Cursor) Next() (key, value []byte) {
var k, v C.bolt_val
C.bolt_cursor_next(c.C, &k, &v)
var flags C.uint32_t
C.bolt_cursor_next(c.C, &k, &v, &flags)
return C.GoBytes(k.data, C.int(k.size)), C.GoBytes(v.data, C.int(v.size))
}

View File

@ -1,123 +1,107 @@
package c
package c_test
import (
"fmt"
"io/ioutil"
"os"
// "sort"
"testing"
// "testing/quick"
"github.com/boltdb/bolt"
. "github.com/boltdb/bolt/c"
"github.com/stretchr/testify/assert"
)
// Test when cursor hits the end
// Implement seek; binary search within the page (branch page and element page)
// Ensure that a cursor can iterate over all elements in a bucket.
// func TestIterate(t *testing.T) {
// if testing.Short() {
// t.Skip("skipping test in short mode.")
// }
// f := func(items testdata) bool {
// withOpenDB(func(db *bolt.DB, path string) {
// // Bulk insert all values.
// tx, _ := db.Begin(true)
// tx.CreateBucket("widgets")
// b := tx.Bucket("widgets")
// for _, item := range items {
// assert.NoError(t, b.Put(item.Key, item.Value))
// }
// assert.NoError(t, tx.Commit())
// // Sort test data.
// sort.Sort(items)
// // Iterate over all items and check consistency.
// var index = 0
// tx, _ = db.Begin(false)
// c := NewCursor(tx.Bucket("widgets"))
// for key, value := first(c); key != nil && index < len(items); key, value = next(c) {
// assert.Equal(t, key, items[index].Key)
// assert.Equal(t, value, items[index].Value)
// index++
// }
// assert.Equal(t, len(items), index)
// assert.Equal(t, len(items), index)
// tx.Rollback()
// })
// return true
// }
// if err := quick.Check(f, qconfig()); err != nil {
// t.Error(err)
// }
// fmt.Fprint(os.Stderr, "\n")
// }
func TestCursorFirst(t *testing.T) {
withOpenDB(func(db *bolt.DB, path string) {
// Bulk insert all values.
tx, _ := db.Begin(true)
b, _ := tx.CreateBucket([]byte("widgets"))
assert.NoError(t, b.Put([]byte("foo"), []byte("barz")))
assert.NoError(t, tx.Commit())
// Get first and check consistency
tx, _ = db.Begin(false)
c := NewCursor(tx.Bucket([]byte("widgets")))
key, value := first(c)
assert.Equal(t, key, []byte("foo"))
assert.Equal(t, value, []byte("barz"))
tx.Rollback()
// Ensure that the C cursor can
func TestCursor_First(t *testing.T) {
withDB(func(db *bolt.DB) {
db.Update(func(tx *bolt.Tx) error {
b, _ := tx.CreateBucket([]byte("widgets"))
return b.Put([]byte("foo"), []byte("barz"))
})
db.View(func(tx *bolt.Tx) error {
c := NewCursor(tx.Bucket([]byte("widgets")))
key, value := c.First()
assert.Equal(t, []byte("foo"), key)
assert.Equal(t, []byte("barz"), value)
return nil
})
})
}
// withTempPath executes a function with a database reference.
func withTempPath(fn func(string)) {
f, _ := ioutil.TempFile("", "bolt-")
path := f.Name()
// Ensure that a C cursor can iterate over a single root with a couple elements.
func TestCursor_Iterate_Leaf(t *testing.T) {
withDB(func(db *bolt.DB) {
db.Update(func(tx *bolt.Tx) error {
tx.CreateBucket([]byte("widgets"))
tx.Bucket([]byte("widgets")).Put([]byte("baz"), []byte{})
tx.Bucket([]byte("widgets")).Put([]byte("foo"), []byte{0})
tx.Bucket([]byte("widgets")).Put([]byte("bar"), []byte{1})
return nil
})
db.View(func(tx *bolt.Tx) error {
c := NewCursor(tx.Bucket([]byte("widgets")))
k, v := c.First()
assert.Equal(t, string(k), "bar")
assert.Equal(t, []byte{1}, v)
k, v = c.Next()
assert.Equal(t, string(k), "baz")
assert.Equal(t, []byte{}, v)
k, v = c.Next()
assert.Equal(t, string(k), "foo")
assert.Equal(t, []byte{0}, v)
k, v = c.Next()
assert.Equal(t, []byte{}, k)
assert.Equal(t, []byte{}, v)
k, v = c.Next()
assert.Equal(t, []byte{}, k)
assert.Equal(t, []byte{}, v)
return nil
})
})
}
// tempfile returns a temporary path.
func tempfile() string {
f, _ := ioutil.TempFile("", "bolt-c-")
f.Close()
os.Remove(path)
defer os.RemoveAll(path)
fn(path)
os.Remove(f.Name())
return f.Name()
}
// withOpenDB executes a function with an already opened database.
func withOpenDB(fn func(*bolt.DB, string)) {
withTempPath(func(path string) {
db, err := bolt.Open(path, 0666)
if err != nil {
panic("cannot open db: " + err.Error())
}
defer db.Close()
fn(db, path)
// withDB executes a function with an already opened database.
func withDB(fn func(*bolt.DB)) {
path := tempfile()
db, err := bolt.Open(path, 0666)
if err != nil {
panic("cannot open db: " + err.Error())
}
defer os.Remove(path)
defer db.Close()
// Log statistics.
// if *statsFlag {
// logStats(db)
// }
fn(db)
// Check database consistency after every test.
mustCheck(db)
})
// Check database consistency after every test.
mustCheck(db)
}
// mustCheck runs a consistency check on the database and panics if any errors are found.
func mustCheck(db *bolt.DB) {
if err := db.Check(); err != nil {
// Copy db off first.
db.CopyFile("/tmp/check.db", 0600)
var path = tempfile()
db.CopyFile(path, 0600)
if errors, ok := err.(bolt.ErrorList); ok {
for _, err := range errors {
warn(err)
fmt.Println(err)
}
}
warn(err)
panic("check failure: see /tmp/check.db")
fmt.Println(err)
panic("check failure: " + path)
}
}

4
c/doc.go Normal file
View File

@ -0,0 +1,4 @@
/*
Package c provides a C interface to Bolt.
*/
package c

View File

@ -1,77 +0,0 @@
package c
import (
"bytes"
"flag"
"math/rand"
"reflect"
"testing/quick"
"time"
)
// testing/quick defaults to 5 iterations and a random seed.
// You can override these settings from the command line:
//
// -quick.count The number of iterations to perform.
// -quick.seed The seed to use for randomizing.
// -quick.maxitems The maximum number of items to insert into a DB.
// -quick.maxksize The maximum size of a key.
// -quick.maxvsize The maximum size of a value.
//
var qcount, qseed, qmaxitems, qmaxksize, qmaxvsize int
func init() {
flag.IntVar(&qcount, "quick.count", 5, "")
flag.IntVar(&qseed, "quick.seed", int(time.Now().UnixNano())%100000, "")
flag.IntVar(&qmaxitems, "quick.maxitems", 1000, "")
flag.IntVar(&qmaxksize, "quick.maxksize", 1024, "")
flag.IntVar(&qmaxvsize, "quick.maxvsize", 1024, "")
flag.Parse()
warn("seed:", qseed)
warnf("quick settings: count=%v, items=%v, ksize=%v, vsize=%v", qcount, qmaxitems, qmaxksize, qmaxvsize)
}
func qconfig() *quick.Config {
return &quick.Config{
MaxCount: qcount,
Rand: rand.New(rand.NewSource(int64(qseed))),
}
}
type testdata []testdataitem
func (t testdata) Len() int { return len(t) }
func (t testdata) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
func (t testdata) Less(i, j int) bool { return bytes.Compare(t[i].Key, t[j].Key) == -1 }
func (t testdata) Generate(rand *rand.Rand, size int) reflect.Value {
n := rand.Intn(qmaxitems-1) + 1
items := make(testdata, n)
for i := 0; i < n; i++ {
item := &items[i]
item.Key = randByteSlice(rand, 1, qmaxksize)
item.Value = randByteSlice(rand, 0, qmaxvsize)
}
return reflect.ValueOf(items)
}
type revtestdata []testdataitem
func (t revtestdata) Len() int { return len(t) }
func (t revtestdata) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
func (t revtestdata) Less(i, j int) bool { return bytes.Compare(t[i].Key, t[j].Key) == 1 }
type testdataitem struct {
Key []byte
Value []byte
}
func randByteSlice(rand *rand.Rand, minSize, maxSize int) []byte {
n := rand.Intn(maxSize-minSize) + minSize
b := make([]byte, n)
for i := 0; i < n; i++ {
b[i] = byte(rand.Intn(255))
}
return b
}

View File

@ -89,7 +89,7 @@ func TestCursor_EmptyBucketReverse(t *testing.T) {
}
// Ensure that a Tx cursor can iterate over a single root with a couple elements.
func TestCursor_LeafRoot(t *testing.T) {
func TestCursor_Iterate_Leaf(t *testing.T) {
withOpenDB(func(db *DB, path string) {
db.Update(func(tx *Tx) error {
tx.CreateBucket([]byte("widgets"))
@ -192,7 +192,7 @@ func TestCursor_Restart(t *testing.T) {
}
// Ensure that a Tx can iterate over all elements in a bucket.
func TestCursor_Iterate(t *testing.T) {
func TestCursor_QuickCheck(t *testing.T) {
f := func(items testdata) bool {
withOpenDB(func(db *DB, path string) {
// Bulk insert all values.
@ -227,7 +227,7 @@ func TestCursor_Iterate(t *testing.T) {
}
// Ensure that a transaction can iterate over all elements in a bucket in reverse.
func TestCursor_Iterate_Reverse(t *testing.T) {
func TestCursor_QuickCheck_Reverse(t *testing.T) {
f := func(items testdata) bool {
withOpenDB(func(db *DB, path string) {
// Bulk insert all values.
@ -262,7 +262,7 @@ func TestCursor_Iterate_Reverse(t *testing.T) {
}
// Ensure that a Tx cursor can iterate over subbuckets.
func TestCursor_Iterate_BucketsOnly(t *testing.T) {
func TestCursor_QuickCheck_BucketsOnly(t *testing.T) {
withOpenDB(func(db *DB, path string) {
db.Update(func(tx *Tx) error {
b, err := tx.CreateBucket([]byte("widgets"))
@ -289,7 +289,7 @@ func TestCursor_Iterate_BucketsOnly(t *testing.T) {
}
// Ensure that a Tx cursor can reverse iterate over subbuckets.
func TestCursor_Iterate_BucketsOnly_Reverse(t *testing.T) {
func TestCursor_QuickCheck_BucketsOnly_Reverse(t *testing.T) {
withOpenDB(func(db *DB, path string) {
db.Update(func(tx *Tx) error {
b, err := tx.CreateBucket([]byte("widgets"))