Merge branch 'master' into nested-keys

Conflicts:
	db_test.go
	tx_test.go

.gitignore

@@ -1,2 +1,3 @@
 *.prof
 *.test
+/bin/

Makefile

@@ -1,6 +1,9 @@
 TEST=.
 BENCH=.
 COVERPROFILE=/tmp/c.out
+BRANCH=`git rev-parse --abbrev-ref HEAD`
+COMMIT=`git rev-parse --short HEAD`
+GOLDFLAGS="-X main.branch $(BRANCH) -X main.commit $(COMMIT)"
 
 bench: benchpreq
 	go test -v -test.bench=$(BENCH)
@@ -26,11 +29,23 @@ errcheck:
 fmt:
 	@go fmt ./...
 
+get:
+	@go get -d ./...
+
+build: get
+	@mkdir -p bin
+	@go build -ldflags=$(GOLDFLAGS) -a -o bin/bolt-`git rev-parse --short HEAD` ./cmd/bolt
+
 test: fmt errcheck
+	@go get github.com/stretchr/testify/assert
 	@echo "=== TESTS ==="
 	@go test -v -cover -test.run=$(TEST)
 	@echo ""
 	@echo ""
+	@echo "=== CLI ==="
+	@go test -v -test.run=$(TEST) ./cmd/bolt
+	@echo ""
+	@echo ""
 	@echo "=== RACE DETECTOR ==="
 	@go test -v -race -test.run="TestSimulate_(100op|1000op|10000op)"
 

README.md

@@ -14,7 +14,7 @@ Since Bolt is meant to be used as such a low-level piece of functionality, simpl
 
 ## Project Status
 
-Bolt is functionally complete and has nearly full unit test coverage. The library test suite also includes randomized black box testing to ensure database consistency and thread safety. Bolt is currently in use in a few project, however, it is still at a beta stage so please use with caution and report any bugs found.
+Bolt is functionally complete and has nearly full unit test coverage. The library test suite also includes randomized black box testing to ensure database consistency and thread safety. Bolt is currently in use in a few projects, however, it is still at a beta stage so please use with caution and report any bugs found.
 
 
 ## Comparing Bolt vs LMDB

cmd/bolt/buckets_test.go

@@ -11,14 +11,15 @@ import (
 // Ensure that a list of buckets can be retrieved.
 func TestBuckets(t *testing.T) {
 	SetTestMode(true)
-	open(func(db *bolt.DB) {
+	open(func(db *bolt.DB, path string) {
 		db.Update(func(tx *bolt.Tx) error {
 			tx.CreateBucket("woojits")
 			tx.CreateBucket("widgets")
 			tx.CreateBucket("whatchits")
 			return nil
 		})
-		output := run("buckets", db.Path())
+		db.Close()
+		output := run("buckets", path)
 		assert.Equal(t, "whatchits\nwidgets\nwoojits", output)
 	})
 }

cmd/bolt/export.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/boltdb/bolt"
+)
+
+// Export exports the entire database as a JSON document.
+func Export(path string) {
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		fatal(err)
+		return
+	}
+
+	// Open the database.
+	db, err := bolt.Open(path, 0600)
+	if err != nil {
+		fatal(err)
+		return
+	}
+	defer db.Close()
+
+	db.View(func(tx *bolt.Tx) error {
+		// Loop over every bucket and export it as a raw message.
+		var root []*rawMessage
+		for _, b := range tx.Buckets() {
+			message, err := exportBucket(b)
+			if err != nil {
+				fatal(err)
+			}
+			root = append(root, message)
+		}
+
+		// Encode all buckets into JSON.
+		output, err := json.Marshal(root)
+		if err != nil {
+			fatal("encode: ", err)
+		}
+		print(string(output))
+		return nil
+	})
+}
+
+func exportBucket(b *bolt.Bucket) (*rawMessage, error) {
+	// Encode individual key/value pairs into raw messages.
+	var children = make([]*rawMessage, 0)
+	err := b.ForEach(func(k, v []byte) error {
+		var err error
+
+		var child = &rawMessage{Key: k}
+		if child.Value, err = json.Marshal(v); err != nil {
+			return fmt.Errorf("value: %s", err)
+		}
+
+		children = append(children, child)
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// Encode bucket into a raw message.
+	var root = rawMessage{Type: "bucket"}
+	root.Key = []byte(b.Name())
+	if root.Value, err = json.Marshal(children); err != nil {
+		return nil, fmt.Errorf("children: %s", err)
+	}
+
+	return &root, nil
+}

cmd/bolt/export_test.go

@@ -0,0 +1,37 @@
+package main_test
+
+import (
+	"testing"
+
+	"github.com/boltdb/bolt"
+	. "github.com/boltdb/bolt/cmd/bolt"
+	"github.com/stretchr/testify/assert"
+)
+
+// Ensure that a database can be exported.
+func TestExport(t *testing.T) {
+	SetTestMode(true)
+	open(func(db *bolt.DB, path string) {
+		db.Update(func(tx *bolt.Tx) error {
+			tx.CreateBucket("widgets")
+			b := tx.Bucket("widgets")
+			b.Put([]byte("foo"), []byte("0000"))
+			b.Put([]byte("bar"), []byte(""))
+
+			tx.CreateBucket("woojits")
+			b = tx.Bucket("woojits")
+			b.Put([]byte("baz"), []byte("XXXX"))
+			return nil
+		})
+		db.Close()
+		output := run("export", path)
+		assert.Equal(t, `[{"type":"bucket","key":"d2lkZ2V0cw==","value":[{"key":"YmFy","value":""},{"key":"Zm9v","value":"MDAwMA=="}]},{"type":"bucket","key":"d29vaml0cw==","value":[{"key":"YmF6","value":"WFhYWA=="}]}]`, output)
+	})
+}
+
+// Ensure that an error is reported if the database is not found.
+func TestExport_NotFound(t *testing.T) {
+	SetTestMode(true)
+	output := run("export", "no/such/db")
+	assert.Equal(t, "stat no/such/db: no such file or directory", output)
+}

cmd/bolt/get_test.go

@@ -11,13 +11,14 @@ import (
 // Ensure that a value can be retrieved from the CLI.
 func TestGet(t *testing.T) {
 	SetTestMode(true)
-	open(func(db *bolt.DB) {
+	open(func(db *bolt.DB, path string) {
 		db.Update(func(tx *bolt.Tx) error {
 			tx.CreateBucket("widgets")
 			tx.Bucket("widgets").Put([]byte("foo"), []byte("bar"))
 			return nil
 		})
-		output := run("get", db.Path(), "widgets", "foo")
+		db.Close()
+		output := run("get", path, "widgets", "foo")
 		assert.Equal(t, "bar", output)
 	})
 }
@@ -32,8 +33,9 @@ func TestGetDBNotFound(t *testing.T) {
 // Ensure that an error is reported if the bucket is not found.
 func TestGetBucketNotFound(t *testing.T) {
 	SetTestMode(true)
-	open(func(db *bolt.DB) {
-		output := run("get", db.Path(), "widgets", "foo")
+	open(func(db *bolt.DB, path string) {
+		db.Close()
+		output := run("get", path, "widgets", "foo")
 		assert.Equal(t, "bucket not found: widgets", output)
 	})
 }
@@ -41,11 +43,12 @@ func TestGetBucketNotFound(t *testing.T) {
 // Ensure that an error is reported if the key is not found.
 func TestGetKeyNotFound(t *testing.T) {
 	SetTestMode(true)
-	open(func(db *bolt.DB) {
+	open(func(db *bolt.DB, path string) {
 		db.Update(func(tx *bolt.Tx) error {
 			return tx.CreateBucket("widgets")
 		})
-		output := run("get", db.Path(), "widgets", "foo")
+		db.Close()
+		output := run("get", path, "widgets", "foo")
 		assert.Equal(t, "key not found: foo", output)
 	})
 }

cmd/bolt/import.go

@@ -0,0 +1,82 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/boltdb/bolt"
+)
+
+// Import converts an exported database dump into a new database.
+func Import(path string, input string) {
+	f, err := os.Open(input)
+	if err != nil {
+		fatal(err)
+		return
+	}
+	defer f.Close()
+
+	// Read in entire dump.
+	var root []*rawMessage
+	if err := json.NewDecoder(f).Decode(&root); err != nil {
+		fatal(err)
+	}
+
+	// Open the database.
+	db, err := bolt.Open(path, 0600)
+	if err != nil {
+		fatal(err)
+		return
+	}
+	defer db.Close()
+
+	// Insert entire dump into database.
+	err = db.Update(func(tx *bolt.Tx) error {
+		// Loop over every message and create a bucket.
+		for _, message := range root {
+			// Validate that root messages are buckets.
+			if message.Type != "bucket" {
+				return fmt.Errorf("invalid root type: %q", message.Type)
+			}
+
+			// Create the bucket if it doesn't exist.
+			if err := tx.CreateBucketIfNotExists(string(message.Key)); err != nil {
+				return fmt.Errorf("create bucket: %s", err)
+			}
+
+			// Decode child messages.
+			var children []*rawMessage
+			if err := json.Unmarshal(message.Value, &children); err != nil {
+				return fmt.Errorf("decode children: %s", err)
+			}
+
+			// Import all the values into the bucket.
+			b := tx.Bucket(string(message.Key))
+			if err := importBucket(b, children); err != nil {
+				return fmt.Errorf("import bucket: %s", err)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		fatal("update: ", err)
+	}
+}
+
+func importBucket(b *bolt.Bucket, children []*rawMessage) error {
+	// Decode each message into a key/value pair.
+	for _, child := range children {
+		// Decode the base64 value.
+		var value []byte
+		if err := json.Unmarshal(child.Value, &value); err != nil {
+			return fmt.Errorf("decode value: %s", err)
+		}
+
+		// Insert key/value into bucket.
+		if err := b.Put(child.Key, value); err != nil {
+			return fmt.Errorf("put: %s", err)
+		}
+	}
+	return nil
+}

cmd/bolt/import_test.go

@@ -0,0 +1,50 @@
+package main_test
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/boltdb/bolt"
+	. "github.com/boltdb/bolt/cmd/bolt"
+	"github.com/stretchr/testify/assert"
+)
+
+// Ensure that a database can be imported.
+func TestImport(t *testing.T) {
+	SetTestMode(true)
+
+	// Write input file.
+	input := tempfile()
+	assert.NoError(t, ioutil.WriteFile(input, []byte(`[{"type":"bucket","key":"d2lkZ2V0cw==","value":[{"key":"YmFy","value":""},{"key":"Zm9v","value":"MDAwMA=="}]},{"type":"bucket","key":"d29vaml0cw==","value":[{"key":"YmF6","value":"WFhYWA=="}]}]`), 0600))
+
+	// Import database.
+	path := tempfile()
+	output := run("import", path, "--input", input)
+	assert.Equal(t, ``, output)
+
+	// Open database and verify contents.
+	db, err := bolt.Open(path, 0600)
+	assert.NoError(t, err)
+	db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket("widgets")
+		if assert.NotNil(t, b) {
+			assert.Equal(t, []byte("0000"), b.Get([]byte("foo")))
+			assert.Equal(t, []byte(""), b.Get([]byte("bar")))
+		}
+
+		b = tx.Bucket("woojits")
+		if assert.NotNil(t, b) {
+			assert.Equal(t, []byte("XXXX"), b.Get([]byte("baz")))
+		}
+
+		return nil
+	})
+	db.Close()
+}
+
+// Ensure that an error is reported if the database is not found.
+func TestImport_NotFound(t *testing.T) {
+	SetTestMode(true)
+	output := run("import", "path/to/db", "--input", "no/such/file")
+	assert.Equal(t, "open no/such/file: no such file or directory", output)
+}

cmd/bolt/keys_test.go

@@ -11,7 +11,7 @@ import (
 // Ensure that a list of keys can be retrieved for a given bucket.
 func TestKeys(t *testing.T) {
 	SetTestMode(true)
-	open(func(db *bolt.DB) {
+	open(func(db *bolt.DB, path string) {
 		db.Update(func(tx *bolt.Tx) error {
 			tx.CreateBucket("widgets")
 			tx.Bucket("widgets").Put([]byte("0002"), []byte(""))
@@ -19,7 +19,8 @@ func TestKeys(t *testing.T) {
 			tx.Bucket("widgets").Put([]byte("0003"), []byte(""))
 			return nil
 		})
-		output := run("keys", db.Path(), "widgets")
+		db.Close()
+		output := run("keys", path, "widgets")
 		assert.Equal(t, "0001\n0002\n0003", output)
 	})
 }
@@ -34,8 +35,9 @@ func TestKeysDBNotFound(t *testing.T) {
 // Ensure that an error is reported if the bucket is not found.
 func TestKeysBucketNotFound(t *testing.T) {
 	SetTestMode(true)
-	open(func(db *bolt.DB) {
-		output := run("keys", db.Path(), "widgets")
+	open(func(db *bolt.DB, path string) {
+		db.Close()
+		output := run("keys", path, "widgets")
 		assert.Equal(t, "bucket not found: widgets", output)
 	})
 }

cmd/bolt/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"log"
 	"os"
@@ -9,6 +10,8 @@ import (
 	"github.com/codegangsta/cli"
 )
 
+var branch, commit string
+
 func main() {
 	log.SetFlags(0)
 	NewApp().Run(os.Args)
@@ -19,7 +22,7 @@ func NewApp() *cli.App {
 	app := cli.NewApp()
 	app.Name = "bolt"
 	app.Usage = "BoltDB toolkit"
-	app.Version = "0.1.0"
+	app.Version = fmt.Sprintf("0.1.0 (%s %s)", branch, commit)
 	app.Commands = []cli.Command{
 		{
 			Name:  "get",
@@ -53,6 +56,24 @@ func NewApp() *cli.App {
 				Buckets(path)
 			},
 		},
+		{
+			Name:  "import",
+			Usage: "Imports from a JSON dump into a database",
+			Flags: []cli.Flag{
+				&cli.StringFlag{Name: "input"},
+			},
+			Action: func(c *cli.Context) {
+				Import(c.Args().Get(0), c.String("input"))
+			},
+		},
+		{
+			Name:  "export",
+			Usage: "Exports a database to JSON",
+			Action: func(c *cli.Context) {
+				path := c.Args().Get(0)
+				Export(path)
+			},
+		},
 		{
 			Name:  "pages",
 			Usage: "Dumps page information for a database",
@@ -142,3 +163,10 @@ func SetTestMode(value bool) {
 		logger = log.New(os.Stderr, "", 0)
 	}
 }
+
+// rawMessage represents a JSON element in the import/export document.
+type rawMessage struct {
+	Type  string          `json:"type,omitempty"`
+	Key   []byte          `json:"key"`
+	Value json.RawMessage `json:"value"`
+}

cmd/bolt/main_test.go

@@ -10,17 +10,15 @@ import (
 )
 
 // open creates and opens a Bolt database in the temp directory.
-func open(fn func(*bolt.DB)) {
-	f, _ := ioutil.TempFile("", "bolt-")
-	f.Close()
-	os.Remove(f.Name())
-	defer os.RemoveAll(f.Name())
+func open(fn func(*bolt.DB, string)) {
+	path := tempfile()
+	defer os.RemoveAll(path)
 
-	db, err := bolt.Open(f.Name(), 0600)
+	db, err := bolt.Open(path, 0600)
 	if err != nil {
 		panic("db open error: " + err.Error())
 	}
-	fn(db)
+	fn(db, path)
 }
 
 // run executes a command against the CLI and returns the output.
@@ -29,3 +27,11 @@ func run(args ...string) string {
 	NewApp().Run(args)
 	return strings.TrimSpace(LogBuffer())
 }
+
+// tempfile returns a temporary file path.
+func tempfile() string {
+	f, _ := ioutil.TempFile("", "bolt-")
+	f.Close()
+	os.Remove(f.Name())
+	return f.Name()
+}

cmd/bolt/set_test.go

@@ -11,13 +11,14 @@ import (
 // Ensure that a value can be set from the CLI.
 func TestSet(t *testing.T) {
 	SetTestMode(true)
-	open(func(db *bolt.DB) {
+	open(func(db *bolt.DB, path string) {
 		db.Update(func(tx *bolt.Tx) error {
 			tx.CreateBucket("widgets")
 			return nil
 		})
-		assert.Equal(t, "", run("set", db.Path(), "widgets", "foo", "bar"))
-		assert.Equal(t, "bar", run("get", db.Path(), "widgets", "foo"))
+		db.Close()
+		assert.Equal(t, "", run("set", path, "widgets", "foo", "bar"))
+		assert.Equal(t, "bar", run("get", path, "widgets", "foo"))
 	})
 }
 
@@ -31,8 +32,9 @@ func TestSetDBNotFound(t *testing.T) {
 // Ensure that an error is reported if the bucket is not found.
 func TestSetBucketNotFound(t *testing.T) {
 	SetTestMode(true)
-	open(func(db *bolt.DB) {
-		output := run("set", db.Path(), "widgets", "foo", "bar")
+	open(func(db *bolt.DB, path string) {
+		db.Close()
+		output := run("set", path, "widgets", "foo", "bar")
 		assert.Equal(t, "bucket not found: widgets", output)
 	})
 }

tx_test.go

@@ -241,12 +241,16 @@ func TestTx_OnCommit_Rollback(t *testing.T) {
 }
 
 // Benchmark the performance iterating over a cursor.
-func BenchmarkTxCursor(b *testing.B) {
-	var total = 50000
+func BenchmarkTxCursor1(b *testing.B)     { benchmarkTxCursor(b, 1) }
+func BenchmarkTxCursor10(b *testing.B)    { benchmarkTxCursor(b, 10) }
+func BenchmarkTxCursor100(b *testing.B)   { benchmarkTxCursor(b, 100) }
+func BenchmarkTxCursor1000(b *testing.B)  { benchmarkTxCursor(b, 1000) }
+func BenchmarkTxCursor10000(b *testing.B) { benchmarkTxCursor(b, 10000) }
+
+func benchmarkTxCursor(b *testing.B, total int) {
 	indexes := rand.Perm(total)
 	value := []byte(strings.Repeat("0", 100))
 
-	warn("X", b.N)
 	withOpenDB(func(db *DB, path string) {
 		// Write data to bucket.
 		db.Update(func(tx *Tx) error {
@@ -277,8 +281,14 @@ func BenchmarkTxCursor(b *testing.B) {
 }
 
 // Benchmark the performance of bulk put transactions in random order.
-func BenchmarkTxPutRandom(b *testing.B) {
-	indexes := rand.Perm(b.N)
+func BenchmarkTxPutRandom1(b *testing.B)     { benchmarkTxPutRandom(b, 1) }
+func BenchmarkTxPutRandom10(b *testing.B)    { benchmarkTxPutRandom(b, 10) }
+func BenchmarkTxPutRandom100(b *testing.B)   { benchmarkTxPutRandom(b, 100) }
+func BenchmarkTxPutRandom1000(b *testing.B)  { benchmarkTxPutRandom(b, 1000) }
+func BenchmarkTxPutRandom10000(b *testing.B) { benchmarkTxPutRandom(b, 10000) }
+
+func benchmarkTxPutRandom(b *testing.B, total int) {
+	indexes := rand.Perm(total)
 	value := []byte(strings.Repeat("0", 64))
 	withOpenDB(func(db *DB, path string) {
 		db.Update(func(tx *Tx) error {
@@ -286,22 +296,30 @@ func BenchmarkTxPutRandom(b *testing.B) {
 		})
 		var tx *Tx
 		var bucket *Bucket
-		for i := 0; i < b.N; i++ {
-			if i%1000 == 0 {
-				if tx != nil {
-					tx.Commit()
+		for j := 0; j < b.N; j++ {
+			for i := 0; i < total; i++ {
+				if i%1000 == 0 {
+					if tx != nil {
+						tx.Commit()
+					}
+					tx, _ = db.Begin(true)
+					bucket = tx.Bucket([]byte("widgets"))
 				}
-				tx, _ = db.Begin(true)
-				bucket = tx.Bucket([]byte("widgets"))
+				bucket.Put([]byte(strconv.Itoa(indexes[i])), value)
 			}
-			bucket.Put([]byte(strconv.Itoa(indexes[i])), value)
 		}
 		tx.Commit()
 	})
 }
 
 // Benchmark the performance of bulk put transactions in sequential order.
-func BenchmarkTxPutSequential(b *testing.B) {
+func BenchmarkTxPutSequential1(b *testing.B)     { benchmarkTxPutSequential(b, 1) }
+func BenchmarkTxPutSequential10(b *testing.B)    { benchmarkTxPutSequential(b, 10) }
+func BenchmarkTxPutSequential100(b *testing.B)   { benchmarkTxPutSequential(b, 100) }
+func BenchmarkTxPutSequential1000(b *testing.B)  { benchmarkTxPutSequential(b, 1000) }
+func BenchmarkTxPutSequential10000(b *testing.B) { benchmarkTxPutSequential(b, 10000) }
+
+func benchmarkTxPutSequential(b *testing.B, total int) {
 	value := []byte(strings.Repeat("0", 64))
 	withOpenDB(func(db *DB, path string) {
 		db.Update(func(tx *Tx) error {
@@ -309,8 +327,10 @@ func BenchmarkTxPutSequential(b *testing.B) {
 		})
 		db.Update(func(tx *Tx) error {
 			bucket := tx.Bucket([]byte("widgets"))
-			for i := 0; i < b.N; i++ {
-				bucket.Put([]byte(strconv.Itoa(i)), value)
+			for j := 0; j < b.N; j++ {
+				for i := 0; i < total; i++ {
+					bucket.Put([]byte(strconv.Itoa(i)), value)
+				}
 			}
 			return nil
 		})