Merge pull request #166 from benbjohnson/fill-percent

Add option to adjust fill percentage.

cmd/bolt/bench.go

@@ -46,6 +46,7 @@ func Bench(options *BenchOptions) {
 		fatal(err)
 		return
 	}
+	db.FillPercent = options.FillPercent
 	defer db.Close()
 
 	// Enable streaming stats.
@@ -280,6 +281,7 @@ type BenchOptions struct {
 	MemProfile    string
 	BlockProfile  string
 	StatsInterval time.Duration
+	FillPercent   float64
 	Clean         bool
 }
 

cmd/bolt/main.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"time"
 
+	"github.com/boltdb/bolt"
 	"github.com/codegangsta/cli"
 )
 
@@ -114,6 +115,7 @@ func NewApp() *cli.App {
 				&cli.StringFlag{Name: "memprofile", Usage: "Memory profile output path"},
 				&cli.StringFlag{Name: "blockprofile", Usage: "Block profile output path"},
 				&cli.StringFlag{Name: "stats-interval", Value: "0s", Usage: "Continuous stats interval"},
+				&cli.Float64Flag{Name: "fill-percent", Value: bolt.DefaultFillPercent, Usage: "Fill percentage"},
 				&cli.BoolFlag{Name: "work", Usage: "Print the temp db and do not delete on exit"},
 			},
 			Action: func(c *cli.Context) {
@@ -134,6 +136,7 @@ func NewApp() *cli.App {
 					MemProfile:    c.String("memprofile"),
 					BlockProfile:  c.String("blockprofile"),
 					StatsInterval: statsInterval,
+					FillPercent:   c.Float64("fill-percent"),
 					Clean:         !c.Bool("work"),
 				})
 			},

db.go

@@ -24,6 +24,15 @@ const version = 2
 // Represents a marker value to indicate that a file is a Bolt DB.
 const magic uint32 = 0xED0CDAED
 
+const (
+	minFillPercent = 0.1
+	maxFillPercent = 1.0
+)
+
+// DefaultFillPercent is the percentage that split pages are filled.
+// This value can be changed by setting DB.FillPercent.
+const DefaultFillPercent = 0.5
+
 var (
 	// ErrDatabaseNotOpen is returned when a DB instance is accessed before it
 	// is opened or after it is closed.
@@ -54,6 +63,11 @@ type DB struct {
 	// debugging purposes.
 	StrictMode bool
 
+	// Sets the threshold for filling nodes when they split. By default,
+	// the database will fill to 50% but it can be useful to increase this
+	// amount if you know that your write workloads are mostly append-only.
+	FillPercent float64
+
 	path     string
 	file     *os.File
 	data     []byte
@@ -94,7 +108,7 @@ func (db *DB) String() string {
 // Open creates and opens a database at the given path.
 // If the file does not exist then it will be created automatically.
 func Open(path string, mode os.FileMode) (*DB, error) {
-	var db = &DB{opened: true}
+	var db = &DB{opened: true, FillPercent: DefaultFillPercent}
 
 	// Open data file and separate sync handler for metadata writes.
 	db.path = path

node.go

@@ -215,8 +215,14 @@ func (n *node) split(pageSize int) []*node {
 		return nodes
 	}
 
-	// Set fill threshold to 50%.
-	threshold := pageSize / 2
+	// Determine the threshold before starting a new node.
+	var fillPercent = n.bucket.tx.db.FillPercent
+	if fillPercent < minFillPercent {
+		fillPercent = minFillPercent
+	} else if fillPercent > maxFillPercent {
+		fillPercent = maxFillPercent
+	}
+	threshold := int(float64(pageSize) * fillPercent)
 
 	// Group into smaller pages and target a given fill size.
 	size := pageHeaderSize

node_test.go

@@ -58,7 +58,7 @@ func TestNode_read_LeafPage(t *testing.T) {
 // Ensure that a node can serialize into a leaf page.
 func TestNode_write_LeafPage(t *testing.T) {
 	// Create a node.
-	n := &node{isLeaf: true, inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{meta: &meta{pgid: 1}}}}
+	n := &node{isLeaf: true, inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{db: &DB{}, meta: &meta{pgid: 1}}}}
 	n.put([]byte("susy"), []byte("susy"), []byte("que"), 0, 0)
 	n.put([]byte("ricki"), []byte("ricki"), []byte("lake"), 0, 0)
 	n.put([]byte("john"), []byte("john"), []byte("johnson"), 0, 0)
@@ -85,7 +85,7 @@ func TestNode_write_LeafPage(t *testing.T) {
 // Ensure that a node can split into appropriate subgroups.
 func TestNode_split(t *testing.T) {
 	// Create a node.
-	n := &node{inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{meta: &meta{pgid: 1}}}}
+	n := &node{inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{db: &DB{}, meta: &meta{pgid: 1}}}}
 	n.put([]byte("00000001"), []byte("00000001"), []byte("0123456701234567"), 0, 0)
 	n.put([]byte("00000002"), []byte("00000002"), []byte("0123456701234567"), 0, 0)
 	n.put([]byte("00000003"), []byte("00000003"), []byte("0123456701234567"), 0, 0)
@@ -104,7 +104,7 @@ func TestNode_split(t *testing.T) {
 // Ensure that a page with the minimum number of inodes just returns a single node.
 func TestNode_split_MinKeys(t *testing.T) {
 	// Create a node.
-	n := &node{inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{meta: &meta{pgid: 1}}}}
+	n := &node{inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{db: &DB{}, meta: &meta{pgid: 1}}}}
 	n.put([]byte("00000001"), []byte("00000001"), []byte("0123456701234567"), 0, 0)
 	n.put([]byte("00000002"), []byte("00000002"), []byte("0123456701234567"), 0, 0)
 
@@ -116,7 +116,7 @@ func TestNode_split_MinKeys(t *testing.T) {
 // Ensure that a node that has keys that all fit on a page just returns one leaf.
 func TestNode_split_SinglePage(t *testing.T) {
 	// Create a node.
-	n := &node{inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{meta: &meta{pgid: 1}}}}
+	n := &node{inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{db: &DB{}, meta: &meta{pgid: 1}}}}
 	n.put([]byte("00000001"), []byte("00000001"), []byte("0123456701234567"), 0, 0)
 	n.put([]byte("00000002"), []byte("00000002"), []byte("0123456701234567"), 0, 0)
 	n.put([]byte("00000003"), []byte("00000003"), []byte("0123456701234567"), 0, 0)