diff --git a/bolt_unix.go b/bolt_unix.go index 251680b..4b0723a 100644 --- a/bolt_unix.go +++ b/bolt_unix.go @@ -46,17 +46,6 @@ func funlock(f *os.File) error { // mmap memory maps a DB's data file. func mmap(db *DB, sz int) error { - // Truncate and fsync to ensure file size metadata is flushed. - // https://github.com/boltdb/bolt/issues/284 - if !db.NoGrowSync && !db.readOnly { - if err := db.file.Truncate(int64(sz)); err != nil { - return fmt.Errorf("file resize error: %s", err) - } - if err := db.file.Sync(); err != nil { - return fmt.Errorf("file sync error: %s", err) - } - } - // Map the data file to memory. b, err := syscall.Mmap(int(db.file.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED|db.MmapFlags) if err != nil { diff --git a/bolt_unix_solaris.go b/bolt_unix_solaris.go index 214009b..1c4e48d 100644 --- a/bolt_unix_solaris.go +++ b/bolt_unix_solaris.go @@ -56,17 +56,6 @@ func funlock(f *os.File) error { // mmap memory maps a DB's data file. func mmap(db *DB, sz int) error { - // Truncate and fsync to ensure file size metadata is flushed. - // https://github.com/boltdb/bolt/issues/284 - if !db.NoGrowSync && !db.readOnly { - if err := db.file.Truncate(int64(sz)); err != nil { - return fmt.Errorf("file resize error: %s", err) - } - if err := db.file.Sync(); err != nil { - return fmt.Errorf("file sync error: %s", err) - } - } - // Map the data file to memory. b, err := unix.Mmap(int(db.file.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED|db.MmapFlags) if err != nil { diff --git a/db.go b/db.go index f559ee5..911055d 100644 --- a/db.go +++ b/db.go @@ -33,6 +33,7 @@ const IgnoreNoSync = runtime.GOOS == "openbsd" const ( DefaultMaxBatchSize int = 1000 DefaultMaxBatchDelay = 10 * time.Millisecond + DefaultAllocSize = 16 * 1024 * 1024 ) // DB represents a collection of buckets persisted to a file on disk. @@ -85,11 +86,17 @@ type DB struct { // Do not change concurrently with calls to Batch. MaxBatchDelay time.Duration + // AllocSize is the amount of space allocated when the database + // needs to create new pages. This is done to amortize the cost + // of truncate() and fsync() when growing the data file. + AllocSize int + path string file *os.File dataref []byte // mmap'ed readonly, write throws SEGV data *[maxMapSize]byte datasz int + filesz int // current on disk file size meta0 *meta meta1 *meta pageSize int @@ -147,6 +154,7 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) { // Set default values for later DB operations. db.MaxBatchSize = DefaultMaxBatchSize db.MaxBatchDelay = DefaultMaxBatchDelay + db.AllocSize = DefaultAllocSize flag := os.O_RDWR if options.ReadOnly { @@ -798,6 +806,36 @@ func (db *DB) allocate(count int) (*page, error) { return p, nil } +// grow grows the size of the database to the given sz. +func (db *DB) grow(sz int) error { + // Ignore if the new size is less than available file size. + if sz <= db.filesz { + return nil + } + + // If the data is smaller than the alloc size then only allocate what's needed. + // Once it goes over the allocation size then allocate in chunks. + if db.datasz < db.AllocSize { + sz = db.datasz + } else { + sz += db.AllocSize + } + + // Truncate and fsync to ensure file size metadata is flushed. + // https://github.com/boltdb/bolt/issues/284 + if !db.NoGrowSync && !db.readOnly { + if err := db.file.Truncate(int64(sz)); err != nil { + return fmt.Errorf("file resize error: %s", err) + } + if err := db.file.Sync(); err != nil { + return fmt.Errorf("file sync error: %s", err) + } + } + + db.filesz = sz + return nil +} + func (db *DB) IsReadOnly() bool { return db.readOnly } diff --git a/db_test.go b/db_test.go index c1346b8..7077ef2 100644 --- a/db_test.go +++ b/db_test.go @@ -229,6 +229,8 @@ func TestOpen_Size(t *testing.T) { path := db.Path() defer db.MustClose() + pagesize := db.Info().PageSize + // Insert until we get above the minimum 4MB size. if err := db.Update(func(tx *bolt.Tx) error { b, _ := tx.CreateBucketIfNotExists([]byte("data")) @@ -273,7 +275,8 @@ func TestOpen_Size(t *testing.T) { } // Compare the original size with the new size. - if sz != newSz { + // db size might increase by a few page sizes due to the new small update. + if sz < newSz-5*int64(pagesize) { t.Fatalf("unexpected file growth: %d => %d", sz, newSz) } } @@ -290,6 +293,8 @@ func TestOpen_Size_Large(t *testing.T) { path := db.Path() defer db.MustClose() + pagesize := db.Info().PageSize + // Insert until we get above the minimum 4MB size. var index uint64 for i := 0; i < 10000; i++ { @@ -338,7 +343,8 @@ func TestOpen_Size_Large(t *testing.T) { } // Compare the original size with the new size. - if sz != newSz { + // db size might increase by a few page sizes due to the new small update. + if sz < newSz-5*int64(pagesize) { t.Fatalf("unexpected file growth: %d => %d", sz, newSz) } } diff --git a/tx.go b/tx.go index 5f4c9f0..52b68ef 100644 --- a/tx.go +++ b/tx.go @@ -168,6 +168,8 @@ func (tx *Tx) Commit() error { // Free the old root bucket. tx.meta.root.root = tx.root.root + opgid := tx.meta.pgid + // Free the freelist and allocate new pages for it. This will overestimate // the size of the freelist but not underestimate the size (which would be bad). tx.db.freelist.free(tx.meta.txid, tx.db.page(tx.meta.freelist)) @@ -182,6 +184,14 @@ func (tx *Tx) Commit() error { } tx.meta.freelist = p.id + // If the high water mark has moved up then attempt to grow the database. + if tx.meta.pgid > opgid { + if err := tx.db.grow(int(tx.meta.pgid+1) * tx.db.pageSize); err != nil { + tx.rollback() + return err + } + } + // Write dirty pages to disk. startTime = time.Now() if err := tx.write(); err != nil {