From 220b61e988ecf0a1667943fde98513583b8c1338 Mon Sep 17 00:00:00 2001
From: Ben Johnson <benbjohnson@yahoo.com>
Date: Mon, 29 Feb 2016 22:58:44 -0700
Subject: [PATCH 1/6] move to separate lock file on windows
---
bolt_unix.go | 8 ++++----
bolt_unix_solaris.go | 8 ++++----
bolt_windows.go | 22 ++++++++++++++++++----
db.go | 15 +++++++++++----
4 files changed, 37 insertions(+), 16 deletions(-)
diff --git a/bolt_unix.go b/bolt_unix.go
index 4b0723a..cad62dd 100644
--- a/bolt_unix.go
+++ b/bolt_unix.go
@@ -11,7 +11,7 @@ import (
)
// flock acquires an advisory lock on a file descriptor.
-func flock(f *os.File, exclusive bool, timeout time.Duration) error {
+func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error {
var t time.Time
for {
// If we're beyond our timeout then return an error.
@@ -27,7 +27,7 @@ func flock(f *os.File, exclusive bool, timeout time.Duration) error {
}
// Otherwise attempt to obtain an exclusive lock.
- err := syscall.Flock(int(f.Fd()), flag|syscall.LOCK_NB)
+ err := syscall.Flock(int(db.file.Fd()), flag|syscall.LOCK_NB)
if err == nil {
return nil
} else if err != syscall.EWOULDBLOCK {
@@ -40,8 +40,8 @@ func flock(f *os.File, exclusive bool, timeout time.Duration) error {
}
// funlock releases an advisory lock on a file descriptor.
-func funlock(f *os.File) error {
- return syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
+func funlock(db *DB) error {
+ return syscall.Flock(int(db.file.Fd()), syscall.LOCK_UN)
}
// mmap memory maps a DB's data file.
diff --git a/bolt_unix_solaris.go b/bolt_unix_solaris.go
index 1c4e48d..307bf2b 100644
--- a/bolt_unix_solaris.go
+++ b/bolt_unix_solaris.go
@@ -11,7 +11,7 @@ import (
)
// flock acquires an advisory lock on a file descriptor.
-func flock(f *os.File, exclusive bool, timeout time.Duration) error {
+func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error {
var t time.Time
for {
// If we're beyond our timeout then return an error.
@@ -32,7 +32,7 @@ func flock(f *os.File, exclusive bool, timeout time.Duration) error {
} else {
lock.Type = syscall.F_RDLCK
}
- err := syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &lock)
+ err := syscall.FcntlFlock(db.file.Fd(), syscall.F_SETLK, &lock)
if err == nil {
return nil
} else if err != syscall.EAGAIN {
@@ -45,13 +45,13 @@ func flock(f *os.File, exclusive bool, timeout time.Duration) error {
}
// funlock releases an advisory lock on a file descriptor.
-func funlock(f *os.File) error {
+func funlock(db *DB) error {
var lock syscall.Flock_t
lock.Start = 0
lock.Len = 0
lock.Type = syscall.F_UNLCK
lock.Whence = 0
- return syscall.FcntlFlock(uintptr(f.Fd()), syscall.F_SETLK, &lock)
+ return syscall.FcntlFlock(uintptr(db.file.Fd()), syscall.F_SETLK, &lock)
}
// mmap memory maps a DB's data file.
diff --git a/bolt_windows.go b/bolt_windows.go
index 91c4968..d538e6a 100644
--- a/bolt_windows.go
+++ b/bolt_windows.go
@@ -16,6 +16,8 @@ var (
)
const (
+ lockExt = ".lock"
+
// see https://msdn.microsoft.com/en-us/library/windows/desktop/aa365203(v=vs.85).aspx
flagLockExclusive = 2
flagLockFailImmediately = 1
@@ -46,7 +48,16 @@ func fdatasync(db *DB) error {
}
// flock acquires an advisory lock on a file descriptor.
-func flock(f *os.File, exclusive bool, timeout time.Duration) error {
+func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error {
+ // Create a separate lock file on windows because a process
+ // cannot share an exclusive lock on the same file. This is
+ // needed during Tx.WriteTo().
+ f, err := os.OpenFile(db.path+lockExt, os.O_CREATE, mode)
+ if err != nil {
+ return err
+ }
+ db.lockfile = f
+
var t time.Time
for {
// If we're beyond our timeout then return an error.
@@ -62,7 +73,7 @@ func flock(f *os.File, exclusive bool, timeout time.Duration) error {
flag |= flagLockExclusive
}
- err := lockFileEx(syscall.Handle(f.Fd()), flag, 0, 1, 0, &syscall.Overlapped{})
+ err := lockFileEx(syscall.Handle(db.lockfile.Fd()), flag, 0, 1, 0, &syscall.Overlapped{})
if err == nil {
return nil
} else if err != errLockViolation {
@@ -75,8 +86,11 @@ func flock(f *os.File, exclusive bool, timeout time.Duration) error {
}
// funlock releases an advisory lock on a file descriptor.
-func funlock(f *os.File) error {
- return unlockFileEx(syscall.Handle(f.Fd()), 0, 1, 0, &syscall.Overlapped{})
+func funlock(db *DB) error {
+ err := unlockFileEx(syscall.Handle(db.lockfile.Fd()), 0, 1, 0, &syscall.Overlapped{})
+ db.lockfile.Close()
+ os.Remove(db.path+lockExt)
+ return err
}
// mmap memory maps a DB's data file.
diff --git a/db.go b/db.go
index 0f1e1bc..501d36a 100644
--- a/db.go
+++ b/db.go
@@ -93,6 +93,7 @@ type DB struct {
path string
file *os.File
+ lockfile *os.File // windows only
dataref []byte // mmap'ed readonly, write throws SEGV
data *[maxMapSize]byte
datasz int
@@ -177,7 +178,7 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
// if !options.ReadOnly.
// The database file is locked using the shared lock (more than one process may
// hold a lock at the same time) otherwise (options.ReadOnly is set).
- if err := flock(db.file, !db.readOnly, options.Timeout); err != nil {
+ if err := flock(db, mode, !db.readOnly, options.Timeout); err != nil {
_ = db.close()
return nil, err
}
@@ -379,6 +380,10 @@ func (db *DB) Close() error {
}
func (db *DB) close() error {
+ if !db.opened {
+ return nil
+ }
+
db.opened = false
db.freelist = nil
@@ -397,7 +402,7 @@ func (db *DB) close() error {
// No need to unlock read-only file.
if !db.readOnly {
// Unlock the file.
- if err := funlock(db.file); err != nil {
+ if err := funlock(db); err != nil {
log.Printf("bolt.Close(): funlock error: %s", err)
}
}
@@ -824,8 +829,10 @@ func (db *DB) grow(sz int) error {
// Truncate and fsync to ensure file size metadata is flushed.
// https://github.com/boltdb/bolt/issues/284
if !db.NoGrowSync && !db.readOnly {
- if err := db.file.Truncate(int64(sz)); err != nil {
- return fmt.Errorf("file resize error: %s", err)
+ if runtime.GOOS != "windows" {
+ if err := db.file.Truncate(int64(sz)); err != nil {
+ return fmt.Errorf("file resize error: %s", err)
+ }
}
if err := db.file.Sync(); err != nil {
return fmt.Errorf("file sync error: %s", err)
From 522043366c6e60fc92776c09658a20b7e22ac120 Mon Sep 17 00:00:00 2001
From: Ben Johnson <benbjohnson@yahoo.com>
Date: Fri, 19 Feb 2016 09:18:07 -0700
Subject: [PATCH 2/6] use tx.meta during Tx.WriteTo()
This commit changes `Tx.WriteTo()` to use the transaction's
in-memory meta page instead of copying from the disk. This is
needed because the transaction uses the size from its meta page
but writes the current meta page on disk which may have allocated
additional pages since the transaction started.
Fixes #513
---
tx.go | 32 +++++++++++++++++++++++++++-----
tx_test.go | 2 +-
2 files changed, 28 insertions(+), 6 deletions(-)
diff --git a/tx.go b/tx.go
index e74d2ca..299c073 100644
--- a/tx.go
+++ b/tx.go
@@ -297,12 +297,34 @@ func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) {
}
defer func() { _ = f.Close() }()
- // Copy the meta pages.
- tx.db.metalock.Lock()
- n, err = io.CopyN(w, f, int64(tx.db.pageSize*2))
- tx.db.metalock.Unlock()
+ // Generate a meta page. We use the same page data for both meta pages.
+ buf := make([]byte, tx.db.pageSize)
+ page := (*page)(unsafe.Pointer(&buf[0]))
+ page.flags = metaPageFlag
+ *page.meta() = *tx.meta
+
+ // Write meta 0.
+ page.id = 0
+ page.meta().checksum = page.meta().sum64()
+ nn, err := w.Write(buf)
+ n += int64(nn)
if err != nil {
- return n, fmt.Errorf("meta copy: %s", err)
+ return n, fmt.Errorf("meta 0 copy: %s", err)
+ }
+
+ // Write meta 1 with a lower transaction id.
+ page.id = 1
+ page.meta().txid -= 1
+ page.meta().checksum = page.meta().sum64()
+ nn, err = w.Write(buf)
+ n += int64(nn)
+ if err != nil {
+ return n, fmt.Errorf("meta 1 copy: %s", err)
+ }
+
+ // Move past the meta pages in the file.
+ if _, err := f.Seek(int64(tx.db.pageSize*2), os.SEEK_SET); err != nil {
+ return n, fmt.Errorf("seek: %s", err)
}
// Copy data pages.
diff --git a/tx_test.go b/tx_test.go
index 18ff166..2201e79 100644
--- a/tx_test.go
+++ b/tx_test.go
@@ -570,7 +570,7 @@ func TestTx_CopyFile_Error_Meta(t *testing.T) {
if err := db.View(func(tx *bolt.Tx) error {
return tx.Copy(&failWriter{})
- }); err == nil || err.Error() != "meta copy: error injected for tests" {
+ }); err == nil || err.Error() != "meta 0 copy: error injected for tests" {
t.Fatalf("unexpected error: %v", err)
}
}
From 7db0126c4b7e17080ccc1a759c60f95726fc6a5e Mon Sep 17 00:00:00 2001
From: Abhinav Ajgaonkar <abhinav316@gmail.com>
Date: Mon, 21 Mar 2016 00:58:27 -0400
Subject: [PATCH 3/6] Adding "lru" to list of projects using Bolt.
---
README.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/README.md b/README.md
index 8a21c6e..66b19ac 100644
--- a/README.md
+++ b/README.md
@@ -839,5 +839,6 @@ Below is a list of public, open source projects that use Bolt:
* [Request Baskets](https://github.com/darklynx/request-baskets) - A web service to collect arbitrary HTTP requests and inspect them via REST API or simple web UI, similar to [RequestBin](http://requestb.in/) service
* [Go Report Card](https://goreportcard.com/) - Go code quality report cards as a (free and open source) service.
* [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners.
+* [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores.
If you are using Bolt in a project please send a pull request to add it to the list.
From 37e96de68dcc7f3ee75e2a96410b2688553d9b82 Mon Sep 17 00:00:00 2001
From: Ben Johnson <benbjohnson@yahoo.com>
Date: Mon, 21 Mar 2016 08:49:39 -0600
Subject: [PATCH 4/6] fix strict mode
This commits fixes a timing bug where `DB.StrictMode` can panic
before the goroutine reading the database can finish. If an error
is found in strict mode then it now finishes reading the entire
database before panicking.
---
tx.go | 14 ++++++++++++--
1 file changed, 12 insertions(+), 2 deletions(-)
diff --git a/tx.go b/tx.go
index 299c073..b8510fd 100644
--- a/tx.go
+++ b/tx.go
@@ -5,6 +5,7 @@ import (
"io"
"os"
"sort"
+ "strings"
"time"
"unsafe"
)
@@ -202,8 +203,17 @@ func (tx *Tx) Commit() error {
// If strict mode is enabled then perform a consistency check.
// Only the first consistency error is reported in the panic.
if tx.db.StrictMode {
- if err, ok := <-tx.Check(); ok {
- panic("check fail: " + err.Error())
+ ch := tx.Check()
+ var errs []string
+ for {
+ err, ok := <-ch
+ if !ok {
+ break
+ }
+ errs = append(errs, err.Error())
+ }
+ if len(errs) > 0 {
+ panic("check fail: " + strings.Join(errs, "\n"))
}
}
From c0934840fdd3462df8ea9f408f791a65957598ac Mon Sep 17 00:00:00 2001
From: Ben Johnson <benbjohnson@yahoo.com>
Date: Mon, 21 Mar 2016 08:53:28 -0600
Subject: [PATCH 5/6] fix rebalance bug
This commit fixes a rare issue where a page can become accessible
when it has already been freed. This occurs when the first two
child pages of a parent both have deletions and the first page
has 1 remaining children and the second page has 2 remaining
children. During rebalancing the first page pulls an element from
the second page and then the second page pulls the same element
back from the first. The child page was not being freed properly.
I resolved this issue by removing this part of the rebalancing.
I made this choice for two reasons:
1. Moving a single item between pages has negligible benefit. The
page will eventually be cleaned up when it reaches zero elements.
2. This is an infrequently executed branch of code which increases
the likelihood of bugs occurring and it makes it more difficult
to test properly.
Fixes #348
---
node.go | 37 -------------------------------------
1 file changed, 37 deletions(-)
diff --git a/node.go b/node.go
index c9fb21c..e9d64af 100644
--- a/node.go
+++ b/node.go
@@ -463,43 +463,6 @@ func (n *node) rebalance() {
target = n.prevSibling()
}
- // If target node has extra nodes then just move one over.
- if target.numChildren() > target.minKeys() {
- if useNextSibling {
- // Reparent and move node.
- if child, ok := n.bucket.nodes[target.inodes[0].pgid]; ok {
- child.parent.removeChild(child)
- child.parent = n
- child.parent.children = append(child.parent.children, child)
- }
- n.inodes = append(n.inodes, target.inodes[0])
- target.inodes = target.inodes[1:]
-
- // Update target key on parent.
- target.parent.put(target.key, target.inodes[0].key, nil, target.pgid, 0)
- target.key = target.inodes[0].key
- _assert(len(target.key) > 0, "rebalance(1): zero-length node key")
- } else {
- // Reparent and move node.
- if child, ok := n.bucket.nodes[target.inodes[len(target.inodes)-1].pgid]; ok {
- child.parent.removeChild(child)
- child.parent = n
- child.parent.children = append(child.parent.children, child)
- }
- n.inodes = append(n.inodes, inode{})
- copy(n.inodes[1:], n.inodes)
- n.inodes[0] = target.inodes[len(target.inodes)-1]
- target.inodes = target.inodes[:len(target.inodes)-1]
- }
-
- // Update parent key for node.
- n.parent.put(n.key, n.inodes[0].key, nil, n.pgid, 0)
- n.key = n.inodes[0].key
- _assert(len(n.key) > 0, "rebalance(2): zero-length node key")
-
- return
- }
-
// If both this node and the target node are too small then merge them.
if useNextSibling {
// Reparent all child nodes being moved.