support injecting failpoints and add failpoint github workflow

Implemented first demo "TestFailpoint_MapFail"

Signed-off-by: Benjamin Wang <wachao@vmware.com>

.github/workflows/failpoint_test.yaml

@@ -0,0 +1,18 @@
+name: Failpoint test
+on: [push, pull_request]
+permissions: read-all
+jobs:
+  test:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: "1.17.13"
+      - run: |
+          make gofail-enable
+          make test-failpoint
+

Makefile

@@ -13,12 +13,15 @@ ifdef CPU
 endif
 TESTFLAGS = $(TESTFLAGS_RACE) $(TESTFLAGS_CPU) $(EXTRA_TESTFLAGS)
 
+.PHONY: fmt
 fmt:
 	!(gofmt -l -s -d $(shell find . -name \*.go) | grep '[a-z]')
 
+.PHONY: lint
 lint:
 	golangci-lint run ./...
 
+.PHONY: test
 test:
 	@echo "hashmap freelist test"
 	TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} -timeout 30m
@@ -28,6 +31,7 @@ test:
 	TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} -timeout 30m
 	TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} ./cmd/bbolt
 
+.PHONY: coverage
 coverage:
 	@echo "hashmap freelist test"
 	TEST_FREELIST_TYPE=hashmap go test -v -timeout 30m \
@@ -37,4 +41,23 @@ coverage:
 	TEST_FREELIST_TYPE=array go test -v -timeout 30m \
 		-coverprofile cover-freelist-array.out -covermode atomic
 
-.PHONY: fmt test lint
+.PHONY: gofail-enable
+gofail-enable: install-gofail
+	gofail enable .
+
+.PHONY: gofail-disable
+gofail-disable:
+	gofail disable .
+
+.PHONY: install-gofail
+install-gofail:
+	go install go.etcd.io/gofail
+
+.PHONY: test-failpoint
+test-failpoint:
+	@echo "[failpoint] hashmap freelist test"
+	TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} -timeout 30m ./tests/failpoint
+
+	@echo "[failpoint] array freelist test"
+	TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} -timeout 30m ./tests/failpoint
+

db.go

@@ -81,7 +81,7 @@ type DB struct {
 	NoFreelistSync bool
 
 	// FreelistType sets the backend freelist type. There are two options. Array which is simple but endures
-	// dramatic performance degradation if database is large and framentation in freelist is common.
+	// dramatic performance degradation if database is large and fragmentation in freelist is common.
 	// The alternative one is using hashmap, it is faster in almost all circumstances
 	// but it doesn't guarantee that it offers the smallest page id available. In normal case it is safe.
 	// The default type is array
@@ -464,6 +464,8 @@ func (db *DB) mmap(minsz int) error {
 	}
 
 	// Memory-map the data file as a byte slice.
+	// gofail: var mapError string
+	// return errors.New(mapError)
 	if err := mmap(db, size); err != nil {
 		return err
 	}
@@ -504,9 +506,12 @@ func (db *DB) invalidate() {
 func (db *DB) munmap() error {
 	defer db.invalidate()
 
+	// gofail: var unmapError string
+	// return errors.New(unmapError)
 	if err := munmap(db); err != nil {
 		return fmt.Errorf("unmap error: " + err.Error())
 	}
+
 	return nil
 }
 
@@ -1207,7 +1212,7 @@ type Options struct {
 	PreLoadFreelist bool
 
 	// FreelistType sets the backend freelist type. There are two options. Array which is simple but endures
-	// dramatic performance degradation if database is large and framentation in freelist is common.
+	// dramatic performance degradation if database is large and fragmentation in freelist is common.
 	// The alternative one is using hashmap, it is faster in almost all circumstances
 	// but it doesn't guarantee that it offers the smallest page id available. In normal case it is safe.
 	// The default type is array

go.mod

@@ -4,6 +4,7 @@ go 1.17
 
 require (
 	github.com/stretchr/testify v1.8.1
+	go.etcd.io/gofail v0.1.0
 	golang.org/x/sys v0.3.0
 )
 

go.sum

@@ -10,6 +10,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+go.etcd.io/gofail v0.1.0 h1:XItAMIhOojXFQMgrxjnd2EIIHun/d5qL0Pf7FzVTkFg=
+go.etcd.io/gofail v0.1.0/go.mod h1:VZBCXYGZhHAinaBiiqYvuDynvahNsAyLFwB3kEHKz1M=
 golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

tests/failpoint/db_failpoint_test.go

@@ -0,0 +1,25 @@
+package failpoint
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	bolt "go.etcd.io/bbolt"
+	gofail "go.etcd.io/gofail/runtime"
+)
+
+func TestFailpoint_MapFail(t *testing.T) {
+	err := gofail.Enable("mapError", `return("map somehow failed")`)
+	require.NoError(t, err)
+	defer func() {
+		err = gofail.Disable("mapError")
+		require.NoError(t, err)
+	}()
+
+	f := filepath.Join(t.TempDir(), "db")
+	_, err = bolt.Open(f, 0666, nil)
+	require.Error(t, err)
+	require.ErrorContains(t, err, "map somehow failed")
+}