Add library that allows to perform low-level changes/inspection on bbolt file.

For sake of usage it in tests and CLI (so internal).

Signed-off-by: Piotr Tabor <ptab@google.com>

internal/guts_cli/guts_cli.go

@@ -142,7 +142,7 @@ func (p *Page) Type() string {
 	} else if (p.flags & leafPageFlag) != 0 {
 		return "leaf"
 	} else if (p.flags & metaPageFlag) != 0 {
-		return "Meta"
+		return "meta"
 	} else if (p.flags & freelistPageFlag) != 0 {
 		return "freelist"
 	}

internal/surgeon/surgeon.go

@@ -0,0 +1,50 @@
+package surgeon
+
+import (
+	"fmt"
+	"go.etcd.io/bbolt/internal/guts_cli"
+	"os"
+)
+
+func CopyPage(path string, srcPage guts_cli.Pgid, target guts_cli.Pgid) error {
+	p1, d1, err1 := guts_cli.ReadPage(path, uint64(srcPage))
+	if err1 != nil {
+		return err1
+	}
+	p1.SetId(target)
+	return WritePage(path, d1)
+}
+
+func WritePage(path string, pageBuf []byte) error {
+	page := guts_cli.LoadPage(pageBuf)
+	pageSize, _, err := guts_cli.ReadPageAndHWMSize(path)
+	if err != nil {
+		return err
+	}
+	if pageSize != uint64(len(pageBuf)) {
+		return fmt.Errorf("WritePage: len(buf)=%d != pageSize=%d", len(pageBuf), pageSize)
+	}
+	f, err := os.OpenFile(path, os.O_WRONLY, 0)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	_, err = f.WriteAt(pageBuf, int64(page.Id())*int64(pageSize))
+	return err
+}
+
+// RevertMetaPage replaces the newer metadata page with the older.
+// It usually means that one transaction is being lost. But frequently
+// data corruption happens on the last transaction pages and the
+// previous state is consistent.
+func RevertMetaPage(path string) error {
+	_, activeMetaPage, err := guts_cli.GetRootPage(path)
+	if err != nil {
+		return err
+	}
+	if activeMetaPage == 0 {
+		return CopyPage(path, 1, 0)
+	} else {
+		return CopyPage(path, 0, 1)
+	}
+}

internal/surgeon/surgeon_test.go

@@ -0,0 +1,55 @@
+package surgeon_test
+
+import (
+	"fmt"
+	"github.com/stretchr/testify/assert"
+	bolt "go.etcd.io/bbolt"
+	"go.etcd.io/bbolt/internal/btesting"
+	"go.etcd.io/bbolt/internal/surgeon"
+	"testing"
+)
+
+func TestRevertMetaPage(t *testing.T) {
+	db := btesting.MustCreateDB(t)
+	assert.NoError(t,
+		db.Fill([]byte("data"), 1, 500,
+			func(tx int, k int) []byte { return []byte(fmt.Sprintf("%04d", k)) },
+			func(tx int, k int) []byte { return make([]byte, 100) },
+		))
+	assert.NoError(t,
+		db.Update(
+			func(tx *bolt.Tx) error {
+				b := tx.Bucket([]byte("data"))
+				assert.NoError(t, b.Put([]byte("0123"), []byte("new Value for 123")))
+				assert.NoError(t, b.Put([]byte("1234b"), []byte("additional object")))
+				assert.NoError(t, b.Delete([]byte("0246")))
+				return nil
+			}))
+
+	assert.NoError(t,
+		db.View(
+			func(tx *bolt.Tx) error {
+				b := tx.Bucket([]byte("data"))
+				assert.Equal(t, []byte("new Value for 123"), b.Get([]byte("0123")))
+				assert.Equal(t, []byte("additional object"), b.Get([]byte("1234b")))
+				assert.Nil(t, b.Get([]byte("0246")))
+				return nil
+			}))
+
+	db.Close()
+
+	// This causes the whole tree to be linked to the previous state
+	assert.NoError(t, surgeon.RevertMetaPage(db.Path()))
+
+	db.MustReopen()
+	db.MustCheck()
+	assert.NoError(t,
+		db.View(
+			func(tx *bolt.Tx) error {
+				b := tx.Bucket([]byte("data"))
+				assert.Equal(t, make([]byte, 100), b.Get([]byte("0123")))
+				assert.Nil(t, b.Get([]byte("1234b")))
+				assert.Equal(t, make([]byte, 100), b.Get([]byte("0246")))
+				return nil
+			}))
+}

internal/surgeon/xray.go

@@ -0,0 +1,99 @@
+package surgeon
+
+// Library contains raw access to bbolt files for sake of testing or fixing of corrupted files.
+//
+// The library must not be used bbolt btree - just by CLI or tests.
+// It's not optimized for performance.
+
+import (
+	"bytes"
+	"fmt"
+	"go.etcd.io/bbolt/internal/guts_cli"
+)
+
+type XRay struct {
+	path string
+}
+
+func NewXRay(path string) XRay {
+	return XRay{path}
+}
+
+func (n XRay) traverseInternal(stack []guts_cli.Pgid, callback func(page *guts_cli.Page, stack []guts_cli.Pgid) error) error {
+	p, data, err := guts_cli.ReadPage(n.path, uint64(stack[len(stack)-1]))
+	if err != nil {
+		return fmt.Errorf("failed reading page (stack %v): %w", stack, err)
+	}
+	err = callback(p, stack)
+	if err != nil {
+		return fmt.Errorf("failed callback for page (stack %v): %w", stack, err)
+	}
+	switch p.Type() {
+	case "meta":
+		{
+			m := guts_cli.LoadPageMeta(data)
+			r := m.RootBucket().RootPage()
+			return n.traverseInternal(append(stack, r), callback)
+		}
+	case "branch":
+		{
+			for i := uint16(0); i < p.Count(); i++ {
+				bpe := p.BranchPageElement(i)
+				if err := n.traverseInternal(append(stack, bpe.PgId()), callback); err != nil {
+					return err
+				}
+			}
+		}
+	case "leaf":
+		for i := uint16(0); i < p.Count(); i++ {
+			lpe := p.LeafPageElement(i)
+			if lpe.IsBucketEntry() {
+				pgid := lpe.Bucket().RootPage()
+				if pgid > 0 {
+					if err := n.traverseInternal(append(stack, pgid), callback); err != nil {
+						return err
+					}
+				} else {
+					inlinePage := lpe.Bucket().InlinePage(lpe.Value())
+					if err := callback(inlinePage, stack); err != nil {
+						return fmt.Errorf("failed callback for inline page  (stack %v): %w", stack, err)
+					}
+				}
+			}
+		}
+	case "freelist":
+		return nil
+		// Free does not have children.
+	}
+	return nil
+}
+
+// FindPathToPagesWithKey finds a path (from root to the page that contains the given key.
+// As it traverses multiple buckets, in theory there might be multiple keys with given name.
+// Note: For simplicity it's currently implemented as traversing whole reachable tree.
+func (n XRay) FindPathToPagesWithKey(key []byte) ([][]guts_cli.Pgid, error) {
+	var found [][]guts_cli.Pgid
+
+	rootPage, _, err := guts_cli.GetRootPage(n.path)
+	if err != nil {
+		return nil, err
+	}
+	err = n.traverseInternal([]guts_cli.Pgid{rootPage},
+		func(page *guts_cli.Page, stack []guts_cli.Pgid) error {
+			if page.Type() == "leaf" {
+				for i := uint16(0); i < page.Count(); i++ {
+					if bytes.Equal(page.LeafPageElement(i).Key(), key) {
+						var copyPath []guts_cli.Pgid
+						copyPath = append(copyPath, stack...)
+						found = append(found, copyPath)
+					}
+				}
+			}
+			return nil
+		})
+	if err != nil {
+		return nil, err
+	} else {
+		return found, nil
+	}
+}

internal/surgeon/xray_test.go

@@ -0,0 +1,31 @@
+package surgeon_test
+
+import (
+	"fmt"
+	"github.com/stretchr/testify/assert"
+	"go.etcd.io/bbolt/internal/btesting"
+	"go.etcd.io/bbolt/internal/guts_cli"
+	"go.etcd.io/bbolt/internal/surgeon"
+	"testing"
+)
+
+func TestNavigator_FindPathToPagesWithKey(t *testing.T) {
+	db := btesting.MustCreateDB(t)
+	assert.NoError(t,
+		db.Fill([]byte("data"), 1, 500,
+			func(tx int, k int) []byte { return []byte(fmt.Sprintf("%04d", k)) },
+			func(tx int, k int) []byte { return make([]byte, 100) },
+		))
+	assert.NoError(t, db.Close())
+
+	navigator := surgeon.NewXRay(db.Path())
+	path1, err := navigator.FindPathToPagesWithKey([]byte("0451"))
+	assert.NoError(t, err)
+	assert.NotEmpty(t, path1)
+
+	page := path1[0][len(path1[0])-1]
+	p, _, err := guts_cli.ReadPage(db.Path(), uint64(page))
+	assert.NoError(t, err)
+	assert.GreaterOrEqual(t, []byte("0451"), p.LeafPageElement(0).Key())
+	assert.LessOrEqual(t, []byte("0451"), p.LeafPageElement(p.Count()-1).Key())
+}