mirror of https://github.com/etcd-io/bbolt.git
get the page size from the second meta page if the first one is invalid
Signed-off-by: Benjamin Wang <wachao@vmware.com>pull/294/head
ahrtr
Benjamin Wang
parent
d4831e6217
commit
8e6a1168ae
107
db.go
107
db.go
|
|
@ -252,21 +252,9 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
|
|||
return nil, err
|
||||
}
|
||||
} else {
|
||||
// Read the first meta page to determine the page size.
|
||||
var buf [0x1000]byte
|
||||
// If we can't read the page size, but can read a page, assume
|
||||
// it's the same as the OS or one given -- since that's how the
|
||||
// page size was chosen in the first place.
|
||||
//
|
||||
// If the first page is invalid and this OS uses a different
|
||||
// page size than what the database was created with then we
|
||||
// are out of luck and cannot access the database.
|
||||
//
|
||||
// TODO: scan for next page
|
||||
if bw, err := db.file.ReadAt(buf[:], 0); err == nil && bw == len(buf) {
|
||||
if m := db.pageInBuffer(buf[:], 0).meta(); m.validate() == nil {
|
||||
db.pageSize = int(m.pageSize)
|
||||
}
|
||||
// try to get the page size from the metadata pages
|
||||
if pgSize, err := db.getPageSize(); err == nil {
|
||||
db.pageSize = pgSize
|
||||
} else {
|
||||
_ = db.close()
|
||||
return nil, ErrInvalid
|
||||
|
|
@ -309,6 +297,95 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
|
|||
return db, nil
|
||||
}
|
||||
|
||||
// getPageSize reads the pageSize from the meta pages. It tries
|
||||
// to read the first meta page firstly. If the first page is invalid,
|
||||
// then it tries to read the second page using the default page size.
|
||||
func (db *DB) getPageSize() (int, error) {
|
||||
var (
|
||||
meta0CanRead, meta1CanRead bool
|
||||
)
|
||||
|
||||
// Read the first meta page to determine the page size.
|
||||
if pgSize, canRead, err := db.getPageSizeFromFirstMeta(); err != nil {
|
||||
// We cannot read the page size from page 0, but can read page 0.
|
||||
meta0CanRead = canRead
|
||||
} else {
|
||||
return pgSize, nil
|
||||
}
|
||||
|
||||
// Read the second meta page to determine the page size.
|
||||
if pgSize, canRead, err := db.getPageSizeFromSecondMeta(); err != nil {
|
||||
// We cannot read the page size from page 1, but can read page 1.
|
||||
meta1CanRead = canRead
|
||||
} else {
|
||||
return pgSize, nil
|
||||
}
|
||||
|
||||
// If we can't read the page size from both pages, but can read
|
||||
// either page, then we assume it's the same as the OS or the one
|
||||
// given, since that's how the page size was chosen in the first place.
|
||||
//
|
||||
// If both pages are invalid, and (this OS uses a different page size
|
||||
// from what the database was created with or the given page size is
|
||||
// different from what the database was created with), then we are out
|
||||
// of luck and cannot access the database.
|
||||
if meta0CanRead || meta1CanRead {
|
||||
return db.pageSize, nil
|
||||
}
|
||||
|
||||
return 0, ErrInvalid
|
||||
}
|
||||
|
||||
// getPageSizeFromFirstMeta reads the pageSize from the first meta page
|
||||
func (db *DB) getPageSizeFromFirstMeta() (int, bool, error) {
|
||||
var buf [0x1000]byte
|
||||
var metaCanRead bool
|
||||
if bw, err := db.file.ReadAt(buf[:], 0); err == nil && bw == len(buf) {
|
||||
metaCanRead = true
|
||||
if m := db.pageInBuffer(buf[:], 0).meta(); m.validate() == nil {
|
||||
return int(m.pageSize), metaCanRead, nil
|
||||
}
|
||||
}
|
||||
return 0, metaCanRead, ErrInvalid
|
||||
}
|
||||
|
||||
// getPageSizeFromSecondMeta reads the pageSize from the second meta page
|
||||
func (db *DB) getPageSizeFromSecondMeta() (int, bool, error) {
|
||||
var (
|
||||
fileSize int64
|
||||
metaCanRead bool
|
||||
)
|
||||
|
||||
// get the db file size
|
||||
if info, err := db.file.Stat(); err != nil {
|
||||
return 0, metaCanRead, err
|
||||
} else {
|
||||
fileSize = info.Size()
|
||||
}
|
||||
|
||||
// We need to read the second meta page, so we should skip the first page;
|
||||
// but we don't know the exact page size yet, it's chicken & egg problem.
|
||||
// The solution is to try all the possible page sizes, which starts from 1KB
|
||||
// and until 16MB (1024<<14) or the end of the db file
|
||||
//
|
||||
// TODO: should we support larger page size?
|
||||
for i := 0; i <= 14; i++ {
|
||||
var buf [0x1000]byte
|
||||
var pos int64 = 1024 << uint(i)
|
||||
if pos >= fileSize-1024 {
|
||||
break
|
||||
}
|
||||
if bw, err := db.file.ReadAt(buf[:], pos); err == nil && bw == len(buf) {
|
||||
metaCanRead = true
|
||||
if m := db.pageInBuffer(buf[:], 0).meta(); m.validate() == nil {
|
||||
return int(m.pageSize), metaCanRead, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 0, metaCanRead, ErrInvalid
|
||||
}
|
||||
|
||||
// loadFreelist reads the freelist if it is synced, or reconstructs it
|
||||
// by scanning the DB if it is not synced. It assumes there are no
|
||||
// concurrent accesses being made to the freelist.
|
||||
|
|
|
|||
82
db_test.go
82
db_test.go
|
|
@ -214,6 +214,88 @@ func TestOpen_ErrChecksum(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Ensure that it can read the page size from the second meta page if the first one is invalid.
|
||||
// The page size is expected to be the OS's page size in this case.
|
||||
func TestOpen_ReadPageSize_FromMeta1_OS(t *testing.T) {
|
||||
// Create empty database.
|
||||
db := MustOpenDB()
|
||||
path := db.Path()
|
||||
defer db.MustClose()
|
||||
|
||||
// Close database.
|
||||
if err := db.DB.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Read data file.
|
||||
buf, err := ioutil.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Rewrite first meta page.
|
||||
meta0 := (*meta)(unsafe.Pointer(&buf[pageHeaderSize]))
|
||||
meta0.pgid++
|
||||
if err := ioutil.WriteFile(path, buf, 0666); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Reopen data file.
|
||||
if db, err := bolt.Open(path, 0666, nil); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
} else {
|
||||
if db.Info().PageSize != os.Getpagesize() {
|
||||
t.Fatalf("The page size is expected to be %d, but actually is %d", os.Getpagesize(), db.Info().PageSize)
|
||||
}
|
||||
if err := db.Close(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure that it can read the page size from the second meta page if the first one is invalid.
|
||||
// The page size is expected to be the given page size in this case.
|
||||
func TestOpen_ReadPageSize_FromMeta1_Given(t *testing.T) {
|
||||
// test page size from 1KB (1024<<0) to 16MB(1024<<14)
|
||||
for i := 0; i <= 14; i++ {
|
||||
givenPageSize := 1024 << uint(i)
|
||||
// Create empty database.
|
||||
db := MustOpenWithOption(&bolt.Options{PageSize: givenPageSize})
|
||||
path := db.Path()
|
||||
defer db.MustClose()
|
||||
|
||||
// Close database.
|
||||
if err := db.DB.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Read data file.
|
||||
buf, err := ioutil.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Rewrite meta pages.
|
||||
meta0 := (*meta)(unsafe.Pointer(&buf[pageHeaderSize]))
|
||||
meta0.pgid++
|
||||
if err := ioutil.WriteFile(path, buf, 0666); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Reopen data file.
|
||||
if db, err := bolt.Open(path, 0666, &bolt.Options{PageSize: givenPageSize}); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
} else {
|
||||
if db.Info().PageSize != givenPageSize {
|
||||
t.Fatalf("The page size is expected to be %d, but actually is %d", givenPageSize, db.Info().PageSize)
|
||||
}
|
||||
if err := db.Close(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure that opening a database does not increase its size.
|
||||
// https://github.com/boltdb/bolt/issues/291
|
||||
func TestOpen_Size(t *testing.T) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue