Merge pull request #294 from ahrtr/enhance_check_command

Get the page size from the second meta page if the first one is invalid

db.go

@@ -254,21 +254,9 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 			return nil, err
 		}
 	} else {
-		// Read the first meta page to determine the page size.
-		var buf [0x1000]byte
-		// If we can't read the page size, but can read a page, assume
-		// it's the same as the OS or one given -- since that's how the
-		// page size was chosen in the first place.
-		//
-		// If the first page is invalid and this OS uses a different
-		// page size than what the database was created with then we
-		// are out of luck and cannot access the database.
-		//
-		// TODO: scan for next page
-		if bw, err := db.file.ReadAt(buf[:], 0); err == nil && bw == len(buf) {
-			if m := db.pageInBuffer(buf[:], 0).meta(); m.validate() == nil {
-				db.pageSize = int(m.pageSize)
-			}
+		// try to get the page size from the metadata pages
+		if pgSize, err := db.getPageSize(); err == nil {
+			db.pageSize = pgSize
 		} else {
 			_ = db.close()
 			return nil, ErrInvalid
@@ -311,6 +299,95 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 	return db, nil
 }
 
+// getPageSize reads the pageSize from the meta pages. It tries
+// to read the first meta page firstly. If the first page is invalid,
+// then it tries to read the second page using the default page size.
+func (db *DB) getPageSize() (int, error) {
+	var (
+		meta0CanRead, meta1CanRead bool
+	)
+
+	// Read the first meta page to determine the page size.
+	if pgSize, canRead, err := db.getPageSizeFromFirstMeta(); err != nil {
+		// We cannot read the page size from page 0, but can read page 0.
+		meta0CanRead = canRead
+	} else {
+		return pgSize, nil
+	}
+
+	// Read the second meta page to determine the page size.
+	if pgSize, canRead, err := db.getPageSizeFromSecondMeta(); err != nil {
+		// We cannot read the page size from page 1, but can read page 1.
+		meta1CanRead = canRead
+	} else {
+		return pgSize, nil
+	}
+
+	// If we can't read the page size from both pages, but can read
+	// either page, then we assume it's the same as the OS or the one
+	// given, since that's how the page size was chosen in the first place.
+	//
+	// If both pages are invalid, and (this OS uses a different page size
+	// from what the database was created with or the given page size is
+	// different from what the database was created with), then we are out
+	// of luck and cannot access the database.
+	if meta0CanRead || meta1CanRead {
+		return db.pageSize, nil
+	}
+
+	return 0, ErrInvalid
+}
+
+// getPageSizeFromFirstMeta reads the pageSize from the first meta page
+func (db *DB) getPageSizeFromFirstMeta() (int, bool, error) {
+	var buf [0x1000]byte
+	var metaCanRead bool
+	if bw, err := db.file.ReadAt(buf[:], 0); err == nil && bw == len(buf) {
+		metaCanRead = true
+		if m := db.pageInBuffer(buf[:], 0).meta(); m.validate() == nil {
+			return int(m.pageSize), metaCanRead, nil
+		}
+	}
+	return 0, metaCanRead, ErrInvalid
+}
+
+// getPageSizeFromSecondMeta reads the pageSize from the second meta page
+func (db *DB) getPageSizeFromSecondMeta() (int, bool, error) {
+	var (
+		fileSize    int64
+		metaCanRead bool
+	)
+
+	// get the db file size
+	if info, err := db.file.Stat(); err != nil {
+		return 0, metaCanRead, err
+	} else {
+		fileSize = info.Size()
+	}
+
+	// We need to read the second meta page, so we should skip the first page;
+	// but we don't know the exact page size yet, it's chicken & egg problem.
+	// The solution is to try all the possible page sizes, which starts from 1KB
+	// and until 16MB (1024<<14) or the end of the db file
+	//
+	// TODO: should we support larger page size?
+	for i := 0; i <= 14; i++ {
+		var buf [0x1000]byte
+		var pos int64 = 1024 << uint(i)
+		if pos >= fileSize-1024 {
+			break
+		}
+		if bw, err := db.file.ReadAt(buf[:], pos); err == nil && bw == len(buf) {
+			metaCanRead = true
+			if m := db.pageInBuffer(buf[:], 0).meta(); m.validate() == nil {
+				return int(m.pageSize), metaCanRead, nil
+			}
+		}
+	}
+
+	return 0, metaCanRead, ErrInvalid
+}
+
 // loadFreelist reads the freelist if it is synced, or reconstructs it
 // by scanning the DB if it is not synced. It assumes there are no
 // concurrent accesses being made to the freelist.

db_test.go

@@ -211,6 +211,88 @@ func TestOpen_ErrChecksum(t *testing.T) {
 	}
 }
 
+// Ensure that it can read the page size from the second meta page if the first one is invalid.
+// The page size is expected to be the OS's page size in this case.
+func TestOpen_ReadPageSize_FromMeta1_OS(t *testing.T) {
+	// Create empty database.
+	db := MustOpenDB()
+	path := db.Path()
+	defer db.MustClose()
+
+	// Close database.
+	if err := db.DB.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Read data file.
+	buf, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Rewrite first meta page.
+	meta0 := (*meta)(unsafe.Pointer(&buf[pageHeaderSize]))
+	meta0.pgid++
+	if err := os.WriteFile(path, buf, 0666); err != nil {
+		t.Fatal(err)
+	}
+
+	// Reopen data file.
+	if db, err := bolt.Open(path, 0666, nil); err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	} else {
+		if db.Info().PageSize != os.Getpagesize() {
+			t.Fatalf("The page size is expected to be %d, but actually is %d", os.Getpagesize(), db.Info().PageSize)
+		}
+		if err := db.Close(); err != nil {
+			panic(err)
+		}
+	}
+}
+
+// Ensure that it can read the page size from the second meta page if the first one is invalid.
+// The page size is expected to be the given page size in this case.
+func TestOpen_ReadPageSize_FromMeta1_Given(t *testing.T) {
+	// test page size from 1KB (1024<<0) to 16MB(1024<<14)
+	for i := 0; i <= 14; i++ {
+		givenPageSize := 1024 << uint(i)
+		// Create empty database.
+		db := MustOpenWithOption(&bolt.Options{PageSize: givenPageSize})
+		path := db.Path()
+		defer db.MustClose()
+
+		// Close database.
+		if err := db.DB.Close(); err != nil {
+			t.Fatal(err)
+		}
+
+		// Read data file.
+		buf, err := os.ReadFile(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Rewrite meta pages.
+		meta0 := (*meta)(unsafe.Pointer(&buf[pageHeaderSize]))
+		meta0.pgid++
+		if err := os.WriteFile(path, buf, 0666); err != nil {
+			t.Fatal(err)
+		}
+
+		// Reopen data file.
+		if db, err := bolt.Open(path, 0666, &bolt.Options{PageSize: givenPageSize}); err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		} else {
+			if db.Info().PageSize != givenPageSize {
+				t.Fatalf("The page size is expected to be %d, but actually is %d", givenPageSize, db.Info().PageSize)
+			}
+			if err := db.Close(); err != nil {
+				panic(err)
+			}
+		}
+	}
+}
+
 // Ensure that opening a database does not increase its size.
 // https://github.com/boltdb/bolt/issues/291
 func TestOpen_Size(t *testing.T) {