Merge d6e3c19dab into a8c943f09d

db.go

@@ -110,6 +110,12 @@ type DB struct {
 	// of truncate() and fsync() when growing the data file.
 	AllocSize int
 
+	// MaxSize is the maximum amount of space allowed for the data file.
+	// If a caller's attempt to add data results in the need to grow
+	// the data file, an error will be returned and the data file will not grow.
+	// 0 means no maximum.
+	MaxSize int
+
 	// Mlock locks database file in memory when set to true.
 	// It prevents major page faults, however used memory can't be reclaimed.
 	//
@@ -191,6 +197,7 @@ func Open(path string, mode os.FileMode, options *Options) (db *DB, err error) {
 	db.PreLoadFreelist = options.PreLoadFreelist
 	db.FreelistType = options.FreelistType
 	db.Mlock = options.Mlock
+	db.MaxSize = options.MaxSize
 
 	// Set default values for later DB operations.
 	db.MaxBatchSize = common.DefaultMaxBatchSize
@@ -469,6 +476,12 @@ func (db *DB) mmap(minsz int) (err error) {
 		return err
 	}
 
+	if db.MaxSize != 0 && fileSize < size && db.MaxSize < size {
+		// if we are mapping past the end of the file, this could cause the file to grow
+		// If the planned size is larger than the max size, then we should error out
+		return berrors.ErrMaxSizeReached
+	}
+
 	if db.Mlock {
 		// Unlock db memory
 		if err := db.munlock(fileSize); err != nil {
@@ -1166,7 +1179,11 @@ func (db *DB) allocate(txid common.Txid, count int) (*common.Page, error) {
 	var minsz = int((p.Id()+common.Pgid(count))+1) * db.pageSize
 	if minsz >= db.datasz {
 		if err := db.mmap(minsz); err != nil {
-			return nil, fmt.Errorf("mmap allocate error: %s", err)
+			if err == berrors.ErrMaxSizeReached {
+				return nil, err
+			} else {
+				return nil, fmt.Errorf("mmap allocate error: %s", err)
+			}
 		}
 	}
 
@@ -1201,6 +1218,11 @@ func (db *DB) grow(sz int) error {
 	// Truncate and fsync to ensure file size metadata is flushed.
 	// https://github.com/boltdb/bolt/issues/284
 	if !db.NoGrowSync && !db.readOnly {
+		if db.MaxSize != 0 && sz > db.MaxSize {
+			lg.Errorf("[GOOS: %s, GOARCH: %s] growing file failed, size: %d, db.MaxSize: %d", runtime.GOOS, runtime.GOARCH, sz, db.MaxSize)
+			return berrors.ErrMaxSizeReached
+		}
+
 		if runtime.GOOS != "windows" {
 			// gofail: var resizeFileError string
 			// return errors.New(resizeFileError)
@@ -1314,6 +1336,9 @@ type Options struct {
 	// PageSize overrides the default OS page size.
 	PageSize int
 
+	// MaxSize sets the maximum size of the data file. 0 means no maximum.
+	MaxSize int
+
 	// NoSync sets the initial value of DB.NoSync. Normally this can just be
 	// set directly on the DB itself when returned from Open(), but this option
 	// is useful in APIs which expose Options but not the underlying DB.
@@ -1337,8 +1362,8 @@ func (o *Options) String() string {
 		return "{}"
 	}
 
-	return fmt.Sprintf("{Timeout: %s, NoGrowSync: %t, NoFreelistSync: %t, PreLoadFreelist: %t, FreelistType: %s, ReadOnly: %t, MmapFlags: %x, InitialMmapSize: %d, PageSize: %d, NoSync: %t, OpenFile: %p, Mlock: %t, Logger: %p}",
-		o.Timeout, o.NoGrowSync, o.NoFreelistSync, o.PreLoadFreelist, o.FreelistType, o.ReadOnly, o.MmapFlags, o.InitialMmapSize, o.PageSize, o.NoSync, o.OpenFile, o.Mlock, o.Logger)
+	return fmt.Sprintf("{Timeout: %s, NoGrowSync: %t, NoFreelistSync: %t, PreLoadFreelist: %t, FreelistType: %s, ReadOnly: %t, MmapFlags: %x, InitialMmapSize: %d, PageSize: %d, MaxSize: %d, NoSync: %t, OpenFile: %p, Mlock: %t, Logger: %p}",
+		o.Timeout, o.NoGrowSync, o.NoFreelistSync, o.PreLoadFreelist, o.FreelistType, o.ReadOnly, o.MmapFlags, o.InitialMmapSize, o.PageSize, o.MaxSize, o.NoSync, o.OpenFile, o.Mlock, o.Logger)
 
 }
 

db_test.go

@@ -1373,6 +1373,85 @@ func TestDBUnmap(t *testing.T) {
 	db.DB = nil
 }
 
+// Ensure that a database cannot exceed its maximum size
+// https://github.com/boltdb/bolt/issues/928
+func TestDB_MaxSizeNotExceeded(t *testing.T) {
+	// Open a data file.
+	db := btesting.MustCreateDBWithOption(t, &bolt.Options{
+		MaxSize: 5 * 1024 * 1024, // 5 MiB
+	})
+	db.AllocSize = 4 * 1024 * 1024 // adjust allocation jumps to 4 MiB
+
+	path := db.Path()
+
+	// Insert a reasonable amount of data below the max size.
+	err := db.Fill([]byte("data"), 1, 2000,
+		func(tx int, k int) []byte { return []byte(fmt.Sprintf("%04d", k)) },
+		func(tx int, k int) []byte { return make([]byte, 1000) },
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = db.Sync()
+	assert.NoError(t, err, "Sync should succeed")
+
+	// The data file should be 4 MiB now (expanded once from zero).
+	// It should have space for roughly 16 more entries before trying to grow
+	// Keep inserting until grow is required
+	err = db.Fill([]byte("data"), 1, 100,
+		func(tx int, k int) []byte { return []byte(fmt.Sprintf("%04d", k)) },
+		func(tx int, k int) []byte { return make([]byte, 1000) },
+	)
+	assert.ErrorIs(t, err, berrors.ErrMaxSizeReached)
+
+	newSz := fileSize(path)
+	if newSz == 0 {
+		t.Fatalf("unexpected new file size: %d", newSz)
+	}
+	assert.LessOrEqual(t, newSz, int64(db.MaxSize), "The size of the data file should not exceed db.MaxSize")
+}
+
+// Ensure that opening a database that is beyond the maximum size succeeds
+// The maximum size should only apply to growing the data file
+// https://github.com/boltdb/bolt/issues/928
+func TestDB_MaxSizeExceededCanOpen(t *testing.T) {
+	// Open a data file.
+	db := btesting.MustCreateDB(t)
+	db.AllocSize = 4 * 1024 * 1024 // adjust allocation jumps to 4 MiB
+
+	path := db.Path()
+
+	// Insert a reasonable amount of data below the max size.
+	err := db.Fill([]byte("data"), 1, 2000,
+		func(tx int, k int) []byte { return []byte(fmt.Sprintf("%04d", k)) },
+		func(tx int, k int) []byte { return make([]byte, 1000) },
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = db.Close()
+	assert.NoError(t, err, "Close should succeed")
+
+	// The data file should be 4 MiB now (expanded once from zero).
+	minimumSizeForTest := int64(1024 * 1024)
+	newSz := fileSize(path)
+	if newSz < minimumSizeForTest {
+		t.Fatalf("unexpected new file size: %d. Expected at least %d", newSz, minimumSizeForTest)
+	}
+
+	// Now try to re-open the database with an extremely small max size
+	t.Logf("Reopening bbolt DB at: %s", path)
+	db, err = btesting.OpenDBWithOption(t, path, &bolt.Options{
+		MaxSize: 1,
+	})
+	assert.NoError(t, err, "Should be able to open database bigger than MaxSize")
+
+	err = db.Close()
+	assert.NoError(t, err, "Closing the re-opened database should succeed")
+}
+
 func ExampleDB_Update() {
 	// Open the database.
 	db, err := bolt.Open(tempfile(), 0600, nil)

errors/errors.go

@@ -69,6 +69,9 @@ var (
 	// ErrValueTooLarge is returned when inserting a value that is larger than MaxValueSize.
 	ErrValueTooLarge = errors.New("value too large")
 
+	// ErrMaxSizeReached is returned when the configured maximum size of the data file is reached.
+	ErrMaxSizeReached = errors.New("database reached maximum size")
+
 	// ErrIncompatibleValue is returned when trying to create or delete a bucket
 	// on an existing non-bucket key or when trying to create or delete a
 	// non-bucket key on an existing bucket key.

internal/btesting/btesting.go

@@ -44,6 +44,13 @@ func MustCreateDBWithOption(t testing.TB, o *bolt.Options) *DB {
 }
 
 func MustOpenDBWithOption(t testing.TB, f string, o *bolt.Options) *DB {
+	db, err := OpenDBWithOption(t, f, o)
+	require.NoError(t, err)
+	require.NotNil(t, db)
+	return db
+}
+
+func OpenDBWithOption(t testing.TB, f string, o *bolt.Options) (*DB, error) {
 	t.Logf("Opening bbolt DB at: %s", f)
 	if o == nil {
 		o = bolt.DefaultOptions
@@ -57,7 +64,9 @@ func MustOpenDBWithOption(t testing.TB, f string, o *bolt.Options) *DB {
 	o.FreelistType = freelistType
 
 	db, err := bolt.Open(f, 0600, o)
-	require.NoError(t, err)
+	if err != nil {
+		return nil, err
+	}
 	resDB := &DB{
 		DB: db,
 		f:  f,
@@ -66,7 +75,7 @@ func MustOpenDBWithOption(t testing.TB, f string, o *bolt.Options) *DB {
 	}
 	resDB.strictModeEnabledDefault()
 	t.Cleanup(resDB.PostTestCleanup)
-	return resDB
+	return resDB, nil
 }
 
 func (db *DB) PostTestCleanup() {