fix goose memory consumption on large migrations

migration_sql.go

@@ -8,9 +8,17 @@ import (
 	"log"
 	"os"
 	"strings"
+	"sync"
 )
 
 const sqlCmdPrefix = "-- +goose "
+const scanBufSize = 4 * 1024 * 1024
+
+var bufferPool = sync.Pool{
+	New: func() interface{} {
+		return make([]byte, scanBufSize)
+	},
+}
 
 // Checks the line to see if the line has a statement-ending semicolon
 // or if the line contains a double-dash comment.
@@ -18,6 +26,7 @@ func endsWithSemicolon(line string) bool {
 
 	prev := ""
 	scanner := bufio.NewScanner(strings.NewReader(line))
+	scanner.Buffer(bufferPool.Get().([]byte), scanBufSize)
 	scanner.Split(bufio.ScanWords)
 
 	for scanner.Scan() {
@@ -43,6 +52,7 @@ func endsWithSemicolon(line string) bool {
 func getSQLStatements(r io.Reader, direction bool) (stmts []string, tx bool) {
 	var buf bytes.Buffer
 	scanner := bufio.NewScanner(r)
+	scanner.Buffer(bufferPool.Get().([]byte), scanBufSize)
 
 	// track the count of each section
 	// so we can diagnose scripts with no annotations