Merge pull request #120 from pressly/timestamp_versioning

add `fix` and timestamped migrations by default

README.md

@@ -22,9 +22,7 @@ Goose is a database migration tool. Manage your database schema by creating incr
     - goose pkg doesn't register any SQL drivers anymore,
       thus no driver `panic()` conflict within your codebase!
     - goose pkg doesn't have any vendor dependencies anymore
-- We encourage using sequential versioning of migration files
-    (rather than timestamps-based versioning) to prevent version
-    mismatch and migration colissions
+- We use timestamped migrations by default but recommend a hybrid approach of using timestamps in the development process and sequential versions in production.  
 
 # Install
 
@@ -51,7 +49,7 @@ Commands:
     redo                 Re-run the latest migration
     status               Dump the migration status for the current DB
     version              Print the current version of the database
-    create NAME [sql|go] Creates new migration file with next version
+    create NAME [sql|go] Creates new migration file with the current timestamp
 
 Options:
     -dir string
@@ -74,14 +72,14 @@ Examples:
 Create a new SQL migration.
 
     $ goose create add_some_column sql
-    $ Created new file: 00001_add_some_column.sql
+    $ Created new file: 20170506082420_add_some_column.sql
 
 Edit the newly created file to define the behavior of your migration.
 
 You can also create a Go migration, if you then invoke it with [your own goose binary](#go-migrations):
 
     $ goose create fetch_user_data go
-    $ Created new file: 00002_fetch_user_data.go
+    $ Created new file: 20170506082421_fetch_user_data.go
 
 ## up
 
@@ -241,6 +239,11 @@ func Down(tx *sql.Tx) error {
 }
 ```
 
+# Hybrid Versioning
+We strongly recommend adopting a hybrid versioning approach, using both timestamps and sequential numbers. Migrations created during the development process are timestamped and sequential versions are ran on production. We believe this method will prevent the problem of conflicting versions when writing software in a team environment.
+
+To help you adopt this approach, `create` will use the current timestamp as the migration version. When you're ready to deploy your migrations in a production environment, we also provide a helpful `fix` command to convert your migrations into sequential order, while preserving the timestamp ordering. We recommend running `fix` in the CI pipeline, and only when the migrations are ready for production.
+
 ## License
 
 Licensed under [MIT License](./LICENSE)

cmd/goose/main.go

@@ -33,6 +33,14 @@ func main() {
 		return
 	}
 
+	// TODO clean up arg/flag parsing flow
+	if args[0] == "fix" {
+		if err := goose.Run("fix", nil, *dir); err != nil {
+			log.Fatalf("goose run: %v", err)
+		}
+		return
+	}
+
 	if len(args) < 3 {
 		flags.Usage()
 		return
@@ -117,6 +125,7 @@ Commands:
     reset                Roll back all migrations
     status               Dump the migration status for the current DB
     version              Print the current version of the database
-    create NAME [sql|go] Creates new migration file with next version
+		create NAME [sql|go] Creates new migration file with the current timestamp
+		fix                  Apply sequential ordering to migrations
 `
 )

create.go

@@ -6,22 +6,12 @@ import (
 	"os"
 	"path/filepath"
 	"text/template"
+	"time"
 )
 
 // Create writes a new blank migration file.
 func CreateWithTemplate(db *sql.DB, dir string, migrationTemplate *template.Template, name, migrationType string) error {
-	migrations, err := CollectMigrations(dir, minVersion, maxVersion)
-	if err != nil {
-		return err
-	}
-
-	// Initial version.
-	version := "00001"
-
-	if last, err := migrations.Last(); err == nil {
-		version = fmt.Sprintf("%05v", last.Version+1)
-	}
-
+	version := time.Now().Format(timestampFormat)
 	filename := fmt.Sprintf("%v_%v.%v", version, name, migrationType)
 
 	fpath := filepath.Join(dir, filename)

dialect.go

@@ -10,6 +10,7 @@ import (
 type SQLDialect interface {
 	createVersionTableSQL() string // sql string to create the db version table
 	insertVersionSQL() string      // sql string to insert the initial version table row
+	updateVersionSQL() string      // sql string to update version
 	dbVersionQuery(db *sql.DB) (*sql.Rows, error)
 }
 
@@ -61,6 +62,10 @@ func (pg PostgresDialect) insertVersionSQL() string {
 	return fmt.Sprintf("INSERT INTO %s (version_id, is_applied) VALUES ($1, $2);", TableName())
 }
 
+func (pg PostgresDialect) updateVersionSQL() string {
+	return fmt.Sprintf("UPDATE %s SET version_id=? WHERE version_id=?;", TableName())
+}
+
 func (pg PostgresDialect) dbVersionQuery(db *sql.DB) (*sql.Rows, error) {
 	rows, err := db.Query(fmt.Sprintf("SELECT version_id, is_applied from %s ORDER BY id DESC", TableName()))
 	if err != nil {
@@ -91,6 +96,10 @@ func (m MySQLDialect) insertVersionSQL() string {
 	return fmt.Sprintf("INSERT INTO %s (version_id, is_applied) VALUES (?, ?);", TableName())
 }
 
+func (m MySQLDialect) updateVersionSQL() string {
+	return fmt.Sprintf("UPDATE %s SET version_id=? WHERE version_id=?;", TableName())
+}
+
 func (m MySQLDialect) dbVersionQuery(db *sql.DB) (*sql.Rows, error) {
 	rows, err := db.Query(fmt.Sprintf("SELECT version_id, is_applied from %s ORDER BY id DESC", TableName()))
 	if err != nil {
@@ -120,6 +129,10 @@ func (m Sqlite3Dialect) insertVersionSQL() string {
 	return fmt.Sprintf("INSERT INTO %s (version_id, is_applied) VALUES (?, ?);", TableName())
 }
 
+func (m Sqlite3Dialect) updateVersionSQL() string {
+	return fmt.Sprintf("UPDATE %s SET version_id=? WHERE version_id=?;", TableName())
+}
+
 func (m Sqlite3Dialect) dbVersionQuery(db *sql.DB) (*sql.Rows, error) {
 	rows, err := db.Query(fmt.Sprintf("SELECT version_id, is_applied from %s ORDER BY id DESC", TableName()))
 	if err != nil {
@@ -150,6 +163,10 @@ func (rs RedshiftDialect) insertVersionSQL() string {
 	return fmt.Sprintf("INSERT INTO %s (version_id, is_applied) VALUES ($1, $2);", TableName())
 }
 
+func (rs RedshiftDialect) updateVersionSQL() string {
+	return fmt.Sprintf("UPDATE %s SET version_id=? WHERE version_id=?;", TableName())
+}
+
 func (rs RedshiftDialect) dbVersionQuery(db *sql.DB) (*sql.Rows, error) {
 	rows, err := db.Query(fmt.Sprintf("SELECT version_id, is_applied from %s ORDER BY id DESC", TableName()))
 	if err != nil {
@@ -180,6 +197,10 @@ func (m TiDBDialect) insertVersionSQL() string {
 	return fmt.Sprintf("INSERT INTO %s (version_id, is_applied) VALUES (?, ?);", TableName())
 }
 
+func (m TiDBDialect) updateVersionSQL() string {
+	return fmt.Sprintf("UPDATE %s SET version_id=? WHERE version_id=?;", TableName())
+}
+
 func (m TiDBDialect) dbVersionQuery(db *sql.DB) (*sql.Rows, error) {
 	rows, err := db.Query(fmt.Sprintf("SELECT version_id, is_applied from %s ORDER BY id DESC", TableName()))
 	if err != nil {

examples/go-migrations/main.go

@@ -33,6 +33,14 @@ func main() {
 		return
 	}
 
+	// TODO clean up arg/flag parsing flow
+	if args[0] == "fix" {
+		if err := goose.Run("fix", nil, *dir); err != nil {
+			log.Fatalf("goose run: %v", err)
+		}
+		return
+	}
+
 	if len(args) < 3 {
 		flags.Usage()
 		return
@@ -117,6 +125,7 @@ Commands:
     redo                 Re-run the latest migration
     status               Dump the migration status for the current DB
     version              Print the current version of the database
-    create NAME [sql|go] Creates new migration file with next version
+    create NAME [sql|go] Creates new migration file with the current timestamp
+		fix                  Apply sequential ordering to migrations
 `
 )

fix.go

@@ -0,0 +1,46 @@
+package goose
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func Fix(dir string) error {
+	migrations, err := CollectMigrations(dir, minVersion, maxVersion)
+	if err != nil {
+		return err
+	}
+
+	// split into timestamped and versioned migrations
+	tsMigrations, err := migrations.timestamped()
+	if err != nil {
+		return err
+	}
+
+	vMigrations, err := migrations.versioned()
+	if err != nil {
+		return err
+	}
+	// Initial version.
+	version := int64(1)
+	if last, err := vMigrations.Last(); err == nil {
+		version = last.Version + 1
+	}
+
+	// fix filenames by replacing timestamps with sequential versions
+	for _, tsm := range tsMigrations {
+		oldPath := tsm.Source
+		newPath := strings.Replace(oldPath, fmt.Sprintf("%d", tsm.Version), fmt.Sprintf("%05v", version), 1)
+
+		if err := os.Rename(oldPath, newPath); err != nil {
+			return err
+		}
+
+		log.Printf("RENAMED %s => %s", filepath.Base(oldPath), filepath.Base(newPath))
+		version++
+	}
+
+	return nil
+}

fix_test.go

@@ -0,0 +1,81 @@
+package goose
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestFix(t *testing.T) {
+	dir, err := ioutil.TempDir("", "tmptest")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer os.RemoveAll(dir)  // clean up
+	defer os.Remove("goose") // clean up
+
+	commands := []string{
+		"go build -i -o goose ./cmd/goose",
+		fmt.Sprintf("./goose -dir=%s create create_table", dir),
+		fmt.Sprintf("./goose -dir=%s create add_users", dir),
+		fmt.Sprintf("./goose -dir=%s create add_indices", dir),
+		fmt.Sprintf("./goose -dir=%s create update_users", dir),
+		fmt.Sprintf("./goose -dir=%s fix", dir),
+	}
+
+	for _, cmd := range commands {
+		args := strings.Split(cmd, " ")
+		time.Sleep(1 * time.Second)
+		out, err := exec.Command(args[0], args[1:]...).CombinedOutput()
+		if err != nil {
+			t.Fatalf("%s:\n%v\n\n%s", err, cmd, out)
+		}
+	}
+
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// check that the files are in order
+	for i, f := range files {
+		expected := fmt.Sprintf("%05v", i+1)
+		if !strings.HasPrefix(f.Name(), expected) {
+			t.Errorf("failed to find %s prefix in %s", expected, f.Name())
+		}
+	}
+
+	// add more migrations and then fix it
+	commands = []string{
+		fmt.Sprintf("./goose -dir=%s create remove_column", dir),
+		fmt.Sprintf("./goose -dir=%s create create_books_table", dir),
+		fmt.Sprintf("./goose -dir=%s fix", dir),
+	}
+
+	for _, cmd := range commands {
+		args := strings.Split(cmd, " ")
+		time.Sleep(1 * time.Second)
+		out, err := exec.Command(args[0], args[1:]...).CombinedOutput()
+		if err != nil {
+			t.Fatalf("%s:\n%v\n\n%s", err, cmd, out)
+		}
+	}
+
+	files, err = ioutil.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// check that the files still in order
+	for i, f := range files {
+		expected := fmt.Sprintf("%05v", i+1)
+		if !strings.HasPrefix(f.Name(), expected) {
+			t.Errorf("failed to find %s prefix in %s", expected, f.Name())
+		}
+	}
+}

goose.go

@@ -11,6 +11,7 @@ var (
 	duplicateCheckOnce sync.Once
 	minVersion         = int64(0)
 	maxVersion         = int64((1 << 63) - 1)
+	timestampFormat    = "20060102150405"
 )
 
 // Run runs a goose command.
@@ -64,6 +65,10 @@ func Run(command string, db *sql.DB, dir string, args ...string) error {
 		if err := DownTo(db, dir, version); err != nil {
 			return err
 		}
+	case "fix":
+		if err := Fix(dir); err != nil {
+			return err
+		}
 	case "redo":
 		if err := Redo(db, dir); err != nil {
 			return err

migrate.go

@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"sort"
+	"time"
 )
 
 var (
@@ -76,6 +77,43 @@ func (ms Migrations) Last() (*Migration, error) {
 	return ms[len(ms)-1], nil
 }
 
+// Versioned gets versioned migrations.
+func (ms Migrations) versioned() (Migrations, error) {
+	var migrations Migrations
+
+	// assume that the user will never have more than 19700101000000 migrations
+	for _, m := range ms {
+		// parse version as timestmap
+		versionTime, err := time.Parse(timestampFormat, fmt.Sprintf("%d", m.Version))
+
+		if versionTime.Before(time.Unix(0, 0)) || err != nil {
+			migrations = append(migrations, m)
+		}
+	}
+
+	return migrations, nil
+}
+
+// Timestamped gets the timestamped migrations.
+func (ms Migrations) timestamped() (Migrations, error) {
+	var migrations Migrations
+
+	// assume that the user will never have more than 19700101000000 migrations
+	for _, m := range ms {
+		// parse version as timestmap
+		versionTime, err := time.Parse(timestampFormat, fmt.Sprintf("%d", m.Version))
+		if err != nil {
+			// probably not a timestamp
+			continue
+		}
+
+		if versionTime.After(time.Unix(0, 0)) {
+			migrations = append(migrations, m)
+		}
+	}
+	return migrations, nil
+}
+
 func (ms Migrations) String() string {
 	str := ""
 	for _, m := range ms {