From 45eeb19d7d952531c3102add91cad61f1d02a410 Mon Sep 17 00:00:00 2001 From: 1vn Date: Tue, 30 Oct 2018 16:45:45 -0400 Subject: [PATCH 1/7] add fix and timestamp default --- create.go | 14 ++------- dialect.go | 20 +++++++++++++ fix.go | 45 +++++++++++++++++++++++++++++ fix_test.go | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++ goose.go | 5 ++++ migrate.go | 38 +++++++++++++++++++++++++ 6 files changed, 191 insertions(+), 12 deletions(-) create mode 100644 fix.go create mode 100644 fix_test.go diff --git a/create.go b/create.go index b863076..6761dec 100644 --- a/create.go +++ b/create.go @@ -6,22 +6,12 @@ import ( "os" "path/filepath" "text/template" + "time" ) // Create writes a new blank migration file. func CreateWithTemplate(db *sql.DB, dir string, migrationTemplate *template.Template, name, migrationType string) error { - migrations, err := CollectMigrations(dir, minVersion, maxVersion) - if err != nil { - return err - } - - // Initial version. - version := "00001" - - if last, err := migrations.Last(); err == nil { - version = fmt.Sprintf("%05v", last.Version+1) - } - + version := time.Now().Format(timestampFormat) filename := fmt.Sprintf("%v_%v.%v", version, name, migrationType) fpath := filepath.Join(dir, filename) diff --git a/dialect.go b/dialect.go index 488f5e0..edc2c35 100644 --- a/dialect.go +++ b/dialect.go @@ -61,6 +61,10 @@ func (pg PostgresDialect) insertVersionSQL() string { return fmt.Sprintf("INSERT INTO %s (version_id, is_applied) VALUES ($1, $2);", TableName()) } +func (pg PostgresDialect) updateVersionSQL() string { + return fmt.Sprintf("UPDATE %s SET version_id=? WHERE version_id=?;", TableName()) +} + func (pg PostgresDialect) dbVersionQuery(db *sql.DB) (*sql.Rows, error) { rows, err := db.Query(fmt.Sprintf("SELECT version_id, is_applied from %s ORDER BY id DESC", TableName())) if err != nil { @@ -91,6 +95,10 @@ func (m MySQLDialect) insertVersionSQL() string { return fmt.Sprintf("INSERT INTO %s (version_id, is_applied) VALUES (?, ?);", TableName()) } +func (m MySQLDialect) updateVersionSQL() string { + return fmt.Sprintf("UPDATE %s SET version_id=? WHERE version_id=?;", TableName()) +} + func (m MySQLDialect) dbVersionQuery(db *sql.DB) (*sql.Rows, error) { rows, err := db.Query(fmt.Sprintf("SELECT version_id, is_applied from %s ORDER BY id DESC", TableName())) if err != nil { @@ -120,6 +128,10 @@ func (m Sqlite3Dialect) insertVersionSQL() string { return fmt.Sprintf("INSERT INTO %s (version_id, is_applied) VALUES (?, ?);", TableName()) } +func (m Sqlite3Dialect) updateVersionSQL() string { + return fmt.Sprintf("UPDATE %s SET version_id=? WHERE version_id=?;", TableName()) +} + func (m Sqlite3Dialect) dbVersionQuery(db *sql.DB) (*sql.Rows, error) { rows, err := db.Query(fmt.Sprintf("SELECT version_id, is_applied from %s ORDER BY id DESC", TableName())) if err != nil { @@ -150,6 +162,10 @@ func (rs RedshiftDialect) insertVersionSQL() string { return fmt.Sprintf("INSERT INTO %s (version_id, is_applied) VALUES ($1, $2);", TableName()) } +func (rs RedshiftDialect) updateVersionSQL() string { + return fmt.Sprintf("UPDATE %s SET version_id=? WHERE version_id=?;", TableName()) +} + func (rs RedshiftDialect) dbVersionQuery(db *sql.DB) (*sql.Rows, error) { rows, err := db.Query(fmt.Sprintf("SELECT version_id, is_applied from %s ORDER BY id DESC", TableName())) if err != nil { @@ -180,6 +196,10 @@ func (m TiDBDialect) insertVersionSQL() string { return fmt.Sprintf("INSERT INTO %s (version_id, is_applied) VALUES (?, ?);", TableName()) } +func (m TiDBDialect) updateVersionSQL() string { + return fmt.Sprintf("UPDATE %s SET version_id=? WHERE version_id=?;", TableName()) +} + func (m TiDBDialect) dbVersionQuery(db *sql.DB) (*sql.Rows, error) { rows, err := db.Query(fmt.Sprintf("SELECT version_id, is_applied from %s ORDER BY id DESC", TableName())) if err != nil { diff --git a/fix.go b/fix.go new file mode 100644 index 0000000..abb9d19 --- /dev/null +++ b/fix.go @@ -0,0 +1,45 @@ +package goose + +import ( + "database/sql" + "fmt" + "os" + "strings" +) + +func Fix(db *sql.DB, dir string) error { + migrations, err := CollectMigrations(dir, minVersion, maxVersion) + if err != nil { + return err + } + + // split into timestamped and versioned migrations + tsMigrations, err := migrations.Timestamped() + if err != nil { + return err + } + + vMigrations, err := migrations.Versioned() + if err != nil { + return err + } + // Initial version. + version := int64(1) + if last, err := vMigrations.Last(); err == nil { + version = last.Version + 1 + } + + // fix filenames by replacing timestamps with sequential versions + for _, tsm := range tsMigrations { + oldPath := tsm.Source + newPath := strings.Replace(oldPath, fmt.Sprintf("%d", tsm.Version), fmt.Sprintf("%05v", version), 1) + + if err := os.Rename(oldPath, newPath); err != nil { + return err + } + + version++ + } + + return nil +} diff --git a/fix_test.go b/fix_test.go new file mode 100644 index 0000000..e2f9fc9 --- /dev/null +++ b/fix_test.go @@ -0,0 +1,81 @@ +package goose + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +func TestFix(t *testing.T) { + dir, err := ioutil.TempDir("", "tmptest") + if err != nil { + t.Fatal(err) + } + + defer os.RemoveAll(dir) // clean up + defer os.Remove("goose") // clean up + + commands := []string{ + "go build -i -o goose ./cmd/goose", + fmt.Sprintf("./goose -dir=%s create yolo", dir), + fmt.Sprintf("./goose -dir=%s create yolo", dir), + fmt.Sprintf("./goose -dir=%s create yolo", dir), + fmt.Sprintf("./goose -dir=%s create yolo", dir), + fmt.Sprintf("./goose -dir=%s sqlite3 sql.db fix", dir), + } + + for _, cmd := range commands { + args := strings.Split(cmd, " ") + time.Sleep(1 * time.Second) + out, err := exec.Command(args[0], args[1:]...).CombinedOutput() + if err != nil { + t.Fatalf("%s:\n%v\n\n%s", err, cmd, out) + } + } + + files, err := ioutil.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + // check that the files are in order + for i, f := range files { + expected := fmt.Sprintf("%05v", i+1) + if !strings.HasPrefix(f.Name(), expected) { + t.Errorf("failed to find %s prefix in %s", expected, f.Name()) + } + } + + // add more migrations and then fix it + commands = []string{ + fmt.Sprintf("./goose -dir=%s create yolo", dir), + fmt.Sprintf("./goose -dir=%s create yolo", dir), + fmt.Sprintf("./goose -dir=%s sqlite3 sql.db fix", dir), + } + + for _, cmd := range commands { + args := strings.Split(cmd, " ") + time.Sleep(1 * time.Second) + out, err := exec.Command(args[0], args[1:]...).CombinedOutput() + if err != nil { + t.Fatalf("%s:\n%v\n\n%s", err, cmd, out) + } + } + + files, err = ioutil.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + // check that the files still in order + for i, f := range files { + expected := fmt.Sprintf("%05v", i+1) + if !strings.HasPrefix(f.Name(), expected) { + t.Errorf("failed to find %s prefix in %s", expected, f.Name()) + } + } +} diff --git a/goose.go b/goose.go index 51ca6ec..1c492b0 100644 --- a/goose.go +++ b/goose.go @@ -11,6 +11,7 @@ var ( duplicateCheckOnce sync.Once minVersion = int64(0) maxVersion = int64((1 << 63) - 1) + timestampFormat = "20060102150405" ) // Run runs a goose command. @@ -64,6 +65,10 @@ func Run(command string, db *sql.DB, dir string, args ...string) error { if err := DownTo(db, dir, version); err != nil { return err } + case "fix": + if err := Fix(db, dir); err != nil { + return err + } case "redo": if err := Redo(db, dir); err != nil { return err diff --git a/migrate.go b/migrate.go index 4774af0..f998b27 100644 --- a/migrate.go +++ b/migrate.go @@ -8,6 +8,7 @@ import ( "path/filepath" "runtime" "sort" + "time" ) var ( @@ -76,6 +77,43 @@ func (ms Migrations) Last() (*Migration, error) { return ms[len(ms)-1], nil } +// Versioned gets versioned migrations. +func (ms Migrations) Versioned() (Migrations, error) { + var migrations Migrations + + // assume that the user will never have more than 19700101000000 migrations + for _, m := range ms { + // parse version as timestmap + versionTime, err := time.Parse(timestampFormat, fmt.Sprintf("%d", m.Version)) + + if versionTime.Before(time.Unix(0, 0)) || err != nil { + migrations = append(migrations, m) + } + } + + return migrations, nil +} + +// Timestamped gets the timestamped migrations. +func (ms Migrations) Timestamped() (Migrations, error) { + var migrations Migrations + + // assume that the user will never have more than 19700101000000 migrations + for _, m := range ms { + // parse version as timestmap + versionTime, err := time.Parse(timestampFormat, fmt.Sprintf("%d", m.Version)) + if err != nil { + // probably not a timestamp + continue + } + + if versionTime.After(time.Unix(0, 0)) { + migrations = append(migrations, m) + } + } + return migrations, nil +} + func (ms Migrations) String() string { str := "" for _, m := range ms { From 286883adf54fb4d6e749d95a4596bebfe44d191d Mon Sep 17 00:00:00 2001 From: 1vn Date: Tue, 30 Oct 2018 16:51:07 -0400 Subject: [PATCH 2/7] add db fix as well --- dialect.go | 1 + fix.go | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/dialect.go b/dialect.go index edc2c35..1f21aa6 100644 --- a/dialect.go +++ b/dialect.go @@ -10,6 +10,7 @@ import ( type SQLDialect interface { createVersionTableSQL() string // sql string to create the db version table insertVersionSQL() string // sql string to insert the initial version table row + updateVersionSQL() string // sql string to update version dbVersionQuery(db *sql.DB) (*sql.Rows, error) } diff --git a/fix.go b/fix.go index abb9d19..caa9ff8 100644 --- a/fix.go +++ b/fix.go @@ -29,6 +29,12 @@ func Fix(db *sql.DB, dir string) error { version = last.Version + 1 } + // fix db table as well + tx, err := db.Begin() + if err != nil { + log.Fatal("db.Begin: ", err) + } + // fix filenames by replacing timestamps with sequential versions for _, tsm := range tsMigrations { oldPath := tsm.Source @@ -38,8 +44,13 @@ func Fix(db *sql.DB, dir string) error { return err } + if _, err := tx.Exec(GetDialect().updateVersionSQL(), version, tsm.Version); err != nil { + tx.Rollback() + return err + } + version++ } - return nil + return tx.Commit() } From 22d96449ec7edf128453482eec3a30a5bf7817ca Mon Sep 17 00:00:00 2001 From: 1vn Date: Tue, 30 Oct 2018 17:02:57 -0400 Subject: [PATCH 3/7] remove db fix, unexport stuff --- cmd/goose/main.go | 4 ++-- fix.go | 20 ++++---------------- goose.go | 2 +- migrate.go | 4 ++-- 4 files changed, 9 insertions(+), 21 deletions(-) diff --git a/cmd/goose/main.go b/cmd/goose/main.go index dcc5910..87b9506 100644 --- a/cmd/goose/main.go +++ b/cmd/goose/main.go @@ -26,8 +26,8 @@ func main() { args := flags.Args() - if len(args) > 1 && args[0] == "create" { - if err := goose.Run("create", nil, *dir, args[1:]...); err != nil { + if len(args) > 1 && (args[0] == "create" || args[0] == "fix") { + if err := goose.Run(args[0], nil, *dir, args[1:]...); err != nil { log.Fatalf("goose run: %v", err) } return diff --git a/fix.go b/fix.go index caa9ff8..34f5705 100644 --- a/fix.go +++ b/fix.go @@ -1,25 +1,24 @@ package goose import ( - "database/sql" "fmt" "os" "strings" ) -func Fix(db *sql.DB, dir string) error { +func Fix(dir string) error { migrations, err := CollectMigrations(dir, minVersion, maxVersion) if err != nil { return err } // split into timestamped and versioned migrations - tsMigrations, err := migrations.Timestamped() + tsMigrations, err := migrations.timestamped() if err != nil { return err } - vMigrations, err := migrations.Versioned() + vMigrations, err := migrations.versioned() if err != nil { return err } @@ -29,12 +28,6 @@ func Fix(db *sql.DB, dir string) error { version = last.Version + 1 } - // fix db table as well - tx, err := db.Begin() - if err != nil { - log.Fatal("db.Begin: ", err) - } - // fix filenames by replacing timestamps with sequential versions for _, tsm := range tsMigrations { oldPath := tsm.Source @@ -44,13 +37,8 @@ func Fix(db *sql.DB, dir string) error { return err } - if _, err := tx.Exec(GetDialect().updateVersionSQL(), version, tsm.Version); err != nil { - tx.Rollback() - return err - } - version++ } - return tx.Commit() + return nil } diff --git a/goose.go b/goose.go index 1c492b0..ff4b5e8 100644 --- a/goose.go +++ b/goose.go @@ -66,7 +66,7 @@ func Run(command string, db *sql.DB, dir string, args ...string) error { return err } case "fix": - if err := Fix(db, dir); err != nil { + if err := Fix(dir); err != nil { return err } case "redo": diff --git a/migrate.go b/migrate.go index f998b27..5529442 100644 --- a/migrate.go +++ b/migrate.go @@ -78,7 +78,7 @@ func (ms Migrations) Last() (*Migration, error) { } // Versioned gets versioned migrations. -func (ms Migrations) Versioned() (Migrations, error) { +func (ms Migrations) versioned() (Migrations, error) { var migrations Migrations // assume that the user will never have more than 19700101000000 migrations @@ -95,7 +95,7 @@ func (ms Migrations) Versioned() (Migrations, error) { } // Timestamped gets the timestamped migrations. -func (ms Migrations) Timestamped() (Migrations, error) { +func (ms Migrations) timestamped() (Migrations, error) { var migrations Migrations // assume that the user will never have more than 19700101000000 migrations From c6c49ff9953c8f541d027cb679c7b357a5da770c Mon Sep 17 00:00:00 2001 From: 1vn Date: Tue, 30 Oct 2018 17:08:25 -0400 Subject: [PATCH 4/7] remove yolo --- fix_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fix_test.go b/fix_test.go index e2f9fc9..263ebf7 100644 --- a/fix_test.go +++ b/fix_test.go @@ -21,10 +21,10 @@ func TestFix(t *testing.T) { commands := []string{ "go build -i -o goose ./cmd/goose", - fmt.Sprintf("./goose -dir=%s create yolo", dir), - fmt.Sprintf("./goose -dir=%s create yolo", dir), - fmt.Sprintf("./goose -dir=%s create yolo", dir), - fmt.Sprintf("./goose -dir=%s create yolo", dir), + fmt.Sprintf("./goose -dir=%s create create_table", dir), + fmt.Sprintf("./goose -dir=%s create add_users", dir), + fmt.Sprintf("./goose -dir=%s create add_indices", dir), + fmt.Sprintf("./goose -dir=%s create update_users", dir), fmt.Sprintf("./goose -dir=%s sqlite3 sql.db fix", dir), } @@ -52,8 +52,8 @@ func TestFix(t *testing.T) { // add more migrations and then fix it commands = []string{ - fmt.Sprintf("./goose -dir=%s create yolo", dir), - fmt.Sprintf("./goose -dir=%s create yolo", dir), + fmt.Sprintf("./goose -dir=%s create remove_column", dir), + fmt.Sprintf("./goose -dir=%s create create_books_table", dir), fmt.Sprintf("./goose -dir=%s sqlite3 sql.db fix", dir), } From 3c2c9d9076bc96c4ad57b3ccc69fa623e5ff4372 Mon Sep 17 00:00:00 2001 From: 1vn Date: Wed, 31 Oct 2018 12:27:15 -0400 Subject: [PATCH 5/7] add output --- cmd/goose/main.go | 12 ++++++++++-- fix.go | 2 ++ fix_test.go | 4 ++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/cmd/goose/main.go b/cmd/goose/main.go index 87b9506..cfb9043 100644 --- a/cmd/goose/main.go +++ b/cmd/goose/main.go @@ -26,8 +26,16 @@ func main() { args := flags.Args() - if len(args) > 1 && (args[0] == "create" || args[0] == "fix") { - if err := goose.Run(args[0], nil, *dir, args[1:]...); err != nil { + if len(args) > 1 && args[0] == "create" { + if err := goose.Run("create", nil, *dir, args[1:]...); err != nil { + log.Fatalf("goose run: %v", err) + } + return + } + + // TODO clean up arg/flag parsing flow + if args[0] == "fix" { + if err := goose.Run("fix", nil, *dir); err != nil { log.Fatalf("goose run: %v", err) } return diff --git a/fix.go b/fix.go index 34f5705..dcd8010 100644 --- a/fix.go +++ b/fix.go @@ -3,6 +3,7 @@ package goose import ( "fmt" "os" + "path/filepath" "strings" ) @@ -37,6 +38,7 @@ func Fix(dir string) error { return err } + log.Printf("RENAMED %s => %s", filepath.Base(oldPath), filepath.Base(newPath)) version++ } diff --git a/fix_test.go b/fix_test.go index 263ebf7..956070c 100644 --- a/fix_test.go +++ b/fix_test.go @@ -25,7 +25,7 @@ func TestFix(t *testing.T) { fmt.Sprintf("./goose -dir=%s create add_users", dir), fmt.Sprintf("./goose -dir=%s create add_indices", dir), fmt.Sprintf("./goose -dir=%s create update_users", dir), - fmt.Sprintf("./goose -dir=%s sqlite3 sql.db fix", dir), + fmt.Sprintf("./goose -dir=%s fix", dir), } for _, cmd := range commands { @@ -54,7 +54,7 @@ func TestFix(t *testing.T) { commands = []string{ fmt.Sprintf("./goose -dir=%s create remove_column", dir), fmt.Sprintf("./goose -dir=%s create create_books_table", dir), - fmt.Sprintf("./goose -dir=%s sqlite3 sql.db fix", dir), + fmt.Sprintf("./goose -dir=%s fix", dir), } for _, cmd := range commands { From a4a53bdf99114be929c0a80e08997c0824c7129f Mon Sep 17 00:00:00 2001 From: 1vn Date: Tue, 6 Nov 2018 14:14:09 -0500 Subject: [PATCH 6/7] add to example as well --- examples/go-migrations/main.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/examples/go-migrations/main.go b/examples/go-migrations/main.go index 64226ea..1f1ea5a 100644 --- a/examples/go-migrations/main.go +++ b/examples/go-migrations/main.go @@ -33,6 +33,14 @@ func main() { return } + // TODO clean up arg/flag parsing flow + if args[0] == "fix" { + if err := goose.Run("fix", nil, *dir); err != nil { + log.Fatalf("goose run: %v", err) + } + return + } + if len(args) < 3 { flags.Usage() return From 90f0cf504b284bfa2b294586e631a4fde3cf3179 Mon Sep 17 00:00:00 2001 From: 1vn Date: Tue, 6 Nov 2018 16:54:31 -0500 Subject: [PATCH 7/7] update documentation --- README.md | 15 +++++++++------ cmd/goose/main.go | 3 ++- examples/go-migrations/main.go | 3 ++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index efedf5b..e60d33c 100644 --- a/README.md +++ b/README.md @@ -22,9 +22,7 @@ Goose is a database migration tool. Manage your database schema by creating incr - goose pkg doesn't register any SQL drivers anymore, thus no driver `panic()` conflict within your codebase! - goose pkg doesn't have any vendor dependencies anymore -- We encourage using sequential versioning of migration files - (rather than timestamps-based versioning) to prevent version - mismatch and migration colissions +- We use timestamped migrations by default but recommend a hybrid approach of using timestamps in the development process and sequential versions in production. # Install @@ -51,7 +49,7 @@ Commands: redo Re-run the latest migration status Dump the migration status for the current DB version Print the current version of the database - create NAME [sql|go] Creates new migration file with next version + create NAME [sql|go] Creates new migration file with the current timestamp Options: -dir string @@ -74,14 +72,14 @@ Examples: Create a new SQL migration. $ goose create add_some_column sql - $ Created new file: 00001_add_some_column.sql + $ Created new file: 20170506082420_add_some_column.sql Edit the newly created file to define the behavior of your migration. You can also create a Go migration, if you then invoke it with [your own goose binary](#go-migrations): $ goose create fetch_user_data go - $ Created new file: 00002_fetch_user_data.go + $ Created new file: 20170506082421_fetch_user_data.go ## up @@ -241,6 +239,11 @@ func Down(tx *sql.Tx) error { } ``` +# Hybrid Versioning +We strongly recommend adopting a hybrid versioning approach, using both timestamps and sequential numbers. Migrations created during the development process are timestamped and sequential versions are ran on production. We believe this method will prevent the problem of conflicting versions when writing software in a team environment. + +To help you adopt this approach, `create` will use the current timestamp as the migration version. When you're ready to deploy your migrations in a production environment, we also provide a helpful `fix` command to convert your migrations into sequential order, while preserving the timestamp ordering. We recommend running `fix` in the CI pipeline, and only when the migrations are ready for production. + ## License Licensed under [MIT License](./LICENSE) diff --git a/cmd/goose/main.go b/cmd/goose/main.go index cfb9043..15b4caf 100644 --- a/cmd/goose/main.go +++ b/cmd/goose/main.go @@ -125,6 +125,7 @@ Commands: reset Roll back all migrations status Dump the migration status for the current DB version Print the current version of the database - create NAME [sql|go] Creates new migration file with next version + create NAME [sql|go] Creates new migration file with the current timestamp + fix Apply sequential ordering to migrations ` ) diff --git a/examples/go-migrations/main.go b/examples/go-migrations/main.go index 1f1ea5a..bb323af 100644 --- a/examples/go-migrations/main.go +++ b/examples/go-migrations/main.go @@ -125,6 +125,7 @@ Commands: redo Re-run the latest migration status Dump the migration status for the current DB version Print the current version of the database - create NAME [sql|go] Creates new migration file with next version + create NAME [sql|go] Creates new migration file with the current timestamp + fix Apply sequential ordering to migrations ` )