diff --git a/.travis.yml b/.travis.yml index 2ac59d9..ba67774 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,5 +12,11 @@ install: script: - go test -- go run ./cmd/goose/main.go sqlite3 foo.db up - +- go run ./cmd/goose/main.go -dir=example/migrations sqlite3 sql.db up +- go run ./cmd/goose/main.go -dir=example/migrations sqlite3 sql.db version +- go run ./cmd/goose/main.go -dir=example/migrations sqlite3 sql.db down +- go run ./cmd/goose/main.go -dir=example/migrations sqlite3 sql.db status +- go run ./example/migrations-go/cmd/main.go -dir=example/migrations-go sqlite3 go.db up +- go run ./example/migrations-go/cmd/main.go -dir=example/migrations-go sqlite3 go.db version +- go run ./example/migrations-go/cmd/main.go -dir=example/migrations-go sqlite3 go.db down +- go run ./example/migrations-go/cmd/main.go -dir=example/migrations-go sqlite3 go.db status diff --git a/README.md b/README.md index c6897d4..c76da3f 100644 --- a/README.md +++ b/README.md @@ -4,17 +4,12 @@ Goose is a database migration tool. Manage your database's evolution by creating [![GoDoc Widget]][GoDoc] [![Travis Widget]][Travis] -This is a fork of https://bitbucket.org/liamstask/goose with following differences: -- No config files -- Meant to be imported by your application, so you can run complex Go migration functions with your own DB driver -- Standalone goose binary can only run SQL files -- we dropped building .go files in favor of the above +### Goals of this fork -# Goals -- [x] Move lib/goose to top level directory -- [x] Remove all config files -- [x] Commands should be part of the API -- [x] Update & finish README -- [ ] Registry for Go migration functions +This is a fork of https://bitbucket.org/liamstask/goose with the following changes: +- No config files +- Default binary can migrate SQL files only, we dropped building .go files on-the-fly +- Import goose pkg to run complex Go migrations with your own `*sql.DB` connection (no pkg dependency hell anymore) # Install @@ -22,17 +17,27 @@ This is a fork of https://bitbucket.org/liamstask/goose with following differenc This will install the `goose` binary to your `$GOPATH/bin` directory. -*Note: A standalone goose binary can only run pure SQL migrations. To run complex Go migrations, you have to import this pkg and register functions.* - # Usage -`goose [OPTIONS] DRIVER DBSTRING COMMAND` +``` +Usage: goose [OPTIONS] DRIVER DBSTRING COMMAND Examples: + goose postgres "user=postgres dbname=postgres sslmode=disable" up + goose mysql "user:password@/dbname" down + goose sqlite3 ./foo.db status - $ goose postgres "user=postgres dbname=postgres sslmode=disable" up - $ goose mysql "user:password@/dbname" down - $ goose sqlite3 ./foo.db status +Options: + -dir string + directory with migration files (default ".") + +Commands: + up Migrate the DB to the most recent version available + down Roll back the version by 1 + redo Re-run the latest migration + status Dump the migration status for the current DB + dbversion Print the current version of the database +``` ## create @@ -58,16 +63,6 @@ Apply all available migrations. $ OK 002_next.sql $ OK 003_and_again.go -### option: pgschema - -Use the `pgschema` flag with the `up` command specify a postgres schema. - - $ goose -pgschema=my_schema_name up - $ goose: migrating db environment 'development', current version: 0, target: 3 - $ OK 001_basics.sql - $ OK 002_next.sql - $ OK 003_and_again.go - ## down Roll back a single migration from the current version. @@ -105,10 +100,6 @@ Print the current version of the database: $ goose dbversion $ goose: dbversion 002 - -`goose -h` provides more detailed info on each command. - - # Migrations goose supports migrations written in SQL or in Go - see the `goose create` command above for details on how to generate them. @@ -164,31 +155,40 @@ language plpgsql; ## Go Migrations -A sample Go migration looks like: +Import `github.com/pressly/goose` from your own project (see [example](./example/migrations-go/cmd/main.go)), register migration functions and run goose command (ie. `goose.Up(db *sql.DB, dir string)`). + +A [sample Go migration 00002_users_add_email.go file](./example/migrations-go/00002_users_add_email.go) looks like: ```go -package main +package migrations import ( - "database/sql" - "fmt" + "database/sql" + + "github.com/pressly/goose" ) -func Up_20130106222315(txn *sql.Tx) { - fmt.Println("Hello from migration 20130106222315 Up!") +func init() { + goose.AddMigration(Up, Down) } -func Down_20130106222315(txn *sql.Tx) { - fmt.Println("Hello from migration 20130106222315 Down!") +func Up(tx *sql.Tx) error { + _, err := tx.Query("ALTER TABLE users ADD COLUMN email text DEFAULT '' NOT NULL;") + if err != nil { + return err + } + return nil +} + +func Down(tx *sql.Tx) error { + _, err := tx.Query("ALTER TABLE users DROP COLUMN email;") + if err != nil { + return err + } + return nil } ``` -`Up_20130106222315()` will be executed as part of a forward migration, and `Down_20130106222315()` will be executed as part of a rollback. - -The numeric portion of the function name (`20130106222315`) must be the leading portion of migration's filename, such as `20130106222315_descriptive_name.go`. `goose create` does this by default. - -A transaction is provided, rather than the DB instance directly, since goose also needs to record the schema version within the same transaction. Each migration should run as a single transaction to ensure DB integrity, so it's good practice anyway. - ## License Licensed under [MIT License](./LICENSE) diff --git a/example/migrations-go/00001_create_users_table.sql b/example/migrations-go/00001_create_users_table.sql new file mode 100644 index 0000000..efce49f --- /dev/null +++ b/example/migrations-go/00001_create_users_table.sql @@ -0,0 +1,14 @@ +-- +goose Up +CREATE TABLE users ( + id int NOT NULL PRIMARY KEY, + username text, + name text, + surname text +); + +INSERT INTO users VALUES +(0, 'root', '', ''), +(1, 'vojtechvitek', 'Vojtech', 'Vitek'); + +-- +goose Down +DROP TABLE users; diff --git a/example/migrations-go/00002_rename_root.go b/example/migrations-go/00002_rename_root.go new file mode 100644 index 0000000..7a645aa --- /dev/null +++ b/example/migrations-go/00002_rename_root.go @@ -0,0 +1,27 @@ +package migrations + +import ( + "database/sql" + + "github.com/pressly/goose" +) + +func init() { + goose.AddMigration(Up, Down) +} + +func Up(tx *sql.Tx) error { + _, err := tx.Exec("UPDATE users SET username='admin' WHERE username='root';") + if err != nil { + return err + } + return nil +} + +func Down(tx *sql.Tx) error { + _, err := tx.Exec("UPDATE users SET username='root' WHERE username='admin';") + if err != nil { + return err + } + return nil +} diff --git a/example/migrations-go/cmd/main.go b/example/migrations-go/cmd/main.go new file mode 100644 index 0000000..797b471 --- /dev/null +++ b/example/migrations-go/cmd/main.go @@ -0,0 +1,92 @@ +package main + +import ( + "database/sql" + "flag" + "fmt" + "log" + "os" + + "github.com/pressly/goose" + + _ "github.com/pressly/goose/example/migrations-go" + + _ "github.com/go-sql-driver/mysql" + _ "github.com/lib/pq" + _ "github.com/mattn/go-sqlite3" + _ "github.com/ziutek/mymysql/godrv" +) + +var ( + flags = flag.NewFlagSet("goose", flag.ExitOnError) + dir = flags.String("dir", ".", "directory with migration files") +) + +func main() { + flags.Usage = usage + flags.Parse(os.Args[1:]) + + args := flags.Args() + if len(args) != 3 { + flags.Usage() + return + } + + if args[0] == "-h" || args[0] == "--help" { + flags.Usage() + return + } + + driver, dbstring, command := args[0], args[1], args[2] + + switch driver { + case "postgres", "mysql", "sqlite3": + if err := goose.SetDialect(driver); err != nil { + log.Fatal(err) + } + default: + log.Fatalf("%q driver not supported\n", driver) + } + + switch dbstring { + case "": + log.Fatalf("-dbstring=%q not supported\n", dbstring) + default: + } + + db, err := sql.Open(driver, dbstring) + if err != nil { + log.Fatalf("-dbstring=%q: %v\n", dbstring, err) + } + + if err := goose.Run(command, db, *dir); err != nil { + log.Fatalf("goose run: %v", err) + } +} + +func usage() { + fmt.Print(usagePrefix) + flags.PrintDefaults() + fmt.Print(usageCommands) +} + +var ( + usagePrefix = `Usage: goose [OPTIONS] DRIVER DBSTRING COMMAND + +Examples: + goose postgres "user=postgres dbname=postgres sslmode=disable" up + goose mysql "user:password@/dbname" down + goose sqlite3 ./foo.db status + +Options: +` + + usageCommands = ` +Commands: + up Migrate the DB to the most recent version available + down Roll back the version by 1 + redo Re-run the latest migration + status Dump the migration status for the current DB + dbversion Print the current version of the database +` +) diff --git a/example/migrations/00001_create_post_table.sql b/example/migrations/00001_create_post_table.sql deleted file mode 100644 index b962691..0000000 --- a/example/migrations/00001_create_post_table.sql +++ /dev/null @@ -1,10 +0,0 @@ --- +goose Up -CREATE TABLE post ( - id int NOT NULL, - title text, - body text, - PRIMARY KEY(id) -); - --- +goose Down -DROP TABLE post; diff --git a/example/migrations/00001_create_users_table.sql b/example/migrations/00001_create_users_table.sql new file mode 100644 index 0000000..efce49f --- /dev/null +++ b/example/migrations/00001_create_users_table.sql @@ -0,0 +1,14 @@ +-- +goose Up +CREATE TABLE users ( + id int NOT NULL PRIMARY KEY, + username text, + name text, + surname text +); + +INSERT INTO users VALUES +(0, 'root', '', ''), +(1, 'vojtechvitek', 'Vojtech', 'Vitek'); + +-- +goose Down +DROP TABLE users; diff --git a/example/migrations/00002_next.sql b/example/migrations/00002_next.sql deleted file mode 100644 index 4cea847..0000000 --- a/example/migrations/00002_next.sql +++ /dev/null @@ -1,11 +0,0 @@ --- +goose Up -CREATE TABLE fancier_post ( - id int NOT NULL, - title text, - body text, - created_on timestamp without time zone, - PRIMARY KEY(id) -); - --- +goose Down -DROP TABLE fancier_post; diff --git a/example/migrations/00002_rename_root.sql b/example/migrations/00002_rename_root.sql new file mode 100644 index 0000000..203d54e --- /dev/null +++ b/example/migrations/00002_rename_root.sql @@ -0,0 +1,9 @@ +-- +goose Up +-- +goose StatementBegin +UPDATE users SET username='admin' WHERE username='root'; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +UPDATE users SET username='root' WHERE username='admin'; +-- +goose StatementEnd diff --git a/migrate.go b/migrate.go index ef97ac5..0a6a8dd 100644 --- a/migrate.go +++ b/migrate.go @@ -7,6 +7,7 @@ import ( "log" "os" "path/filepath" + "runtime" "sort" "strconv" "strings" @@ -17,6 +18,7 @@ import ( var ( ErrTableDoesNotExist = errors.New("table does not exist") ErrNoPreviousVersion = errors.New("no previous version found") + goMigrations []*Migration ) type MigrationRecord struct { @@ -27,9 +29,11 @@ type MigrationRecord struct { type Migration struct { Version int64 - Next int64 // next version, or -1 if none - Previous int64 // previous version, -1 if none - Source string // path to .go or .sql script + Next int64 // next version, or -1 if none + Previous int64 // previous version, -1 if none + Source string // path to .sql script + Up func(*sql.Tx) error // Up go migration function + Down func(*sql.Tx) error // Down go migration function } type migrationSorter []*Migration @@ -39,8 +43,12 @@ func (ms migrationSorter) Len() int { return len(ms) } func (ms migrationSorter) Swap(i, j int) { ms[i], ms[j] = ms[j], ms[i] } func (ms migrationSorter) Less(i, j int) bool { return ms[i].Version < ms[j].Version } -func newMigration(v int64, src string) *Migration { - return &Migration{v, -1, -1, src} +func AddMigration(up func(*sql.Tx) error, down func(*sql.Tx) error) { + _, filename, _, _ := runtime.Caller(1) + v, _ := NumericComponent(filename) + migration := &Migration{Version: v, Next: -1, Previous: -1, Up: up, Down: down, Source: filename} + + goMigrations = append(goMigrations, migration) } func RunMigrations(db *sql.DB, dir string, target int64) (err error) { @@ -69,13 +77,33 @@ func RunMigrations(db *sql.DB, dir string, target int64) (err error) { switch filepath.Ext(m.Source) { case ".sql": - default: - continue - } + if err = runSQLMigration(db, m.Source, m.Version, direction); err != nil { + return errors.New(fmt.Sprintf("FAIL %v, quitting migration", err)) + } - err = runSQLMigration(db, m.Source, m.Version, direction) - if err != nil { - return errors.New(fmt.Sprintf("FAIL %v, quitting migration", err)) + case ".go": + fn := m.Up + if !direction { + fn = m.Down + } + if fn == nil { + continue + } + + tx, err := db.Begin() + if err != nil { + log.Fatal("db.Begin: ", err) + } + + if err := fn(tx); err != nil { + tx.Rollback() + log.Fatalf("FAIL %s (%v), quitting migration.", filepath.Base(m.Source), err) + return err + } + + if err = FinalizeMigration(tx, direction, m.Version); err != nil { + log.Fatalf("error finalizing migration %s, quitting. (%v)", filepath.Base(m.Source), err) + } } fmt.Println("OK ", filepath.Base(m.Source)) @@ -85,30 +113,37 @@ func RunMigrations(db *sql.DB, dir string, target int64) (err error) { } // collect all the valid looking migration scripts in the -// migrations folder, and key them by version +// migrations folder and go func registry, and key them by version func CollectMigrations(dirpath string, current, target int64) (m []*Migration, err error) { // extract the numeric component of each migration, // filter out any uninteresting files, // and ensure we only have one file per migration version. - filepath.Walk(dirpath, func(name string, info os.FileInfo, err error) error { + sqlMigrations, err := filepath.Glob(dirpath + "/*.sql") + if err != nil { + return nil, err + } - if v, e := NumericComponent(name); e == nil { - - for _, g := range m { - if v == g.Version { - log.Fatalf("more than one file specifies the migration for version %d (%s and %s)", - v, g.Source, filepath.Join(dirpath, name)) - } - } - - if versionFilter(v, current, target) { - m = append(m, newMigration(v, name)) - } + for _, file := range sqlMigrations { + v, err := NumericComponent(file) + if err != nil { + return nil, err } + if versionFilter(v, current, target) { + migration := &Migration{Version: v, Next: -1, Previous: -1, Source: file} + m = append(m, migration) + } + } - return nil - }) + for _, migration := range goMigrations { + v, err := NumericComponent(migration.Source) + if err != nil { + return nil, err + } + if versionFilter(v, current, target) { + m = append(m, migration) + } + } return m, nil } @@ -341,16 +376,16 @@ func CreateMigration(name, migrationType, dir string, t time.Time) (path string, // Update the version table for the given migration, // and finalize the transaction. -func FinalizeMigration(txn *sql.Tx, direction bool, v int64) error { +func FinalizeMigration(tx *sql.Tx, direction bool, v int64) error { // XXX: drop goose_db_version table on some minimum version number? stmt := GetDialect().insertVersionSql() - if _, err := txn.Exec(stmt, v, direction); err != nil { - txn.Rollback() + if _, err := tx.Exec(stmt, v, direction); err != nil { + tx.Rollback() return err } - return txn.Commit() + return tx.Commit() } var sqlMigrationTemplate = template.Must(template.New("goose.sql-migration").Parse(` diff --git a/migrate_test.go b/migrate_test.go index 6dbffa3..f0ef2c1 100644 --- a/migrate_test.go +++ b/migrate_test.go @@ -4,6 +4,10 @@ import ( "testing" ) +func newMigration(v int64, src string) *Migration { + return &Migration{Version: v, Previous: -1, Next: -1, Source: src} +} + func TestMigrationMapSortUp(t *testing.T) { ms := migrationSorter{} diff --git a/migration_sql.go b/migration_sql.go index 8c55ac4..e31826f 100644 --- a/migration_sql.go +++ b/migration_sql.go @@ -137,7 +137,7 @@ func splitSQLStatements(r io.Reader, direction bool) (stmts []string) { // until another direction directive is found. func runSQLMigration(db *sql.DB, scriptFile string, v int64, direction bool) error { - txn, err := db.Begin() + tx, err := db.Begin() if err != nil { log.Fatal("db.Begin:", err) } @@ -153,14 +153,14 @@ func runSQLMigration(db *sql.DB, scriptFile string, v int64, direction bool) err // records the version into the version table or returns an error and // rolls back the transaction. for _, query := range splitSQLStatements(f, direction) { - if _, err = txn.Exec(query); err != nil { - txn.Rollback() + if _, err = tx.Exec(query); err != nil { + tx.Rollback() log.Fatalf("FAIL %s (%v), quitting migration.", filepath.Base(scriptFile), err) return err } } - if err = FinalizeMigration(txn, direction, v); err != nil { + if err = FinalizeMigration(tx, direction, v); err != nil { log.Fatalf("error finalizing migration %s, quitting. (%v)", filepath.Base(scriptFile), err) }