Register Go functions as complex Go migrations

.travis.yml

@@ -12,5 +12,11 @@ install:
 
 script:
 - go test
-- go run ./cmd/goose/main.go sqlite3 foo.db up
+- go run ./cmd/goose/main.go -dir=example/migrations sqlite3 sql.db up
-
+- go run ./cmd/goose/main.go -dir=example/migrations sqlite3 sql.db version
+- go run ./cmd/goose/main.go -dir=example/migrations sqlite3 sql.db down
+- go run ./cmd/goose/main.go -dir=example/migrations sqlite3 sql.db status
+- go run ./example/migrations-go/cmd/main.go -dir=example/migrations-go sqlite3 go.db up
+- go run ./example/migrations-go/cmd/main.go -dir=example/migrations-go sqlite3 go.db version
+- go run ./example/migrations-go/cmd/main.go -dir=example/migrations-go sqlite3 go.db down
+- go run ./example/migrations-go/cmd/main.go -dir=example/migrations-go sqlite3 go.db status

README.md

@@ -4,17 +4,12 @@ Goose is a database migration tool. Manage your database's evolution by creating
 
 [![GoDoc Widget]][GoDoc] [![Travis Widget]][Travis]
 
-This is a fork of https://bitbucket.org/liamstask/goose with following differences:
+### Goals of this fork
-- No config files
-- Meant to be imported by your application, so you can run complex Go migration functions with your own DB driver
-- Standalone goose binary can only run SQL files -- we dropped building .go files in favor of the above
 
-# Goals
+This is a fork of https://bitbucket.org/liamstask/goose with the following changes:
-- [x] Move lib/goose to top level directory
+- No config files
-- [x] Remove all config files
+- Default binary can migrate SQL files only, we dropped building .go files on-the-fly
-- [x] Commands should be part of the API
+- Import goose pkg to run complex Go migrations with your own `*sql.DB` connection (no pkg dependency hell anymore)
-- [x] Update & finish README
-- [ ] Registry for Go migration functions
 
 # Install
 
@@ -22,17 +17,27 @@ This is a fork of https://bitbucket.org/liamstask/goose with following differenc
 
 This will install the `goose` binary to your `$GOPATH/bin` directory.
 
-*Note: A standalone goose binary can only run pure SQL migrations. To run complex Go migrations, you have to import this pkg and register functions.*
-
 # Usage
 
-`goose [OPTIONS] DRIVER DBSTRING COMMAND`
+```
+Usage: goose [OPTIONS] DRIVER DBSTRING COMMAND
 
 Examples:
+    goose postgres "user=postgres dbname=postgres sslmode=disable" up
+    goose mysql "user:password@/dbname" down
+    goose sqlite3 ./foo.db status
 
-    $ goose postgres "user=postgres dbname=postgres sslmode=disable" up
+Options:
-    $ goose mysql "user:password@/dbname" down
+  -dir string
-    $ goose sqlite3 ./foo.db status
+    	directory with migration files (default ".")
+
+Commands:
+    up         Migrate the DB to the most recent version available
+    down       Roll back the version by 1
+    redo       Re-run the latest migration
+    status     Dump the migration status for the current DB
+    dbversion  Print the current version of the database
+```
 
 ## create
 
@@ -58,16 +63,6 @@ Apply all available migrations.
     $ OK    002_next.sql
     $ OK    003_and_again.go
 
-### option: pgschema
-
-Use the `pgschema` flag with the `up` command specify a postgres schema.
-
-    $ goose -pgschema=my_schema_name up
-    $ goose: migrating db environment 'development', current version: 0, target: 3
-    $ OK    001_basics.sql
-    $ OK    002_next.sql
-    $ OK    003_and_again.go
-
 ## down
 
 Roll back a single migration from the current version.
@@ -105,10 +100,6 @@ Print the current version of the database:
     $ goose dbversion
     $ goose: dbversion 002
 
-
-`goose -h` provides more detailed info on each command.
-
-
 # Migrations
 
 goose supports migrations written in SQL or in Go - see the `goose create` command above for details on how to generate them.
@@ -164,31 +155,40 @@ language plpgsql;
 
 ## Go Migrations
 
-A sample Go migration looks like:
+Import `github.com/pressly/goose` from your own project (see [example](./example/migrations-go/cmd/main.go)), register migration functions and run goose command (ie. `goose.Up(db *sql.DB, dir string)`).
+
+A [sample Go migration 00002_users_add_email.go file](./example/migrations-go/00002_users_add_email.go) looks like:
 
 ```go
-package main
+package migrations
 
 import (
-    "database/sql"
+	"database/sql"
-    "fmt"
+
+	"github.com/pressly/goose"
 )
 
-func Up_20130106222315(txn *sql.Tx) {
+func init() {
-    fmt.Println("Hello from migration 20130106222315 Up!")
+	goose.AddMigration(Up, Down)
 }
 
-func Down_20130106222315(txn *sql.Tx) {
+func Up(tx *sql.Tx) error {
-    fmt.Println("Hello from migration 20130106222315 Down!")
+	_, err := tx.Query("ALTER TABLE users ADD COLUMN email text DEFAULT '' NOT NULL;")
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func Down(tx *sql.Tx) error {
+	_, err := tx.Query("ALTER TABLE users DROP COLUMN email;")
+	if err != nil {
+		return err
+	}
+	return nil
 }
 ```
 
-`Up_20130106222315()` will be executed as part of a forward migration, and `Down_20130106222315()` will be executed as part of a rollback.
-
-The numeric portion of the function name (`20130106222315`) must be the leading portion of migration's filename, such as `20130106222315_descriptive_name.go`. `goose create` does this by default.
-
-A transaction is provided, rather than the DB instance directly, since goose also needs to record the schema version within the same transaction. Each migration should run as a single transaction to ensure DB integrity, so it's good practice anyway.
-
 ## License
 
 Licensed under [MIT License](./LICENSE)

example/migrations-go/00001_create_users_table.sql

@@ -0,0 +1,14 @@
+-- +goose Up
+CREATE TABLE users (
+    id int NOT NULL PRIMARY KEY,
+    username text,
+    name text,
+    surname text
+);
+
+INSERT INTO users VALUES
+(0, 'root', '', ''),
+(1, 'vojtechvitek', 'Vojtech', 'Vitek');
+
+-- +goose Down
+DROP TABLE users;

example/migrations-go/00002_rename_root.go

@@ -0,0 +1,27 @@
+package migrations
+
+import (
+	"database/sql"
+
+	"github.com/pressly/goose"
+)
+
+func init() {
+	goose.AddMigration(Up, Down)
+}
+
+func Up(tx *sql.Tx) error {
+	_, err := tx.Exec("UPDATE users SET username='admin' WHERE username='root';")
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func Down(tx *sql.Tx) error {
+	_, err := tx.Exec("UPDATE users SET username='root' WHERE username='admin';")
+	if err != nil {
+		return err
+	}
+	return nil
+}

example/migrations-go/cmd/main.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"database/sql"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/pressly/goose"
+
+	_ "github.com/pressly/goose/example/migrations-go"
+
+	_ "github.com/go-sql-driver/mysql"
+	_ "github.com/lib/pq"
+	_ "github.com/mattn/go-sqlite3"
+	_ "github.com/ziutek/mymysql/godrv"
+)
+
+var (
+	flags = flag.NewFlagSet("goose", flag.ExitOnError)
+	dir   = flags.String("dir", ".", "directory with migration files")
+)
+
+func main() {
+	flags.Usage = usage
+	flags.Parse(os.Args[1:])
+
+	args := flags.Args()
+	if len(args) != 3 {
+		flags.Usage()
+		return
+	}
+
+	if args[0] == "-h" || args[0] == "--help" {
+		flags.Usage()
+		return
+	}
+
+	driver, dbstring, command := args[0], args[1], args[2]
+
+	switch driver {
+	case "postgres", "mysql", "sqlite3":
+		if err := goose.SetDialect(driver); err != nil {
+			log.Fatal(err)
+		}
+	default:
+		log.Fatalf("%q driver not supported\n", driver)
+	}
+
+	switch dbstring {
+	case "":
+		log.Fatalf("-dbstring=%q not supported\n", dbstring)
+	default:
+	}
+
+	db, err := sql.Open(driver, dbstring)
+	if err != nil {
+		log.Fatalf("-dbstring=%q: %v\n", dbstring, err)
+	}
+
+	if err := goose.Run(command, db, *dir); err != nil {
+		log.Fatalf("goose run: %v", err)
+	}
+}
+
+func usage() {
+	fmt.Print(usagePrefix)
+	flags.PrintDefaults()
+	fmt.Print(usageCommands)
+}
+
+var (
+	usagePrefix = `Usage: goose [OPTIONS] DRIVER DBSTRING COMMAND
+
+Examples:
+    goose postgres "user=postgres dbname=postgres sslmode=disable" up
+    goose mysql "user:password@/dbname" down
+    goose sqlite3 ./foo.db status
+
+Options:
+`
+
+	usageCommands = `
+Commands:
+    up         Migrate the DB to the most recent version available
+    down       Roll back the version by 1
+    redo       Re-run the latest migration
+    status     Dump the migration status for the current DB
+    dbversion  Print the current version of the database
+`
+)

example/migrations/00001_create_post_table.sql

@@ -1,10 +0,0 @@
--- +goose Up
-CREATE TABLE post (
-    id int NOT NULL,
-    title text,
-    body text,
-    PRIMARY KEY(id)
-);
-
--- +goose Down
-DROP TABLE post;

example/migrations/00001_create_users_table.sql

@@ -0,0 +1,14 @@
+-- +goose Up
+CREATE TABLE users (
+    id int NOT NULL PRIMARY KEY,
+    username text,
+    name text,
+    surname text
+);
+
+INSERT INTO users VALUES
+(0, 'root', '', ''),
+(1, 'vojtechvitek', 'Vojtech', 'Vitek');
+
+-- +goose Down
+DROP TABLE users;

example/migrations/00002_next.sql

@@ -1,11 +0,0 @@
--- +goose Up
-CREATE TABLE fancier_post (
-    id int NOT NULL,
-    title text,
-    body text,
-    created_on timestamp without time zone,
-    PRIMARY KEY(id)
-);
-
--- +goose Down
-DROP TABLE fancier_post;

example/migrations/00002_rename_root.sql

@@ -0,0 +1,9 @@
+-- +goose Up
+-- +goose StatementBegin
+UPDATE users SET username='admin' WHERE username='root';
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+UPDATE users SET username='root' WHERE username='admin';
+-- +goose StatementEnd

migrate.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -17,6 +18,7 @@ import (
 var (
 	ErrTableDoesNotExist = errors.New("table does not exist")
 	ErrNoPreviousVersion = errors.New("no previous version found")
+	goMigrations         []*Migration
 )
 
 type MigrationRecord struct {
@@ -27,9 +29,11 @@ type MigrationRecord struct {
 
 type Migration struct {
 	Version  int64
-	Next     int64  // next version, or -1 if none
+	Next     int64               // next version, or -1 if none
-	Previous int64  // previous version, -1 if none
+	Previous int64               // previous version, -1 if none
-	Source   string // path to .go or .sql script
+	Source   string              // path to .sql script
+	Up       func(*sql.Tx) error // Up go migration function
+	Down     func(*sql.Tx) error // Down go migration function
 }
 
 type migrationSorter []*Migration
@@ -39,8 +43,12 @@ func (ms migrationSorter) Len() int           { return len(ms) }
 func (ms migrationSorter) Swap(i, j int)      { ms[i], ms[j] = ms[j], ms[i] }
 func (ms migrationSorter) Less(i, j int) bool { return ms[i].Version < ms[j].Version }
 
-func newMigration(v int64, src string) *Migration {
+func AddMigration(up func(*sql.Tx) error, down func(*sql.Tx) error) {
-	return &Migration{v, -1, -1, src}
+	_, filename, _, _ := runtime.Caller(1)
+	v, _ := NumericComponent(filename)
+	migration := &Migration{Version: v, Next: -1, Previous: -1, Up: up, Down: down, Source: filename}
+
+	goMigrations = append(goMigrations, migration)
 }
 
 func RunMigrations(db *sql.DB, dir string, target int64) (err error) {
@@ -69,13 +77,33 @@ func RunMigrations(db *sql.DB, dir string, target int64) (err error) {
 
 		switch filepath.Ext(m.Source) {
 		case ".sql":
-		default:
+			if err = runSQLMigration(db, m.Source, m.Version, direction); err != nil {
-			continue
+				return errors.New(fmt.Sprintf("FAIL %v, quitting migration", err))
-		}
+			}
 
-		err = runSQLMigration(db, m.Source, m.Version, direction)
+		case ".go":
-		if err != nil {
+			fn := m.Up
-			return errors.New(fmt.Sprintf("FAIL %v, quitting migration", err))
+			if !direction {
+				fn = m.Down
+			}
+			if fn == nil {
+				continue
+			}
+
+			tx, err := db.Begin()
+			if err != nil {
+				log.Fatal("db.Begin: ", err)
+			}
+
+			if err := fn(tx); err != nil {
+				tx.Rollback()
+				log.Fatalf("FAIL %s (%v), quitting migration.", filepath.Base(m.Source), err)
+				return err
+			}
+
+			if err = FinalizeMigration(tx, direction, m.Version); err != nil {
+				log.Fatalf("error finalizing migration %s, quitting. (%v)", filepath.Base(m.Source), err)
+			}
 		}
 
 		fmt.Println("OK   ", filepath.Base(m.Source))
@@ -85,30 +113,37 @@ func RunMigrations(db *sql.DB, dir string, target int64) (err error) {
 }
 
 // collect all the valid looking migration scripts in the
-// migrations folder, and key them by version
+// migrations folder and go func registry, and key them by version
 func CollectMigrations(dirpath string, current, target int64) (m []*Migration, err error) {
 
 	// extract the numeric component of each migration,
 	// filter out any uninteresting files,
 	// and ensure we only have one file per migration version.
-	filepath.Walk(dirpath, func(name string, info os.FileInfo, err error) error {
+	sqlMigrations, err := filepath.Glob(dirpath + "/*.sql")
+	if err != nil {
+		return nil, err
+	}
 
-		if v, e := NumericComponent(name); e == nil {
+	for _, file := range sqlMigrations {
-
+		v, err := NumericComponent(file)
-			for _, g := range m {
+		if err != nil {
-				if v == g.Version {
+			return nil, err
-					log.Fatalf("more than one file specifies the migration for version %d (%s and %s)",
-						v, g.Source, filepath.Join(dirpath, name))
-				}
-			}
-
-			if versionFilter(v, current, target) {
-				m = append(m, newMigration(v, name))
-			}
 		}
+		if versionFilter(v, current, target) {
+			migration := &Migration{Version: v, Next: -1, Previous: -1, Source: file}
+			m = append(m, migration)
+		}
+	}
 
-		return nil
+	for _, migration := range goMigrations {
-	})
+		v, err := NumericComponent(migration.Source)
+		if err != nil {
+			return nil, err
+		}
+		if versionFilter(v, current, target) {
+			m = append(m, migration)
+		}
+	}
 
 	return m, nil
 }
@@ -341,16 +376,16 @@ func CreateMigration(name, migrationType, dir string, t time.Time) (path string,
 
 // Update the version table for the given migration,
 // and finalize the transaction.
-func FinalizeMigration(txn *sql.Tx, direction bool, v int64) error {
+func FinalizeMigration(tx *sql.Tx, direction bool, v int64) error {
 
 	// XXX: drop goose_db_version table on some minimum version number?
 	stmt := GetDialect().insertVersionSql()
-	if _, err := txn.Exec(stmt, v, direction); err != nil {
+	if _, err := tx.Exec(stmt, v, direction); err != nil {
-		txn.Rollback()
+		tx.Rollback()
 		return err
 	}
 
-	return txn.Commit()
+	return tx.Commit()
 }
 
 var sqlMigrationTemplate = template.Must(template.New("goose.sql-migration").Parse(`

migrate_test.go

@@ -4,6 +4,10 @@ import (
 	"testing"
 )
 
+func newMigration(v int64, src string) *Migration {
+	return &Migration{Version: v, Previous: -1, Next: -1, Source: src}
+}
+
 func TestMigrationMapSortUp(t *testing.T) {
 
 	ms := migrationSorter{}

migration_sql.go

@@ -137,7 +137,7 @@ func splitSQLStatements(r io.Reader, direction bool) (stmts []string) {
 // until another direction directive is found.
 func runSQLMigration(db *sql.DB, scriptFile string, v int64, direction bool) error {
 
-	txn, err := db.Begin()
+	tx, err := db.Begin()
 	if err != nil {
 		log.Fatal("db.Begin:", err)
 	}
@@ -153,14 +153,14 @@ func runSQLMigration(db *sql.DB, scriptFile string, v int64, direction bool) err
 	// records the version into the version table or returns an error and
 	// rolls back the transaction.
 	for _, query := range splitSQLStatements(f, direction) {
-		if _, err = txn.Exec(query); err != nil {
+		if _, err = tx.Exec(query); err != nil {
-			txn.Rollback()
+			tx.Rollback()
 			log.Fatalf("FAIL %s (%v), quitting migration.", filepath.Base(scriptFile), err)
 			return err
 		}
 	}
 
-	if err = FinalizeMigration(txn, direction, v); err != nil {
+	if err = FinalizeMigration(tx, direction, v); err != nil {
 		log.Fatalf("error finalizing migration %s, quitting. (%v)", filepath.Base(scriptFile), err)
 	}