package goose import ( "errors" "fmt" "io/fs" "path/filepath" "sort" "strings" ) // fileSources represents a collection of migration files on the filesystem. type fileSources struct { sqlSources []Source goSources []Source } // collectFilesystemSources scans the file system for migration files that have a numeric prefix // (greater than one) followed by an underscore and a file extension of either .go or .sql. fsys may // be nil, in which case an empty fileSources is returned. // // If strict is true, then any error parsing the numeric component of the filename will result in an // error. The file is skipped otherwise. // // This function DOES NOT parse SQL migrations or merge registered Go migrations. It only collects // migration sources from the filesystem. func collectFilesystemSources( fsys fs.FS, strict bool, excludePaths map[string]bool, excludeVersions map[int64]bool, ) (*fileSources, error) { if fsys == nil { return new(fileSources), nil } sources := new(fileSources) versionToBaseLookup := make(map[int64]string) // map[version]filepath.Base(fullpath) for _, pattern := range []string{ "*.sql", "*.go", } { files, err := fs.Glob(fsys, pattern) if err != nil { return nil, fmt.Errorf("failed to glob pattern %q: %w", pattern, err) } for _, fullpath := range files { base := filepath.Base(fullpath) if strings.HasSuffix(base, "_test.go") { continue } if excludePaths[base] { // TODO(mf): log this? continue } // If the filename has a valid looking version of the form: NUMBER_.{sql,go}, then use // that as the version. Otherwise, ignore it. This allows users to have arbitrary // filenames, but still have versioned migrations within the same directory. For // example, a user could have a helpers.go file which contains unexported helper // functions for migrations. version, err := NumericComponent(base) if err != nil { if strict { return nil, fmt.Errorf("failed to parse numeric component from %q: %w", base, err) } continue } if excludeVersions[version] { // TODO: log this? continue } // Ensure there are no duplicate versions. if existing, ok := versionToBaseLookup[version]; ok { return nil, fmt.Errorf("found duplicate migration version %d:\n\texisting:%v\n\tcurrent:%v", version, existing, base, ) } switch filepath.Ext(base) { case ".sql": sources.sqlSources = append(sources.sqlSources, Source{ Type: TypeSQL, Path: fullpath, Version: version, }) case ".go": sources.goSources = append(sources.goSources, Source{ Type: TypeGo, Path: fullpath, Version: version, }) default: // Should never happen since we already filtered out all other file types. return nil, fmt.Errorf("invalid file extension: %q", base) } // Add the version to the lookup map. versionToBaseLookup[version] = base } } return sources, nil } func newSQLMigration(source Source) *Migration { return &Migration{ Type: source.Type, Version: source.Version, Source: source.Path, construct: true, Next: -1, Previous: -1, sql: sqlMigration{ Parsed: false, // SQL migrations are parsed lazily. }, } } func merge(sources *fileSources, registered map[int64]*Migration) ([]*Migration, error) { var migrations []*Migration migrationLookup := make(map[int64]*Migration) // Add all SQL migrations to the list of migrations. for _, source := range sources.sqlSources { m := newSQLMigration(source) migrations = append(migrations, m) migrationLookup[source.Version] = m } // If there are no Go files in the filesystem and no registered Go migrations, return early. if len(sources.goSources) == 0 && len(registered) == 0 { return migrations, nil } // Return an error if the given sources contain a versioned Go migration that has not been // registered. This is a sanity check to ensure users didn't accidentally create a valid looking // Go migration file on disk and forget to register it. // // This is almost always a user error. var unregistered []string for _, s := range sources.goSources { m, ok := registered[s.Version] if !ok { unregistered = append(unregistered, s.Path) } else { // Populate the source path for registered Go migrations that have a corresponding file // on disk. m.Source = s.Path } } if len(unregistered) > 0 { return nil, unregisteredError(unregistered) } // Add all registered Go migrations to the list of migrations, checking for duplicate versions. // // Important, users can register Go migrations manually via goose.Add_ functions. These // migrations may not have a corresponding file on disk. Which is fine! We include them // wholesale as part of migrations. This allows users to build a custom binary that only embeds // the SQL migration files. for version, r := range registered { // Ensure there are no duplicate versions. if existing, ok := migrationLookup[version]; ok { fullpath := r.Source if fullpath == "" { fullpath = "no source path" } return nil, fmt.Errorf("found duplicate migration version %d:\n\texisting:%v\n\tcurrent:%v", version, existing.Source, fullpath, ) } migrations = append(migrations, r) migrationLookup[version] = r } // Sort migrations by version in ascending order. sort.Slice(migrations, func(i, j int) bool { return migrations[i].Version < migrations[j].Version }) return migrations, nil } func unregisteredError(unregistered []string) error { const ( hintURL = "https://github.com/pressly/goose/tree/master/examples/go-migrations" ) f := "file" if len(unregistered) > 1 { f += "s" } var b strings.Builder b.WriteString(fmt.Sprintf("error: detected %d unregistered Go %s:\n", len(unregistered), f)) for _, name := range unregistered { b.WriteString("\t" + name + "\n") } hint := fmt.Sprintf("hint: go functions must be registered and built into a custom binary see:\n%s", hintURL) b.WriteString(hint) b.WriteString("\n") return errors.New(b.String()) }