migrations: remove old migrations before 0.12.0 (#6270)

pull/6271/head
ᴜɴᴋɴᴡᴏɴ 2020-08-23 12:51:55 +08:00 committed by GitHub
parent 13ae25b785
commit 3ed8c292c2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 13 additions and 610 deletions

View File

@ -12,6 +12,7 @@ All notable changes to Gogs are documented in this file.
### Removed ### Removed
- ⚠️ Migrations before 0.12 are removed, installations not on 0.12 should upgrade to it to run the migrations and then upgrade to 0.13.
- Configuration section `[mailer]` is no longer used. - Configuration section `[mailer]` is no longer used.
- Configuration section `[service]` is no longer used. - Configuration section `[service]` is no longer used.
- Configuration option `APP_NAME` is no longer used. - Configuration option `APP_NAME` is no longer used.

View File

@ -6,17 +6,12 @@ package migrations
import ( import (
"fmt" "fmt"
"strings"
"time"
"github.com/unknwon/com"
log "unknwon.dev/clog/v2" log "unknwon.dev/clog/v2"
"xorm.io/xorm" "xorm.io/xorm"
"gogs.io/gogs/internal/strutil"
) )
const _MIN_DB_VER = 10 const minDBVersion = 19
type Migration interface { type Migration interface {
Description() string Description() string
@ -52,22 +47,11 @@ type Version struct {
var migrations = []Migration{ var migrations = []Migration{
// v0 -> v4 : before 0.6.0 -> last support 0.7.33 // v0 -> v4 : before 0.6.0 -> last support 0.7.33
// v4 -> v10: before 0.7.0 -> last support 0.9.141 // v4 -> v10: before 0.7.0 -> last support 0.9.141
NewMigration("generate rands and salt for organizations", generateOrgRandsAndSalt), // V10 -> V11:v0.8.5 // v10 -> v19: before 0.11.55 -> last support 0.12.0
NewMigration("convert date to unix timestamp", convertDateToUnix), // V11 -> V12:v0.9.2
NewMigration("convert LDAP UseSSL option to SecurityProtocol", ldapUseSSLToSecurityProtocol), // V12 -> V13:v0.9.37
// v13 -> v14:v0.9.87 // Add new migration here, example:
NewMigration("set comment updated with created", setCommentUpdatedWithCreated),
// v14 -> v15:v0.9.147
NewMigration("generate and migrate Git hooks", generateAndMigrateGitHooks),
// v15 -> v16:v0.10.16
NewMigration("update repository sizes", updateRepositorySizes),
// v16 -> v17:v0.10.31
NewMigration("remove invalid protect branch whitelist", removeInvalidProtectBranchWhitelist),
// v17 -> v18:v0.11.48
NewMigration("store long text in repository description field", updateRepositoryDescriptionField),
// v18 -> v19:v0.11.55 // v18 -> v19:v0.11.55
NewMigration("clean unlinked webhook and hook_tasks", cleanUnlinkedWebhookAndHookTasks), // NewMigration("clean unlinked webhook and hook_tasks", cleanUnlinkedWebhookAndHookTasks),
} }
// Migrate database to current version // Migrate database to current version
@ -84,7 +68,7 @@ func Migrate(x *xorm.Engine) error {
// If the version record does not exist we think // If the version record does not exist we think
// it is a fresh installation and we can skip all migrations. // it is a fresh installation and we can skip all migrations.
currentVersion.ID = 0 currentVersion.ID = 0
currentVersion.Version = int64(_MIN_DB_VER + len(migrations)) currentVersion.Version = int64(minDBVersion + len(migrations))
if _, err = x.InsertOne(currentVersion); err != nil { if _, err = x.InsertOne(currentVersion); err != nil {
return fmt.Errorf("insert: %v", err) return fmt.Errorf("insert: %v", err)
@ -92,7 +76,7 @@ func Migrate(x *xorm.Engine) error {
} }
v := currentVersion.Version v := currentVersion.Version
if _MIN_DB_VER > v { if minDBVersion > v {
log.Fatal(` log.Fatal(`
Hi there, thank you for using Gogs for so long! Hi there, thank you for using Gogs for so long!
However, Gogs has stopped supporting auto-migration from your previously installed version. However, Gogs has stopped supporting auto-migration from your previously installed version.
@ -105,8 +89,10 @@ Please save following instructions to somewhere and start working:
https://gogs.io/gogs/releases/tag/v0.7.33 https://gogs.io/gogs/releases/tag/v0.7.33
- If you were using below 0.7.0 (e.g. 0.6.x), download last supported archive from following link: - If you were using below 0.7.0 (e.g. 0.6.x), download last supported archive from following link:
https://gogs.io/gogs/releases/tag/v0.9.141 https://gogs.io/gogs/releases/tag/v0.9.141
- If you were using below 0.11.55 (e.g. 0.9.141), download last supported archive from following link:
https://gogs.io/gogs/releases/tag/v0.12.0
Once finished downloading, Once finished downloading:
1. Extract the archive and to upgrade steps as usual. 1. Extract the archive and to upgrade steps as usual.
2. Run it once. To verify, you should see some migration traces. 2. Run it once. To verify, you should see some migration traces.
@ -118,13 +104,13 @@ In case you're stilling getting this notice, go through instructions again until
return nil return nil
} }
if int(v-_MIN_DB_VER) > len(migrations) { if int(v-minDBVersion) > len(migrations) {
// User downgraded Gogs. // User downgraded Gogs.
currentVersion.Version = int64(len(migrations) + _MIN_DB_VER) currentVersion.Version = int64(len(migrations) + minDBVersion)
_, err = x.Id(1).Update(currentVersion) _, err = x.Id(1).Update(currentVersion)
return err return err
} }
for i, m := range migrations[v-_MIN_DB_VER:] { for i, m := range migrations[v-minDBVersion:] {
log.Info("Migration: %s", m.Description()) log.Info("Migration: %s", m.Description())
if err = m.Migrate(x); err != nil { if err = m.Migrate(x); err != nil {
return fmt.Errorf("do migrate: %v", err) return fmt.Errorf("do migrate: %v", err)
@ -136,255 +122,3 @@ In case you're stilling getting this notice, go through instructions again until
} }
return nil return nil
} }
func generateOrgRandsAndSalt(x *xorm.Engine) (err error) {
type User struct {
ID int64 `xorm:"pk autoincr"`
Rands string `xorm:"VARCHAR(10)"`
Salt string `xorm:"VARCHAR(10)"`
}
orgs := make([]*User, 0, 10)
if err = x.Where("type=1").And("rands=''").Find(&orgs); err != nil {
return fmt.Errorf("select all organizations: %v", err)
}
sess := x.NewSession()
defer sess.Close()
if err = sess.Begin(); err != nil {
return err
}
for _, org := range orgs {
if org.Rands, err = strutil.RandomChars(10); err != nil {
return err
}
if org.Salt, err = strutil.RandomChars(10); err != nil {
return err
}
if _, err = sess.ID(org.ID).Update(org); err != nil {
return err
}
}
return sess.Commit()
}
type TAction struct {
ID int64 `xorm:"pk autoincr"`
CreatedUnix int64
}
func (t *TAction) TableName() string { return "action" }
type TNotice struct {
ID int64 `xorm:"pk autoincr"`
CreatedUnix int64
}
func (t *TNotice) TableName() string { return "notice" }
type TComment struct {
ID int64 `xorm:"pk autoincr"`
CreatedUnix int64
}
func (t *TComment) TableName() string { return "comment" }
type TIssue struct {
ID int64 `xorm:"pk autoincr"`
DeadlineUnix int64
CreatedUnix int64
UpdatedUnix int64
}
func (t *TIssue) TableName() string { return "issue" }
type TMilestone struct {
ID int64 `xorm:"pk autoincr"`
DeadlineUnix int64
ClosedDateUnix int64
}
func (t *TMilestone) TableName() string { return "milestone" }
type TAttachment struct {
ID int64 `xorm:"pk autoincr"`
CreatedUnix int64
}
func (t *TAttachment) TableName() string { return "attachment" }
type TLoginSource struct {
ID int64 `xorm:"pk autoincr"`
CreatedUnix int64
UpdatedUnix int64
}
func (t *TLoginSource) TableName() string { return "login_source" }
type TPull struct {
ID int64 `xorm:"pk autoincr"`
MergedUnix int64
}
func (t *TPull) TableName() string { return "pull_request" }
type TRelease struct {
ID int64 `xorm:"pk autoincr"`
CreatedUnix int64
}
func (t *TRelease) TableName() string { return "release" }
type TRepo struct {
ID int64 `xorm:"pk autoincr"`
CreatedUnix int64
UpdatedUnix int64
}
func (t *TRepo) TableName() string { return "repository" }
type TMirror struct {
ID int64 `xorm:"pk autoincr"`
UpdatedUnix int64
NextUpdateUnix int64
}
func (t *TMirror) TableName() string { return "mirror" }
type TPublicKey struct {
ID int64 `xorm:"pk autoincr"`
CreatedUnix int64
UpdatedUnix int64
}
func (t *TPublicKey) TableName() string { return "public_key" }
type TDeployKey struct {
ID int64 `xorm:"pk autoincr"`
CreatedUnix int64
UpdatedUnix int64
}
func (t *TDeployKey) TableName() string { return "deploy_key" }
type TAccessToken struct {
ID int64 `xorm:"pk autoincr"`
CreatedUnix int64
UpdatedUnix int64
}
func (t *TAccessToken) TableName() string { return "access_token" }
type TUser struct {
ID int64 `xorm:"pk autoincr"`
CreatedUnix int64
UpdatedUnix int64
}
func (t *TUser) TableName() string { return "user" }
type TWebhook struct {
ID int64 `xorm:"pk autoincr"`
CreatedUnix int64
UpdatedUnix int64
}
func (t *TWebhook) TableName() string { return "webhook" }
func convertDateToUnix(x *xorm.Engine) (err error) {
log.Info("This migration could take up to minutes, please be patient.")
type Bean struct {
ID int64 `xorm:"pk autoincr"`
Created time.Time
Updated time.Time
Merged time.Time
Deadline time.Time
ClosedDate time.Time
NextUpdate time.Time
}
var tables = []struct {
name string
cols []string
bean interface{}
}{
{"action", []string{"created"}, new(TAction)},
{"notice", []string{"created"}, new(TNotice)},
{"comment", []string{"created"}, new(TComment)},
{"issue", []string{"deadline", "created", "updated"}, new(TIssue)},
{"milestone", []string{"deadline", "closed_date"}, new(TMilestone)},
{"attachment", []string{"created"}, new(TAttachment)},
{"login_source", []string{"created", "updated"}, new(TLoginSource)},
{"pull_request", []string{"merged"}, new(TPull)},
{"release", []string{"created"}, new(TRelease)},
{"repository", []string{"created", "updated"}, new(TRepo)},
{"mirror", []string{"updated", "next_update"}, new(TMirror)},
{"public_key", []string{"created", "updated"}, new(TPublicKey)},
{"deploy_key", []string{"created", "updated"}, new(TDeployKey)},
{"access_token", []string{"created", "updated"}, new(TAccessToken)},
{"user", []string{"created", "updated"}, new(TUser)},
{"webhook", []string{"created", "updated"}, new(TWebhook)},
}
for _, table := range tables {
log.Info("Converting table: %s", table.name)
if err = x.Sync2(table.bean); err != nil {
return fmt.Errorf("Sync [table: %s]: %v", table.name, err)
}
offset := 0
for {
beans := make([]*Bean, 0, 100)
if err = x.SQL(fmt.Sprintf("SELECT * FROM `%s` ORDER BY id ASC LIMIT 100 OFFSET %d",
table.name, offset)).Find(&beans); err != nil {
return fmt.Errorf("select beans [table: %s, offset: %d]: %v", table.name, offset, err)
}
log.Trace("Table [%s]: offset: %d, beans: %d", table.name, offset, len(beans))
if len(beans) == 0 {
break
}
offset += 100
baseSQL := "UPDATE `" + table.name + "` SET "
for _, bean := range beans {
valSQLs := make([]string, 0, len(table.cols))
for _, col := range table.cols {
fieldSQL := ""
fieldSQL += col + "_unix = "
switch col {
case "deadline":
if bean.Deadline.IsZero() {
continue
}
fieldSQL += com.ToStr(bean.Deadline.Unix())
case "created":
fieldSQL += com.ToStr(bean.Created.Unix())
case "updated":
fieldSQL += com.ToStr(bean.Updated.Unix())
case "closed_date":
fieldSQL += com.ToStr(bean.ClosedDate.Unix())
case "merged":
fieldSQL += com.ToStr(bean.Merged.Unix())
case "next_update":
fieldSQL += com.ToStr(bean.NextUpdate.Unix())
}
valSQLs = append(valSQLs, fieldSQL)
}
if len(valSQLs) == 0 {
continue
}
if _, err = x.Exec(baseSQL + strings.Join(valSQLs, ",") + " WHERE id = " + com.ToStr(bean.ID)); err != nil {
return fmt.Errorf("update bean [table: %s, id: %d]: %v", table.name, bean.ID, err)
}
}
}
}
return nil
}

View File

@ -1,52 +0,0 @@
// Copyright 2016 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"fmt"
"strings"
"github.com/unknwon/com"
"xorm.io/xorm"
"github.com/json-iterator/go"
)
func ldapUseSSLToSecurityProtocol(x *xorm.Engine) error {
results, err := x.Query("SELECT `id`,`cfg` FROM `login_source` WHERE `type` = 2 OR `type` = 5")
if err != nil {
if strings.Contains(err.Error(), "no such column") {
return nil
}
return fmt.Errorf("select LDAP login sources: %v", err)
}
sess := x.NewSession()
defer sess.Close()
if err = sess.Begin(); err != nil {
return err
}
for _, result := range results {
cfg := map[string]interface{}{}
if err = jsoniter.Unmarshal(result["cfg"], &cfg); err != nil {
return fmt.Errorf("unmarshal JSON config: %v", err)
}
if com.ToStr(cfg["UseSSL"]) == "true" {
cfg["SecurityProtocol"] = 1 // LDAPS
}
delete(cfg, "UseSSL")
data, err := jsoniter.Marshal(&cfg)
if err != nil {
return fmt.Errorf("marshal JSON config: %v", err)
}
if _, err = sess.Exec("UPDATE `login_source` SET `cfg`=? WHERE `id`=?",
string(data), com.StrTo(result["id"]).MustInt64()); err != nil {
return fmt.Errorf("update config column: %v", err)
}
}
return sess.Commit()
}

View File

@ -1,24 +0,0 @@
// Copyright 2016 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"fmt"
"xorm.io/xorm"
)
func setCommentUpdatedWithCreated(x *xorm.Engine) (err error) {
type Comment struct {
UpdatedUnix int64
}
if err = x.Sync2(new(Comment)); err != nil {
return fmt.Errorf("Sync2: %v", err)
} else if _, err = x.Exec("UPDATE comment SET updated_unix = created_unix"); err != nil {
return fmt.Errorf("set update_unix: %v", err)
}
return nil
}

View File

@ -1,105 +0,0 @@
// Copyright 2017 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"github.com/unknwon/com"
log "unknwon.dev/clog/v2"
"xorm.io/xorm"
"gogs.io/gogs/internal/conf"
"gogs.io/gogs/internal/osutil"
)
func generateAndMigrateGitHooks(x *xorm.Engine) (err error) {
type Repository struct {
ID int64
OwnerID int64
Name string
}
type User struct {
ID int64
Name string
}
var (
hookNames = []string{"pre-receive", "update", "post-receive"}
hookTpls = []string{
fmt.Sprintf("#!/usr/bin/env %s\n\"%s\" hook --config='%s' pre-receive\n", conf.Repository.ScriptType, conf.AppPath(), conf.CustomConf),
fmt.Sprintf("#!/usr/bin/env %s\n\"%s\" hook --config='%s' update $1 $2 $3\n", conf.Repository.ScriptType, conf.AppPath(), conf.CustomConf),
fmt.Sprintf("#!/usr/bin/env %s\n\"%s\" hook --config='%s' post-receive\n", conf.Repository.ScriptType, conf.AppPath(), conf.CustomConf),
}
)
// Cleanup old update.log and http.log files.
_ = filepath.Walk(conf.Log.RootPath, func(path string, info os.FileInfo, err error) error {
if !info.IsDir() &&
(strings.HasPrefix(filepath.Base(path), "update.log") ||
strings.HasPrefix(filepath.Base(path), "http.log")) {
_ = os.Remove(path)
}
return nil
})
return x.Where("id > 0").Iterate(new(Repository),
func(idx int, bean interface{}) error {
repo := bean.(*Repository)
if repo.Name == "." || repo.Name == ".." {
return nil
}
user := new(User)
has, err := x.Where("id = ?", repo.OwnerID).Get(user)
if err != nil {
return fmt.Errorf("query owner of repository [repo_id: %d, owner_id: %d]: %v", repo.ID, repo.OwnerID, err)
} else if !has {
return nil
}
repoBase := filepath.Join(conf.Repository.Root, strings.ToLower(user.Name), strings.ToLower(repo.Name))
repoPath := repoBase + ".git"
wikiPath := repoBase + ".wiki.git"
log.Trace("[%04d]: %s", idx, repoPath)
// Note: we should not create hookDir here because update hook file should already exists inside this direcotry,
// if this directory does not exist, the current setup is not correct anyway.
hookDir := filepath.Join(repoPath, "hooks")
customHookDir := filepath.Join(repoPath, "custom_hooks")
wikiHookDir := filepath.Join(wikiPath, "hooks")
for i, hookName := range hookNames {
oldHookPath := filepath.Join(hookDir, hookName)
newHookPath := filepath.Join(customHookDir, hookName)
// Gogs didn't allow user to set custom update hook thus no migration for it.
// In case user runs this migration multiple times, and custom hook exists,
// we assume it's been migrated already.
if hookName != "update" && osutil.IsFile(oldHookPath) && !com.IsExist(customHookDir) {
_ = os.MkdirAll(customHookDir, os.ModePerm)
if err = os.Rename(oldHookPath, newHookPath); err != nil {
return fmt.Errorf("move hook file to custom directory '%s' -> '%s': %v", oldHookPath, newHookPath, err)
}
}
if err = ioutil.WriteFile(oldHookPath, []byte(hookTpls[i]), os.ModePerm); err != nil {
return fmt.Errorf("write hook file '%s': %v", oldHookPath, err)
}
if com.IsDir(wikiPath) {
_ = os.MkdirAll(wikiHookDir, os.ModePerm)
wikiHookPath := filepath.Join(wikiHookDir, hookName)
if err = ioutil.WriteFile(wikiHookPath, []byte(hookTpls[i]), os.ModePerm); err != nil {
return fmt.Errorf("write wiki hook file '%s': %v", wikiHookPath, err)
}
}
}
return nil
})
}

View File

@ -1,77 +0,0 @@
// Copyright 2017 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"fmt"
"path/filepath"
"strings"
log "unknwon.dev/clog/v2"
"xorm.io/xorm"
"github.com/gogs/git-module"
"gogs.io/gogs/internal/conf"
)
func updateRepositorySizes(x *xorm.Engine) (err error) {
log.Info("[migrations.v16] This migration could take up to minutes, please be patient.")
type Repository struct {
ID int64
OwnerID int64
Name string
Size int64
}
type User struct {
ID int64
Name string
}
if err = x.Sync2(new(Repository)); err != nil {
return fmt.Errorf("Sync2: %v", err)
}
// For the sake of SQLite3, we can't use x.Iterate here.
offset := 0
for {
repos := make([]*Repository, 0, 10)
if err = x.SQL(fmt.Sprintf("SELECT * FROM `repository` ORDER BY id ASC LIMIT 10 OFFSET %d", offset)).
Find(&repos); err != nil {
return fmt.Errorf("select repos [offset: %d]: %v", offset, err)
}
log.Trace("[migrations.v16] Select [offset: %d, repos: %d]", offset, len(repos))
if len(repos) == 0 {
break
}
offset += 10
for _, repo := range repos {
if repo.Name == "." || repo.Name == ".." {
continue
}
user := new(User)
has, err := x.Where("id = ?", repo.OwnerID).Get(user)
if err != nil {
return fmt.Errorf("query owner of repository [repo_id: %d, owner_id: %d]: %v", repo.ID, repo.OwnerID, err)
} else if !has {
continue
}
repoPath := strings.ToLower(filepath.Join(conf.Repository.Root, user.Name, repo.Name)) + ".git"
countObject, err := git.RepoCountObjects(repoPath)
if err != nil {
log.Warn("[migrations.v16] Count repository objects: %v", err)
continue
}
repo.Size = countObject.Size + countObject.SizePack
if _, err = x.Id(repo.ID).Cols("size").Update(repo); err != nil {
return fmt.Errorf("update size: %v", err)
}
}
}
return nil
}

View File

@ -1,22 +0,0 @@
// Copyright 2017 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"fmt"
"xorm.io/xorm"
)
func removeInvalidProtectBranchWhitelist(x *xorm.Engine) error {
exist, err := x.IsTableExist("protect_branch_whitelist")
if err != nil {
return fmt.Errorf("IsTableExist: %v", err)
} else if !exist {
return nil
}
_, err = x.Exec("DELETE FROM protect_branch_whitelist WHERE protect_branch_id = 0")
return err
}

View File

@ -1,34 +0,0 @@
// Copyright 2018 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"fmt"
"xorm.io/xorm"
"gogs.io/gogs/internal/conf"
)
func updateRepositoryDescriptionField(x *xorm.Engine) error {
exist, err := x.IsTableExist("repository")
if err != nil {
return fmt.Errorf("IsTableExist: %v", err)
} else if !exist {
return nil
}
switch {
case conf.UseMySQL:
_, err = x.Exec("ALTER TABLE `repository` MODIFY `description` VARCHAR(512);")
case conf.UseMSSQL:
_, err = x.Exec("ALTER TABLE `repository` ALTER COLUMN `description` VARCHAR(512);")
case conf.UsePostgreSQL:
_, err = x.Exec("ALTER TABLE `repository` ALTER COLUMN `description` TYPE VARCHAR(512);")
case conf.UseSQLite3:
// Sqlite3 uses TEXT type by default for any string type field.
// Keep this comment to mention that we don't missed any option.
}
return err
}

View File

@ -1,18 +0,0 @@
// Copyright 2018 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"xorm.io/xorm"
)
func cleanUnlinkedWebhookAndHookTasks(x *xorm.Engine) error {
_, err := x.Exec(`DELETE FROM webhook WHERE repo_id NOT IN (SELECT id FROM repository);`)
if err != nil {
return err
}
_, err = x.Exec(`DELETE FROM hook_task WHERE repo_id NOT IN (SELECT id FROM repository);`)
return err
}