From b538c5345e1271fcaf413b82ba1ece555dd635a5 Mon Sep 17 00:00:00 2001 From: Unknwon Date: Sat, 9 Jun 2018 17:21:58 +0800 Subject: [PATCH] restore: reset original created_unix after insert (#5264) --- cmd/backup.go | 3 ++- cmd/restore.go | 30 +++++++++++++++++++++--------- gogs.go | 2 +- models/models.go | 40 ++++++++++++++++++++++++++++------------ models/webhook.go | 8 ++++---- templates/.VERSION | 2 +- 6 files changed, 57 insertions(+), 28 deletions(-) diff --git a/cmd/backup.go b/cmd/backup.go index 4ff62887d..ad635a4d8 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -39,6 +39,7 @@ portable among all supported database engines.`, }, } +const _CURRENT_BACKUP_FORMAT_VERSION = 1 const _ARCHIVE_ROOT_DIR = "gogs-backup" func runBackup(c *cli.Context) error { @@ -63,7 +64,7 @@ func runBackup(c *cli.Context) error { // Metadata metaFile := path.Join(rootDir, "metadata.ini") metadata := ini.Empty() - metadata.Section("").Key("VERSION").SetValue("1") + metadata.Section("").Key("VERSION").SetValue(com.ToStr(_CURRENT_BACKUP_FORMAT_VERSION)) metadata.Section("").Key("DATE_TIME").SetValue(time.Now().String()) metadata.Section("").Key("GOGS_VERSION").SetValue(setting.AppVer) if err = metadata.SaveTo(metaFile); err != nil { diff --git a/cmd/restore.go b/cmd/restore.go index db81b8b08..9591b044c 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -39,6 +39,10 @@ be skipped and remain unchanged.`, }, } +// lastSupportedVersionOfFormat returns the last supported version of the backup archive +// format that is able to import. +var lastSupportedVersionOfFormat = map[int]string{} + func runRestore(c *cli.Context) error { zip.Verbose = c.Bool("verbose") @@ -49,9 +53,10 @@ func runRestore(c *cli.Context) error { log.Info("Restore backup from: %s", c.String("from")) if err := zip.ExtractTo(c.String("from"), tmpDir); err != nil { - log.Fatal(0, "Fail to extract backup archive: %v", err) + log.Fatal(0, "Failed to extract backup archive: %v", err) } archivePath := path.Join(tmpDir, _ARCHIVE_ROOT_DIR) + defer os.RemoveAll(archivePath) // Check backup version metaFile := path.Join(archivePath, "metadata.ini") @@ -60,12 +65,20 @@ func runRestore(c *cli.Context) error { } metadata, err := ini.Load(metaFile) if err != nil { - log.Fatal(0, "Fail to load metadata '%s': %v", metaFile, err) + log.Fatal(0, "Failed to load metadata '%s': %v", metaFile, err) } backupVersion := metadata.Section("").Key("GOGS_VERSION").MustString("999.0") if version.Compare(setting.AppVer, backupVersion, "<") { log.Fatal(0, "Current Gogs version is lower than backup version: %s < %s", setting.AppVer, backupVersion) } + formatVersion := metadata.Section("").Key("VERSION").MustInt() + if formatVersion == 0 { + log.Fatal(0, "Failed to determine the backup format version from metadata '%s': %s", metaFile, "VERSION is not presented") + } + if formatVersion != _CURRENT_BACKUP_FORMAT_VERSION { + log.Fatal(0, "Backup format version found is %d but this binary only supports %d\nThe last known version that is able to import your backup is %s", + formatVersion, _CURRENT_BACKUP_FORMAT_VERSION, lastSupportedVersionOfFormat[formatVersion]) + } // If config file is not present in backup, user must set this file via flag. // Otherwise, it's optional to set config file flag. @@ -84,18 +97,18 @@ func runRestore(c *cli.Context) error { // Database dbDir := path.Join(archivePath, "db") if err = models.ImportDatabase(dbDir, c.Bool("verbose")); err != nil { - log.Fatal(0, "Fail to import database: %v", err) + log.Fatal(0, "Failed to import database: %v", err) } // Custom files if !c.Bool("database-only") { if com.IsExist(setting.CustomPath) { if err = os.Rename(setting.CustomPath, setting.CustomPath+".bak"); err != nil { - log.Fatal(0, "Fail to backup current 'custom': %v", err) + log.Fatal(0, "Failed to backup current 'custom': %v", err) } } if err = os.Rename(path.Join(archivePath, "custom"), setting.CustomPath); err != nil { - log.Fatal(0, "Fail to import 'custom': %v", err) + log.Fatal(0, "Failed to import 'custom': %v", err) } } @@ -112,11 +125,11 @@ func runRestore(c *cli.Context) error { dirPath := path.Join(setting.AppDataPath, dir) if com.IsExist(dirPath) { if err = os.Rename(dirPath, dirPath+".bak"); err != nil { - log.Fatal(0, "Fail to backup current 'data': %v", err) + log.Fatal(0, "Failed to backup current 'data': %v", err) } } if err = os.Rename(srcPath, dirPath); err != nil { - log.Fatal(0, "Fail to import 'data': %v", err) + log.Fatal(0, "Failed to import 'data': %v", err) } } } @@ -125,11 +138,10 @@ func runRestore(c *cli.Context) error { reposPath := path.Join(archivePath, "repositories.zip") if !c.Bool("exclude-repos") && !c.Bool("database-only") && com.IsExist(reposPath) { if err := zip.ExtractTo(reposPath, path.Dir(setting.RepoRootPath)); err != nil { - log.Fatal(0, "Fail to extract 'repositories.zip': %v", err) + log.Fatal(0, "Failed to extract 'repositories.zip': %v", err) } } - os.RemoveAll(path.Join(tmpDir, _ARCHIVE_ROOT_DIR)) log.Info("Restore succeed!") log.Shutdown() return nil diff --git a/gogs.go b/gogs.go index 11b1caf34..d45c1904a 100644 --- a/gogs.go +++ b/gogs.go @@ -16,7 +16,7 @@ import ( "github.com/gogs/gogs/pkg/setting" ) -const APP_VER = "0.11.54.0609" +const APP_VER = "0.11.55.0609" func init() { setting.AppVer = APP_VER diff --git a/models/models.go b/models/models.go index f33c32d0b..4392bb764 100644 --- a/models/models.go +++ b/models/models.go @@ -7,7 +7,6 @@ package models import ( "bufio" "database/sql" - "encoding/json" "errors" "fmt" "net/url" @@ -20,6 +19,7 @@ import ( _ "github.com/go-sql-driver/mysql" "github.com/go-xorm/core" "github.com/go-xorm/xorm" + "github.com/json-iterator/go" _ "github.com/lib/pq" log "gopkg.in/clog.v1" @@ -285,8 +285,7 @@ func DumpDatabase(dirPath string) (err error) { } if err = x.Asc("id").Iterate(table, func(idx int, bean interface{}) (err error) { - enc := json.NewEncoder(f) - return enc.Encode(bean) + return jsoniter.NewEncoder(f).Encode(bean) }); err != nil { f.Close() return fmt.Errorf("fail to dump table '%s': %v", tableName, err) @@ -300,6 +299,10 @@ func DumpDatabase(dirPath string) (err error) { func ImportDatabase(dirPath string, verbose bool) (err error) { snakeMapper := core.SnakeMapper{} + skipInsertProcessors := map[string]bool{ + "mirror": true, + } + // Purposely create a local variable to not modify global variable tables := append(tables, new(Version)) for _, table := range tables { @@ -314,22 +317,24 @@ func ImportDatabase(dirPath string, verbose bool) (err error) { } if err = x.DropTables(table); err != nil { - return fmt.Errorf("fail to drop table '%s': %v", tableName, err) + return fmt.Errorf("drop table '%s': %v", tableName, err) } else if err = x.Sync2(table); err != nil { - return fmt.Errorf("fail to sync table '%s': %v", tableName, err) + return fmt.Errorf("sync table '%s': %v", tableName, err) } f, err := os.Open(tableFile) if err != nil { - return fmt.Errorf("fail to open JSON file: %v", err) + return fmt.Errorf("open JSON file: %v", err) } + rawTableName := x.TableName(table) + _, isInsertProcessor := table.(xorm.BeforeInsertProcessor) scanner := bufio.NewScanner(f) for scanner.Scan() { switch bean := table.(type) { case *LoginSource: meta := make(map[string]interface{}) - if err = json.Unmarshal(scanner.Bytes(), &meta); err != nil { - return fmt.Errorf("fail to unmarshal to map: %v", err) + if err = jsoniter.Unmarshal(scanner.Bytes(), &meta); err != nil { + return fmt.Errorf("unmarshal to map: %v", err) } tp := LoginType(com.StrTo(com.ToStr(meta["Type"])).MustInt64()) @@ -346,12 +351,23 @@ func ImportDatabase(dirPath string, verbose bool) (err error) { table = bean } - if err = json.Unmarshal(scanner.Bytes(), table); err != nil { - return fmt.Errorf("fail to unmarshal to struct: %v", err) + if err = jsoniter.Unmarshal(scanner.Bytes(), table); err != nil { + return fmt.Errorf("unmarshal to struct: %v", err) } if _, err = x.Insert(table); err != nil { - return fmt.Errorf("fail to insert strcut: %v", err) + return fmt.Errorf("insert strcut: %v", err) + } + + // Reset created_unix back to the date save in archive because Insert method updates its value + if isInsertProcessor && !skipInsertProcessors[rawTableName] { + meta := make(map[string]interface{}) + if err = jsoniter.Unmarshal(scanner.Bytes(), &meta); err != nil { + log.Error(2, "Failed to unmarshal to map: %v", err) + } + if _, err = x.Exec("UPDATE "+rawTableName+" SET created_unix=? WHERE id=?", meta["CreatedUnix"], meta["ID"]); err != nil { + log.Error(2, "Failed to reset 'created_unix': %v", err) + } } } @@ -360,7 +376,7 @@ func ImportDatabase(dirPath string, verbose bool) (err error) { rawTableName := snakeMapper.Obj2Table(tableName) seqName := rawTableName + "_id_seq" if _, err = x.Exec(fmt.Sprintf(`SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM "%s"), 1), false);`, seqName, rawTableName)); err != nil { - return fmt.Errorf("fail to reset table '%s' sequence: %v", rawTableName, err) + return fmt.Errorf("reset table '%s' sequence: %v", rawTableName, err) } } } diff --git a/models/webhook.go b/models/webhook.go index 7d0287c5d..ca2ee5883 100644 --- a/models/webhook.go +++ b/models/webhook.go @@ -97,10 +97,10 @@ type Webhook struct { OrgID int64 URL string `xorm:"url TEXT"` ContentType HookContentType - Secret string `xorm:"TEXT"` - Events string `xorm:"TEXT"` - *HookEvent `xorm:"-" json:"-"` - IsSSL bool `xorm:"is_ssl"` + Secret string `xorm:"TEXT"` + Events string `xorm:"TEXT"` + *HookEvent `xorm:"-"` // LEGACY [1.0]: Cannot ignore JSON here, it breaks old backup archive + IsSSL bool `xorm:"is_ssl"` IsActive bool HookTaskType HookTaskType Meta string `xorm:"TEXT"` // store hook-specific attributes diff --git a/templates/.VERSION b/templates/.VERSION index 267b4b0b8..73c982ebb 100644 --- a/templates/.VERSION +++ b/templates/.VERSION @@ -1 +1 @@ -0.11.54.0609 \ No newline at end of file +0.11.55.0609 \ No newline at end of file