db: use GORM to backup and restore non-legacy tables (#6142)

CHANGELOG.md

@@ -47,6 +47,7 @@ All notable changes to Gogs are documented in this file.
 - [Security] Potential RCE on mirror repositories. [#5767](https://github.com/gogs/gogs/issues/5767)
 - [Security] Potential XSS attack with raw markdown API. [#5907](https://github.com/gogs/gogs/pull/5907)
 - File both modified and renamed within a commit treated as separate files. [#5056](https://github.com/gogs/gogs/issues/5056)
+- Unable to restore the database backup to MySQL 8.0 with syntax error. [#5602](https://github.com/gogs/gogs/issues/5602)
 - Open/close milestone redirects to a 404 page. [#5677](https://github.com/gogs/gogs/issues/5677)
 - Disallow multiple tokens with same name. [#5587](https://github.com/gogs/gogs/issues/5587) [#5820](https://github.com/gogs/gogs/pull/5820)
 - Enable Federated Avatar Lookup could cause server to crash. [#5848](https://github.com/gogs/gogs/issues/5848)

go.sum

@@ -39,8 +39,6 @@ github.com/denisenkom/go-mssqldb v0.0.0-20200206145737-bbfc9a55622e/go.mod h1:xb
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
-github.com/editorconfig/editorconfig-core-go/v2 v2.3.1 h1:8+L7G4cCtuYprGaNawfTBq20m8+VpPCH2O0vwKS7r84=
-github.com/editorconfig/editorconfig-core-go/v2 v2.3.1/go.mod h1:mJYZ8yC2PWr+pabYXwHMfcEe45fh2w2sxk8cudJdLPM=
 github.com/editorconfig/editorconfig-core-go/v2 v2.3.2 h1:j9GLz0kWF9+1T3IX0MOhhvzLtqhFOvIKLhZFxtY95Qc=
 github.com/editorconfig/editorconfig-core-go/v2 v2.3.2/go.mod h1:+u4rFiKVvlbukHyJM76GYXqQcnHScxvQCuKpMLRtJVw=
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
@@ -405,7 +403,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE=
 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
 gopkg.in/ini.v1 v1.46.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/ini.v1 v1.54.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.55.0 h1:E8yzL5unfpW3M6fz/eB7Cb5MQAYSZ7GKo4Qth+N2sgQ=
 gopkg.in/ini.v1 v1.55.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ldap.v2 v2.5.1 h1:wiu0okdNfjlBzg6UWvd1Hn8Y+Ux17/u/4nlk4CQr6tU=

internal/cmd/admin.go

@@ -147,7 +147,7 @@ func runCreateUser(c *cli.Context) error {
 	}
 	conf.InitLogging(true)
 
-	if err = db.SetEngine(); err != nil {
+	if _, err = db.SetEngine(); err != nil {
 		return errors.Wrap(err, "set engine")
 	}
 
@@ -173,7 +173,7 @@ func adminDashboardOperation(operation func() error, successMessage string) func
 		}
 		conf.InitLogging(true)
 
-		if err = db.SetEngine(); err != nil {
+		if _, err = db.SetEngine(); err != nil {
 			return errors.Wrap(err, "set engine")
 		}
 

internal/cmd/backup.go

@@ -42,8 +42,8 @@ portable among all supported database engines.`,
 	},
 }
 
-const _CURRENT_BACKUP_FORMAT_VERSION = 1
+const currentBackupFormatVersion = 1
-const _ARCHIVE_ROOT_DIR = "gogs-backup"
+const archiveRootDir = "gogs-backup"
 
 func runBackup(c *cli.Context) error {
 	zip.Verbose = c.Bool("verbose")
@@ -54,7 +54,8 @@ func runBackup(c *cli.Context) error {
 	}
 	conf.InitLogging(true)
 
-	if err = db.SetEngine(); err != nil {
+	conn, err := db.SetEngine()
+	if err != nil {
 		return errors.Wrap(err, "set engine")
 	}
 
@@ -71,7 +72,7 @@ func runBackup(c *cli.Context) error {
 	// Metadata
 	metaFile := path.Join(rootDir, "metadata.ini")
 	metadata := ini.Empty()
-	metadata.Section("").Key("VERSION").SetValue(com.ToStr(_CURRENT_BACKUP_FORMAT_VERSION))
+	metadata.Section("").Key("VERSION").SetValue(com.ToStr(currentBackupFormatVersion))
 	metadata.Section("").Key("DATE_TIME").SetValue(time.Now().String())
 	metadata.Section("").Key("GOGS_VERSION").SetValue(conf.App.Version)
 	if err = metadata.SaveTo(metaFile); err != nil {
@@ -85,22 +86,22 @@ func runBackup(c *cli.Context) error {
 	if err != nil {
 		log.Fatal("Failed to create backup archive '%s': %v", archiveName, err)
 	}
-	if err = z.AddFile(_ARCHIVE_ROOT_DIR+"/metadata.ini", metaFile); err != nil {
+	if err = z.AddFile(archiveRootDir+"/metadata.ini", metaFile); err != nil {
 		log.Fatal("Failed to include 'metadata.ini': %v", err)
 	}
 
 	// Database
 	dbDir := filepath.Join(rootDir, "db")
-	if err = db.DumpDatabase(dbDir); err != nil {
+	if err = db.DumpDatabase(conn, dbDir, c.Bool("verbose")); err != nil {
 		log.Fatal("Failed to dump database: %v", err)
 	}
-	if err = z.AddDir(_ARCHIVE_ROOT_DIR+"/db", dbDir); err != nil {
+	if err = z.AddDir(archiveRootDir+"/db", dbDir); err != nil {
 		log.Fatal("Failed to include 'db': %v", err)
 	}
 
 	// Custom files
 	if !c.Bool("database-only") {
-		if err = z.AddDir(_ARCHIVE_ROOT_DIR+"/custom", conf.CustomDir()); err != nil {
+		if err = z.AddDir(archiveRootDir+"/custom", conf.CustomDir()); err != nil {
 			log.Fatal("Failed to include 'custom': %v", err)
 		}
 	}
@@ -113,7 +114,7 @@ func runBackup(c *cli.Context) error {
 				continue
 			}
 
-			if err = z.AddDir(path.Join(_ARCHIVE_ROOT_DIR+"/data", dir), dirPath); err != nil {
+			if err = z.AddDir(path.Join(archiveRootDir+"/data", dir), dirPath); err != nil {
 				log.Fatal("Failed to include 'data': %v", err)
 			}
 		}
@@ -149,7 +150,7 @@ func runBackup(c *cli.Context) error {
 		}
 		log.Info("Repositories dumped to: %s", reposDump)
 
-		if err = z.AddFile(_ARCHIVE_ROOT_DIR+"/repositories.zip", reposDump); err != nil {
+		if err = z.AddFile(archiveRootDir+"/repositories.zip", reposDump); err != nil {
 			log.Fatal("Failed to include %q: %v", reposDump, err)
 		}
 	}
@@ -158,7 +159,7 @@ func runBackup(c *cli.Context) error {
 		log.Fatal("Failed to save backup archive '%s': %v", archiveName, err)
 	}
 
-	os.RemoveAll(rootDir)
+	_ = os.RemoveAll(rootDir)
 	log.Info("Backup succeed! Archive is located at: %s", archiveName)
 	log.Stop()
 	return nil

internal/cmd/restore.go

@@ -57,7 +57,7 @@ func runRestore(c *cli.Context) error {
 	if err := zip.ExtractTo(c.String("from"), tmpDir); err != nil {
 		log.Fatal("Failed to extract backup archive: %v", err)
 	}
-	archivePath := path.Join(tmpDir, _ARCHIVE_ROOT_DIR)
+	archivePath := path.Join(tmpDir, archiveRootDir)
 	defer os.RemoveAll(archivePath)
 
 	// Check backup version
@@ -77,9 +77,9 @@ func runRestore(c *cli.Context) error {
 	if formatVersion == 0 {
 		log.Fatal("Failed to determine the backup format version from metadata '%s': %s", metaFile, "VERSION is not presented")
 	}
-	if formatVersion != _CURRENT_BACKUP_FORMAT_VERSION {
+	if formatVersion != currentBackupFormatVersion {
 		log.Fatal("Backup format version found is %d but this binary only supports %d\nThe last known version that is able to import your backup is %s",
-			formatVersion, _CURRENT_BACKUP_FORMAT_VERSION, lastSupportedVersionOfFormat[formatVersion])
+			formatVersion, currentBackupFormatVersion, lastSupportedVersionOfFormat[formatVersion])
 	}
 
 	// If config file is not present in backup, user must set this file via flag.
@@ -100,13 +100,14 @@ func runRestore(c *cli.Context) error {
 	}
 	conf.InitLogging(true)
 
-	if err = db.SetEngine(); err != nil {
+	conn, err := db.SetEngine()
+	if err != nil {
 		return errors.Wrap(err, "set engine")
 	}
 
 	// Database
 	dbDir := path.Join(archivePath, "db")
-	if err = db.ImportDatabase(dbDir, c.Bool("verbose")); err != nil {
+	if err = db.ImportDatabase(conn, dbDir, c.Bool("verbose")); err != nil {
 		log.Fatal("Failed to import database: %v", err)
 	}
 

internal/cmd/serv.go

@@ -93,7 +93,7 @@ func setup(c *cli.Context, logPath string, connectDB bool) {
 		_ = os.Chdir(conf.WorkDir())
 	}
 
-	if err := db.SetEngine(); err != nil {
+	if _, err := db.SetEngine(); err != nil {
 		fail("Internal error", "Failed to set database engine: %v", err)
 	}
 }

internal/cryptoutil/sha1.go

@@ -0,0 +1,17 @@
+// Copyright 2020 The Gogs Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cryptoutil
+
+import (
+	"crypto/sha1"
+	"encoding/hex"
+)
+
+// SHA1 encodes string to hexadecimal of SHA1 checksum.
+func SHA1(str string) string {
+	h := sha1.New()
+	_, _ = h.Write([]byte(str))
+	return hex.EncodeToString(h.Sum(nil))
+}

internal/cryptoutil/sha1_test.go

@@ -0,0 +1,27 @@
+// Copyright 2020 The Gogs Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cryptoutil
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSHA1(t *testing.T) {
+	tests := []struct {
+		input  string
+		output string
+	}{
+		{input: "", output: "da39a3ee5e6b4b0d3255bfef95601890afd80709"},
+		{input: "The quick brown fox jumps over the lazy dog", output: "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12"},
+		{input: "The quick brown fox jumps over the lazy dog.", output: "408d94384216f890ff7a0c3528e8bed1e0b01621"},
+	}
+	for _, test := range tests {
+		t.Run(test.input, func(t *testing.T) {
+			assert.Equal(t, test.output, SHA1(test.input))
+		})
+	}
+}

internal/db/access_tokens.go

@@ -11,8 +11,8 @@ import (
 	"github.com/jinzhu/gorm"
 	gouuid "github.com/satori/go.uuid"
 
+	"gogs.io/gogs/internal/cryptoutil"
 	"gogs.io/gogs/internal/errutil"
-	"gogs.io/gogs/internal/tool"
 )
 
 // AccessTokensStore is the persistent interface for access tokens.
@@ -56,6 +56,9 @@ type AccessToken struct {
 
 // NOTE: This is a GORM create hook.
 func (t *AccessToken) BeforeCreate() {
+	if t.CreatedUnix > 0 {
+		return
+	}
 	t.CreatedUnix = gorm.NowFunc().Unix()
 }
 
@@ -102,7 +105,7 @@ func (db *accessTokens) Create(userID int64, name string) (*AccessToken, error)
 	token := &AccessToken{
 		UserID: userID,
 		Name:   name,
-		Sha1:   tool.SHA1(gouuid.NewV4().String()),
+		Sha1:   cryptoutil.SHA1(gouuid.NewV4().String()),
 	}
 	return token, db.DB.Create(token).Error
 }

internal/db/access_tokens_test.go

@@ -14,6 +14,22 @@ import (
 	"gogs.io/gogs/internal/errutil"
 )
 
+func TestAccessToken_BeforeCreate(t *testing.T) {
+	t.Run("CreatedUnix has been set", func(t *testing.T) {
+		token := &AccessToken{CreatedUnix: 1}
+		token.BeforeCreate()
+		assert.Equal(t, int64(1), token.CreatedUnix)
+		assert.Equal(t, int64(0), token.UpdatedUnix)
+	})
+
+	t.Run("CreatedUnix has not been set", func(t *testing.T) {
+		token := &AccessToken{}
+		token.BeforeCreate()
+		assert.Equal(t, gorm.NowFunc().Unix(), token.CreatedUnix)
+		assert.Equal(t, int64(0), token.UpdatedUnix)
+	})
+}
+
 func Test_accessTokens(t *testing.T) {
 	if testing.Short() {
 		t.Skip()
@@ -64,7 +80,7 @@ func test_accessTokens_Create(t *testing.T, db *accessTokens) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	assert.Equal(t, gorm.NowFunc().Format(time.RFC3339), token.Created.Format(time.RFC3339))
+	assert.Equal(t, gorm.NowFunc().Format(time.RFC3339), token.Created.UTC().Format(time.RFC3339))
 
 	// Try create second access token with same name should fail
 	_, err = db.Create(token.UserID, token.Name)
@@ -177,5 +193,5 @@ func test_accessTokens_Save(t *testing.T, db *accessTokens) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	assert.Equal(t, gorm.NowFunc().Format(time.RFC3339), token.Updated.Format(time.RFC3339))
+	assert.Equal(t, gorm.NowFunc().Format(time.RFC3339), token.Updated.UTC().Format(time.RFC3339))
 }

internal/db/backup.go

@@ -0,0 +1,268 @@
+package db
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+
+	"github.com/jinzhu/gorm"
+	jsoniter "github.com/json-iterator/go"
+	"github.com/pkg/errors"
+	log "unknwon.dev/clog/v2"
+	"xorm.io/core"
+	"xorm.io/xorm"
+
+	"gogs.io/gogs/internal/conf"
+	"gogs.io/gogs/internal/osutil"
+)
+
+// getTableType returns the type name of a table definition without package name,
+// e.g. *db.LFSObject -> LFSObject.
+func getTableType(t interface{}) string {
+	return strings.TrimPrefix(fmt.Sprintf("%T", t), "*db.")
+}
+
+// DumpDatabase dumps all data from database to file system in JSON Lines format.
+func DumpDatabase(db *gorm.DB, dirPath string, verbose bool) error {
+	err := os.MkdirAll(dirPath, os.ModePerm)
+	if err != nil {
+		return err
+	}
+
+	err = dumpLegacyTables(dirPath, verbose)
+	if err != nil {
+		return errors.Wrap(err, "dump legacy tables")
+	}
+
+	for _, table := range tables {
+		tableName := getTableType(table)
+		if verbose {
+			log.Trace("Dumping table %q...", tableName)
+		}
+
+		err := func() error {
+			tableFile := filepath.Join(dirPath, tableName+".json")
+			f, err := os.Create(tableFile)
+			if err != nil {
+				return errors.Wrap(err, "create table file")
+			}
+			defer func() { _ = f.Close() }()
+
+			return dumpTable(db, table, f)
+		}()
+		if err != nil {
+			return errors.Wrapf(err, "dump table %q", tableName)
+		}
+	}
+
+	return nil
+}
+
+func dumpTable(db *gorm.DB, table interface{}, w io.Writer) error {
+	query := db.Model(table).Order("id ASC")
+	switch table.(type) {
+	case *LFSObject:
+		query = db.Model(table).Order("repo_id, oid ASC")
+	}
+
+	rows, err := query.Rows()
+	if err != nil {
+		return errors.Wrap(err, "select rows")
+	}
+	defer func() { _ = rows.Close() }()
+
+	for rows.Next() {
+		elem := reflect.New(reflect.TypeOf(table).Elem()).Interface()
+		err = db.ScanRows(rows, elem)
+		if err != nil {
+			return errors.Wrap(err, "scan rows")
+		}
+
+		err = jsoniter.NewEncoder(w).Encode(elem)
+		if err != nil {
+			return errors.Wrap(err, "encode JSON")
+		}
+	}
+	return nil
+}
+
+func dumpLegacyTables(dirPath string, verbose bool) error {
+	// Purposely create a local variable to not modify global variable
+	legacyTables := append(legacyTables, new(Version))
+	for _, table := range legacyTables {
+		tableName := getTableType(table)
+		if verbose {
+			log.Trace("Dumping table %q...", tableName)
+		}
+
+		tableFile := filepath.Join(dirPath, tableName+".json")
+		f, err := os.Create(tableFile)
+		if err != nil {
+			return fmt.Errorf("create JSON file: %v", err)
+		}
+
+		if err = x.Asc("id").Iterate(table, func(idx int, bean interface{}) (err error) {
+			return jsoniter.NewEncoder(f).Encode(bean)
+		}); err != nil {
+			_ = f.Close()
+			return fmt.Errorf("dump table '%s': %v", tableName, err)
+		}
+		_ = f.Close()
+	}
+	return nil
+}
+
+// ImportDatabase imports data from backup archive in JSON Lines format.
+func ImportDatabase(db *gorm.DB, dirPath string, verbose bool) error {
+	err := importLegacyTables(dirPath, verbose)
+	if err != nil {
+		return errors.Wrap(err, "import legacy tables")
+	}
+
+	for _, table := range tables {
+		tableName := strings.TrimPrefix(fmt.Sprintf("%T", table), "*db.")
+		err := func() error {
+			tableFile := filepath.Join(dirPath, tableName+".json")
+			if !osutil.IsFile(tableFile) {
+				log.Info("Skipped table %q", tableName)
+				return nil
+			}
+
+			if verbose {
+				log.Trace("Importing table %q...", tableName)
+			}
+
+			f, err := os.Open(tableFile)
+			if err != nil {
+				return errors.Wrap(err, "open table file")
+			}
+			defer func() { _ = f.Close() }()
+
+			return importTable(db, table, f)
+		}()
+		if err != nil {
+			return errors.Wrapf(err, "import table %q", tableName)
+		}
+	}
+
+	return nil
+}
+
+func importTable(db *gorm.DB, table interface{}, r io.Reader) error {
+	err := db.DropTableIfExists(table).Error
+	if err != nil {
+		return errors.Wrap(err, "drop table")
+	}
+
+	err = db.AutoMigrate(table).Error
+	if err != nil {
+		return errors.Wrap(err, "auto migrate")
+	}
+
+	rawTableName := db.NewScope(table).TableName()
+	skipResetIDSeq := map[string]bool{
+		"lfs_object": true,
+	}
+
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		elem := reflect.New(reflect.TypeOf(table).Elem()).Interface()
+		err = jsoniter.Unmarshal(scanner.Bytes(), elem)
+		if err != nil {
+			return errors.Wrap(err, "unmarshal JSON to struct")
+		}
+
+		err = db.Create(elem).Error
+		if err != nil {
+			return errors.Wrap(err, "create row")
+		}
+	}
+
+	// PostgreSQL needs manually reset table sequence for auto increment keys
+	if conf.UsePostgreSQL && !skipResetIDSeq[rawTableName] {
+		seqName := rawTableName + "_id_seq"
+		if _, err = x.Exec(fmt.Sprintf(`SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM "%s"), 1), false);`, seqName, rawTableName)); err != nil {
+			return errors.Wrapf(err, "reset table %q.%q", rawTableName, seqName)
+		}
+	}
+	return nil
+}
+
+func importLegacyTables(dirPath string, verbose bool) error {
+	snakeMapper := core.SnakeMapper{}
+
+	skipInsertProcessors := map[string]bool{
+		"mirror":    true,
+		"milestone": true,
+	}
+
+	// Purposely create a local variable to not modify global variable
+	legacyTables := append(legacyTables, new(Version))
+	for _, table := range legacyTables {
+		tableName := strings.TrimPrefix(fmt.Sprintf("%T", table), "*db.")
+		tableFile := filepath.Join(dirPath, tableName+".json")
+		if !osutil.IsFile(tableFile) {
+			continue
+		}
+
+		if verbose {
+			log.Trace("Importing table %q...", tableName)
+		}
+
+		if err := x.DropTables(table); err != nil {
+			return fmt.Errorf("drop table %q: %v", tableName, err)
+		} else if err = x.Sync2(table); err != nil {
+			return fmt.Errorf("sync table %q: %v", tableName, err)
+		}
+
+		f, err := os.Open(tableFile)
+		if err != nil {
+			return fmt.Errorf("open JSON file: %v", err)
+		}
+		rawTableName := x.TableName(table)
+		_, isInsertProcessor := table.(xorm.BeforeInsertProcessor)
+		scanner := bufio.NewScanner(f)
+		for scanner.Scan() {
+			if err = jsoniter.Unmarshal(scanner.Bytes(), table); err != nil {
+				return fmt.Errorf("unmarshal to struct: %v", err)
+			}
+
+			if _, err = x.Insert(table); err != nil {
+				return fmt.Errorf("insert strcut: %v", err)
+			}
+
+			meta := make(map[string]interface{})
+			if err = jsoniter.Unmarshal(scanner.Bytes(), &meta); err != nil {
+				log.Error("Failed to unmarshal to map: %v", err)
+			}
+
+			// Reset created_unix back to the date save in archive because Insert method updates its value
+			if isInsertProcessor && !skipInsertProcessors[rawTableName] {
+				if _, err = x.Exec("UPDATE `"+rawTableName+"` SET created_unix=? WHERE id=?", meta["CreatedUnix"], meta["ID"]); err != nil {
+					log.Error("Failed to reset 'created_unix': %v", err)
+				}
+			}
+
+			switch rawTableName {
+			case "milestone":
+				if _, err = x.Exec("UPDATE `"+rawTableName+"` SET deadline_unix=?, closed_date_unix=? WHERE id=?", meta["DeadlineUnix"], meta["ClosedDateUnix"], meta["ID"]); err != nil {
+					log.Error("Failed to reset 'milestone.deadline_unix', 'milestone.closed_date_unix': %v", err)
+				}
+			}
+		}
+
+		// PostgreSQL needs manually reset table sequence for auto increment keys
+		if conf.UsePostgreSQL {
+			rawTableName := snakeMapper.Obj2Table(tableName)
+			seqName := rawTableName + "_id_seq"
+			if _, err = x.Exec(fmt.Sprintf(`SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM "%s"), 1), false);`, seqName, rawTableName)); err != nil {
+				return fmt.Errorf("reset table %q' sequence: %v", rawTableName, err)
+			}
+		}
+	}
+	return nil
+}

internal/db/backup_test.go

@@ -0,0 +1,146 @@
+// Copyright 2020 The Gogs Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package db
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/jinzhu/gorm"
+	"github.com/pkg/errors"
+
+	"gogs.io/gogs/internal/cryptoutil"
+	"gogs.io/gogs/internal/lfsutil"
+	"gogs.io/gogs/internal/testutil"
+)
+
+func Test_dumpAndImport(t *testing.T) {
+	if testing.Short() {
+		t.Skip()
+	}
+
+	t.Parallel()
+
+	if len(tables) != 3 {
+		t.Fatalf("New table has added (want 3 got %d), please add new tests for the table and update this check", len(tables))
+	}
+
+	db := initTestDB(t, "dumpAndImport", tables...)
+	setupDBToDump(t, db)
+	dumpTables(t, db)
+	importTables(t, db)
+
+	// Dump and assert golden again to make sure data aren't changed.
+	dumpTables(t, db)
+}
+
+func setupDBToDump(t *testing.T, db *gorm.DB) {
+	t.Helper()
+
+	vals := []interface{}{
+		&AccessToken{
+			UserID:      1,
+			Name:        "test1",
+			Sha1:        cryptoutil.SHA1("2910d03d-c0b5-4f71-bad5-c4086e4efae3"),
+			CreatedUnix: 1588568886,
+			UpdatedUnix: 1588572486, // 1 hour later
+		},
+		&AccessToken{
+			UserID:      1,
+			Name:        "test2",
+			Sha1:        cryptoutil.SHA1("84117e17-7e67-4024-bd04-1c23e6e809d4"),
+			CreatedUnix: 1588568886,
+		},
+		&AccessToken{
+			UserID:      2,
+			Name:        "test1",
+			Sha1:        cryptoutil.SHA1("da2775ce-73dd-47ba-b9d2-bbcc346585c4"),
+			CreatedUnix: 1588568886,
+		},
+
+		&LFSObject{
+			RepoID:    1,
+			OID:       "ef797c8118f02dfb649607dd5d3f8c7623048c9c063d532cc95c5ed7a898a64f",
+			Size:      100,
+			Storage:   lfsutil.StorageLocal,
+			CreatedAt: time.Unix(1588568886, 0).UTC(),
+		},
+		&LFSObject{
+			RepoID:    2,
+			OID:       "ef797c8118f02dfb649607dd5d3f8c7623048c9c063d532cc95c5ed7a898a64f",
+			Size:      100,
+			Storage:   lfsutil.StorageLocal,
+			CreatedAt: time.Unix(1588568886, 0).UTC(),
+		},
+
+		&LoginSource{
+			Type:      LoginPAM,
+			Name:      "My PAM",
+			IsActived: true,
+			Config: &PAMConfig{
+				ServiceName: "PAM service",
+			},
+			CreatedUnix: 1588568886,
+			UpdatedUnix: 1588572486, // 1 hour later
+		},
+		&LoginSource{
+			Type:      LoginGitHub,
+			Name:      "GitHub.com",
+			IsActived: true,
+			Config: &GitHubConfig{
+				APIEndpoint: "https://api.github.com",
+			},
+			CreatedUnix: 1588568886,
+		},
+	}
+	for _, val := range vals {
+		err := db.Create(val).Error
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func dumpTables(t *testing.T, db *gorm.DB) {
+	t.Helper()
+
+	for _, table := range tables {
+		tableName := getTableType(table)
+
+		var buf bytes.Buffer
+		err := dumpTable(db, table, &buf)
+		if err != nil {
+			t.Fatalf("%s: %v", tableName, err)
+		}
+
+		golden := filepath.Join("testdata", "backup", tableName+".golden.json")
+		testutil.AssertGolden(t, golden, testutil.Update("Test_dumpAndImport"), buf.String())
+	}
+}
+
+func importTables(t *testing.T, db *gorm.DB) {
+	t.Helper()
+
+	for _, table := range tables {
+		tableName := getTableType(table)
+
+		err := func() error {
+			golden := filepath.Join("testdata", "backup", tableName+".golden.json")
+			f, err := os.Open(golden)
+			if err != nil {
+				return errors.Wrap(err, "open table file")
+			}
+			defer func() { _ = f.Close() }()
+
+			return importTable(db, table, f)
+		}()
+		if err != nil {
+			t.Fatalf("%s: %v", tableName, err)
+		}
+	}
+}

internal/db/db.go

@@ -122,15 +122,16 @@ func getLogWriter() (io.Writer, error) {
 	return w, nil
 }
 
+// NOTE: Lines are sorted in alphabetical order, each letter in its own line.
 var tables = []interface{}{
 	new(AccessToken),
 	new(LFSObject), new(LoginSource),
 }
 
-func Init() error {
+func Init() (*gorm.DB, error) {
 	db, err := openDB(conf.Database)
 	if err != nil {
-		return errors.Wrap(err, "open database")
+		return nil, errors.Wrap(err, "open database")
 	}
 	db.SingularTable(true)
 	db.DB().SetMaxOpenConns(conf.Database.MaxOpenConns)
@@ -139,7 +140,7 @@ func Init() error {
 
 	w, err := getLogWriter()
 	if err != nil {
-		return errors.Wrap(err, "get log writer")
+		return nil, errors.Wrap(err, "get log writer")
 	}
 	db.SetLogger(&dbutil.Writer{Writer: w})
 	if !conf.IsProdMode() {
@@ -168,7 +169,7 @@ func Init() error {
 		name := strings.TrimPrefix(fmt.Sprintf("%T", table), "*db.")
 		err = db.AutoMigrate(table).Error
 		if err != nil {
-			return errors.Wrapf(err, "auto migrate %q", name)
+			return nil, errors.Wrapf(err, "auto migrate %q", name)
 		}
 		log.Trace("Auto migrated %q", name)
 	}
@@ -179,7 +180,7 @@ func Init() error {
 
 	sourceFiles, err := loadLoginSourceFiles(filepath.Join(conf.CustomDir(), "conf", "auth.d"))
 	if err != nil {
-		return errors.Wrap(err, "load login source files")
+		return nil, errors.Wrap(err, "load login source files")
 	}
 
 	// Initialize stores, sorted in alphabetical order.
@@ -191,5 +192,5 @@ func Init() error {
 	TwoFactors = &twoFactors{DB: db}
 	Users = &users{DB: db}
 
-	return db.DB().Ping()
+	return db, db.DB().Ping()
 }

internal/db/login_sources.go

@@ -47,8 +47,8 @@ var LoginSources LoginSourcesStore
 type LoginSource struct {
 	ID        int64
 	Type      LoginType
-	Name      string      `xorm:"UNIQUE"`
+	Name      string      `xorm:"UNIQUE" gorm:"UNIQUE"`
-	IsActived bool        `xorm:"NOT NULL DEFAULT false"`
+	IsActived bool        `xorm:"NOT NULL DEFAULT false" gorm:"NOT NULL"`
 	IsDefault bool        `xorm:"DEFAULT false"`
 	Config    interface{} `xorm:"-" gorm:"-"`
 	RawConfig string      `xorm:"TEXT cfg" gorm:"COLUMN:cfg"`
@@ -63,12 +63,18 @@ type LoginSource struct {
 
 // NOTE: This is a GORM save hook.
 func (s *LoginSource) BeforeSave() (err error) {
+	if s.Config == nil {
+		return nil
+	}
 	s.RawConfig, err = jsoniter.MarshalToString(s.Config)
 	return err
 }
 
 // NOTE: This is a GORM create hook.
 func (s *LoginSource) BeforeCreate() {
+	if s.CreatedUnix > 0 {
+		return
+	}
 	s.CreatedUnix = gorm.NowFunc().Unix()
 	s.UpdatedUnix = s.CreatedUnix
 }

internal/db/login_sources_test.go

@@ -14,6 +14,44 @@ import (
 	"gogs.io/gogs/internal/errutil"
 )
 
+func TestLoginSource_BeforeSave(t *testing.T) {
+	t.Run("Config has not been set", func(t *testing.T) {
+		s := &LoginSource{}
+		err := s.BeforeSave()
+		if err != nil {
+			t.Fatal(err)
+		}
+		assert.Empty(t, s.RawConfig)
+	})
+
+	t.Run("Config has been set", func(t *testing.T) {
+		s := &LoginSource{
+			Config: &PAMConfig{ServiceName: "pam_service"},
+		}
+		err := s.BeforeSave()
+		if err != nil {
+			t.Fatal(err)
+		}
+		assert.Equal(t, `{"ServiceName":"pam_service"}`, s.RawConfig)
+	})
+}
+
+func TestLoginSource_BeforeCreate(t *testing.T) {
+	t.Run("CreatedUnix has been set", func(t *testing.T) {
+		s := &LoginSource{CreatedUnix: 1}
+		s.BeforeCreate()
+		assert.Equal(t, int64(1), s.CreatedUnix)
+		assert.Equal(t, int64(0), s.UpdatedUnix)
+	})
+
+	t.Run("CreatedUnix has not been set", func(t *testing.T) {
+		s := &LoginSource{}
+		s.BeforeCreate()
+		assert.Equal(t, gorm.NowFunc().Unix(), s.CreatedUnix)
+		assert.Equal(t, gorm.NowFunc().Unix(), s.UpdatedUnix)
+	})
+}
+
 func Test_loginSources(t *testing.T) {
 	if testing.Short() {
 		t.Skip()
@@ -70,8 +108,8 @@ func test_loginSources_Create(t *testing.T, db *loginSources) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	assert.Equal(t, gorm.NowFunc().Format(time.RFC3339), source.Created.Format(time.RFC3339))
+	assert.Equal(t, gorm.NowFunc().Format(time.RFC3339), source.Created.UTC().Format(time.RFC3339))
-	assert.Equal(t, gorm.NowFunc().Format(time.RFC3339), source.Updated.Format(time.RFC3339))
+	assert.Equal(t, gorm.NowFunc().Format(time.RFC3339), source.Updated.UTC().Format(time.RFC3339))
 
 	// Try create second login source with same name should fail
 	_, err = db.Create(CreateLoginSourceOpts{Name: source.Name})

internal/db/main_test.go

@@ -35,7 +35,7 @@ func TestMain(m *testing.M) {
 		}
 	}
 
-	now := time.Now().Truncate(time.Second)
+	now := time.Now().UTC().Truncate(time.Second)
 	gorm.NowFunc = func() time.Time { return now }
 
 	os.Exit(m.Run())

internal/db/models.go

@@ -5,7 +5,6 @@
 package db
 
 import (
-	"bufio"
 	"database/sql"
 	"fmt"
 	"net/url"
@@ -14,8 +13,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/json-iterator/go"
+	"github.com/jinzhu/gorm"
-	"github.com/unknwon/com"
 	log "unknwon.dev/clog/v2"
 	"xorm.io/core"
 	"xorm.io/xorm"
@@ -123,10 +121,11 @@ func NewTestEngine() error {
 	return x.StoreEngine("InnoDB").Sync2(legacyTables...)
 }
 
-func SetEngine() (err error) {
+func SetEngine() (*gorm.DB, error) {
+	var err error
 	x, err = getEngine()
 	if err != nil {
-		return fmt.Errorf("connect to database: %v", err)
+		return nil, fmt.Errorf("connect to database: %v", err)
 	}
 
 	x.SetMapper(core.GonicMapper{})
@@ -142,7 +141,7 @@ func SetEngine() (err error) {
 			MaxDays: sec.Key("MAX_DAYS").MustInt64(3),
 		})
 	if err != nil {
-		return fmt.Errorf("create 'xorm.log': %v", err)
+		return nil, fmt.Errorf("create 'xorm.log': %v", err)
 	}
 
 	x.SetMaxOpenConns(conf.Database.MaxOpenConns)
@@ -159,7 +158,7 @@ func SetEngine() (err error) {
 }
 
 func NewEngine() (err error) {
-	if err = SetEngine(); err != nil {
+	if _, err = SetEngine(); err != nil {
 		return err
 	}
 
@@ -219,131 +218,3 @@ type Version struct {
 	ID      int64
 	Version int64
 }
-
-// DumpDatabase dumps all data from database to file system in JSON format.
-func DumpDatabase(dirPath string) error {
-	if err := os.MkdirAll(dirPath, os.ModePerm); err != nil {
-		return err
-	}
-
-	// Purposely create a local variable to not modify global variable
-	allTables := append(legacyTables, new(Version))
-	allTables = append(allTables, tables...)
-	for _, table := range allTables {
-		tableName := strings.TrimPrefix(fmt.Sprintf("%T", table), "*db.")
-		tableFile := path.Join(dirPath, tableName+".json")
-		f, err := os.Create(tableFile)
-		if err != nil {
-			return fmt.Errorf("create JSON file: %v", err)
-		}
-
-		if err = x.Asc("id").Iterate(table, func(idx int, bean interface{}) (err error) {
-			return jsoniter.NewEncoder(f).Encode(bean)
-		}); err != nil {
-			_ = f.Close()
-			return fmt.Errorf("dump table '%s': %v", tableName, err)
-		}
-		_ = f.Close()
-	}
-	return nil
-}
-
-// ImportDatabase imports data from backup archive.
-func ImportDatabase(dirPath string, verbose bool) (err error) {
-	snakeMapper := core.SnakeMapper{}
-
-	skipInsertProcessors := map[string]bool{
-		"mirror":    true,
-		"milestone": true,
-	}
-
-	// Purposely create a local variable to not modify global variable
-	allTables := append(legacyTables, new(Version))
-	allTables = append(allTables, tables...)
-	for _, table := range allTables {
-		tableName := strings.TrimPrefix(fmt.Sprintf("%T", table), "*db.")
-		tableFile := path.Join(dirPath, tableName+".json")
-		if !com.IsExist(tableFile) {
-			continue
-		}
-
-		if verbose {
-			log.Trace("Importing table '%s'...", tableName)
-		}
-
-		if err = x.DropTables(table); err != nil {
-			return fmt.Errorf("drop table '%s': %v", tableName, err)
-		} else if err = x.Sync2(table); err != nil {
-			return fmt.Errorf("sync table '%s': %v", tableName, err)
-		}
-
-		f, err := os.Open(tableFile)
-		if err != nil {
-			return fmt.Errorf("open JSON file: %v", err)
-		}
-		rawTableName := x.TableName(table)
-		_, isInsertProcessor := table.(xorm.BeforeInsertProcessor)
-		scanner := bufio.NewScanner(f)
-		for scanner.Scan() {
-			switch bean := table.(type) {
-			case *LoginSource:
-				meta := make(map[string]interface{})
-				if err = jsoniter.Unmarshal(scanner.Bytes(), &meta); err != nil {
-					return fmt.Errorf("unmarshal to map: %v", err)
-				}
-
-				tp := LoginType(com.StrTo(com.ToStr(meta["Type"])).MustInt64())
-				switch tp {
-				case LoginLDAP, LoginDLDAP:
-					bean.Config = new(LDAPConfig)
-				case LoginSMTP:
-					bean.Config = new(SMTPConfig)
-				case LoginPAM:
-					bean.Config = new(PAMConfig)
-				case LoginGitHub:
-					bean.Config = new(GitHubConfig)
-				default:
-					return fmt.Errorf("unrecognized login source type:: %v", tp)
-				}
-				table = bean
-			}
-
-			if err = jsoniter.Unmarshal(scanner.Bytes(), table); err != nil {
-				return fmt.Errorf("unmarshal to struct: %v", err)
-			}
-
-			if _, err = x.Insert(table); err != nil {
-				return fmt.Errorf("insert strcut: %v", err)
-			}
-
-			meta := make(map[string]interface{})
-			if err = jsoniter.Unmarshal(scanner.Bytes(), &meta); err != nil {
-				log.Error("Failed to unmarshal to map: %v", err)
-			}
-
-			// Reset created_unix back to the date save in archive because Insert method updates its value
-			if isInsertProcessor && !skipInsertProcessors[rawTableName] {
-				if _, err = x.Exec("UPDATE "+rawTableName+" SET created_unix=? WHERE id=?", meta["CreatedUnix"], meta["ID"]); err != nil {
-					log.Error("Failed to reset 'created_unix': %v", err)
-				}
-			}
-
-			switch rawTableName {
-			case "milestone":
-				if _, err = x.Exec("UPDATE "+rawTableName+" SET deadline_unix=?, closed_date_unix=? WHERE id=?", meta["DeadlineUnix"], meta["ClosedDateUnix"], meta["ID"]); err != nil {
-					log.Error("Failed to reset 'milestone.deadline_unix', 'milestone.closed_date_unix': %v", err)
-				}
-			}
-		}
-
-		// PostgreSQL needs manually reset table sequence for auto increment keys
-		if conf.UsePostgreSQL {
-			rawTableName := snakeMapper.Obj2Table(tableName)
-			seqName := rawTableName + "_id_seq"
-			if _, err = x.Exec(fmt.Sprintf(`SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM "%s"), 1), false);`, seqName, rawTableName)); err != nil {
-				return fmt.Errorf("reset table '%s' sequence: %v", rawTableName, err)
-			}
-		}
-	}
-	return nil
-}

internal/db/repos_test.go

@@ -80,7 +80,7 @@ func test_repos_create(t *testing.T, db *repos) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	assert.Equal(t, gorm.NowFunc().Format(time.RFC3339), repo.Created.Format(time.RFC3339))
+	assert.Equal(t, gorm.NowFunc().Format(time.RFC3339), repo.Created.UTC().Format(time.RFC3339))
 }
 
 func test_repos_GetByName(t *testing.T, db *repos) {

internal/db/testdata/backup/AccessToken.golden.json

@@ -0,0 +1,3 @@
+{"ID":1,"UserID":1,"Name":"test1","Sha1":"56ed62d55225e9ae1275b1c4aa6e3de62f44e730","CreatedUnix":1588568886,"UpdatedUnix":1588572486}
+{"ID":2,"UserID":1,"Name":"test2","Sha1":"16fb74941e834e057d11c59db5d81cdae15be794","CreatedUnix":1588568886,"UpdatedUnix":0}
+{"ID":3,"UserID":2,"Name":"test1","Sha1":"09f170f4ee70ba035587f7df8319b2a3a3d2b74a","CreatedUnix":1588568886,"UpdatedUnix":0}

internal/db/testdata/backup/LFSObject.golden.json

@@ -0,0 +1,2 @@
+{"RepoID":1,"OID":"ef797c8118f02dfb649607dd5d3f8c7623048c9c063d532cc95c5ed7a898a64f","Size":100,"Storage":"local","CreatedAt":"2020-05-04T05:08:06Z"}
+{"RepoID":2,"OID":"ef797c8118f02dfb649607dd5d3f8c7623048c9c063d532cc95c5ed7a898a64f","Size":100,"Storage":"local","CreatedAt":"2020-05-04T05:08:06Z"}

internal/db/testdata/backup/LoginSource.golden.json

@@ -0,0 +1,2 @@
+{"ID":1,"Type":4,"Name":"My PAM","IsActived":true,"IsDefault":false,"Config":null,"RawConfig":"{\"ServiceName\":\"PAM service\"}","CreatedUnix":1588568886,"UpdatedUnix":1588572486}
+{"ID":2,"Type":6,"Name":"GitHub.com","IsActived":true,"IsDefault":false,"Config":null,"RawConfig":"{\"APIEndpoint\":\"https://api.github.com\"}","CreatedUnix":1588568886,"UpdatedUnix":0}

internal/db/two_factors_test.go

@@ -58,7 +58,7 @@ func test_twoFactors_Create(t *testing.T, db *twoFactors) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	assert.Equal(t, gorm.NowFunc().Format(time.RFC3339), tf.Created.Format(time.RFC3339))
+	assert.Equal(t, gorm.NowFunc().Format(time.RFC3339), tf.Created.UTC().Format(time.RFC3339))
 
 	// Verify there are 10 recover codes generated
 	var count int64

internal/db/users_test.go

@@ -129,8 +129,8 @@ func test_users_Create(t *testing.T, db *users) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	assert.Equal(t, gorm.NowFunc().Format(time.RFC3339), user.Created.Format(time.RFC3339))
+	assert.Equal(t, gorm.NowFunc().Format(time.RFC3339), user.Created.UTC().Format(time.RFC3339))
-	assert.Equal(t, gorm.NowFunc().Format(time.RFC3339), user.Updated.Format(time.RFC3339))
+	assert.Equal(t, gorm.NowFunc().Format(time.RFC3339), user.Updated.UTC().Format(time.RFC3339))
 }
 
 func test_users_GetByEmail(t *testing.T, db *users) {

internal/template/template.go

@@ -23,6 +23,7 @@ import (
 	"github.com/gogs/git-module"
 
 	"gogs.io/gogs/internal/conf"
+	"gogs.io/gogs/internal/cryptoutil"
 	"gogs.io/gogs/internal/db"
 	"gogs.io/gogs/internal/gitutil"
 	"gogs.io/gogs/internal/markup"
@@ -149,7 +150,7 @@ func NewLine2br(raw string) string {
 }
 
 func Sha1(str string) string {
-	return tool.SHA1(str)
+	return cryptoutil.SHA1(str)
 }
 
 func ToUTF8WithErr(content []byte) (error, string) {

internal/testutil/golden.go

@@ -8,6 +8,8 @@ import (
 	"encoding/json"
 	"flag"
 	"io/ioutil"
+	"os"
+	"path/filepath"
 	"regexp"
 	"runtime"
 	"testing"
@@ -17,7 +19,7 @@ import (
 
 var updateRegex = flag.String("update", "", "Update testdata of tests matching the given regex")
 
-// Update returns true if update regex mathces given test name.
+// Update returns true if update regex matches given test name.
 func Update(name string) bool {
 	if updateRegex == nil || *updateRegex == "" {
 		return false
@@ -38,14 +40,20 @@ func AssertGolden(t testing.TB, path string, update bool, got interface{}) {
 	data := marshal(t, got)
 
 	if update {
-		if err := ioutil.WriteFile(path, data, 0640); err != nil {
+		err := os.MkdirAll(filepath.Dir(path), os.ModePerm)
-			t.Fatalf("update golden file %q: %s", path, err)
+		if err != nil {
+			t.Fatalf("create directories for golden file %q: %v", path, err)
+		}
+
+		err = ioutil.WriteFile(path, data, 0640)
+		if err != nil {
+			t.Fatalf("update golden file %q: %v", path, err)
 		}
 	}
 
 	golden, err := ioutil.ReadFile(path)
 	if err != nil {
-		t.Fatalf("read golden file %q: %s", path, err)
+		t.Fatalf("read golden file %q: %v", path, err)
 	}
 
 	assert.Equal(t, string(golden), string(data))

internal/tool/tool.go

@@ -25,14 +25,6 @@ import (
 	"gogs.io/gogs/internal/conf"
 )
 
-// SHA1 encodes string to SHA1 hex value.
-// TODO: Move to cryptoutil.SHA1.
-func SHA1(str string) string {
-	h := sha1.New()
-	_, _ = h.Write([]byte(str))
-	return hex.EncodeToString(h.Sum(nil))
-}
-
 // ShortSHA1 truncates SHA1 string length to at most 10.
 func ShortSHA1(sha1 string) string {
 	if len(sha1) > 10 {