mirror of https://github.com/gogs/gogs.git
cmd: able to backup and restore
Not very robust, must execute under correct workdir. Addresses #2072, #3708, #648pull/4224/head^2
Unknwon
parent
b06f299748
commit
ca2cfaf71e
|
|
@ -0,0 +1,135 @@
|
|||
// Copyright 2017 The Gogs Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/Unknwon/cae/zip"
|
||||
"github.com/Unknwon/com"
|
||||
"github.com/urfave/cli"
|
||||
log "gopkg.in/clog.v1"
|
||||
"gopkg.in/ini.v1"
|
||||
|
||||
"github.com/gogits/gogs/models"
|
||||
"github.com/gogits/gogs/modules/setting"
|
||||
)
|
||||
|
||||
var Backup = cli.Command{
|
||||
Name: "backup",
|
||||
Usage: "Backup files and database",
|
||||
Description: `Backup dumps and compresses all related files and database into zip file,
|
||||
which can be used for migrating Gogs to another server. The output format is meant to be
|
||||
portable among all supported database engines.`,
|
||||
Action: runBackup,
|
||||
Flags: []cli.Flag{
|
||||
stringFlag("config, c", "custom/conf/app.ini", "Custom configuration file path"),
|
||||
boolFlag("verbose, v", "Show process details"),
|
||||
stringFlag("tempdir, t", os.TempDir(), "Temporary directory path"),
|
||||
stringFlag("target", "./", "Target directory path to save backup archive"),
|
||||
boolFlag("database-only", "Only dump database"),
|
||||
boolFlag("exclude-repos", "Exclude repositories"),
|
||||
},
|
||||
}
|
||||
|
||||
const _ARCHIVE_ROOT_DIR = "gogs-backup"
|
||||
|
||||
func runBackup(c *cli.Context) error {
|
||||
zip.Verbose = c.Bool("verbose")
|
||||
if c.IsSet("config") {
|
||||
setting.CustomConf = c.String("config")
|
||||
}
|
||||
setting.NewContext()
|
||||
models.LoadConfigs()
|
||||
models.SetEngine()
|
||||
|
||||
tmpDir := c.String("tempdir")
|
||||
if !com.IsExist(tmpDir) {
|
||||
log.Fatal(0, "'--tempdir' does not exist: %s", tmpDir)
|
||||
}
|
||||
rootDir, err := ioutil.TempDir(tmpDir, "gogs-backup-")
|
||||
if err != nil {
|
||||
log.Fatal(0, "Fail to create backup root directory '%s': %v", rootDir, err)
|
||||
}
|
||||
log.Info("Backup root directory: %s", rootDir)
|
||||
|
||||
// Metadata
|
||||
metaFile := path.Join(rootDir, "metadata.ini")
|
||||
metadata := ini.Empty()
|
||||
metadata.Section("").Key("VERSION").SetValue("1")
|
||||
metadata.Section("").Key("DATE_TIME").SetValue(time.Now().String())
|
||||
metadata.Section("").Key("GOGS_VERSION").SetValue(setting.AppVer)
|
||||
if err = metadata.SaveTo(metaFile); err != nil {
|
||||
log.Fatal(0, "Fail to save metadata '%s': %v", metaFile, err)
|
||||
}
|
||||
|
||||
archiveName := path.Join(c.String("target"), fmt.Sprintf("gogs-backup-%d.zip", time.Now().Unix()))
|
||||
log.Info("Packing backup files to: %s", archiveName)
|
||||
|
||||
z, err := zip.Create(archiveName)
|
||||
if err != nil {
|
||||
log.Fatal(0, "Fail to create backup archive '%s': %v", archiveName, err)
|
||||
}
|
||||
if err = z.AddFile(_ARCHIVE_ROOT_DIR+"/metadata.ini", metaFile); err != nil {
|
||||
log.Fatal(0, "Fail to include 'metadata.ini': %v", err)
|
||||
}
|
||||
|
||||
// Database
|
||||
dbDir := path.Join(rootDir, "db")
|
||||
if err = models.DumpDatabase(dbDir); err != nil {
|
||||
log.Fatal(0, "Fail to dump database: %v", err)
|
||||
}
|
||||
if err = z.AddDir(_ARCHIVE_ROOT_DIR+"/db", dbDir); err != nil {
|
||||
log.Fatal(0, "Fail to include 'db': %v", err)
|
||||
}
|
||||
|
||||
// Custom files
|
||||
if !c.Bool("database-only") {
|
||||
if err = z.AddDir(_ARCHIVE_ROOT_DIR+"/custom", setting.CustomPath); err != nil {
|
||||
log.Fatal(0, "Fail to include 'custom': %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Data files
|
||||
if !c.Bool("database-only") {
|
||||
for _, dir := range []string{"attachments", "avatars"} {
|
||||
dirPath := path.Join(setting.AppDataPath, dir)
|
||||
if !com.IsDir(dirPath) {
|
||||
continue
|
||||
}
|
||||
|
||||
if err = z.AddDir(path.Join(_ARCHIVE_ROOT_DIR+"/data", dir), dirPath); err != nil {
|
||||
log.Fatal(0, "Fail to include 'data': %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Repositories
|
||||
if !c.Bool("exclude-repos") && !c.Bool("database-only") {
|
||||
reposDump := path.Join(rootDir, "repositories.zip")
|
||||
log.Info("Dumping repositories in '%s'", setting.RepoRootPath)
|
||||
if err = zip.PackTo(setting.RepoRootPath, reposDump, true); err != nil {
|
||||
log.Fatal(0, "Fail to dump repositories: %v", err)
|
||||
}
|
||||
log.Info("Repositories dumped to: %s", reposDump)
|
||||
|
||||
if err = z.AddFile(_ARCHIVE_ROOT_DIR+"/repositories.zip", reposDump); err != nil {
|
||||
log.Fatal(0, "Fail to include 'repositories.zip': %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err = z.Close(); err != nil {
|
||||
log.Fatal(0, "Fail to save backup archive '%s': %v", archiveName, err)
|
||||
}
|
||||
|
||||
os.RemoveAll(rootDir)
|
||||
log.Info("Backup succeed! Archive is located at: %s", archiveName)
|
||||
log.Shutdown()
|
||||
return nil
|
||||
}
|
||||
122
cmd/dump.go
122
cmd/dump.go
|
|
@ -1,122 +0,0 @@
|
|||
// Copyright 2014 The Gogs Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"io/ioutil"
|
||||
|
||||
"github.com/Unknwon/cae/zip"
|
||||
"github.com/Unknwon/com"
|
||||
"github.com/urfave/cli"
|
||||
|
||||
"github.com/gogits/gogs/models"
|
||||
"github.com/gogits/gogs/modules/setting"
|
||||
)
|
||||
|
||||
var Dump = cli.Command{
|
||||
Name: "dump",
|
||||
Usage: "Dump Gogs files and database",
|
||||
Description: `Dump compresses all related files and database into zip file.
|
||||
It can be used for backup and capture Gogs server image to send to maintainer`,
|
||||
Action: runDump,
|
||||
Flags: []cli.Flag{
|
||||
stringFlag("config, c", "custom/conf/app.ini", "Custom configuration file path"),
|
||||
boolFlag("verbose, v", "Show process details"),
|
||||
stringFlag("tempdir, t", os.TempDir(), "Temporary dir path"),
|
||||
},
|
||||
}
|
||||
|
||||
func runDump(ctx *cli.Context) error {
|
||||
if ctx.IsSet("config") {
|
||||
setting.CustomConf = ctx.String("config")
|
||||
}
|
||||
setting.NewContext()
|
||||
models.LoadConfigs()
|
||||
models.SetEngine()
|
||||
|
||||
tmpDir := ctx.String("tempdir")
|
||||
if _, err := os.Stat(tmpDir); os.IsNotExist(err) {
|
||||
log.Fatalf("Path does not exist: %s", tmpDir)
|
||||
}
|
||||
TmpWorkDir, err := ioutil.TempDir(tmpDir, "gogs-dump-")
|
||||
if err != nil {
|
||||
log.Fatalf("Fail to create tmp work directory: %v", err)
|
||||
}
|
||||
log.Printf("Creating tmp work dir: %s", TmpWorkDir)
|
||||
|
||||
reposDump := path.Join(TmpWorkDir, "gogs-repo.zip")
|
||||
dbDump := path.Join(TmpWorkDir, "gogs-db.sql")
|
||||
|
||||
log.Printf("Dumping local repositories...%s", setting.RepoRootPath)
|
||||
zip.Verbose = ctx.Bool("verbose")
|
||||
if err := zip.PackTo(setting.RepoRootPath, reposDump, true); err != nil {
|
||||
log.Fatalf("Fail to dump local repositories: %v", err)
|
||||
}
|
||||
|
||||
log.Printf("Dumping database...")
|
||||
if err := models.DumpDatabase(dbDump); err != nil {
|
||||
log.Fatalf("Fail to dump database: %v", err)
|
||||
}
|
||||
|
||||
fileName := fmt.Sprintf("gogs-dump-%d.zip", time.Now().Unix())
|
||||
log.Printf("Packing dump files...")
|
||||
z, err := zip.Create(fileName)
|
||||
if err != nil {
|
||||
os.Remove(fileName)
|
||||
log.Fatalf("Fail to create %s: %v", fileName, err)
|
||||
}
|
||||
|
||||
if err := z.AddFile("gogs-repo.zip", reposDump); err != nil {
|
||||
log.Fatalf("Fail to include gogs-repo.zip: %v", err)
|
||||
}
|
||||
if err := z.AddFile("gogs-db.sql", dbDump); err != nil {
|
||||
log.Fatalf("Fail to include gogs-db.sql: %v", err)
|
||||
}
|
||||
customDir, err := os.Stat(setting.CustomPath)
|
||||
if err == nil && customDir.IsDir() {
|
||||
if err := z.AddDir("custom", setting.CustomPath); err != nil {
|
||||
log.Fatalf("Fail to include custom: %v", err)
|
||||
}
|
||||
} else {
|
||||
log.Printf("Custom dir %s doesn't exist, skipped", setting.CustomPath)
|
||||
}
|
||||
|
||||
if err := z.AddDir("log", setting.LogRootPath); err != nil {
|
||||
log.Fatalf("Fail to include log: %v", err)
|
||||
}
|
||||
|
||||
for _, dir := range []string{"attachments", "avatars"} {
|
||||
dirPath := path.Join(setting.AppDataPath, dir)
|
||||
if !com.IsDir(dirPath) {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := z.AddDir(path.Join("data", dir), dirPath); err != nil {
|
||||
log.Fatalf("Fail to include '%s': %v", dirPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
// FIXME: SSH key file.
|
||||
if err = z.Close(); err != nil {
|
||||
os.Remove(fileName)
|
||||
log.Fatalf("Fail to save %s: %v", fileName, err)
|
||||
}
|
||||
|
||||
if err := os.Chmod(fileName, 0600); err != nil {
|
||||
log.Printf("Can't change file access permissions mask to 0600: %v", err)
|
||||
}
|
||||
|
||||
log.Printf("Removing tmp work dir: %s", TmpWorkDir)
|
||||
os.RemoveAll(TmpWorkDir)
|
||||
log.Printf("Finish dumping in file %s", fileName)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,129 @@
|
|||
// Copyright 2017 The Gogs Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"github.com/Unknwon/cae/zip"
|
||||
"github.com/Unknwon/com"
|
||||
"github.com/mcuadros/go-version"
|
||||
"github.com/urfave/cli"
|
||||
log "gopkg.in/clog.v1"
|
||||
"gopkg.in/ini.v1"
|
||||
|
||||
"github.com/gogits/gogs/models"
|
||||
"github.com/gogits/gogs/modules/setting"
|
||||
)
|
||||
|
||||
var Restore = cli.Command{
|
||||
Name: "restore",
|
||||
Usage: "Restore files and database from backup",
|
||||
Description: `Restore imports all related files and database from a backup archive.
|
||||
The backup version must lower or equal to current Gogs version. You can also import
|
||||
backup from other database engines, which is useful for database migrating.
|
||||
|
||||
If corresponding files or database tables are not presented in the archive, they will
|
||||
be skipped and remian unchanged.`,
|
||||
Action: runRestore,
|
||||
Flags: []cli.Flag{
|
||||
stringFlag("config, c", "custom/conf/app.ini", "Custom configuration file path"),
|
||||
boolFlag("verbose, v", "Show process details"),
|
||||
stringFlag("tempdir, t", os.TempDir(), "Temporary directory path"),
|
||||
stringFlag("from", "", "Path to backup archive"),
|
||||
boolFlag("database-only", "Only import database"),
|
||||
boolFlag("exclude-repos", "Exclude repositories"),
|
||||
},
|
||||
}
|
||||
|
||||
func runRestore(c *cli.Context) error {
|
||||
zip.Verbose = c.Bool("verbose")
|
||||
|
||||
tmpDir := c.String("tempdir")
|
||||
if !com.IsExist(tmpDir) {
|
||||
log.Fatal(0, "'--tempdir' does not exist: %s", tmpDir)
|
||||
}
|
||||
|
||||
log.Info("Restore backup from: %s", c.String("from"))
|
||||
if err := zip.ExtractTo(c.String("from"), tmpDir); err != nil {
|
||||
log.Fatal(0, "Fail to extract backup archive: %v", err)
|
||||
}
|
||||
archivePath := path.Join(tmpDir, _ARCHIVE_ROOT_DIR)
|
||||
|
||||
// Check backup version
|
||||
metaFile := path.Join(archivePath, "metadata.ini")
|
||||
if !com.IsExist(metaFile) {
|
||||
log.Fatal(0, "File 'metadata.ini' is missing")
|
||||
}
|
||||
metadata, err := ini.Load(metaFile)
|
||||
if err != nil {
|
||||
log.Fatal(0, "Fail to load metadata '%s': %v", metaFile, err)
|
||||
}
|
||||
backupVersion := metadata.Section("").Key("GOGS_VERSION").MustString("999.0")
|
||||
if version.Compare(setting.AppVer, backupVersion, "<") {
|
||||
log.Fatal(0, "Current Gogs version is lower than backup version: %s < %s", setting.AppVer, backupVersion)
|
||||
}
|
||||
|
||||
// If config file is not present in backup, user must set this file via flag.
|
||||
// Otherwise, it's optional to set config file flag.
|
||||
configFile := path.Join(archivePath, "custom/conf/app.ini")
|
||||
if c.IsSet("config") {
|
||||
setting.CustomConf = c.String("config")
|
||||
} else if !com.IsExist(configFile) {
|
||||
log.Fatal(0, "'--config' is not specified and custom config file is not found in backup")
|
||||
} else {
|
||||
setting.CustomConf = configFile
|
||||
}
|
||||
setting.NewContext()
|
||||
models.LoadConfigs()
|
||||
models.SetEngine()
|
||||
|
||||
// Database
|
||||
dbDir := path.Join(archivePath, "db")
|
||||
if err = models.ImportDatabase(dbDir); err != nil {
|
||||
log.Fatal(0, "Fail to import database: %v", err)
|
||||
}
|
||||
|
||||
// Custom files
|
||||
if !c.Bool("database-only") {
|
||||
if com.IsExist(setting.CustomPath) {
|
||||
if err = os.Rename(setting.CustomPath, setting.CustomPath+".bak"); err != nil {
|
||||
log.Fatal(0, "Fail to backup current 'custom': %v", err)
|
||||
}
|
||||
}
|
||||
if err = os.Rename(path.Join(archivePath, "custom"), setting.CustomPath); err != nil {
|
||||
log.Fatal(0, "Fail to import 'custom': %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Data files
|
||||
if !c.Bool("database-only") {
|
||||
for _, dir := range []string{"attachments", "avatars"} {
|
||||
dirPath := path.Join(setting.AppDataPath, dir)
|
||||
if com.IsExist(dirPath) {
|
||||
if err = os.Rename(dirPath, dirPath+".bak"); err != nil {
|
||||
log.Fatal(0, "Fail to backup current 'data': %v", err)
|
||||
}
|
||||
}
|
||||
if err = os.Rename(path.Join(archivePath, "data", dir), dirPath); err != nil {
|
||||
log.Fatal(0, "Fail to import 'data': %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Repositories
|
||||
reposPath := path.Join(archivePath, "repositories.zip")
|
||||
if !c.Bool("exclude-repos") && !c.Bool("database-only") && com.IsExist(reposPath) {
|
||||
if err := zip.ExtractTo(reposPath, path.Dir(setting.RepoRootPath)); err != nil {
|
||||
log.Fatal(0, "Fail to extract 'repositories.zip': %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
os.RemoveAll(path.Join(tmpDir, _ARCHIVE_ROOT_DIR))
|
||||
log.Info("Restore succeed!")
|
||||
log.Shutdown()
|
||||
return nil
|
||||
}
|
||||
|
|
@ -51,7 +51,7 @@ import (
|
|||
|
||||
var Web = cli.Command{
|
||||
Name: "web",
|
||||
Usage: "Start Gogs web server",
|
||||
Usage: "Start web server",
|
||||
Description: `Gogs web server is the only thing you need to run,
|
||||
and it takes care of all the other things for you`,
|
||||
Action: runWeb,
|
||||
|
|
|
|||
3
gogs.go
3
gogs.go
|
|
@ -31,10 +31,11 @@ func main() {
|
|||
cmd.Web,
|
||||
cmd.Serv,
|
||||
cmd.Hook,
|
||||
cmd.Dump,
|
||||
cmd.Cert,
|
||||
cmd.Admin,
|
||||
cmd.Import,
|
||||
cmd.Backup,
|
||||
cmd.Restore,
|
||||
}
|
||||
app.Flags = append(app.Flags, []cli.Flag{}...)
|
||||
app.Run(os.Args)
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ func (m *migration) Migrate(x *xorm.Engine) error {
|
|||
|
||||
// The version table. Should have only one row with id==1
|
||||
type Version struct {
|
||||
ID int64 `xorm:"pk autoincr"`
|
||||
ID int64
|
||||
Version int64
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,9 @@
|
|||
package models
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
|
|
@ -13,6 +15,7 @@ import (
|
|||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/Unknwon/com"
|
||||
_ "github.com/denisenkom/go-mssqldb"
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
"github.com/go-xorm/core"
|
||||
|
|
@ -262,7 +265,89 @@ func Ping() error {
|
|||
return x.Ping()
|
||||
}
|
||||
|
||||
// DumpDatabase dumps all data from database to file system.
|
||||
func DumpDatabase(filePath string) error {
|
||||
return x.DumpAllToFile(filePath)
|
||||
// The version table. Should have only one row with id==1
|
||||
type Version struct {
|
||||
ID int64
|
||||
Version int64
|
||||
}
|
||||
|
||||
// DumpDatabase dumps all data from database to file system in JSON format.
|
||||
func DumpDatabase(dirPath string) (err error) {
|
||||
os.MkdirAll(dirPath, os.ModePerm)
|
||||
// Purposely create a local variable to not modify global variable
|
||||
tables := append(tables, new(Version))
|
||||
for _, table := range tables {
|
||||
tableName := strings.TrimPrefix(fmt.Sprintf("%T", table), "*models.")
|
||||
tableFile := path.Join(dirPath, tableName+".json")
|
||||
f, err := os.Create(tableFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fail to create JSON file: %v", err)
|
||||
}
|
||||
|
||||
if err = x.Asc("id").Iterate(table, func(idx int, bean interface{}) (err error) {
|
||||
enc := json.NewEncoder(f)
|
||||
return enc.Encode(bean)
|
||||
}); err != nil {
|
||||
f.Close()
|
||||
return fmt.Errorf("fail to dump table '%s': %v", tableName, err)
|
||||
}
|
||||
f.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ImportDatabase imports data from backup archive.
|
||||
func ImportDatabase(dirPath string) (err error) {
|
||||
// Purposely create a local variable to not modify global variable
|
||||
tables := append(tables, new(Version))
|
||||
for _, table := range tables {
|
||||
tableName := strings.TrimPrefix(fmt.Sprintf("%T", table), "*models.")
|
||||
tableFile := path.Join(dirPath, tableName+".json")
|
||||
if !com.IsExist(tableFile) {
|
||||
continue
|
||||
}
|
||||
|
||||
if err = x.DropTables(table); err != nil {
|
||||
return fmt.Errorf("fail to drop table '%s': %v", tableName, err)
|
||||
} else if err = x.Sync2(table); err != nil {
|
||||
return fmt.Errorf("fail to sync table '%s': %v", tableName, err)
|
||||
}
|
||||
|
||||
f, err := os.Open(tableFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fail to open JSON file: %v", err)
|
||||
}
|
||||
scanner := bufio.NewScanner(f)
|
||||
for scanner.Scan() {
|
||||
switch bean := table.(type) {
|
||||
case *LoginSource:
|
||||
meta := make(map[string]interface{})
|
||||
if err = json.Unmarshal(scanner.Bytes(), &meta); err != nil {
|
||||
return fmt.Errorf("fail to unmarshal to map: %v", err)
|
||||
}
|
||||
|
||||
tp := LoginType(com.StrTo(com.ToStr(meta["Type"])).MustInt64())
|
||||
switch tp {
|
||||
case LOGIN_LDAP, LOGIN_DLDAP:
|
||||
bean.Cfg = new(LDAPConfig)
|
||||
case LOGIN_SMTP:
|
||||
bean.Cfg = new(SMTPConfig)
|
||||
case LOGIN_PAM:
|
||||
bean.Cfg = new(PAMConfig)
|
||||
default:
|
||||
return fmt.Errorf("unrecognized login source type:: %v", tp)
|
||||
}
|
||||
table = bean
|
||||
}
|
||||
|
||||
if err = json.Unmarshal(scanner.Bytes(), table); err != nil {
|
||||
return fmt.Errorf("fail to unmarshal to struct: %v", err)
|
||||
}
|
||||
|
||||
if _, err = x.Insert(table); err != nil {
|
||||
return fmt.Errorf("fail to insert strcut: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue