mirror: show sync feeds on dashboard (#2017)

conf/locale/locale_en-US.ini

@@ -823,12 +823,14 @@ settings.event_push = Push
 settings.event_push_desc = Git push to a repository
 settings.event_issues = Issues
 settings.event_issues_desc = Issue opened, closed, reopened, edited, assigned, unassigned, label updated, label cleared, milestoned, or demilestoned.
-settings.event_issue_comment = Issue Comment
-settings.event_issue_comment_desc = Issue comment created, edited, or deleted.
 settings.event_pull_request = Pull Request
 settings.event_pull_request_desc = Pull request opened, closed, reopened, edited, assigned, unassigned, label updated, label cleared, milestoned, demilestoned, or synchronized.
+settings.event_issue_comment = Issue Comment
+settings.event_issue_comment_desc = Issue comment created, edited, or deleted.
 settings.event_release = Release
 settings.event_release_desc = Release published in a repository.
+settings.event_mirror_sync = Mirror Sync
+settings.event_mirror_sync_desc = Mirror commits pushed, reference created or deleted from upstream.
 settings.active = Active
 settings.active_helper = Details regarding the event which triggered the hook will be delivered as well.
 settings.add_hook_success = New webhook has been added.
@@ -1287,7 +1289,6 @@ notices.delete_success = System notices have been deleted successfully.
 
 [action]
 create_repo = created repository <a href="%s">%s</a>
-fork_repo = forked a repository to <a href="%s">%s</a>
 rename_repo = renamed repository from <code>%[1]s</code> to <a href="%[2]s">%[3]s</a>
 commit_repo = pushed to <a href="%[1]s/src/%[2]s">%[3]s</a> at <a href="%[1]s">%[4]s</a>
 compare_commits = View comparison for these %d commits
@@ -1304,6 +1305,10 @@ create_branch = created new branch <a href="%[1]s/src/%[2]s">%[3]s</a> at <a hre
 delete_branch = deleted branch <code>%[2]s</code> at <a href="%[1]s">%[3]s</a>
 push_tag = pushed tag <a href="%s/src/%s">%[2]s</a> to <a href="%[1]s">%[3]s</a>
 delete_tag = deleted tag <code>%[2]s</code> at <a href="%[1]s">%[3]s</a>
+fork_repo = forked a repository to <a href="%s">%s</a>
+mirror_sync_push = synced commits to <a href="%[1]s/src/%[2]s">%[3]s</a> at <a href="%[1]s">%[4]s</a> from mirror
+mirror_sync_create = synced new reference <a href="%s/src/%s">%[2]s</a> to <a href="%[1]s">%[3]s</a> from mirror
+mirror_sync_delete = synced and deleted reference <code>%[2]s</code> at <a href="%[1]s">%[3]s</a> from mirror
 
 [tool]
 ago = ago

gogs.go

@@ -16,7 +16,7 @@ import (
 	"github.com/gogs/gogs/pkg/setting"
 )
 
-const APP_VER = "0.11.49.0521"
+const APP_VER = "0.11.50.0530"
 
 func init() {
 	setting.AppVer = APP_VER

models/action.go

@@ -27,7 +27,7 @@ import (
 
 type ActionType int
 
-// To maintain backward compatibility only append to the end of list
+// Note: To maintain backward compatibility only append to the end of list
 const (
 	ACTION_CREATE_REPO         ActionType = iota + 1 // 1
 	ACTION_RENAME_REPO                               // 2
@@ -48,6 +48,9 @@ const (
 	ACTION_DELETE_BRANCH                             // 17
 	ACTION_DELETE_TAG                                // 18
 	ACTION_FORK_REPO                                 // 19
+	ACTION_MIRROR_SYNC_PUSH                          // 20
+	ACTION_MIRROR_SYNC_CREATE                        // 21
+	ACTION_MIRROR_SYNC_DELETE                        // 22
 )
 
 var (
@@ -667,6 +670,40 @@ func MergePullRequestAction(actUser *User, repo *Repository, pull *Issue) error
 	return mergePullRequestAction(x, actUser, repo, pull)
 }
 
+func mirrorSyncAction(opType ActionType, repo *Repository, refName string, data []byte) error {
+	return NotifyWatchers(&Action{
+		ActUserID:    repo.OwnerID,
+		ActUserName:  repo.MustOwner().Name,
+		OpType:       opType,
+		Content:      string(data),
+		RepoID:       repo.ID,
+		RepoUserName: repo.MustOwner().Name,
+		RepoName:     repo.Name,
+		RefName:      refName,
+		IsPrivate:    repo.IsPrivate,
+	})
+}
+
+// MirrorSyncPushAction adds new action for mirror synchronization of pushed commits.
+func MirrorSyncPushAction(repo *Repository, refName string, commits *PushCommits) error {
+	data, err := json.Marshal(commits)
+	if err != nil {
+		return err
+	}
+
+	return mirrorSyncAction(ACTION_MIRROR_SYNC_PUSH, repo, refName, data)
+}
+
+// MirrorSyncCreateAction adds new action for mirror synchronization of new reference.
+func MirrorSyncCreateAction(repo *Repository, refName string) error {
+	return mirrorSyncAction(ACTION_MIRROR_SYNC_CREATE, repo, refName, nil)
+}
+
+// MirrorSyncCreateAction adds new action for mirror synchronization of delete reference.
+func MirrorSyncDeleteAction(repo *Repository, refName string) error {
+	return mirrorSyncAction(ACTION_MIRROR_SYNC_DELETE, repo, refName, nil)
+}
+
 // GetFeeds returns action list of given user in given context.
 // actorID is the user who's requesting, ctxUserID is the user/org that is requested.
 // actorID can be -1 when isProfile is true or to skip the permission check.

models/mirror.go

@@ -33,22 +33,22 @@ type Mirror struct {
 	Interval    int         // Hour.
 	EnablePrune bool        `xorm:"NOT NULL DEFAULT true"`
 
-	Updated        time.Time `xorm:"-"`
-	UpdatedUnix    int64
-	NextUpdate     time.Time `xorm:"-"`
-	NextUpdateUnix int64
+	// Last and next sync time of Git data from upstream
+	LastSync     time.Time `xorm:"-"`
+	LastSyncUnix int64     `xorm:"updated_unix"`
+	NextSync     time.Time `xorm:"-"`
+	NextSyncUnix int64     `xorm:"next_update_unix"`
 
 	address string `xorm:"-"`
 }
 
 func (m *Mirror) BeforeInsert() {
-	m.UpdatedUnix = time.Now().Unix()
-	m.NextUpdateUnix = m.NextUpdate.Unix()
+	m.NextSyncUnix = m.NextSync.Unix()
 }
 
 func (m *Mirror) BeforeUpdate() {
-	m.UpdatedUnix = time.Now().Unix()
-	m.NextUpdateUnix = m.NextUpdate.Unix()
+	m.LastSyncUnix = m.LastSync.Unix()
+	m.NextSyncUnix = m.NextSync.Unix()
 }
 
 func (m *Mirror) AfterSet(colName string, _ xorm.Cell) {
@@ -60,15 +60,15 @@ func (m *Mirror) AfterSet(colName string, _ xorm.Cell) {
 			log.Error(3, "GetRepositoryByID [%d]: %v", m.ID, err)
 		}
 	case "updated_unix":
-		m.Updated = time.Unix(m.UpdatedUnix, 0).Local()
+		m.LastSync = time.Unix(m.LastSyncUnix, 0).Local()
 	case "next_update_unix":
-		m.NextUpdate = time.Unix(m.NextUpdateUnix, 0).Local()
+		m.NextSync = time.Unix(m.NextSyncUnix, 0).Local()
 	}
 }
 
-// ScheduleNextUpdate calculates and sets next update time.
-func (m *Mirror) ScheduleNextUpdate() {
-	m.NextUpdate = time.Now().Add(time.Duration(m.Interval) * time.Hour)
+// ScheduleNextSync calculates and sets next sync time based on repostiroy mirror setting.
+func (m *Mirror) ScheduleNextSync() {
+	m.NextSync = time.Now().Add(time.Duration(m.Interval) * time.Hour)
 }
 
 // findPasswordInMirrorAddress returns start (inclusive) and end index (exclusive)
@@ -188,8 +188,67 @@ func (m *Mirror) SaveAddress(addr string) error {
 	return cfg.SaveToIndent(configPath, "\t")
 }
 
+const GIT_SHORT_EMPTY_SHA = "0000000"
+
+// mirrorSyncResult contains information of a updated reference.
+// If the oldCommitID is "0000000", it means a new reference, the value of newCommitID is empty.
+// If the newCommitID is "0000000", it means the reference is deleted, the value of oldCommitID is empty.
+type mirrorSyncResult struct {
+	refName     string
+	oldCommitID string
+	newCommitID string
+}
+
+// parseRemoteUpdateOutput detects create, update and delete operations of references from upstream.
+func parseRemoteUpdateOutput(output string) []*mirrorSyncResult {
+	results := make([]*mirrorSyncResult, 0, 3)
+	lines := strings.Split(output, "\n")
+	for i := range lines {
+		// Make sure reference name is presented before continue
+		idx := strings.Index(lines[i], "-> ")
+		if idx == -1 {
+			continue
+		}
+
+		refName := lines[i][idx+3:]
+
+		switch {
+		case strings.HasPrefix(lines[i], " * "): // New reference
+			results = append(results, &mirrorSyncResult{
+				refName:     refName,
+				oldCommitID: GIT_SHORT_EMPTY_SHA,
+			})
+		case strings.HasPrefix(lines[i], " - "): // Delete reference
+			results = append(results, &mirrorSyncResult{
+				refName:     refName,
+				newCommitID: GIT_SHORT_EMPTY_SHA,
+			})
+		case strings.HasPrefix(lines[i], "   "): // New commits of a reference
+			delimIdx := strings.Index(lines[i][3:], " ")
+			if delimIdx == -1 {
+				log.Error(2, "SHA delimiter not found: %q", lines[i])
+				continue
+			}
+			shas := strings.Split(lines[i][3:delimIdx+3], "..")
+			if len(shas) != 2 {
+				log.Error(2, "Expect two SHAs but not what found: %q", lines[i])
+				continue
+			}
+			results = append(results, &mirrorSyncResult{
+				refName:     refName,
+				oldCommitID: shas[0],
+				newCommitID: shas[1],
+			})
+
+		default:
+			log.Warn("parseRemoteUpdateOutput: unexpected update line %q", lines[i])
+		}
+	}
+	return results
+}
+
 // runSync returns true if sync finished without error.
-func (m *Mirror) runSync() bool {
+func (m *Mirror) runSync() ([]*mirrorSyncResult, bool) {
 	repoPath := m.Repo.RepoPath()
 	wikiPath := m.Repo.WikiPath()
 	timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
@@ -204,29 +263,32 @@ func (m *Mirror) runSync() bool {
 		if err := CreateRepositoryNotice(desc); err != nil {
 			log.Error(2, "CreateRepositoryNotice: %v", err)
 		}
-		return false
+		return nil, false
 	}
 
 	gitArgs := []string{"remote", "update"}
 	if m.EnablePrune {
 		gitArgs = append(gitArgs, "--prune")
 	}
-	if _, stderr, err := process.ExecDir(
+	_, stderr, err := process.ExecDir(
 		timeout, repoPath, fmt.Sprintf("Mirror.runSync: %s", repoPath),
-		"git", gitArgs...); err != nil {
+		"git", gitArgs...)
+	if err != nil {
 		desc := fmt.Sprintf("Fail to update mirror repository '%s': %s", repoPath, stderr)
 		log.Error(2, desc)
 		if err = CreateRepositoryNotice(desc); err != nil {
 			log.Error(2, "CreateRepositoryNotice: %v", err)
 		}
-		return false
+		return nil, false
 	}
+	output := stderr
 
 	if err := m.Repo.UpdateSize(); err != nil {
 		log.Error(2, "UpdateSize [repo_id: %d]: %v", m.Repo.ID, err)
 	}
 
 	if m.Repo.HasWiki() {
+		// Even if wiki sync failed, we still want results from the main repository
 		if _, stderr, err := process.ExecDir(
 			timeout, wikiPath, fmt.Sprintf("Mirror.runSync: %s", wikiPath),
 			"git", "remote", "update", "--prune"); err != nil {
@@ -235,11 +297,10 @@ func (m *Mirror) runSync() bool {
 			if err = CreateRepositoryNotice(desc); err != nil {
 				log.Error(2, "CreateRepositoryNotice: %v", err)
 			}
-			return false
 		}
 	}
 
-	return true
+	return parseRemoteUpdateOutput(output), true
 }
 
 func getMirrorByRepoID(e Engine, repoID int64) (*Mirror, error) {
@@ -301,50 +362,88 @@ func MirrorUpdate() {
 func SyncMirrors() {
 	// Start listening on new sync requests.
 	for repoID := range MirrorQueue.Queue() {
-		log.Trace("SyncMirrors [repo_id: %v]", repoID)
+		log.Trace("SyncMirrors [repo_id: %d]", repoID)
 		MirrorQueue.Remove(repoID)
 
 		m, err := GetMirrorByRepoID(com.StrTo(repoID).MustInt64())
 		if err != nil {
-			log.Error(2, "GetMirrorByRepoID [%s]: %v", m.RepoID, err)
+			log.Error(2, "GetMirrorByRepoID [%d]: %v", m.RepoID, err)
 			continue
 		}
 
-		if !m.runSync() {
+		results, ok := m.runSync()
+		if !ok {
 			continue
 		}
 
-		m.ScheduleNextUpdate()
+		m.ScheduleNextSync()
 		if err = UpdateMirror(m); err != nil {
-			log.Error(2, "UpdateMirror [%s]: %v", m.RepoID, err)
+			log.Error(2, "UpdateMirror [%d]: %v", m.RepoID, err)
 			continue
 		}
 
-		// TODO: Work on #2017 and #4528 together
-		// 1. Get commits after last update time
-		// 2. Simulate push event for new commits, see models/repo_editor.go:164
-		// 		This automatically triggers the "push" webhook
+		// TODO:
+		// - Create "Mirror Sync" webhook event
+		// - Create mirror sync (create, push and delete) events and trigger the "mirror sync" webhooks
 
-		// ******************************
-		// EDIT AREA - START
-		// ******************************
+		var gitRepo *git.Repository
+		if len(results) == 0 {
+			log.Trace("SyncMirrors [repo_id: %d]: no commits fetched", m.RepoID)
+		} else {
+			gitRepo, err = git.OpenRepository(m.Repo.RepoPath())
+			if err != nil {
+				log.Error(2, "OpenRepository [%d]: %v", m.RepoID, err)
+				continue
+			}
+		}
+
+		for _, result := range results {
+			// Create reference
+			if result.oldCommitID == GIT_SHORT_EMPTY_SHA {
+				if err = MirrorSyncCreateAction(m.Repo, result.refName); err != nil {
+					log.Error(2, "MirrorSyncCreateAction [repo_id: %d]: %v", m.RepoID, err)
+				}
+				continue
+			}
+
+			// Delete reference
+			if result.newCommitID == GIT_SHORT_EMPTY_SHA {
+				if err = MirrorSyncDeleteAction(m.Repo, result.refName); err != nil {
+					log.Error(2, "MirrorSyncDeleteAction [repo_id: %d]: %v", m.RepoID, err)
+				}
+				continue
+			}
+
+			// Push commits
+			commits, err := gitRepo.CommitsBetweenIDs(result.newCommitID, result.oldCommitID)
+			if err != nil {
+				log.Error(2, "CommitsBetweenIDs [repo_id: %d, new_commit_id: %s, old_commit_id: %s]: %v", m.RepoID, result.newCommitID, result.oldCommitID, err)
+				continue
+			}
+			pushCommits := ListToPushCommits(commits)
+			if err = MirrorSyncPushAction(m.Repo, result.refName, pushCommits); err != nil {
+				log.Error(2, "MirrorSyncPushAction [repo_id: %d]: %v", m.RepoID, err)
+				continue
+			}
+		}
+
+		if _, err = x.Exec("UPDATE mirror SET updated_unix = ? WHERE repo_id = ?", time.Now().Unix(), m.RepoID); err != nil {
+			log.Error(2, "Update 'mirror.updated_unix' [%d]: %v", m.RepoID, err)
+			continue
+		}
 
 		// Get latest commit date and compare to current repository updated time,
 		// update if latest commit date is newer.
 		commitDate, err := git.GetLatestCommitDate(m.Repo.RepoPath(), "")
 		if err != nil {
-			log.Error(2, "GetLatestCommitDate [%s]: %v", m.RepoID, err)
+			log.Error(2, "GetLatestCommitDate [%d]: %v", m.RepoID, err)
 			continue
 		} else if commitDate.Before(m.Repo.Updated) {
 			continue
 		}
 
-		// ******************************
-		// EDIT AREA - END
-		// ******************************
-
 		if _, err = x.Exec("UPDATE repository SET updated_unix = ? WHERE id = ?", commitDate.Unix(), m.RepoID); err != nil {
-			log.Error(2, "Update repository 'updated_unix' [%s]: %v", m.RepoID, err)
+			log.Error(2, "Update 'repository.updated_unix' [%d]: %v", m.RepoID, err)
 			continue
 		}
 	}

models/mirror_test.go

@@ -10,6 +10,40 @@ import (
 	. "github.com/smartystreets/goconvey/convey"
 )
 
+func Test_parseRemoteUpdateOutput(t *testing.T) {
+	Convey("Parse mirror remote update output", t, func() {
+		testCases := []struct {
+			output  string
+			results []*mirrorSyncResult
+		}{
+			{
+				`
+From https://try.gogs.io/unknwon/upsteam
+ * [new branch]      develop    -> develop
+   b0bb24f..1d85a4f  master     -> master
+ - [deleted]         (none)     -> bugfix
+`,
+				[]*mirrorSyncResult{
+					{"develop", GIT_SHORT_EMPTY_SHA, ""},
+					{"master", "b0bb24f", "1d85a4f"},
+					{"bugfix", "", GIT_SHORT_EMPTY_SHA},
+				},
+			},
+		}
+
+		for _, tc := range testCases {
+			results := parseRemoteUpdateOutput(tc.output)
+			So(len(results), ShouldEqual, len(tc.results))
+
+			for i := range tc.results {
+				So(tc.results[i].refName, ShouldEqual, results[i].refName)
+				So(tc.results[i].oldCommitID, ShouldEqual, results[i].oldCommitID)
+				So(tc.results[i].newCommitID, ShouldEqual, results[i].newCommitID)
+			}
+		}
+	})
+}
+
 func Test_findPasswordInMirrorAddress(t *testing.T) {
 	Convey("Find password portion in mirror address", t, func() {
 		testCases := []struct {

models/repo.go

@@ -750,7 +750,7 @@ func MigrateRepository(doer, owner *User, opts MigrateRepoOptions) (*Repository,
 			RepoID:      repo.ID,
 			Interval:    setting.Mirror.DefaultInterval,
 			EnablePrune: true,
-			NextUpdate:  time.Now().Add(time.Duration(setting.Mirror.DefaultInterval) * time.Hour),
+			NextSync:    time.Now().Add(time.Duration(setting.Mirror.DefaultInterval) * time.Hour),
 		}); err != nil {
 			return repo, fmt.Errorf("InsertOne: %v", err)
 		}

pkg/template/template.go

@@ -272,6 +272,8 @@ func ActionIcon(opType int) string {
 		return "alert"
 	case 19: // Fork a repository
 		return "repo-forked"
+	case 20, 21, 22: // Mirror sync
+		return "repo-clone"
 	default:
 		return "invalid type"
 	}

public/css/gogs.css

@@ -3039,7 +3039,7 @@ footer .ui.language .menu {
 .feeds .news .push.news .content ul {
   font-size: 13px;
   list-style: none;
-  padding-left: 10px;
+  padding-left: 0;
 }
 .feeds .news .push.news .content ul img {
   margin-bottom: -2px;

public/less/_dashboard.less

@@ -72,7 +72,7 @@
 		.push.news .content ul {
 			font-size: 13px;
 			list-style: none;
-			padding-left: 10px;
+			padding-left: 0;
 
 			img {
 				margin-bottom: -2px;

routes/repo/setting.go

@@ -111,7 +111,7 @@ func SettingsPost(c *context.Context, f form.RepoSetting) {
 		if f.Interval > 0 {
 			c.Repo.Mirror.EnablePrune = f.EnablePrune
 			c.Repo.Mirror.Interval = f.Interval
-			c.Repo.Mirror.NextUpdate = time.Now().Add(time.Duration(f.Interval) * time.Hour)
+			c.Repo.Mirror.NextSync = time.Now().Add(time.Duration(f.Interval) * time.Hour)
 			if err := models.UpdateMirror(c.Repo.Mirror); err != nil {
 				c.ServerError("UpdateMirror", err)
 				return

templates/.VERSION

@@ -1 +1 @@
-0.11.49.0521
+0.11.50.0530

templates/user/dashboard/feeds.tmpl

@@ -5,7 +5,7 @@
 		</div>
 		<div class="ui grid">
 			<div class="ui fifteen wide column">
-				<div class="{{if eq .GetOpType 5}}push news{{end}}">
+				<div class="{{if or (eq .GetOpType 5) (eq .GetOpType 20)}}push news{{end}}">
 					<p>
 						<a href="{{AppSubURL}}/{{.GetActUserName}}">{{.ShortActUserName}}</a>
 						<!-- Reference types to models/action.go -->
@@ -53,9 +53,16 @@
 							{{$.i18n.Tr "action.delete_tag" .GetRepoLink .GetBranch .ShortRepoPath | Str2html}}
 						{{else if eq .GetOpType 19}}
 							{{$.i18n.Tr "action.fork_repo" .GetRepoLink .ShortRepoPath | Str2html}}
+						{{else if eq .GetOpType 20}}
+							{{ $branchLink := .GetBranch | EscapePound}}
+							{{$.i18n.Tr "action.mirror_sync_push" .GetRepoLink $branchLink .GetBranch .ShortRepoPath | Str2html}}
+						{{else if eq .GetOpType 21}}
+							{{$.i18n.Tr "action.mirror_sync_create" .GetRepoLink .GetBranch .ShortRepoPath | Str2html}}
+						{{else if eq .GetOpType 22}}
+							{{$.i18n.Tr "action.mirror_sync_delete" .GetRepoLink .GetBranch .ShortRepoPath | Str2html}}
 						{{end}}
 					</p>
-					{{if eq .GetOpType 5}}
+					{{if or (eq .GetOpType 5) (eq .GetOpType 20)}}
 						<div class="content">
 							<ul>
 								{{ $push := ActionContent2Commits .}}