Add RemoteAddress to mirrors (#26952)

This PR adds a new field `RemoteAddress` to both mirror types which
contains the sanitized remote address for easier (database) access to
that information. Will be used in the audit PR if merged.
This commit is contained in:
KN4CK3R 2023-09-16 18:03:02 +02:00 committed by GitHub
parent 5e039b0580
commit c766140dad
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 194 additions and 53 deletions

View file

@ -532,6 +532,8 @@ var migrations = []Migration{
NewMigration("Add Actions artifacts expiration date", v1_21.AddExpiredUnixColumnInActionArtifactTable), NewMigration("Add Actions artifacts expiration date", v1_21.AddExpiredUnixColumnInActionArtifactTable),
// v275 -> v276 // v275 -> v276
NewMigration("Add ScheduleID for ActionRun", v1_21.AddScheduleIDForActionRun), NewMigration("Add ScheduleID for ActionRun", v1_21.AddScheduleIDForActionRun),
// v276 -> v277
NewMigration("Add RemoteAddress to mirrors", v1_21.AddRemoteAddressToMirrors),
} }
// GetCurrentDBVersion returns the current db version // GetCurrentDBVersion returns the current db version

View file

@ -0,0 +1,132 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_21 //nolint
import (
"context"
"fmt"
"path/filepath"
"strings"
"code.gitea.io/gitea/modules/git"
giturl "code.gitea.io/gitea/modules/git/url"
"code.gitea.io/gitea/modules/setting"
"xorm.io/xorm"
)
func AddRemoteAddressToMirrors(x *xorm.Engine) error {
type Mirror struct {
RemoteAddress string `xorm:"VARCHAR(2048)"`
}
type PushMirror struct {
RemoteAddress string `xorm:"VARCHAR(2048)"`
}
if err := x.Sync(new(Mirror), new(PushMirror)); err != nil {
return err
}
if err := migratePullMirrors(x); err != nil {
return err
}
return migratePushMirrors(x)
}
func migratePullMirrors(x *xorm.Engine) error {
type Mirror struct {
ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"INDEX"`
RemoteAddress string `xorm:"VARCHAR(2048)"`
}
sess := x.NewSession()
defer sess.Close()
if err := sess.Begin(); err != nil {
return err
}
if err := sess.Iterate(new(Mirror), func(_ int, bean any) error {
m := bean.(*Mirror)
remoteAddress, err := getRemoteAddress(sess, m.RepoID, "origin")
if err != nil {
return err
}
m.RemoteAddress = remoteAddress
_, err = sess.ID(m.ID).Cols("remote_address").Update(m)
return err
}); err != nil {
return err
}
return sess.Commit()
}
func migratePushMirrors(x *xorm.Engine) error {
type PushMirror struct {
ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"INDEX"`
RemoteName string
RemoteAddress string `xorm:"VARCHAR(2048)"`
}
sess := x.NewSession()
defer sess.Close()
if err := sess.Begin(); err != nil {
return err
}
if err := sess.Iterate(new(PushMirror), func(_ int, bean any) error {
m := bean.(*PushMirror)
remoteAddress, err := getRemoteAddress(sess, m.RepoID, m.RemoteName)
if err != nil {
return err
}
m.RemoteAddress = remoteAddress
_, err = sess.ID(m.ID).Cols("remote_address").Update(m)
return err
}); err != nil {
return err
}
return sess.Commit()
}
func getRemoteAddress(sess *xorm.Session, repoID int64, remoteName string) (string, error) {
var ownerName string
var repoName string
has, err := sess.
Table("repository").
Cols("owner_name", "lower_name").
Where("id=?", repoID).
Get(&ownerName, &repoName)
if err != nil {
return "", err
} else if !has {
return "", fmt.Errorf("repository [%v] not found", repoID)
}
repoPath := filepath.Join(setting.RepoRootPath, strings.ToLower(ownerName), strings.ToLower(repoName)+".git")
remoteURL, err := git.GetRemoteAddress(context.Background(), repoPath, remoteName)
if err != nil {
return "", err
}
u, err := giturl.Parse(remoteURL)
if err != nil {
return "", err
}
u.User = nil
return u.String(), nil
}

View file

@ -31,7 +31,7 @@ type Mirror struct {
LFS bool `xorm:"lfs_enabled NOT NULL DEFAULT false"` LFS bool `xorm:"lfs_enabled NOT NULL DEFAULT false"`
LFSEndpoint string `xorm:"lfs_endpoint TEXT"` LFSEndpoint string `xorm:"lfs_endpoint TEXT"`
Address string `xorm:"-"` RemoteAddress string `xorm:"VARCHAR(2048)"`
} }
func init() { func init() {

View file

@ -24,6 +24,7 @@ type PushMirror struct {
RepoID int64 `xorm:"INDEX"` RepoID int64 `xorm:"INDEX"`
Repo *Repository `xorm:"-"` Repo *Repository `xorm:"-"`
RemoteName string RemoteName string
RemoteAddress string `xorm:"VARCHAR(2048)"`
SyncOnCommit bool `xorm:"NOT NULL DEFAULT true"` SyncOnCommit bool `xorm:"NOT NULL DEFAULT true"`
Interval time.Duration Interval time.Duration
@ -31,6 +32,7 @@ type PushMirror struct {
LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"` LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"`
LastError string `xorm:"text"` LastError string `xorm:"text"`
} }
type PushMirrorOptions struct { type PushMirrorOptions struct {
ID int64 ID int64
RepoID int64 RepoID int64

View file

@ -191,12 +191,8 @@ func (repo *Repository) SanitizedOriginalURL() string {
if repo.OriginalURL == "" { if repo.OriginalURL == "" {
return "" return ""
} }
u, err := url.Parse(repo.OriginalURL) u, _ := util.SanitizeURL(repo.OriginalURL)
if err != nil { return u
return ""
}
u.User = nil
return u.String()
} }
// text representations to be returned in SizeDetail.Name // text representations to be returned in SizeDetail.Name

View file

@ -180,12 +180,17 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
defer committer.Close() defer committer.Close()
if opts.Mirror { if opts.Mirror {
remoteAddress, err := util.SanitizeURL(opts.CloneAddr)
if err != nil {
return repo, err
}
mirrorModel := repo_model.Mirror{ mirrorModel := repo_model.Mirror{
RepoID: repo.ID, RepoID: repo.ID,
Interval: setting.Mirror.DefaultInterval, Interval: setting.Mirror.DefaultInterval,
EnablePrune: true, EnablePrune: true,
NextUpdateUnix: timeutil.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval), NextUpdateUnix: timeutil.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval),
LFS: opts.LFS, LFS: opts.LFS,
RemoteAddress: remoteAddress,
} }
if opts.LFS { if opts.LFS {
mirrorModel.LFSEndpoint = opts.LFSEndpoint mirrorModel.LFSEndpoint = opts.LFSEndpoint

View file

@ -39,3 +39,12 @@ func URLJoin(base string, elems ...string) string {
} }
return joinedURL return joinedURL
} }
func SanitizeURL(s string) (string, error) {
u, err := url.Parse(s)
if err != nil {
return "", err
}
u.User = nil
return u.String(), nil
}

View file

@ -353,12 +353,19 @@ func CreatePushMirror(ctx *context.APIContext, mirrorOption *api.CreatePushMirro
return return
} }
remoteAddress, err := util.SanitizeURL(mirrorOption.RemoteAddress)
if err != nil {
ctx.ServerError("SanitizeURL", err)
return
}
pushMirror := &repo_model.PushMirror{ pushMirror := &repo_model.PushMirror{
RepoID: repo.ID, RepoID: repo.ID,
Repo: repo, Repo: repo,
RemoteName: fmt.Sprintf("remote_mirror_%s", remoteSuffix), RemoteName: fmt.Sprintf("remote_mirror_%s", remoteSuffix),
Interval: interval, Interval: interval,
SyncOnCommit: mirrorOption.SyncOnCommit, SyncOnCommit: mirrorOption.SyncOnCommit,
RemoteAddress: remoteAddress,
} }
if err = repo_model.InsertPushMirror(ctx, pushMirror); err != nil { if err = repo_model.InsertPushMirror(ctx, pushMirror); err != nil {

View file

@ -243,6 +243,13 @@ func SettingsPost(ctx *context.Context) {
return return
} }
remoteAddress, err := util.SanitizeURL(form.MirrorAddress)
if err != nil {
ctx.ServerError("SanitizeURL", err)
return
}
pullMirror.RemoteAddress = remoteAddress
form.LFS = form.LFS && setting.LFS.StartServer form.LFS = form.LFS && setting.LFS.StartServer
if len(form.LFSEndpoint) > 0 { if len(form.LFSEndpoint) > 0 {
@ -397,12 +404,19 @@ func SettingsPost(ctx *context.Context) {
return return
} }
remoteAddress, err := util.SanitizeURL(form.PushMirrorAddress)
if err != nil {
ctx.ServerError("SanitizeURL", err)
return
}
m := &repo_model.PushMirror{ m := &repo_model.PushMirror{
RepoID: repo.ID, RepoID: repo.ID,
Repo: repo, Repo: repo,
RemoteName: fmt.Sprintf("remote_mirror_%s", remoteSuffix), RemoteName: fmt.Sprintf("remote_mirror_%s", remoteSuffix),
SyncOnCommit: form.PushMirrorSyncOnCommit, SyncOnCommit: form.PushMirrorSyncOnCommit,
Interval: interval, Interval: interval,
RemoteAddress: remoteAddress,
} }
if err := repo_model.InsertPushMirror(ctx, m); err != nil { if err := repo_model.InsertPushMirror(ctx, m); err != nil {
ctx.ServerError("InsertPushMirror", err) ctx.ServerError("InsertPushMirror", err)

View file

@ -628,15 +628,6 @@ func markupRender(ctx *context.Context, renderCtx *markup.RenderContext, input i
return escaped, output, err return escaped, output, err
} }
func safeURL(address string) string {
u, err := url.Parse(address)
if err != nil {
return address
}
u.User = nil
return u.String()
}
func checkHomeCodeViewable(ctx *context.Context) { func checkHomeCodeViewable(ctx *context.Context) {
if len(ctx.Repo.Units) > 0 { if len(ctx.Repo.Units) > 0 {
if ctx.Repo.Repository.IsBeingCreated() { if ctx.Repo.Repository.IsBeingCreated() {
@ -660,7 +651,7 @@ func checkHomeCodeViewable(ctx *context.Context) {
ctx.Data["Repo"] = ctx.Repo ctx.Data["Repo"] = ctx.Repo
ctx.Data["MigrateTask"] = task ctx.Data["MigrateTask"] = task
ctx.Data["CloneAddr"] = safeURL(cfg.CloneAddr) ctx.Data["CloneAddr"], _ = util.SanitizeURL(cfg.CloneAddr)
ctx.Data["Failed"] = task.Status == structs.TaskStatusFailed ctx.Data["Failed"] = task.Status == structs.TaskStatusFailed
ctx.HTML(http.StatusOK, tplMigrating) ctx.HTML(http.StatusOK, tplMigrating)
return return

View file

@ -5,21 +5,16 @@ package convert
import ( import (
repo_model "code.gitea.io/gitea/models/repo" repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/git"
api "code.gitea.io/gitea/modules/structs" api "code.gitea.io/gitea/modules/structs"
) )
// ToPushMirror convert from repo_model.PushMirror and remoteAddress to api.TopicResponse // ToPushMirror convert from repo_model.PushMirror and remoteAddress to api.TopicResponse
func ToPushMirror(pm *repo_model.PushMirror) (*api.PushMirror, error) { func ToPushMirror(pm *repo_model.PushMirror) (*api.PushMirror, error) {
repo := pm.GetRepository() repo := pm.GetRepository()
remoteAddress, err := getRemoteAddress(repo, pm.RemoteName)
if err != nil {
return nil, err
}
return &api.PushMirror{ return &api.PushMirror{
RepoName: repo.Name, RepoName: repo.Name,
RemoteName: pm.RemoteName, RemoteName: pm.RemoteName,
RemoteAddress: remoteAddress, RemoteAddress: pm.RemoteAddress,
CreatedUnix: pm.CreatedUnix.FormatLong(), CreatedUnix: pm.CreatedUnix.FormatLong(),
LastUpdateUnix: pm.LastUpdateUnix.FormatLong(), LastUpdateUnix: pm.LastUpdateUnix.FormatLong(),
LastError: pm.LastError, LastError: pm.LastError,
@ -27,13 +22,3 @@ func ToPushMirror(pm *repo_model.PushMirror) (*api.PushMirror, error) {
SyncOnCommit: pm.SyncOnCommit, SyncOnCommit: pm.SyncOnCommit,
}, nil }, nil
} }
func getRemoteAddress(repo *repo_model.Repository, remoteName string) (string, error) {
url, err := git.GetRemoteURL(git.DefaultContext, repo.RepoPath(), remoteName)
if err != nil {
return "", err
}
// remove confidential information
url.User = nil
return url.String(), nil
}

View file

@ -37,8 +37,7 @@
{{end}} {{end}}
</div> </div>
{{if $.PullMirror}} {{if $.PullMirror}}
{{$address := MirrorRemoteAddress $.Context . $.PullMirror.GetRemoteName false}} <div class="fork-flag">{{$.locale.Tr "repo.mirror_from"}} <a target="_blank" rel="noopener noreferrer" href="{{$.PullMirror.RemoteAddress}}">{{$.PullMirror.RemoteAddress}}</a></div>
<div class="fork-flag">{{$.locale.Tr "repo.mirror_from"}} <a target="_blank" rel="noopener noreferrer" href="{{$address.Address}}">{{$address.Address}}</a></div>
{{end}} {{end}}
{{if .IsFork}}<div class="fork-flag">{{$.locale.Tr "repo.forked_from"}} <a href="{{.BaseRepo.Link}}">{{.BaseRepo.FullName}}</a></div>{{end}} {{if .IsFork}}<div class="fork-flag">{{$.locale.Tr "repo.forked_from"}} <a href="{{.BaseRepo.Link}}">{{.BaseRepo.FullName}}</a></div>{{end}}
{{if .IsGenerated}}<div class="fork-flag">{{$.locale.Tr "repo.generated_from"}} <a href="{{.TemplateRepo.Link}}">{{.TemplateRepo.FullName}}</a></div>{{end}} {{if .IsGenerated}}<div class="fork-flag">{{$.locale.Tr "repo.generated_from"}} <a href="{{.TemplateRepo.Link}}">{{.TemplateRepo.FullName}}</a></div>{{end}}

View file

@ -123,7 +123,7 @@
{{else if $isWorkingPullMirror}} {{else if $isWorkingPullMirror}}
<tbody> <tbody>
<tr> <tr>
<td>{{(MirrorRemoteAddress $.Context .Repository .PullMirror.GetRemoteName false).Address}}</td> <td>{{.PullMirror.RemoteAddress}}</td>
<td>{{$.locale.Tr "repo.settings.mirror_settings.direction.pull"}}</td> <td>{{$.locale.Tr "repo.settings.mirror_settings.direction.pull"}}</td>
<td>{{DateTime "full" .PullMirror.UpdatedUnix}}</td> <td>{{DateTime "full" .PullMirror.UpdatedUnix}}</td>
<td class="right aligned"> <td class="right aligned">
@ -200,8 +200,7 @@
<tbody> <tbody>
{{range .PushMirrors}} {{range .PushMirrors}}
<tr> <tr>
{{$address := MirrorRemoteAddress $.Context $.Repository .GetRemoteName true}} <td class="gt-word-break">{{.RemoteAddress}}</td>
<td class="gt-word-break">{{$address.Address}}</td>
<td>{{$.locale.Tr "repo.settings.mirror_settings.direction.push"}}</td> <td>{{$.locale.Tr "repo.settings.mirror_settings.direction.push"}}</td>
<td>{{if .LastUpdateUnix}}{{DateTime "full" .LastUpdateUnix}}{{else}}{{$.locale.Tr "never"}}{{end}} {{if .LastError}}<div class="ui red label" data-tooltip-content="{{.LastError}}">{{$.locale.Tr "error"}}</div>{{end}}</td> <td>{{if .LastUpdateUnix}}{{DateTime "full" .LastUpdateUnix}}{{else}}{{$.locale.Tr "never"}}{{end}} {{if .LastError}}<div class="ui red label" data-tooltip-content="{{.LastError}}">{{$.locale.Tr "error"}}</div>{{end}}</td>
<td class="right aligned"> <td class="right aligned">
@ -211,7 +210,7 @@
data-tooltip-content="{{$.locale.Tr "repo.settings.mirror_settings.push_mirror.edit_sync_time"}}" data-tooltip-content="{{$.locale.Tr "repo.settings.mirror_settings.push_mirror.edit_sync_time"}}"
data-modal-push-mirror-edit-id="{{.ID}}" data-modal-push-mirror-edit-id="{{.ID}}"
data-modal-push-mirror-edit-interval="{{.Interval}}" data-modal-push-mirror-edit-interval="{{.Interval}}"
data-modal-push-mirror-edit-address="{{$address.Address}}" data-modal-push-mirror-edit-address="{{.RemoteAddress}}"
> >
{{svg "octicon-pencil" 14}} {{svg "octicon-pencil" 14}}
</button> </button>