Allow skipping forks and mirrors from being indexed (#23187)

This PR adds two new options to disable repo/code search indexing of
both forks and mirrors.

Related: #22842
This commit is contained in:
techknowlogick 2023-05-25 04:13:47 -04:00 committed by GitHub
parent cff4e37d43
commit 033d92997f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 51 additions and 16 deletions

View file

@ -1362,6 +1362,10 @@ LEVEL = Info
;; repo indexer by default disabled, since it uses a lot of disk space ;; repo indexer by default disabled, since it uses a lot of disk space
;REPO_INDEXER_ENABLED = false ;REPO_INDEXER_ENABLED = false
;; ;;
;; repo indexer units, the items to index, could be `sources`, `forks`, `mirrors`, `templates` or any combination of them separated by a comma.
;; If empty then it defaults to `sources` only, as if you'd like to disable fully please see REPO_INDEXER_ENABLED.
;REPO_INDEXER_REPO_TYPES = sources,forks,mirrors,templates
;;
;; Code search engine type, could be `bleve` or `elasticsearch`. ;; Code search engine type, could be `bleve` or `elasticsearch`.
;REPO_INDEXER_TYPE = bleve ;REPO_INDEXER_TYPE = bleve
;; ;;

View file

@ -465,6 +465,7 @@ relation to port exhaustion.
- `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: Index file used for issue search; available when ISSUE_INDEXER_TYPE is bleve and elasticsearch. Relative paths will be made absolute against _`AppWorkPath`_. - `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: Index file used for issue search; available when ISSUE_INDEXER_TYPE is bleve and elasticsearch. Relative paths will be made absolute against _`AppWorkPath`_.
- `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space, about 6 times more than the repository size). - `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space, about 6 times more than the repository size).
- `REPO_INDEXER_REPO_TYPES`: **sources,forks,mirrors,templates**: Repo indexer units. The items to index could be `sources`, `forks`, `mirrors`, `templates` or any combination of them separated by a comma. If empty then it defaults to `sources` only, as if you'd like to disable fully please see `REPO_INDEXER_ENABLED`.
- `REPO_INDEXER_TYPE`: **bleve**: Code search engine type, could be `bleve` or `elasticsearch`. - `REPO_INDEXER_TYPE`: **bleve**: Code search engine type, could be `bleve` or `elasticsearch`.
- `REPO_INDEXER_PATH`: **indexers/repos.bleve**: Index file used for code search. - `REPO_INDEXER_PATH`: **indexers/repos.bleve**: Index file used for code search.
- `REPO_INDEXER_CONN_STR`: ****: Code indexer connection string, available when `REPO_INDEXER_TYPE` is elasticsearch. i.e. http://elastic:changeme@localhost:9200 - `REPO_INDEXER_CONN_STR`: ****: Code indexer connection string, available when `REPO_INDEXER_TYPE` is elasticsearch. i.e. http://elastic:changeme@localhost:9200

View file

@ -19,6 +19,7 @@ import (
"code.gitea.io/gitea/modules/queue" "code.gitea.io/gitea/modules/queue"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util"
) )
// SearchResult result of performing a search in a repo // SearchResult result of performing a search in a repo
@ -91,6 +92,32 @@ func index(ctx context.Context, indexer Indexer, repoID int64) error {
return err return err
} }
repoTypes := setting.Indexer.RepoIndexerRepoTypes
if len(repoTypes) == 0 {
repoTypes = []string{"sources"}
}
// skip forks from being indexed if unit is not present
if !util.SliceContains(repoTypes, "forks") && repo.IsFork {
return nil
}
// skip mirrors from being indexed if unit is not present
if !util.SliceContains(repoTypes, "mirrors") && repo.IsMirror {
return nil
}
// skip templates from being indexed if unit is not present
if !util.SliceContains(repoTypes, "templates") && repo.IsTemplate {
return nil
}
// skip regular repos from being indexed if unit is not present
if !util.SliceContains(repoTypes, "sources") && !repo.IsFork && !repo.IsMirror && !repo.IsTemplate {
return nil
}
sha, err := getDefaultBranchSha(ctx, repo) sha, err := getDefaultBranchSha(ctx, repo)
if err != nil { if err != nil {
return err return err

View file

@ -23,15 +23,16 @@ var Indexer = struct {
IssueIndexerName string IssueIndexerName string
StartupTimeout time.Duration StartupTimeout time.Duration
RepoIndexerEnabled bool RepoIndexerEnabled bool
RepoType string RepoIndexerRepoTypes []string
RepoPath string RepoType string
RepoConnStr string RepoPath string
RepoIndexerName string RepoConnStr string
MaxIndexerFileSize int64 RepoIndexerName string
IncludePatterns []glob.Glob MaxIndexerFileSize int64
ExcludePatterns []glob.Glob IncludePatterns []glob.Glob
ExcludeVendored bool ExcludePatterns []glob.Glob
ExcludeVendored bool
}{ }{
IssueType: "bleve", IssueType: "bleve",
IssuePath: "indexers/issues.bleve", IssuePath: "indexers/issues.bleve",
@ -39,13 +40,14 @@ var Indexer = struct {
IssueConnAuth: "", IssueConnAuth: "",
IssueIndexerName: "gitea_issues", IssueIndexerName: "gitea_issues",
RepoIndexerEnabled: false, RepoIndexerEnabled: false,
RepoType: "bleve", RepoIndexerRepoTypes: []string{"sources", "forks", "mirrors", "templates"},
RepoPath: "indexers/repos.bleve", RepoType: "bleve",
RepoConnStr: "", RepoPath: "indexers/repos.bleve",
RepoIndexerName: "gitea_codes", RepoConnStr: "",
MaxIndexerFileSize: 1024 * 1024, RepoIndexerName: "gitea_codes",
ExcludeVendored: true, MaxIndexerFileSize: 1024 * 1024,
ExcludeVendored: true,
} }
func loadIndexerFrom(rootCfg ConfigProvider) { func loadIndexerFrom(rootCfg ConfigProvider) {
@ -71,6 +73,7 @@ func loadIndexerFrom(rootCfg ConfigProvider) {
Indexer.IssueIndexerName = sec.Key("ISSUE_INDEXER_NAME").MustString(Indexer.IssueIndexerName) Indexer.IssueIndexerName = sec.Key("ISSUE_INDEXER_NAME").MustString(Indexer.IssueIndexerName)
Indexer.RepoIndexerEnabled = sec.Key("REPO_INDEXER_ENABLED").MustBool(false) Indexer.RepoIndexerEnabled = sec.Key("REPO_INDEXER_ENABLED").MustBool(false)
Indexer.RepoIndexerRepoTypes = strings.Split(sec.Key("REPO_INDEXER_REPO_TYPES").MustString("sources,forks,mirrors,templates"), ",")
Indexer.RepoType = sec.Key("REPO_INDEXER_TYPE").MustString("bleve") Indexer.RepoType = sec.Key("REPO_INDEXER_TYPE").MustString("bleve")
Indexer.RepoPath = filepath.ToSlash(sec.Key("REPO_INDEXER_PATH").MustString(filepath.ToSlash(filepath.Join(AppDataPath, "indexers/repos.bleve")))) Indexer.RepoPath = filepath.ToSlash(sec.Key("REPO_INDEXER_PATH").MustString(filepath.ToSlash(filepath.Join(AppDataPath, "indexers/repos.bleve"))))
if !filepath.IsAbs(Indexer.RepoPath) { if !filepath.IsAbs(Indexer.RepoPath) {