Share HTML template renderers and create a watcher framework (#20218)

The recovery, API, Web and package frameworks all create their own HTML
Renderers. This increases the memory requirements of Gitea
unnecessarily with duplicate templates being kept in memory.

Further the reloading framework in dev mode for these involves locking
and recompiling all of the templates on each load. This will potentially
hide concurrency issues and it is inefficient.

This PR stores the templates renderer in the context and stores this
context in the NormalRoutes, it then creates a fsnotify.Watcher
framework to watch files.

The watching framework is then extended to the mailer templates which
were previously not being reloaded in dev.

Then the locales are simplified to a similar structure.

Fix #20210 
Fix #20211
Fix #20217

Signed-off-by: Andrew Thornton <art27@cantab.net>
This commit is contained in:
zeripath 2022-08-28 10:43:25 +01:00 committed by GitHub
parent c21d6511a8
commit bb0ff77e46
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
43 changed files with 902 additions and 618 deletions

View file

@ -123,7 +123,7 @@ func initEmbeddedExtractor(c *cli.Context) error {
sections["public"] = &section{Path: "public", Names: public.AssetNames, IsDir: public.AssetIsDir, Asset: public.Asset} sections["public"] = &section{Path: "public", Names: public.AssetNames, IsDir: public.AssetIsDir, Asset: public.Asset}
sections["options"] = &section{Path: "options", Names: options.AssetNames, IsDir: options.AssetIsDir, Asset: options.Asset} sections["options"] = &section{Path: "options", Names: options.AssetNames, IsDir: options.AssetIsDir, Asset: options.Asset}
sections["templates"] = &section{Path: "templates", Names: templates.AssetNames, IsDir: templates.AssetIsDir, Asset: templates.Asset} sections["templates"] = &section{Path: "templates", Names: templates.BuiltinAssetNames, IsDir: templates.BuiltinAssetIsDir, Asset: templates.BuiltinAsset}
for _, sec := range sections { for _, sec := range sections {
assets = append(assets, buildAssetList(sec, pats, c)...) assets = append(assets, buildAssetList(sec, pats, c)...)

View file

@ -126,8 +126,10 @@ func runWeb(ctx *cli.Context) error {
return err return err
} }
} }
c := install.Routes() installCtx, cancel := context.WithCancel(graceful.GetManager().HammerContext())
c := install.Routes(installCtx)
err := listen(c, false) err := listen(c, false)
cancel()
if err != nil { if err != nil {
log.Critical("Unable to open listener for installer. Is Gitea already running?") log.Critical("Unable to open listener for installer. Is Gitea already running?")
graceful.GetManager().DoGracefulShutdown() graceful.GetManager().DoGracefulShutdown()
@ -175,7 +177,7 @@ func runWeb(ctx *cli.Context) error {
} }
// Set up Chi routes // Set up Chi routes
c := routers.NormalRoutes() c := routers.NormalRoutes(graceful.GetManager().HammerContext())
err := listen(c, true) err := listen(c, true)
<-graceful.GetManager().Done() <-graceful.GetManager().Done()
log.Info("PID: %d Gitea Web Finished", os.Getpid()) log.Info("PID: %d Gitea Web Finished", os.Getpid())

View file

@ -27,6 +27,7 @@ import (
"code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/models/unittest"
gitea_git "code.gitea.io/gitea/modules/git" gitea_git "code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/markup/external" "code.gitea.io/gitea/modules/markup/external"
repo_module "code.gitea.io/gitea/modules/repository" repo_module "code.gitea.io/gitea/modules/repository"
@ -117,7 +118,7 @@ func runPR() {
// routers.GlobalInit() // routers.GlobalInit()
external.RegisterRenderers() external.RegisterRenderers()
markup.Init() markup.Init()
c := routers.NormalRoutes() c := routers.NormalRoutes(graceful.GetManager().HammerContext())
log.Printf("[PR] Ready for testing !\n") log.Printf("[PR] Ready for testing !\n")
log.Printf("[PR] Login with user1, user2, user3, ... with pass: password\n") log.Printf("[PR] Login with user1, user2, user3, ... with pass: password\n")

2
go.mod
View file

@ -28,6 +28,7 @@ require (
github.com/emirpasic/gods v1.18.1 github.com/emirpasic/gods v1.18.1
github.com/ethantkoenig/rupture v1.0.1 github.com/ethantkoenig/rupture v1.0.1
github.com/felixge/fgprof v0.9.2 github.com/felixge/fgprof v0.9.2
github.com/fsnotify/fsnotify v1.5.4
github.com/gliderlabs/ssh v0.3.4 github.com/gliderlabs/ssh v0.3.4
github.com/go-ap/activitypub v0.0.0-20220615144428-48208c70483b github.com/go-ap/activitypub v0.0.0-20220615144428-48208c70483b
github.com/go-ap/jsonld v0.0.0-20220615144122-1d862b15410d github.com/go-ap/jsonld v0.0.0-20220615144122-1d862b15410d
@ -161,7 +162,6 @@ require (
github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect
github.com/felixge/httpsnoop v1.0.2 // indirect github.com/felixge/httpsnoop v1.0.2 // indirect
github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
github.com/fsnotify/fsnotify v1.5.4 // indirect
github.com/fullstorydev/grpcurl v1.8.1 // indirect github.com/fullstorydev/grpcurl v1.8.1 // indirect
github.com/fxamacker/cbor/v2 v2.4.0 // indirect github.com/fxamacker/cbor/v2 v2.4.0 // indirect
github.com/go-ap/errors v0.0.0-20220615144307-e8bc4a40ae9f // indirect github.com/go-ap/errors v0.0.0-20220615144307-e8bc4a40ae9f // indirect

View file

@ -23,10 +23,10 @@ import (
func TestActivityPubPerson(t *testing.T) { func TestActivityPubPerson(t *testing.T) {
setting.Federation.Enabled = true setting.Federation.Enabled = true
c = routers.NormalRoutes() c = routers.NormalRoutes(context.TODO())
defer func() { defer func() {
setting.Federation.Enabled = false setting.Federation.Enabled = false
c = routers.NormalRoutes() c = routers.NormalRoutes(context.TODO())
}() }()
onGiteaRun(t, func(*testing.T, *url.URL) { onGiteaRun(t, func(*testing.T, *url.URL) {
@ -60,10 +60,10 @@ func TestActivityPubPerson(t *testing.T) {
func TestActivityPubMissingPerson(t *testing.T) { func TestActivityPubMissingPerson(t *testing.T) {
setting.Federation.Enabled = true setting.Federation.Enabled = true
c = routers.NormalRoutes() c = routers.NormalRoutes(context.TODO())
defer func() { defer func() {
setting.Federation.Enabled = false setting.Federation.Enabled = false
c = routers.NormalRoutes() c = routers.NormalRoutes(context.TODO())
}() }()
onGiteaRun(t, func(*testing.T, *url.URL) { onGiteaRun(t, func(*testing.T, *url.URL) {
@ -75,10 +75,10 @@ func TestActivityPubMissingPerson(t *testing.T) {
func TestActivityPubPersonInbox(t *testing.T) { func TestActivityPubPersonInbox(t *testing.T) {
setting.Federation.Enabled = true setting.Federation.Enabled = true
c = routers.NormalRoutes() c = routers.NormalRoutes(context.TODO())
defer func() { defer func() {
setting.Federation.Enabled = false setting.Federation.Enabled = false
c = routers.NormalRoutes() c = routers.NormalRoutes(context.TODO())
}() }()
srv := httptest.NewServer(c) srv := httptest.NewServer(c)

View file

@ -5,6 +5,7 @@
package integrations package integrations
import ( import (
"context"
"net/http" "net/http"
"net/url" "net/url"
"testing" "testing"
@ -18,10 +19,10 @@ import (
func TestNodeinfo(t *testing.T) { func TestNodeinfo(t *testing.T) {
setting.Federation.Enabled = true setting.Federation.Enabled = true
c = routers.NormalRoutes() c = routers.NormalRoutes(context.TODO())
defer func() { defer func() {
setting.Federation.Enabled = false setting.Federation.Enabled = false
c = routers.NormalRoutes() c = routers.NormalRoutes(context.TODO())
}() }()
onGiteaRun(t, func(*testing.T, *url.URL) { onGiteaRun(t, func(*testing.T, *url.URL) {

View file

@ -5,6 +5,7 @@
package integrations package integrations
import ( import (
"context"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"os" "os"
@ -57,7 +58,7 @@ func TestSessionFileCreation(t *testing.T) {
oldSessionConfig := setting.SessionConfig.ProviderConfig oldSessionConfig := setting.SessionConfig.ProviderConfig
defer func() { defer func() {
setting.SessionConfig.ProviderConfig = oldSessionConfig setting.SessionConfig.ProviderConfig = oldSessionConfig
c = routers.NormalRoutes() c = routers.NormalRoutes(context.TODO())
}() }()
var config session.Options var config session.Options
@ -82,7 +83,7 @@ func TestSessionFileCreation(t *testing.T) {
setting.SessionConfig.ProviderConfig = string(newConfigBytes) setting.SessionConfig.ProviderConfig = string(newConfigBytes)
c = routers.NormalRoutes() c = routers.NormalRoutes(context.TODO())
t.Run("NoSessionOnViewIssue", func(t *testing.T) { t.Run("NoSessionOnViewIssue", func(t *testing.T) {
defer PrintCurrentTest(t)() defer PrintCurrentTest(t)()

View file

@ -89,7 +89,7 @@ func TestMain(m *testing.M) {
defer cancel() defer cancel()
initIntegrationTest() initIntegrationTest()
c = routers.NormalRoutes() c = routers.NormalRoutes(context.TODO())
// integration test settings... // integration test settings...
if setting.Cfg != nil { if setting.Cfg != nil {

View file

@ -133,11 +133,18 @@ then resh (ר), and finally heh (ה) (which should appear leftmost).`,
}, },
} }
type nullLocale struct{}
func (nullLocale) Language() string { return "" }
func (nullLocale) Tr(key string, _ ...interface{}) string { return key }
func (nullLocale) TrN(cnt interface{}, key1, keyN string, args ...interface{}) string { return "" }
var _ (translation.Locale) = nullLocale{}
func TestEscapeControlString(t *testing.T) { func TestEscapeControlString(t *testing.T) {
for _, tt := range escapeControlTests { for _, tt := range escapeControlTests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
locale := translation.NewLocale("en_US") status, result := EscapeControlString(tt.text, nullLocale{})
status, result := EscapeControlString(tt.text, locale)
if !reflect.DeepEqual(*status, tt.status) { if !reflect.DeepEqual(*status, tt.status) {
t.Errorf("EscapeControlString() status = %v, wanted= %v", status, tt.status) t.Errorf("EscapeControlString() status = %v, wanted= %v", status, tt.status)
} }
@ -173,7 +180,7 @@ func TestEscapeControlReader(t *testing.T) {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
input := strings.NewReader(tt.text) input := strings.NewReader(tt.text)
output := &strings.Builder{} output := &strings.Builder{}
status, err := EscapeControlReader(input, output, translation.NewLocale("en_US")) status, err := EscapeControlReader(input, output, nullLocale{})
result := output.String() result := output.String()
if err != nil { if err != nil {
t.Errorf("EscapeControlReader(): err = %v", err) t.Errorf("EscapeControlReader(): err = %v", err)
@ -195,5 +202,5 @@ func TestEscapeControlReader_panic(t *testing.T) {
for i := 0; i < 6826; i++ { for i := 0; i < 6826; i++ {
bs = append(bs, []byte("—")...) bs = append(bs, []byte("—")...)
} }
_, _ = EscapeControlString(string(bs), translation.NewLocale("en_US")) _, _ = EscapeControlString(string(bs), nullLocale{})
} }

View file

@ -658,8 +658,8 @@ func Auth(authMethod auth.Method) func(*Context) {
} }
// Contexter initializes a classic context for a request. // Contexter initializes a classic context for a request.
func Contexter() func(next http.Handler) http.Handler { func Contexter(ctx context.Context) func(next http.Handler) http.Handler {
rnd := templates.HTMLRenderer() _, rnd := templates.HTMLRenderer(ctx)
csrfOpts := getCsrfOpts() csrfOpts := getCsrfOpts()
if !setting.IsProd { if !setting.IsProd {
CsrfTokenRegenerationInterval = 5 * time.Second // in dev, re-generate the tokens more aggressively for debug purpose CsrfTokenRegenerationInterval = 5 * time.Second // in dev, re-generate the tokens more aggressively for debug purpose

View file

@ -5,6 +5,7 @@
package context package context
import ( import (
gocontext "context"
"fmt" "fmt"
"net/http" "net/http"
@ -14,6 +15,7 @@ import (
"code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/models/unit"
user_model "code.gitea.io/gitea/models/user" user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/templates"
) )
// Package contains owner, access mode and optional the package descriptor // Package contains owner, access mode and optional the package descriptor
@ -118,12 +120,14 @@ func packageAssignment(ctx *Context, errCb func(int, string, interface{})) {
} }
// PackageContexter initializes a package context for a request. // PackageContexter initializes a package context for a request.
func PackageContexter() func(next http.Handler) http.Handler { func PackageContexter(ctx gocontext.Context) func(next http.Handler) http.Handler {
_, rnd := templates.HTMLRenderer(ctx)
return func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
ctx := Context{ ctx := Context{
Resp: NewResponse(resp), Resp: NewResponse(resp),
Data: map[string]interface{}{}, Data: map[string]interface{}{},
Render: rnd,
} }
defer ctx.Close() defer ctx.Close()

40
modules/options/base.go Normal file
View file

@ -0,0 +1,40 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package options
import (
"fmt"
"io/fs"
"os"
"path/filepath"
"code.gitea.io/gitea/modules/util"
)
func walkAssetDir(root string, callback func(path, name string, d fs.DirEntry, err error) error) error {
if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
// name is the path relative to the root
name := path[len(root):]
if len(name) > 0 && name[0] == '/' {
name = name[1:]
}
if err != nil {
if os.IsNotExist(err) {
return callback(path, name, d, err)
}
return err
}
if util.CommonSkip(d.Name()) {
if d.IsDir() {
return fs.SkipDir
}
return nil
}
return callback(path, name, d, err)
}); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("unable to get files for assets in %s: %w", root, err)
}
return nil
}

View file

@ -8,8 +8,10 @@ package options
import ( import (
"fmt" "fmt"
"io/fs"
"os" "os"
"path" "path"
"path/filepath"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
@ -45,7 +47,7 @@ func Dir(name string) ([]string, error) {
isDir, err = util.IsDir(staticDir) isDir, err = util.IsDir(staticDir)
if err != nil { if err != nil {
return []string{}, fmt.Errorf("Unabe to check if static directory %s is a directory. %v", staticDir, err) return []string{}, fmt.Errorf("unable to check if static directory %s is a directory. %v", staticDir, err)
} }
if isDir { if isDir {
files, err := util.StatDir(staticDir, true) files, err := util.StatDir(staticDir, true)
@ -64,6 +66,18 @@ func Locale(name string) ([]byte, error) {
return fileFromDir(path.Join("locale", name)) return fileFromDir(path.Join("locale", name))
} }
// WalkLocales reads the content of a specific locale from static or custom path.
func WalkLocales(callback func(path, name string, d fs.DirEntry, err error) error) error {
if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to walk locales. Error: %w", err)
}
if err := walkAssetDir(filepath.Join(setting.CustomPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to walk locales. Error: %w", err)
}
return nil
}
// Readme reads the content of a specific readme from static or custom path. // Readme reads the content of a specific readme from static or custom path.
func Readme(name string) ([]byte, error) { func Readme(name string) ([]byte, error) {
return fileFromDir(path.Join("readme", name)) return fileFromDir(path.Join("readme", name))

View file

@ -9,8 +9,10 @@ package options
import ( import (
"fmt" "fmt"
"io" "io"
"io/fs"
"os" "os"
"path" "path"
"path/filepath"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
@ -74,6 +76,14 @@ func Locale(name string) ([]byte, error) {
return fileFromDir(path.Join("locale", name)) return fileFromDir(path.Join("locale", name))
} }
// WalkLocales reads the content of a specific locale from static or custom path.
func WalkLocales(callback func(path, name string, d fs.DirEntry, err error) error) error {
if err := walkAssetDir(filepath.Join(setting.CustomPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to walk locales. Error: %w", err)
}
return nil
}
// Readme reads the content of a specific readme from bindata or custom path. // Readme reads the content of a specific readme from bindata or custom path.
func Readme(name string) ([]byte, error) { func Readme(name string) ([]byte, error) {
return fileFromDir(path.Join("readme", name)) return fileFromDir(path.Join("readme", name))

View file

@ -5,15 +5,16 @@
package templates package templates
import ( import (
"fmt"
"io/fs"
"os" "os"
"path/filepath"
"strings" "strings"
"time" "time"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/util"
"github.com/unrolled/render"
) )
// Vars represents variables to be render in golang templates // Vars represents variables to be render in golang templates
@ -47,8 +48,16 @@ func BaseVars() Vars {
} }
} }
func getDirAssetNames(dir string) []string { func getDirTemplateAssetNames(dir string) []string {
return getDirAssetNames(dir, false)
}
func getDirAssetNames(dir string, mailer bool) []string {
var tmpls []string var tmpls []string
if mailer {
dir += filepath.Join(dir, "mail")
}
f, err := os.Stat(dir) f, err := os.Stat(dir)
if err != nil { if err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
@ -67,8 +76,13 @@ func getDirAssetNames(dir string) []string {
log.Warn("Failed to read %s templates dir. %v", dir, err) log.Warn("Failed to read %s templates dir. %v", dir, err)
return tmpls return tmpls
} }
prefix := "templates/"
if mailer {
prefix += "mail/"
}
for _, filePath := range files { for _, filePath := range files {
if strings.HasPrefix(filePath, "mail/") { if !mailer && strings.HasPrefix(filePath, "mail/") {
continue continue
} }
@ -76,20 +90,39 @@ func getDirAssetNames(dir string) []string {
continue continue
} }
tmpls = append(tmpls, "templates/"+filePath) tmpls = append(tmpls, prefix+filePath)
} }
return tmpls return tmpls
} }
// HTMLRenderer returns a render. func walkAssetDir(root string, skipMail bool, callback func(path, name string, d fs.DirEntry, err error) error) error {
func HTMLRenderer() *render.Render { mailRoot := filepath.Join(root, "mail")
return render.New(render.Options{ if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
Extensions: []string{".tmpl"}, name := path[len(root):]
Directory: "templates", if len(name) > 0 && name[0] == '/' {
Funcs: NewFuncMap(), name = name[1:]
Asset: GetAsset, }
AssetNames: GetAssetNames, if err != nil {
IsDevelopment: !setting.IsProd, if os.IsNotExist(err) {
DisableHTTPErrorRendering: true, return callback(path, name, d, err)
}) }
return err
}
if skipMail && path == mailRoot && d.IsDir() {
return fs.SkipDir
}
if util.CommonSkip(d.Name()) {
if d.IsDir() {
return fs.SkipDir
}
return nil
}
if strings.HasSuffix(d.Name(), ".tmpl") || d.IsDir() {
return callback(path, name, d, err)
}
return nil
}); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("unable to get files for template assets in %s: %w", root, err)
}
return nil
} }

View file

@ -8,15 +8,12 @@ package templates
import ( import (
"html/template" "html/template"
"io/fs"
"os" "os"
"path"
"path/filepath" "path/filepath"
"strings"
texttmpl "text/template" texttmpl "text/template"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
) )
var ( var (
@ -36,77 +33,42 @@ func GetAsset(name string) ([]byte, error) {
return os.ReadFile(filepath.Join(setting.StaticRootPath, name)) return os.ReadFile(filepath.Join(setting.StaticRootPath, name))
} }
// GetAssetNames returns assets list // walkTemplateFiles calls a callback for each template asset
func GetAssetNames() []string { func walkTemplateFiles(callback func(path, name string, d fs.DirEntry, err error) error) error {
tmpls := getDirAssetNames(filepath.Join(setting.CustomPath, "templates")) if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) {
tmpls2 := getDirAssetNames(filepath.Join(setting.StaticRootPath, "templates")) return err
}
if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) {
return err
}
return nil
}
// GetTemplateAssetNames returns list of template names
func GetTemplateAssetNames() []string {
tmpls := getDirTemplateAssetNames(filepath.Join(setting.CustomPath, "templates"))
tmpls2 := getDirTemplateAssetNames(filepath.Join(setting.StaticRootPath, "templates"))
return append(tmpls, tmpls2...) return append(tmpls, tmpls2...)
} }
// Mailer provides the templates required for sending notification mails. func walkMailerTemplates(callback func(path, name string, d fs.DirEntry, err error) error) error {
func Mailer() (*texttmpl.Template, *template.Template) { if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) {
for _, funcs := range NewTextFuncMap() { return err
subjectTemplates.Funcs(funcs)
} }
for _, funcs := range NewFuncMap() { if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) {
bodyTemplates.Funcs(funcs) return err
} }
return nil
staticDir := path.Join(setting.StaticRootPath, "templates", "mail") }
isDir, err := util.IsDir(staticDir) // BuiltinAsset will read the provided asset from the embedded assets
if err != nil { // (This always returns os.ErrNotExist)
log.Warn("Unable to check if templates dir %s is a directory. Error: %v", staticDir, err) func BuiltinAsset(name string) ([]byte, error) {
} return nil, os.ErrNotExist
if isDir { }
files, err := util.StatDir(staticDir)
// BuiltinAssetNames returns the names of the embedded assets
if err != nil { // (This always returns nil)
log.Warn("Failed to read %s templates dir. %v", staticDir, err) func BuiltinAssetNames() []string {
} else { return nil
for _, filePath := range files {
if !strings.HasSuffix(filePath, ".tmpl") {
continue
}
content, err := os.ReadFile(path.Join(staticDir, filePath))
if err != nil {
log.Warn("Failed to read static %s template. %v", filePath, err)
continue
}
buildSubjectBodyTemplate(subjectTemplates, bodyTemplates, strings.TrimSuffix(filePath, ".tmpl"), content)
}
}
}
customDir := path.Join(setting.CustomPath, "templates", "mail")
isDir, err = util.IsDir(customDir)
if err != nil {
log.Warn("Unable to check if templates dir %s is a directory. Error: %v", customDir, err)
}
if isDir {
files, err := util.StatDir(customDir)
if err != nil {
log.Warn("Failed to read %s templates dir. %v", customDir, err)
} else {
for _, filePath := range files {
if !strings.HasSuffix(filePath, ".tmpl") {
continue
}
content, err := os.ReadFile(path.Join(customDir, filePath))
if err != nil {
log.Warn("Failed to read custom %s template. %v", filePath, err)
continue
}
buildSubjectBodyTemplate(subjectTemplates, bodyTemplates, strings.TrimSuffix(filePath, ".tmpl"), content)
}
}
}
return subjectTemplates, bodyTemplates
} }

View file

@ -0,0 +1,52 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package templates
import (
"context"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/watcher"
"github.com/unrolled/render"
)
var rendererKey interface{} = "templatesHtmlRendereer"
// HTMLRenderer returns the current html renderer for the context or creates and stores one within the context for future use
func HTMLRenderer(ctx context.Context) (context.Context, *render.Render) {
rendererInterface := ctx.Value(rendererKey)
if rendererInterface != nil {
renderer, ok := rendererInterface.(*render.Render)
if ok {
return ctx, renderer
}
}
rendererType := "static"
if !setting.IsProd {
rendererType = "auto-reloading"
}
log.Log(1, log.DEBUG, "Creating "+rendererType+" HTML Renderer")
renderer := render.New(render.Options{
Extensions: []string{".tmpl"},
Directory: "templates",
Funcs: NewFuncMap(),
Asset: GetAsset,
AssetNames: GetTemplateAssetNames,
UseMutexLock: !setting.IsProd,
IsDevelopment: false,
DisableHTTPErrorRendering: true,
})
if !setting.IsProd {
watcher.CreateWatcher(ctx, "HTML Templates", &watcher.CreateWatcherOpts{
PathsCallback: walkTemplateFiles,
BetweenCallback: renderer.CompileTemplates,
})
}
return context.WithValue(ctx, rendererKey, renderer), renderer
}

View file

@ -0,0 +1,92 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package templates
import (
"context"
"html/template"
"io/fs"
"os"
"strings"
texttmpl "text/template"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/watcher"
)
// Mailer provides the templates required for sending notification mails.
func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) {
for _, funcs := range NewTextFuncMap() {
subjectTemplates.Funcs(funcs)
}
for _, funcs := range NewFuncMap() {
bodyTemplates.Funcs(funcs)
}
refreshTemplates := func() {
for _, assetPath := range BuiltinAssetNames() {
if !strings.HasPrefix(assetPath, "mail/") {
continue
}
if !strings.HasSuffix(assetPath, ".tmpl") {
continue
}
content, err := BuiltinAsset(assetPath)
if err != nil {
log.Warn("Failed to read embedded %s template. %v", assetPath, err)
continue
}
assetName := strings.TrimPrefix(strings.TrimSuffix(assetPath, ".tmpl"), "mail/")
log.Trace("Adding built-in mailer template for %s", assetName)
buildSubjectBodyTemplate(subjectTemplates,
bodyTemplates,
assetName,
content)
}
if err := walkMailerTemplates(func(path, name string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
content, err := os.ReadFile(path)
if err != nil {
log.Warn("Failed to read custom %s template. %v", path, err)
return nil
}
assetName := strings.TrimSuffix(name, ".tmpl")
log.Trace("Adding mailer template for %s from %q", assetName, path)
buildSubjectBodyTemplate(subjectTemplates,
bodyTemplates,
assetName,
content)
return nil
}); err != nil && !os.IsNotExist(err) {
log.Warn("Error whilst walking mailer templates directories. %v", err)
}
}
refreshTemplates()
if !setting.IsProd {
// Now subjectTemplates and bodyTemplates are both synchronized
// thus it is safe to call refresh from a different goroutine
watcher.CreateWatcher(ctx, "Mailer Templates", &watcher.CreateWatcherOpts{
PathsCallback: walkMailerTemplates,
BetweenCallback: refreshTemplates,
})
}
return subjectTemplates, bodyTemplates
}

View file

@ -9,6 +9,7 @@ package templates
import ( import (
"html/template" "html/template"
"io" "io"
"io/fs"
"os" "os"
"path" "path"
"path/filepath" "path/filepath"
@ -16,10 +17,8 @@ import (
texttmpl "text/template" texttmpl "text/template"
"time" "time"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util"
) )
var ( var (
@ -40,95 +39,42 @@ func GetAsset(name string) ([]byte, error) {
} else if err == nil { } else if err == nil {
return bs, nil return bs, nil
} }
return Asset(strings.TrimPrefix(name, "templates/")) return BuiltinAsset(strings.TrimPrefix(name, "templates/"))
} }
// GetAssetNames only for chi // GetFiles calls a callback for each template asset
func GetAssetNames() []string { func walkTemplateFiles(callback func(path, name string, d fs.DirEntry, err error) error) error {
if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) {
return err
}
return nil
}
// GetTemplateAssetNames only for chi
func GetTemplateAssetNames() []string {
realFS := Assets.(vfsgen۰FS) realFS := Assets.(vfsgen۰FS)
tmpls := make([]string, 0, len(realFS)) tmpls := make([]string, 0, len(realFS))
for k := range realFS { for k := range realFS {
if strings.HasPrefix(k, "/mail/") {
continue
}
tmpls = append(tmpls, "templates/"+k[1:]) tmpls = append(tmpls, "templates/"+k[1:])
} }
customDir := path.Join(setting.CustomPath, "templates") customDir := path.Join(setting.CustomPath, "templates")
customTmpls := getDirAssetNames(customDir) customTmpls := getDirTemplateAssetNames(customDir)
return append(tmpls, customTmpls...) return append(tmpls, customTmpls...)
} }
// Mailer provides the templates required for sending notification mails. func walkMailerTemplates(callback func(path, name string, d fs.DirEntry, err error) error) error {
func Mailer() (*texttmpl.Template, *template.Template) { if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) {
for _, funcs := range NewTextFuncMap() { return err
subjectTemplates.Funcs(funcs)
} }
for _, funcs := range NewFuncMap() { return nil
bodyTemplates.Funcs(funcs)
}
for _, assetPath := range AssetNames() {
if !strings.HasPrefix(assetPath, "mail/") {
continue
}
if !strings.HasSuffix(assetPath, ".tmpl") {
continue
}
content, err := Asset(assetPath)
if err != nil {
log.Warn("Failed to read embedded %s template. %v", assetPath, err)
continue
}
buildSubjectBodyTemplate(subjectTemplates,
bodyTemplates,
strings.TrimPrefix(
strings.TrimSuffix(
assetPath,
".tmpl",
),
"mail/",
),
content)
}
customDir := path.Join(setting.CustomPath, "templates", "mail")
isDir, err := util.IsDir(customDir)
if err != nil {
log.Warn("Failed to check if custom directory %s is a directory. %v", err)
}
if isDir {
files, err := util.StatDir(customDir)
if err != nil {
log.Warn("Failed to read %s templates dir. %v", customDir, err)
} else {
for _, filePath := range files {
if !strings.HasSuffix(filePath, ".tmpl") {
continue
}
content, err := os.ReadFile(path.Join(customDir, filePath))
if err != nil {
log.Warn("Failed to read custom %s template. %v", filePath, err)
continue
}
buildSubjectBodyTemplate(subjectTemplates,
bodyTemplates,
strings.TrimSuffix(
filePath,
".tmpl",
),
content)
}
}
}
return subjectTemplates, bodyTemplates
} }
func Asset(name string) ([]byte, error) { // BuiltinAsset reads the provided asset from the builtin embedded assets
func BuiltinAsset(name string) ([]byte, error) {
f, err := Assets.Open("/" + name) f, err := Assets.Open("/" + name)
if err != nil { if err != nil {
return nil, err return nil, err
@ -137,7 +83,8 @@ func Asset(name string) ([]byte, error) {
return io.ReadAll(f) return io.ReadAll(f)
} }
func AssetNames() []string { // BuiltinAssetNames returns the names of the built-in embedded assets
func BuiltinAssetNames() []string {
realFS := Assets.(vfsgen۰FS) realFS := Assets.(vfsgen۰FS)
results := make([]string, 0, len(realFS)) results := make([]string, 0, len(realFS))
for k := range realFS { for k := range realFS {
@ -146,7 +93,8 @@ func AssetNames() []string {
return results return results
} }
func AssetIsDir(name string) (bool, error) { // BuiltinAssetIsDir returns if a provided asset is a directory
func BuiltinAssetIsDir(name string) (bool, error) {
if f, err := Assets.Open("/" + name); err != nil { if f, err := Assets.Open("/" + name); err != nil {
return false, err return false, err
} else { } else {

View file

@ -5,6 +5,7 @@
package timeutil package timeutil
import ( import (
"context"
"fmt" "fmt"
"os" "os"
"testing" "testing"
@ -31,7 +32,7 @@ func TestMain(m *testing.M) {
setting.Names = []string{"english"} setting.Names = []string{"english"}
setting.Langs = []string{"en-US"} setting.Langs = []string{"en-US"}
// setup // setup
translation.InitLocales() translation.InitLocales(context.Background())
BaseDate = time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC) BaseDate = time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC)
// run the tests // run the tests

View file

@ -0,0 +1,12 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package i18n
import "errors"
var (
ErrLocaleAlreadyExist = errors.New("lang already exists")
ErrUncertainArguments = errors.New("arguments to i18n should not contain uncertain slices")
)

View file

@ -0,0 +1,42 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package i18n
import (
"fmt"
"reflect"
)
// Format formats provided arguments for a given translated message
func Format(format string, args ...interface{}) (msg string, err error) {
if len(args) == 0 {
return format, nil
}
fmtArgs := make([]interface{}, 0, len(args))
for _, arg := range args {
val := reflect.ValueOf(arg)
if val.Kind() == reflect.Slice {
// Previously, we would accept Tr(lang, key, a, [b, c], d, [e, f]) as Sprintf(msg, a, b, c, d, e, f)
// but this is an unstable behavior.
//
// So we restrict the accepted arguments to either:
//
// 1. Tr(lang, key, [slice-items]) as Sprintf(msg, items...)
// 2. Tr(lang, key, args...) as Sprintf(msg, args...)
if len(args) == 1 {
for i := 0; i < val.Len(); i++ {
fmtArgs = append(fmtArgs, val.Index(i).Interface())
}
} else {
err = ErrUncertainArguments
break
}
} else {
fmtArgs = append(fmtArgs, arg)
}
}
return fmt.Sprintf(format, fmtArgs...), err
}

View file

@ -5,297 +5,48 @@
package i18n package i18n
import ( import (
"errors" "io"
"fmt"
"os"
"reflect"
"sync"
"time"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"gopkg.in/ini.v1"
) )
var ( var DefaultLocales = NewLocaleStore()
ErrLocaleAlreadyExist = errors.New("lang already exists")
DefaultLocales = NewLocaleStore(true) type Locale interface {
) // Tr translates a given key and arguments for a language
Tr(trKey string, trArgs ...interface{}) string
type locale struct { // Has reports if a locale has a translation for a given key
// This mutex will be set if we have live-reload enabled (e.g. dev mode) Has(trKey string) bool
reloadMu *sync.RWMutex
store *LocaleStore
langName string
idxToMsgMap map[int]string // the map idx is generated by store's trKeyToIdxMap
sourceFileName string
sourceFileInfo os.FileInfo
lastReloadCheckTime time.Time
} }
type LocaleStore struct { // LocaleStore provides the functions common to all locale stores
// This mutex will be set if we have live-reload enabled (e.g. dev mode) type LocaleStore interface {
reloadMu *sync.RWMutex io.Closer
langNames []string // Tr translates a given key and arguments for a language
langDescs []string Tr(lang, trKey string, trArgs ...interface{}) string
localeMap map[string]*locale // Has reports if a locale has a translation for a given key
Has(lang, trKey string) bool
// this needs to be locked when live-reloading // SetDefaultLang sets the default language to fall back to
trKeyToIdxMap map[string]int SetDefaultLang(lang string)
// ListLangNameDesc provides paired slices of language names to descriptors
defaultLang string ListLangNameDesc() (names, desc []string)
} // Locale return the locale for the provided language or the default language if not found
Locale(langName string) (Locale, bool)
func NewLocaleStore(isProd bool) *LocaleStore { // HasLang returns whether a given language is present in the store
store := &LocaleStore{localeMap: make(map[string]*locale), trKeyToIdxMap: make(map[string]int)} HasLang(langName string) bool
if !isProd { // AddLocaleByIni adds a new language to the store
store.reloadMu = &sync.RWMutex{} AddLocaleByIni(langName, langDesc string, source interface{}) error
}
return store
}
// AddLocaleByIni adds locale by ini into the store
// if source is a string, then the file is loaded. In dev mode, this file will be checked for live-reloading
// if source is a []byte, then the content is used
// Note: this is not concurrent safe
func (store *LocaleStore) AddLocaleByIni(langName, langDesc string, source interface{}) error {
if _, ok := store.localeMap[langName]; ok {
return ErrLocaleAlreadyExist
}
l := &locale{store: store, langName: langName}
if store.reloadMu != nil {
l.reloadMu = &sync.RWMutex{}
l.reloadMu.Lock() // Arguably this is not necessary as AddLocaleByIni isn't concurrent safe - but for consistency we do this
defer l.reloadMu.Unlock()
}
if fileName, ok := source.(string); ok {
l.sourceFileName = fileName
l.sourceFileInfo, _ = os.Stat(fileName) // live-reload only works for regular files. the error can be ignored
}
var err error
l.idxToMsgMap, err = store.readIniToIdxToMsgMap(source)
if err != nil {
return err
}
store.langNames = append(store.langNames, langName)
store.langDescs = append(store.langDescs, langDesc)
store.localeMap[l.langName] = l
return nil
}
// readIniToIdxToMsgMap will read a provided ini and creates an idxToMsgMap
func (store *LocaleStore) readIniToIdxToMsgMap(source interface{}) (map[int]string, error) {
iniFile, err := ini.LoadSources(ini.LoadOptions{
IgnoreInlineComment: true,
UnescapeValueCommentSymbols: true,
}, source)
if err != nil {
return nil, fmt.Errorf("unable to load ini: %w", err)
}
iniFile.BlockMode = false
idxToMsgMap := make(map[int]string)
if store.reloadMu != nil {
store.reloadMu.Lock()
defer store.reloadMu.Unlock()
}
for _, section := range iniFile.Sections() {
for _, key := range section.Keys() {
var trKey string
if section.Name() == "" || section.Name() == "DEFAULT" {
trKey = key.Name()
} else {
trKey = section.Name() + "." + key.Name()
}
// Instead of storing the key strings in multiple different maps we compute a idx which will act as numeric code for key
// This reduces the size of the locale idxToMsgMaps
idx, ok := store.trKeyToIdxMap[trKey]
if !ok {
idx = len(store.trKeyToIdxMap)
store.trKeyToIdxMap[trKey] = idx
}
idxToMsgMap[idx] = key.Value()
}
}
iniFile = nil
return idxToMsgMap, nil
}
func (store *LocaleStore) idxForTrKey(trKey string) (int, bool) {
if store.reloadMu != nil {
store.reloadMu.RLock()
defer store.reloadMu.RUnlock()
}
idx, ok := store.trKeyToIdxMap[trKey]
return idx, ok
}
// HasLang reports if a language is available in the store
func (store *LocaleStore) HasLang(langName string) bool {
_, ok := store.localeMap[langName]
return ok
}
// ListLangNameDesc reports if a language available in the store
func (store *LocaleStore) ListLangNameDesc() (names, desc []string) {
return store.langNames, store.langDescs
}
// SetDefaultLang sets default language as a fallback
func (store *LocaleStore) SetDefaultLang(lang string) {
store.defaultLang = lang
}
// Tr translates content to target language. fall back to default language.
func (store *LocaleStore) Tr(lang, trKey string, trArgs ...interface{}) string {
l, ok := store.localeMap[lang]
if !ok {
l, ok = store.localeMap[store.defaultLang]
}
if ok {
return l.Tr(trKey, trArgs...)
}
return trKey
}
// reloadIfNeeded will check if the locale needs to be reloaded
// this function will assume that the l.reloadMu has been RLocked if it already exists
func (l *locale) reloadIfNeeded() {
if l.reloadMu == nil {
return
}
now := time.Now()
if now.Sub(l.lastReloadCheckTime) < time.Second || l.sourceFileInfo == nil || l.sourceFileName == "" {
return
}
l.reloadMu.RUnlock()
l.reloadMu.Lock() // (NOTE: a pre-emption can occur between these two locks so we need to recheck)
defer l.reloadMu.RLock()
defer l.reloadMu.Unlock()
if now.Sub(l.lastReloadCheckTime) < time.Second || l.sourceFileInfo == nil || l.sourceFileName == "" {
return
}
l.lastReloadCheckTime = now
sourceFileInfo, err := os.Stat(l.sourceFileName)
if err != nil || sourceFileInfo.ModTime().Equal(l.sourceFileInfo.ModTime()) {
return
}
idxToMsgMap, err := l.store.readIniToIdxToMsgMap(l.sourceFileName)
if err == nil {
l.idxToMsgMap = idxToMsgMap
} else {
log.Error("Unable to live-reload the locale file %q, err: %v", l.sourceFileName, err)
}
// We will set the sourceFileInfo to this file to prevent repeated attempts to re-load this broken file
l.sourceFileInfo = sourceFileInfo
}
// Tr translates content to locale language. fall back to default language.
func (l *locale) Tr(trKey string, trArgs ...interface{}) string {
if l.reloadMu != nil {
l.reloadMu.RLock()
defer l.reloadMu.RUnlock()
l.reloadIfNeeded()
}
msg, _ := l.tryTr(trKey, trArgs...)
return msg
}
func (l *locale) tryTr(trKey string, trArgs ...interface{}) (msg string, found bool) {
trMsg := trKey
// convert the provided trKey to a common idx from the store
idx, ok := l.store.idxForTrKey(trKey)
if ok {
if msg, found = l.idxToMsgMap[idx]; found {
trMsg = msg // use the translation that we have found
} else if l.langName != l.store.defaultLang {
// No translation available in our current language... fallback to the default language
// Attempt to get the default language from the locale store
if def, ok := l.store.localeMap[l.store.defaultLang]; ok {
if def.reloadMu != nil {
def.reloadMu.RLock()
def.reloadIfNeeded()
}
if msg, found = def.idxToMsgMap[idx]; found {
trMsg = msg // use the translation that we have found
}
if def.reloadMu != nil {
def.reloadMu.RUnlock()
}
}
}
}
if !found && !setting.IsProd {
log.Error("missing i18n translation key: %q", trKey)
}
if len(trArgs) == 0 {
return trMsg, found
}
fmtArgs := make([]interface{}, 0, len(trArgs))
for _, arg := range trArgs {
val := reflect.ValueOf(arg)
if val.Kind() == reflect.Slice {
// Previously, we would accept Tr(lang, key, a, [b, c], d, [e, f]) as Sprintf(msg, a, b, c, d, e, f)
// but this is an unstable behavior.
//
// So we restrict the accepted arguments to either:
//
// 1. Tr(lang, key, [slice-items]) as Sprintf(msg, items...)
// 2. Tr(lang, key, args...) as Sprintf(msg, args...)
if len(trArgs) == 1 {
for i := 0; i < val.Len(); i++ {
fmtArgs = append(fmtArgs, val.Index(i).Interface())
}
} else {
log.Error("the args for i18n shouldn't contain uncertain slices, key=%q, args=%v", trKey, trArgs)
break
}
} else {
fmtArgs = append(fmtArgs, arg)
}
}
return fmt.Sprintf(trMsg, fmtArgs...), found
} }
// ResetDefaultLocales resets the current default locales // ResetDefaultLocales resets the current default locales
// NOTE: this is not synchronized // NOTE: this is not synchronized
func ResetDefaultLocales(isProd bool) { func ResetDefaultLocales() {
DefaultLocales = NewLocaleStore(isProd) if DefaultLocales != nil {
_ = DefaultLocales.Close()
}
DefaultLocales = NewLocaleStore()
} }
// Tr use default locales to translate content to target language. // GetLocales returns the locale from the default locales
func Tr(lang, trKey string, trArgs ...interface{}) string { func GetLocale(lang string) (Locale, bool) {
return DefaultLocales.Tr(lang, trKey, trArgs...) return DefaultLocales.Locale(lang)
} }

View file

@ -27,8 +27,7 @@ fmt = %[2]s %[1]s
sub = Changed Sub String sub = Changed Sub String
`) `)
for _, isProd := range []bool{true, false} { ls := NewLocaleStore()
ls := NewLocaleStore(isProd)
assert.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", testData1)) assert.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", testData1))
assert.NoError(t, ls.AddLocaleByIni("lang2", "Lang2", testData2)) assert.NoError(t, ls.AddLocaleByIni("lang2", "Lang2", testData2))
ls.SetDefaultLang("lang1") ls.SetDefaultLang("lang1")
@ -55,8 +54,7 @@ sub = Changed Sub String
assert.Equal(t, []string{"lang1", "lang2"}, langs) assert.Equal(t, []string{"lang1", "lang2"}, langs)
assert.Equal(t, []string{"Lang1", "Lang2"}, descs) assert.Equal(t, []string{"Lang1", "Lang2"}, descs)
result, found := ls.localeMap["lang1"].tryTr("no-such") found := ls.Has("lang1", "no-such")
assert.Equal(t, "no-such", result)
assert.False(t, found) assert.False(t, found)
} assert.NoError(t, ls.Close())
} }

View file

@ -0,0 +1,161 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package i18n
import (
"fmt"
"code.gitea.io/gitea/modules/log"
"gopkg.in/ini.v1"
)
// This file implements the static LocaleStore that will not watch for changes
type locale struct {
store *localeStore
langName string
idxToMsgMap map[int]string // the map idx is generated by store's trKeyToIdxMap
}
type localeStore struct {
// After initializing has finished, these fields are read-only.
langNames []string
langDescs []string
localeMap map[string]*locale
trKeyToIdxMap map[string]int
defaultLang string
}
// NewLocaleStore creates a static locale store
func NewLocaleStore() LocaleStore {
return &localeStore{localeMap: make(map[string]*locale), trKeyToIdxMap: make(map[string]int)}
}
// AddLocaleByIni adds locale by ini into the store
// if source is a string, then the file is loaded
// if source is a []byte, then the content is used
func (store *localeStore) AddLocaleByIni(langName, langDesc string, source interface{}) error {
if _, ok := store.localeMap[langName]; ok {
return ErrLocaleAlreadyExist
}
store.langNames = append(store.langNames, langName)
store.langDescs = append(store.langDescs, langDesc)
l := &locale{store: store, langName: langName, idxToMsgMap: make(map[int]string)}
store.localeMap[l.langName] = l
iniFile, err := ini.LoadSources(ini.LoadOptions{
IgnoreInlineComment: true,
UnescapeValueCommentSymbols: true,
}, source)
if err != nil {
return fmt.Errorf("unable to load ini: %w", err)
}
iniFile.BlockMode = false
for _, section := range iniFile.Sections() {
for _, key := range section.Keys() {
var trKey string
if section.Name() == "" || section.Name() == "DEFAULT" {
trKey = key.Name()
} else {
trKey = section.Name() + "." + key.Name()
}
idx, ok := store.trKeyToIdxMap[trKey]
if !ok {
idx = len(store.trKeyToIdxMap)
store.trKeyToIdxMap[trKey] = idx
}
l.idxToMsgMap[idx] = key.Value()
}
}
iniFile = nil
return nil
}
func (store *localeStore) HasLang(langName string) bool {
_, ok := store.localeMap[langName]
return ok
}
func (store *localeStore) ListLangNameDesc() (names, desc []string) {
return store.langNames, store.langDescs
}
// SetDefaultLang sets default language as a fallback
func (store *localeStore) SetDefaultLang(lang string) {
store.defaultLang = lang
}
// Tr translates content to target language. fall back to default language.
func (store *localeStore) Tr(lang, trKey string, trArgs ...interface{}) string {
l, _ := store.Locale(lang)
return l.Tr(trKey, trArgs...)
}
// Has returns whether the given language has a translation for the provided key
func (store *localeStore) Has(lang, trKey string) bool {
l, _ := store.Locale(lang)
return l.Has(trKey)
}
// Locale returns the locale for the lang or the default language
func (store *localeStore) Locale(lang string) (Locale, bool) {
l, found := store.localeMap[lang]
if !found {
var ok bool
l, ok = store.localeMap[store.defaultLang]
if !ok {
// no default - return an empty locale
l = &locale{store: store, idxToMsgMap: make(map[int]string)}
}
}
return l, found
}
// Close implements io.Closer
func (store *localeStore) Close() error {
return nil
}
// Tr translates content to locale language. fall back to default language.
func (l *locale) Tr(trKey string, trArgs ...interface{}) string {
format := trKey
idx, ok := l.store.trKeyToIdxMap[trKey]
if ok {
if msg, ok := l.idxToMsgMap[idx]; ok {
format = msg // use the found translation
} else if def, ok := l.store.localeMap[l.store.defaultLang]; ok {
// try to use default locale's translation
if msg, ok := def.idxToMsgMap[idx]; ok {
format = msg
}
}
}
msg, err := Format(format, trArgs...)
if err != nil {
log.Error("Error whilst formatting %q in %s: %v", trKey, l.langName, err)
}
return msg
}
// Has returns whether a key is present in this locale or not
func (l *locale) Has(trKey string) bool {
idx, ok := l.store.trKeyToIdxMap[trKey]
if !ok {
return false
}
_, ok = l.idxToMsgMap[idx]
return ok
}

View file

@ -5,15 +5,16 @@
package translation package translation
import ( import (
"path" "context"
"sort" "sort"
"strings" "strings"
"sync"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/options"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/translation/i18n" "code.gitea.io/gitea/modules/translation/i18n"
"code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/watcher"
"golang.org/x/text/language" "golang.org/x/text/language"
) )
@ -31,6 +32,7 @@ type LangType struct {
} }
var ( var (
lock *sync.RWMutex
matcher language.Matcher matcher language.Matcher
allLangs []*LangType allLangs []*LangType
allLangMap map[string]*LangType allLangMap map[string]*LangType
@ -43,8 +45,16 @@ func AllLangs() []*LangType {
} }
// InitLocales loads the locales // InitLocales loads the locales
func InitLocales() { func InitLocales(ctx context.Context) {
i18n.ResetDefaultLocales(setting.IsProd) if lock != nil {
lock.Lock()
defer lock.Unlock()
} else if !setting.IsProd && lock == nil {
lock = &sync.RWMutex{}
}
refreshLocales := func() {
i18n.ResetDefaultLocales()
localeNames, err := options.Dir("locale") localeNames, err := options.Dir("locale")
if err != nil { if err != nil {
log.Fatal("Failed to list locale files: %v", err) log.Fatal("Failed to list locale files: %v", err)
@ -52,27 +62,11 @@ func InitLocales() {
localFiles := make(map[string]interface{}, len(localeNames)) localFiles := make(map[string]interface{}, len(localeNames))
for _, name := range localeNames { for _, name := range localeNames {
if options.IsDynamic() {
// Try to check if CustomPath has the file, otherwise fallback to StaticRootPath
value := path.Join(setting.CustomPath, "options/locale", name)
isFile, err := util.IsFile(value)
if err != nil {
log.Fatal("Failed to load %s locale file. %v", name, err)
}
if isFile {
localFiles[name] = value
} else {
localFiles[name] = path.Join(setting.StaticRootPath, "options/locale", name)
}
} else {
localFiles[name], err = options.Locale(name) localFiles[name], err = options.Locale(name)
if err != nil { if err != nil {
log.Fatal("Failed to load %s locale file. %v", name, err) log.Fatal("Failed to load %s locale file. %v", name, err)
} }
} }
}
supportedTags = make([]language.Tag, len(setting.Langs)) supportedTags = make([]language.Tag, len(setting.Langs))
for i, lang := range setting.Langs { for i, lang := range setting.Langs {
@ -94,6 +88,9 @@ func InitLocales() {
} }
i18n.DefaultLocales.SetDefaultLang(defaultLangName) i18n.DefaultLocales.SetDefaultLang(defaultLangName)
} }
}
refreshLocales()
langs, descs := i18n.DefaultLocales.ListLangNameDesc() langs, descs := i18n.DefaultLocales.ListLangNameDesc()
allLangs = make([]*LangType, 0, len(langs)) allLangs = make([]*LangType, 0, len(langs))
@ -108,6 +105,17 @@ func InitLocales() {
sort.Slice(allLangs, func(i, j int) bool { sort.Slice(allLangs, func(i, j int) bool {
return strings.ToLower(allLangs[i].Name) < strings.ToLower(allLangs[j].Name) return strings.ToLower(allLangs[i].Name) < strings.ToLower(allLangs[j].Name)
}) })
if !setting.IsProd {
watcher.CreateWatcher(ctx, "Locales", &watcher.CreateWatcherOpts{
PathsCallback: options.WalkLocales,
BetweenCallback: func() {
lock.Lock()
defer lock.Unlock()
refreshLocales()
},
})
}
} }
// Match matches accept languages // Match matches accept languages
@ -118,16 +126,24 @@ func Match(tags ...language.Tag) language.Tag {
// locale represents the information of localization. // locale represents the information of localization.
type locale struct { type locale struct {
i18n.Locale
Lang, LangName string // these fields are used directly in templates: .i18n.Lang Lang, LangName string // these fields are used directly in templates: .i18n.Lang
} }
// NewLocale return a locale // NewLocale return a locale
func NewLocale(lang string) Locale { func NewLocale(lang string) Locale {
if lock != nil {
lock.RLock()
defer lock.RUnlock()
}
langName := "unknown" langName := "unknown"
if l, ok := allLangMap[lang]; ok { if l, ok := allLangMap[lang]; ok {
langName = l.Name langName = l.Name
} }
i18nLocale, _ := i18n.GetLocale(lang)
return &locale{ return &locale{
Locale: i18nLocale,
Lang: lang, Lang: lang,
LangName: langName, LangName: langName,
} }
@ -137,11 +153,6 @@ func (l *locale) Language() string {
return l.Lang return l.Lang
} }
// Tr translates content to target language.
func (l *locale) Tr(format string, args ...interface{}) string {
return i18n.Tr(l.Lang, format, args...)
}
// Language specific rules for translating plural texts // Language specific rules for translating plural texts
var trNLangRules = map[string]func(int64) int{ var trNLangRules = map[string]func(int64) int{
// the default rule is "en-US" if a language isn't listed here // the default rule is "en-US" if a language isn't listed here

View file

@ -12,7 +12,6 @@ import (
"path/filepath" "path/filepath"
"regexp" "regexp"
"runtime" "runtime"
"strings"
) )
// EnsureAbsolutePath ensure that a path is absolute, making it // EnsureAbsolutePath ensure that a path is absolute, making it
@ -91,7 +90,7 @@ func statDir(dirPath, recPath string, includeDir, isDirOnly, followSymlinks bool
statList := make([]string, 0) statList := make([]string, 0)
for _, fi := range fis { for _, fi := range fis {
if strings.Contains(fi.Name(), ".DS_Store") { if CommonSkip(fi.Name()) {
continue continue
} }
@ -199,3 +198,21 @@ func HomeDir() (home string, err error) {
return home, nil return home, nil
} }
// CommonSkip will check a provided name to see if it represents file or directory that should not be watched
func CommonSkip(name string) bool {
if name == "" {
return true
}
switch name[0] {
case '.':
return true
case 't', 'T':
return name[1:] == "humbs.db"
case 'd', 'D':
return name[1:] == "esktop.ini"
}
return false
}

115
modules/watcher/watcher.go Normal file
View file

@ -0,0 +1,115 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package watcher
import (
"context"
"io/fs"
"os"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/process"
"github.com/fsnotify/fsnotify"
)
// CreateWatcherOpts are options to configure the watcher
type CreateWatcherOpts struct {
// PathsCallback is used to set the required paths to watch
PathsCallback func(func(path, name string, d fs.DirEntry, err error) error) error
// BeforeCallback is called before any files are watched
BeforeCallback func()
// Between Callback is called between after a watched event has occurred
BetweenCallback func()
// AfterCallback is called as this watcher ends
AfterCallback func()
}
// CreateWatcher creates a watcher labelled with the provided description and running with the provided options.
// The created watcher will create a subcontext from the provided ctx and register it with the process manager.
func CreateWatcher(ctx context.Context, desc string, opts *CreateWatcherOpts) {
go run(ctx, desc, opts)
}
func run(ctx context.Context, desc string, opts *CreateWatcherOpts) {
if opts.BeforeCallback != nil {
opts.BeforeCallback()
}
if opts.AfterCallback != nil {
defer opts.AfterCallback()
}
ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Watcher: "+desc, process.SystemProcessType, true)
defer finished()
log.Trace("Watcher loop starting for %s", desc)
defer log.Trace("Watcher loop ended for %s", desc)
watcher, err := fsnotify.NewWatcher()
if err != nil {
log.Error("Unable to create watcher for %s: %v", desc, err)
return
}
if err := opts.PathsCallback(func(path, _ string, d fs.DirEntry, err error) error {
if err != nil && !os.IsNotExist(err) {
return err
}
log.Trace("Watcher: %s watching %q", desc, path)
_ = watcher.Add(path)
return nil
}); err != nil {
log.Error("Unable to create watcher for %s: %v", desc, err)
_ = watcher.Close()
return
}
// Note we don't call the BetweenCallback here
for {
select {
case event, ok := <-watcher.Events:
if !ok {
_ = watcher.Close()
return
}
log.Debug("Watched file for %s had event: %v", desc, event)
case err, ok := <-watcher.Errors:
if !ok {
_ = watcher.Close()
return
}
log.Error("Error whilst watching files for %s: %v", desc, err)
case <-ctx.Done():
_ = watcher.Close()
return
}
// Recreate the watcher - only call the BetweenCallback after the new watcher is set-up
_ = watcher.Close()
watcher, err = fsnotify.NewWatcher()
if err != nil {
log.Error("Unable to create watcher for %s: %v", desc, err)
return
}
if err := opts.PathsCallback(func(path, _ string, _ fs.DirEntry, err error) error {
if err != nil {
return err
}
_ = watcher.Add(path)
return nil
}); err != nil {
log.Error("Unable to create watcher for %s: %v", desc, err)
_ = watcher.Close()
return
}
// Inform our BetweenCallback that there has been an event
if opts.BetweenCallback != nil {
opts.BetweenCallback()
}
}
}

View file

@ -5,6 +5,7 @@
package packages package packages
import ( import (
gocontext "context"
"net/http" "net/http"
"regexp" "regexp"
"strings" "strings"
@ -38,10 +39,10 @@ func reqPackageAccess(accessMode perm.AccessMode) func(ctx *context.Context) {
} }
} }
func Routes() *web.Route { func Routes(ctx gocontext.Context) *web.Route {
r := web.NewRoute() r := web.NewRoute()
r.Use(context.PackageContexter()) r.Use(context.PackageContexter(ctx))
authMethods := []auth.Method{ authMethods := []auth.Method{
&auth.OAuth2{}, &auth.OAuth2{},
@ -270,10 +271,10 @@ func Routes() *web.Route {
return r return r
} }
func ContainerRoutes() *web.Route { func ContainerRoutes(ctx gocontext.Context) *web.Route {
r := web.NewRoute() r := web.NewRoute()
r.Use(context.PackageContexter()) r.Use(context.PackageContexter(ctx))
authMethods := []auth.Method{ authMethods := []auth.Method{
&auth.Basic{}, &auth.Basic{},

View file

@ -16,7 +16,6 @@ import (
packages_module "code.gitea.io/gitea/modules/packages" packages_module "code.gitea.io/gitea/modules/packages"
pypi_module "code.gitea.io/gitea/modules/packages/pypi" pypi_module "code.gitea.io/gitea/modules/packages/pypi"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/templates"
"code.gitea.io/gitea/modules/validation" "code.gitea.io/gitea/modules/validation"
"code.gitea.io/gitea/routers/api/packages/helper" "code.gitea.io/gitea/routers/api/packages/helper"
packages_service "code.gitea.io/gitea/services/packages" packages_service "code.gitea.io/gitea/services/packages"
@ -58,7 +57,6 @@ func PackageMetadata(ctx *context.Context) {
ctx.Data["RegistryURL"] = setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/pypi" ctx.Data["RegistryURL"] = setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/pypi"
ctx.Data["PackageDescriptor"] = pds[0] ctx.Data["PackageDescriptor"] = pds[0]
ctx.Data["PackageDescriptors"] = pds ctx.Data["PackageDescriptors"] = pds
ctx.Render = templates.HTMLRenderer()
ctx.HTML(http.StatusOK, "api/packages/pypi/simple") ctx.HTML(http.StatusOK, "api/packages/pypi/simple")
} }

View file

@ -65,6 +65,7 @@
package v1 package v1
import ( import (
gocontext "context"
"fmt" "fmt"
"net/http" "net/http"
"reflect" "reflect"
@ -605,7 +606,7 @@ func buildAuthGroup() *auth.Group {
} }
// Routes registers all v1 APIs routes to web application. // Routes registers all v1 APIs routes to web application.
func Routes() *web.Route { func Routes(ctx gocontext.Context) *web.Route {
m := web.NewRoute() m := web.NewRoute()
m.Use(securityHeaders()) m.Use(securityHeaders())
@ -623,7 +624,7 @@ func Routes() *web.Route {
m.Use(context.APIContexter()) m.Use(context.APIContexter())
group := buildAuthGroup() group := buildAuthGroup()
if err := group.Init(); err != nil { if err := group.Init(ctx); err != nil {
log.Error("Could not initialize '%s' auth method, error: %s", group.Name(), err) log.Error("Could not initialize '%s' auth method, error: %s", group.Name(), err)
} }

View file

@ -29,7 +29,7 @@ const (
) )
func createContext(req *http.Request) (*context.Context, *httptest.ResponseRecorder) { func createContext(req *http.Request) (*context.Context, *httptest.ResponseRecorder) {
rnd := templates.HTMLRenderer() _, rnd := templates.HTMLRenderer(req.Context())
resp := httptest.NewRecorder() resp := httptest.NewRecorder()
c := &context.Context{ c := &context.Context{
Req: req, Req: req,

View file

@ -27,6 +27,7 @@ import (
"code.gitea.io/gitea/modules/ssh" "code.gitea.io/gitea/modules/ssh"
"code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/svg" "code.gitea.io/gitea/modules/svg"
"code.gitea.io/gitea/modules/templates"
"code.gitea.io/gitea/modules/translation" "code.gitea.io/gitea/modules/translation"
"code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/modules/web"
@ -110,12 +111,12 @@ func GlobalInitInstalled(ctx context.Context) {
log.Info("Run Mode: %s", util.ToTitleCase(setting.RunMode)) log.Info("Run Mode: %s", util.ToTitleCase(setting.RunMode))
// Setup i18n // Setup i18n
translation.InitLocales() translation.InitLocales(ctx)
setting.NewServices() setting.NewServices()
mustInit(storage.Init) mustInit(storage.Init)
mailer.NewContext() mailer.NewContext(ctx)
mustInit(cache.NewContext) mustInit(cache.NewContext)
notification.NewContext() notification.NewContext()
mustInit(archiver.Init) mustInit(archiver.Init)
@ -163,18 +164,19 @@ func GlobalInitInstalled(ctx context.Context) {
} }
// NormalRoutes represents non install routes // NormalRoutes represents non install routes
func NormalRoutes() *web.Route { func NormalRoutes(ctx context.Context) *web.Route {
ctx, _ = templates.HTMLRenderer(ctx)
r := web.NewRoute() r := web.NewRoute()
for _, middle := range common.Middlewares() { for _, middle := range common.Middlewares() {
r.Use(middle) r.Use(middle)
} }
r.Mount("/", web_routers.Routes()) r.Mount("/", web_routers.Routes(ctx))
r.Mount("/api/v1", apiv1.Routes()) r.Mount("/api/v1", apiv1.Routes(ctx))
r.Mount("/api/internal", private.Routes()) r.Mount("/api/internal", private.Routes())
if setting.Packages.Enabled { if setting.Packages.Enabled {
r.Mount("/api/packages", packages_router.Routes()) r.Mount("/api/packages", packages_router.Routes(ctx))
r.Mount("/v2", packages_router.ContainerRoutes()) r.Mount("/v2", packages_router.ContainerRoutes(ctx))
} }
return r return r
} }

View file

@ -6,6 +6,7 @@
package install package install
import ( import (
goctx "context"
"fmt" "fmt"
"net/http" "net/http"
"os" "os"
@ -51,9 +52,10 @@ func getSupportedDbTypeNames() (dbTypeNames []map[string]string) {
} }
// Init prepare for rendering installation page // Init prepare for rendering installation page
func Init(next http.Handler) http.Handler { func Init(ctx goctx.Context) func(next http.Handler) http.Handler {
rnd := templates.HTMLRenderer() _, rnd := templates.HTMLRenderer(ctx)
dbTypeNames := getSupportedDbTypeNames() dbTypeNames := getSupportedDbTypeNames()
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
if setting.InstallLock { if setting.InstallLock {
resp.Header().Add("Refresh", "1; url="+setting.AppURL+"user/login") resp.Header().Add("Refresh", "1; url="+setting.AppURL+"user/login")
@ -84,6 +86,7 @@ func Init(next http.Handler) http.Handler {
ctx.Req = context.WithContext(req, &ctx) ctx.Req = context.WithContext(req, &ctx)
next.ServeHTTP(resp, ctx.Req) next.ServeHTTP(resp, ctx.Req)
}) })
}
} }
// Install render installation page // Install render installation page

View file

@ -5,6 +5,7 @@
package install package install
import ( import (
goctx "context"
"fmt" "fmt"
"net/http" "net/http"
"path" "path"
@ -29,8 +30,8 @@ func (d *dataStore) GetData() map[string]interface{} {
return *d return *d
} }
func installRecovery() func(next http.Handler) http.Handler { func installRecovery(ctx goctx.Context) func(next http.Handler) http.Handler {
rnd := templates.HTMLRenderer() _, rnd := templates.HTMLRenderer(ctx)
return func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
defer func() { defer func() {
@ -82,7 +83,7 @@ func installRecovery() func(next http.Handler) http.Handler {
} }
// Routes registers the install routes // Routes registers the install routes
func Routes() *web.Route { func Routes(ctx goctx.Context) *web.Route {
r := web.NewRoute() r := web.NewRoute()
for _, middle := range common.Middlewares() { for _, middle := range common.Middlewares() {
r.Use(middle) r.Use(middle)
@ -105,8 +106,8 @@ func Routes() *web.Route {
Domain: setting.SessionConfig.Domain, Domain: setting.SessionConfig.Domain,
})) }))
r.Use(installRecovery()) r.Use(installRecovery(ctx))
r.Use(Init) r.Use(Init(ctx))
r.Get("/", Install) r.Get("/", Install)
r.Post("/", web.Bind(forms.InstallForm{}), SubmitInstall) r.Post("/", web.Bind(forms.InstallForm{}), SubmitInstall)
r.Get("/api/healthz", healthcheck.Check) r.Get("/api/healthz", healthcheck.Check)

View file

@ -5,13 +5,16 @@
package install package install
import ( import (
"context"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
func TestRoutes(t *testing.T) { func TestRoutes(t *testing.T) {
routes := Routes() ctx, cancel := context.WithCancel(context.Background())
defer cancel()
routes := Routes(ctx)
assert.NotNil(t, routes) assert.NotNil(t, routes)
assert.EqualValues(t, "/", routes.R.Routes()[0].Pattern) assert.EqualValues(t, "/", routes.R.Routes()[0].Pattern)
assert.Nil(t, routes.R.Routes()[0].SubRoutes) assert.Nil(t, routes.R.Routes()[0].SubRoutes)

View file

@ -24,7 +24,7 @@ func PreloadSettings(ctx context.Context) bool {
log.Info("Log path: %s", setting.LogRootPath) log.Info("Log path: %s", setting.LogRootPath)
log.Info("Configuration file: %s", setting.CustomConf) log.Info("Configuration file: %s", setting.CustomConf)
log.Info("Prepare to run install page") log.Info("Prepare to run install page")
translation.InitLocales() translation.InitLocales(ctx)
if setting.EnableSQLite3 { if setting.EnableSQLite3 {
log.Info("SQLite3 is supported") log.Info("SQLite3 is supported")
} }

View file

@ -5,6 +5,7 @@
package web package web
import ( import (
goctx "context"
"errors" "errors"
"fmt" "fmt"
"io" "io"
@ -123,8 +124,8 @@ func (d *dataStore) GetData() map[string]interface{} {
// Recovery returns a middleware that recovers from any panics and writes a 500 and a log if so. // Recovery returns a middleware that recovers from any panics and writes a 500 and a log if so.
// This error will be created with the gitea 500 page. // This error will be created with the gitea 500 page.
func Recovery() func(next http.Handler) http.Handler { func Recovery(ctx goctx.Context) func(next http.Handler) http.Handler {
rnd := templates.HTMLRenderer() _, rnd := templates.HTMLRenderer(ctx)
return func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
defer func() { defer func() {

View file

@ -22,6 +22,7 @@ import (
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/templates"
"code.gitea.io/gitea/modules/validation" "code.gitea.io/gitea/modules/validation"
"code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/modules/web"
"code.gitea.io/gitea/modules/web/routing" "code.gitea.io/gitea/modules/web/routing"
@ -97,7 +98,7 @@ func buildAuthGroup() *auth_service.Group {
} }
// Routes returns all web routes // Routes returns all web routes
func Routes() *web.Route { func Routes(ctx gocontext.Context) *web.Route {
routes := web.NewRoute() routes := web.NewRoute()
routes.Use(web.WrapWithPrefix(public.AssetsURLPathPrefix, public.AssetsHandlerFunc(&public.Options{ routes.Use(web.WrapWithPrefix(public.AssetsURLPathPrefix, public.AssetsHandlerFunc(&public.Options{
@ -119,7 +120,9 @@ func Routes() *web.Route {
}) })
routes.Use(sessioner) routes.Use(sessioner)
routes.Use(Recovery()) ctx, _ = templates.HTMLRenderer(ctx)
routes.Use(Recovery(ctx))
// We use r.Route here over r.Use because this prevents requests that are not for avatars having to go through this additional handler // We use r.Route here over r.Use because this prevents requests that are not for avatars having to go through this additional handler
routes.Route("/avatars/*", "GET, HEAD", storageHandler(setting.Avatar.Storage, "avatars", storage.Avatars)) routes.Route("/avatars/*", "GET, HEAD", storageHandler(setting.Avatar.Storage, "avatars", storage.Avatars))
@ -192,10 +195,10 @@ func Routes() *web.Route {
routes.Get("/api/healthz", healthcheck.Check) routes.Get("/api/healthz", healthcheck.Check)
// Removed: toolbox.Toolboxer middleware will provide debug information which seems unnecessary // Removed: toolbox.Toolboxer middleware will provide debug information which seems unnecessary
common = append(common, context.Contexter()) common = append(common, context.Contexter(ctx))
group := buildAuthGroup() group := buildAuthGroup()
if err := group.Init(); err != nil { if err := group.Init(ctx); err != nil {
log.Error("Could not initialize '%s' auth method, error: %s", group.Name(), err) log.Error("Could not initialize '%s' auth method, error: %s", group.Name(), err)
} }

View file

@ -5,6 +5,7 @@
package auth package auth
import ( import (
"context"
"net/http" "net/http"
"reflect" "reflect"
"strings" "strings"
@ -51,14 +52,14 @@ func (b *Group) Name() string {
} }
// Init does nothing as the Basic implementation does not need to allocate any resources // Init does nothing as the Basic implementation does not need to allocate any resources
func (b *Group) Init() error { func (b *Group) Init(ctx context.Context) error {
for _, method := range b.methods { for _, method := range b.methods {
initializable, ok := method.(Initializable) initializable, ok := method.(Initializable)
if !ok { if !ok {
continue continue
} }
if err := initializable.Init(); err != nil { if err := initializable.Init(ctx); err != nil {
return err return err
} }
} }

View file

@ -34,7 +34,7 @@ type Method interface {
type Initializable interface { type Initializable interface {
// Init should be called exactly once before using any of the other methods, // Init should be called exactly once before using any of the other methods,
// in order to allow the plugin to allocate necessary resources // in order to allow the plugin to allocate necessary resources
Init() error Init(ctx context.Context) error
} }
// Named represents a named thing // Named represents a named thing

View file

@ -5,6 +5,7 @@
package auth package auth
import ( import (
"context"
"errors" "errors"
"net/http" "net/http"
"strings" "strings"
@ -52,21 +53,14 @@ type SSPI struct {
} }
// Init creates a new global websspi.Authenticator object // Init creates a new global websspi.Authenticator object
func (s *SSPI) Init() error { func (s *SSPI) Init(ctx context.Context) error {
config := websspi.NewConfig() config := websspi.NewConfig()
var err error var err error
sspiAuth, err = websspi.New(config) sspiAuth, err = websspi.New(config)
if err != nil { if err != nil {
return err return err
} }
s.rnd = render.New(render.Options{ _, s.rnd = templates.HTMLRenderer(ctx)
Extensions: []string{".tmpl"},
Directory: "templates",
Funcs: templates.NewFuncMap(),
Asset: templates.GetAsset,
AssetNames: templates.GetAssetNames,
IsDevelopment: !setting.IsProd,
})
return nil return nil
} }

View file

@ -7,6 +7,7 @@ package mailer
import ( import (
"bytes" "bytes"
"context"
"crypto/tls" "crypto/tls"
"fmt" "fmt"
"hash/fnv" "hash/fnv"
@ -348,7 +349,7 @@ var mailQueue queue.Queue
var Sender gomail.Sender var Sender gomail.Sender
// NewContext start mail queue service // NewContext start mail queue service
func NewContext() { func NewContext(ctx context.Context) {
// Need to check if mailQueue is nil because in during reinstall (user had installed // Need to check if mailQueue is nil because in during reinstall (user had installed
// before but switched install lock off), this function will be called again // before but switched install lock off), this function will be called again
// while mail queue is already processing tasks, and produces a race condition. // while mail queue is already processing tasks, and produces a race condition.
@ -381,7 +382,7 @@ func NewContext() {
go graceful.GetManager().RunWithShutdownFns(mailQueue.Run) go graceful.GetManager().RunWithShutdownFns(mailQueue.Run)
subjectTemplates, bodyTemplates = templates.Mailer() subjectTemplates, bodyTemplates = templates.Mailer(ctx)
} }
// SendAsync send mail asynchronously // SendAsync send mail asynchronously