From 723598b803919bfc074fee05f830421a99881c3b Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 3 May 2023 22:58:43 +0200 Subject: [PATCH] Implement Cargo HTTP index (#24452) This implements the HTTP index [RFC](https://rust-lang.github.io/rfcs/2789-sparse-index.html) for Cargo registries. Currently this is a preview feature and you need to use the nightly of `cargo`: `cargo +nightly -Z sparse-registry update` See https://github.com/rust-lang/cargo/issues/9069 for more information. --------- Co-authored-by: Giteabot --- routers/api/packages/api.go | 6 ++ routers/api/packages/cargo/cargo.go | 30 ++++++ services/packages/cargo/index.go | 36 +++++-- tests/integration/api_packages_cargo_test.go | 102 ++++++++++++++----- 4 files changed, 136 insertions(+), 38 deletions(-) diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 04b0c0ab08..ee1feb1414 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -119,6 +119,12 @@ func CommonRoutes(ctx gocontext.Context) *web.Route { r.Get("/owners", cargo.ListOwners) }) }) + r.Get("/config.json", cargo.RepositoryConfig) + r.Get("/1/{package}", cargo.EnumeratePackageVersions) + r.Get("/2/{package}", cargo.EnumeratePackageVersions) + // Use dummy placeholders because these parts are not of interest + r.Get("/3/{_}/{package}", cargo.EnumeratePackageVersions) + r.Get("/{_}/{__}/{package}", cargo.EnumeratePackageVersions) }, reqPackageAccess(perm.AccessModeRead)) r.Group("/chef", func() { r.Group("/api/v1", func() { diff --git a/routers/api/packages/cargo/cargo.go b/routers/api/packages/cargo/cargo.go index 18c93d328a..b666bdde6c 100644 --- a/routers/api/packages/cargo/cargo.go +++ b/routers/api/packages/cargo/cargo.go @@ -4,6 +4,7 @@ package cargo import ( + "errors" "fmt" "net/http" "strconv" @@ -45,6 +46,35 @@ func apiError(ctx *context.Context, status int, obj interface{}) { }) } +// https://rust-lang.github.io/rfcs/2789-sparse-index.html +func RepositoryConfig(ctx *context.Context) { + ctx.JSON(http.StatusOK, cargo_service.BuildConfig(ctx.Package.Owner)) +} + +func EnumeratePackageVersions(ctx *context.Context) { + p, err := packages_model.GetPackageByName(ctx, ctx.Package.Owner.ID, packages_model.TypeCargo, ctx.Params("package")) + if err != nil { + if errors.Is(err, util.ErrNotExist) { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + b, err := cargo_service.BuildPackageIndex(ctx, p) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if b == nil { + apiError(ctx, http.StatusNotFound, nil) + return + } + + ctx.PlainTextBytes(http.StatusOK, b.Bytes()) +} + type SearchResult struct { Crates []*SearchResultCrate `json:"crates"` Meta SearchResultMeta `json:"meta"` diff --git a/services/packages/cargo/index.go b/services/packages/cargo/index.go index e58a472816..1c2d17b504 100644 --- a/services/packages/cargo/index.go +++ b/services/packages/cargo/index.go @@ -137,21 +137,21 @@ type IndexVersionEntry struct { Links string `json:"links,omitempty"` } -func addOrUpdatePackageIndex(ctx context.Context, t *files_service.TemporaryUploadRepository, p *packages_model.Package) error { +func BuildPackageIndex(ctx context.Context, p *packages_model.Package) (*bytes.Buffer, error) { pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ PackageID: p.ID, Sort: packages_model.SortVersionAsc, }) if err != nil { - return fmt.Errorf("SearchVersions[%s]: %w", p.Name, err) + return nil, fmt.Errorf("SearchVersions[%s]: %w", p.Name, err) } if len(pvs) == 0 { - return nil + return nil, nil } pds, err := packages_model.GetPackageDescriptors(ctx, pvs) if err != nil { - return fmt.Errorf("GetPackageDescriptors[%s]: %w", p.Name, err) + return nil, fmt.Errorf("GetPackageDescriptors[%s]: %w", p.Name, err) } var b bytes.Buffer @@ -179,14 +179,26 @@ func addOrUpdatePackageIndex(ctx context.Context, t *files_service.TemporaryUplo Links: metadata.Links, }) if err != nil { - return err + return nil, err } b.Write(entry) b.WriteString("\n") } - return writeObjectToIndex(t, BuildPackagePath(pds[0].Package.LowerName), &b) + return &b, nil +} + +func addOrUpdatePackageIndex(ctx context.Context, t *files_service.TemporaryUploadRepository, p *packages_model.Package) error { + b, err := BuildPackageIndex(ctx, p) + if err != nil { + return err + } + if b == nil { + return nil + } + + return writeObjectToIndex(t, BuildPackagePath(p.LowerName), b) } func getOrCreateIndexRepository(ctx context.Context, doer, owner *user_model.User) (*repo_model.Repository, error) { @@ -212,6 +224,13 @@ type Config struct { APIURL string `json:"api"` } +func BuildConfig(owner *user_model.User) *Config { + return &Config{ + DownloadURL: setting.AppURL + "api/packages/" + owner.Name + "/cargo/api/v1/crates", + APIURL: setting.AppURL + "api/packages/" + owner.Name + "/cargo", + } +} + func createOrUpdateConfigFile(ctx context.Context, repo *repo_model.Repository, doer, owner *user_model.User) error { return alterRepositoryContent( ctx, @@ -220,10 +239,7 @@ func createOrUpdateConfigFile(ctx context.Context, repo *repo_model.Repository, "Initialize Cargo Config", func(t *files_service.TemporaryUploadRepository) error { var b bytes.Buffer - err := json.NewEncoder(&b).Encode(Config{ - DownloadURL: setting.AppURL + "api/packages/" + owner.Name + "/cargo/api/v1/crates", - APIURL: setting.AppURL + "api/packages/" + owner.Name + "/cargo", - }) + err := json.NewEncoder(&b).Encode(BuildConfig(owner)) if err != nil { return err } diff --git a/tests/integration/api_packages_cargo_test.go b/tests/integration/api_packages_cargo_test.go index 0c542eaf1e..608f192968 100644 --- a/tests/integration/api_packages_cargo_test.go +++ b/tests/integration/api_packages_cargo_test.go @@ -98,7 +98,7 @@ func testPackageCargo(t *testing.T, _ *neturl.URL) { url := fmt.Sprintf("%s/api/v1/crates", root) t.Run("Index", func(t *testing.T) { - t.Run("Config", func(t *testing.T) { + t.Run("Git/Config", func(t *testing.T) { defer tests.PrintCurrentTest(t)() content := readGitContent(t, cargo_service.ConfigFileName) @@ -110,6 +110,20 @@ func testPackageCargo(t *testing.T, _ *neturl.URL) { assert.Equal(t, url, config.DownloadURL) assert.Equal(t, root, config.APIURL) }) + + t.Run("HTTP/Config", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", root+"/"+cargo_service.ConfigFileName) + resp := MakeRequest(t, req, http.StatusOK) + + var config cargo_service.Config + err := json.Unmarshal(resp.Body.Bytes(), &config) + assert.NoError(t, err) + + assert.Equal(t, url, config.DownloadURL) + assert.Equal(t, root, config.APIURL) + }) }) t.Run("Upload", func(t *testing.T) { @@ -192,40 +206,72 @@ func testPackageCargo(t *testing.T, _ *neturl.URL) { MakeRequest(t, req, http.StatusConflict) t.Run("Index", func(t *testing.T) { - t.Run("Entry", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Git", func(t *testing.T) { + t.Run("Entry", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - content := readGitContent(t, cargo_service.BuildPackagePath(packageName)) + content := readGitContent(t, cargo_service.BuildPackagePath(packageName)) - var entry cargo_service.IndexVersionEntry - err := json.Unmarshal([]byte(content), &entry) - assert.NoError(t, err) + var entry cargo_service.IndexVersionEntry + err := json.Unmarshal([]byte(content), &entry) + assert.NoError(t, err) - assert.Equal(t, packageName, entry.Name) - assert.Equal(t, packageVersion, entry.Version) - assert.Equal(t, pb.HashSHA256, entry.FileChecksum) - assert.False(t, entry.Yanked) - assert.Len(t, entry.Dependencies, 1) - dep := entry.Dependencies[0] - assert.Equal(t, "dep", dep.Name) - assert.Equal(t, "1.0", dep.Req) - assert.Equal(t, "normal", dep.Kind) - assert.True(t, dep.DefaultFeatures) - assert.Empty(t, dep.Features) - assert.False(t, dep.Optional) - assert.Nil(t, dep.Target) - assert.NotNil(t, dep.Registry) - assert.Equal(t, "https://gitea.io/user/_cargo-index", *dep.Registry) - assert.Nil(t, dep.Package) + assert.Equal(t, packageName, entry.Name) + assert.Equal(t, packageVersion, entry.Version) + assert.Equal(t, pb.HashSHA256, entry.FileChecksum) + assert.False(t, entry.Yanked) + assert.Len(t, entry.Dependencies, 1) + dep := entry.Dependencies[0] + assert.Equal(t, "dep", dep.Name) + assert.Equal(t, "1.0", dep.Req) + assert.Equal(t, "normal", dep.Kind) + assert.True(t, dep.DefaultFeatures) + assert.Empty(t, dep.Features) + assert.False(t, dep.Optional) + assert.Nil(t, dep.Target) + assert.NotNil(t, dep.Registry) + assert.Equal(t, "https://gitea.io/user/_cargo-index", *dep.Registry) + assert.Nil(t, dep.Package) + }) + + t.Run("Rebuild", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + err := cargo_service.RebuildIndex(db.DefaultContext, user, user) + assert.NoError(t, err) + + _ = readGitContent(t, cargo_service.BuildPackagePath(packageName)) + }) }) - t.Run("Rebuild", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("HTTP", func(t *testing.T) { + t.Run("Entry", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - err := cargo_service.RebuildIndex(db.DefaultContext, user, user) - assert.NoError(t, err) + req := NewRequest(t, "GET", root+"/"+cargo_service.BuildPackagePath(packageName)) + resp := MakeRequest(t, req, http.StatusOK) - _ = readGitContent(t, cargo_service.BuildPackagePath(packageName)) + var entry cargo_service.IndexVersionEntry + err := json.Unmarshal(resp.Body.Bytes(), &entry) + assert.NoError(t, err) + + assert.Equal(t, packageName, entry.Name) + assert.Equal(t, packageVersion, entry.Version) + assert.Equal(t, pb.HashSHA256, entry.FileChecksum) + assert.False(t, entry.Yanked) + assert.Len(t, entry.Dependencies, 1) + dep := entry.Dependencies[0] + assert.Equal(t, "dep", dep.Name) + assert.Equal(t, "1.0", dep.Req) + assert.Equal(t, "normal", dep.Kind) + assert.True(t, dep.DefaultFeatures) + assert.Empty(t, dep.Features) + assert.False(t, dep.Optional) + assert.Nil(t, dep.Target) + assert.NotNil(t, dep.Registry) + assert.Equal(t, "https://gitea.io/user/_cargo-index", *dep.Registry) + assert.Nil(t, dep.Package) + }) }) }) })