From 36932e55b4afb1c884632d77780c0a99db60ee8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20N=C3=A4f?= Date: Fri, 16 Oct 2026 18:32:38 +0200 Subject: [PATCH 1/2] feat: Enhance schema generation and dependency handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cyrill Näf --- apis/dev/v1alpha1/project_types.go | 16 + apis/dev/v1alpha1/validate.go | 15 + apis/dev/v1alpha1/validate_test.go | 45 + apis/dev/v1alpha1/zz_generated.deepcopy.go | 5 + cmd/crossplane/composition/generate.go | 45 +- cmd/crossplane/dependency/add.go | 41 +- cmd/crossplane/dependency/cache.go | 96 +- cmd/crossplane/dependency/cache_test.go | 80 + cmd/crossplane/dependency/help/add.md | 24 +- cmd/crossplane/dependency/help/clean-cache.md | 8 +- .../dependency/help/update-cache.md | 14 +- cmd/crossplane/function/generate.go | 49 +- cmd/crossplane/function/generate_test.go | 93 +- cmd/crossplane/main.go | 2 + cmd/crossplane/project/build.go | 36 +- cmd/crossplane/project/help/build.md | 14 +- cmd/crossplane/project/run.go | 36 +- cmd/crossplane/render/op/cmd.go | 15 +- cmd/crossplane/render/xr/cmd.go | 13 +- cmd/crossplane/schema/clean.go | 53 + cmd/crossplane/schema/env.go | 77 + cmd/crossplane/schema/generate.go | 73 + cmd/crossplane/schema/help/clean.md | 3 + cmd/crossplane/schema/help/generate.md | 8 + cmd/crossplane/schema/help/status.md | 6 + cmd/crossplane/schema/help/update.md | 22 + .../{dependency/auth.go => schema/schema.go} | 28 +- cmd/crossplane/schema/schema_test.go | 208 +++ cmd/crossplane/schema/status.go | 76 + cmd/crossplane/schema/update.go | 84 + internal/dependency/manager.go | 716 ++++++-- internal/dependency/manager_test.go | 1559 ++++++++++++++--- internal/project/build.go | 70 +- internal/project/build_test.go | 320 ++++ internal/project/projectfile/projectfile.go | 17 + .../project/projectfile/projectfile_test.go | 100 ++ internal/project/schemas.go | 162 ++ internal/schemas/manager/filter.go | 159 ++ internal/schemas/manager/filter_test.go | 190 ++ internal/schemas/manager/lock.go | 87 +- internal/schemas/manager/manager.go | 632 ++++++- internal/schemas/manager/manager_test.go | 1303 ++++++++++++-- internal/schemas/manager/source.go | 229 ++- internal/schemas/manager/source_test.go | 336 +++- internal/xpkg/resolver.go | 31 + internal/xpkg/resolver_test.go | 80 + 46 files changed, 6249 insertions(+), 1027 deletions(-) create mode 100644 cmd/crossplane/dependency/cache_test.go create mode 100644 cmd/crossplane/schema/clean.go create mode 100644 cmd/crossplane/schema/env.go create mode 100644 cmd/crossplane/schema/generate.go create mode 100644 cmd/crossplane/schema/help/clean.md create mode 100644 cmd/crossplane/schema/help/generate.md create mode 100644 cmd/crossplane/schema/help/status.md create mode 100644 cmd/crossplane/schema/help/update.md rename cmd/crossplane/{dependency/auth.go => schema/schema.go} (50%) create mode 100644 cmd/crossplane/schema/schema_test.go create mode 100644 cmd/crossplane/schema/status.go create mode 100644 cmd/crossplane/schema/update.go create mode 100644 internal/project/schemas.go create mode 100644 internal/schemas/manager/filter.go create mode 100644 internal/schemas/manager/filter_test.go diff --git a/apis/dev/v1alpha1/project_types.go b/apis/dev/v1alpha1/project_types.go index c696e97a..b49b513e 100644 --- a/apis/dev/v1alpha1/project_types.go +++ b/apis/dev/v1alpha1/project_types.go @@ -136,6 +136,13 @@ type ProjectSchemas struct { // Supported values are "go", "json", "kcl", and "python". If not // specified, schemas are generated for all supported languages. Languages []string `json:"languages,omitempty"` + + // APIGroups restricts schema generation to resources whose API group + // matches one of the listed globs, e.g. "*.aws.upbound.io" or + // "kubernetes.crossplane.io". Use it to avoid generating thousands of files + // for provider APIs the project never uses. If not specified, schemas are + // generated for all API groups. + APIGroups []string `json:"apiGroups,omitempty"` } // GetLanguages returns the configured schema languages, or nil if no Schemas @@ -147,6 +154,15 @@ func (s *ProjectSchemas) GetLanguages() []string { return s.Languages } +// GetAPIGroups returns the configured API group globs, or nil if no Schemas +// config is set. It is safe to call on a nil receiver. +func (s *ProjectSchemas) GetAPIGroups() []string { + if s == nil { + return nil + } + return s.APIGroups +} + // ProjectPaths configures the locations of various parts of the project, for // use at build time. All paths must be relative to the project root. type ProjectPaths struct { diff --git a/apis/dev/v1alpha1/validate.go b/apis/dev/v1alpha1/validate.go index fe40a985..df824cce 100644 --- a/apis/dev/v1alpha1/validate.go +++ b/apis/dev/v1alpha1/validate.go @@ -18,6 +18,7 @@ package v1alpha1 import ( "fmt" + "path" "path/filepath" "slices" "strings" @@ -109,6 +110,20 @@ func (s *ProjectSchemas) Validate() []error { if s == nil { return nil } + errs := s.validateLanguages() + for i, glob := range s.APIGroups { + if glob == "" { + errs = append(errs, errors.Errorf("schemas.apiGroups[%d]: glob must not be empty", i)) + continue + } + if _, err := path.Match(glob, ""); err != nil { + errs = append(errs, errors.Wrapf(err, "schemas.apiGroups[%d]: %q is not a valid glob", i, glob)) + } + } + return errs +} + +func (s *ProjectSchemas) validateLanguages() []error { if s.Languages == nil { return nil } diff --git a/apis/dev/v1alpha1/validate_test.go b/apis/dev/v1alpha1/validate_test.go index d7457c56..e3c871af 100644 --- a/apis/dev/v1alpha1/validate_test.go +++ b/apis/dev/v1alpha1/validate_test.go @@ -185,6 +185,51 @@ func TestValidate(t *testing.T) { `schemas.languages[1]: "fortran" is not a supported schema language`, }, }, + "ValidAPIGroupGlobs": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-project", + }, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Schemas: &ProjectSchemas{ + APIGroups: []string{"*.m.upbound.io", "kubernetes.crossplane.io"}, + }, + }, + }, + }, + "EmptyAPIGroupGlob": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-project", + }, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Schemas: &ProjectSchemas{ + APIGroups: []string{""}, + }, + }, + }, + expectedErrors: []string{ + "schemas.apiGroups[0]: glob must not be empty", + }, + }, + "MalformedAPIGroupGlob": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-project", + }, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Schemas: &ProjectSchemas{ + APIGroups: []string{"*.aws.upbound.io", "[a-"}, + }, + }, + }, + expectedErrors: []string{ + `schemas.apiGroups[1]: "[a-" is not a valid glob`, + }, + }, "ValidAPIDependency": { input: &Project{ ObjectMeta: metav1.ObjectMeta{ diff --git a/apis/dev/v1alpha1/zz_generated.deepcopy.go b/apis/dev/v1alpha1/zz_generated.deepcopy.go index dc5d5532..17fbe617 100644 --- a/apis/dev/v1alpha1/zz_generated.deepcopy.go +++ b/apis/dev/v1alpha1/zz_generated.deepcopy.go @@ -225,6 +225,11 @@ func (in *ProjectSchemas) DeepCopyInto(out *ProjectSchemas) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.APIGroups != nil { + in, out := &in.APIGroups, &out.APIGroups + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProjectSchemas. diff --git a/cmd/crossplane/composition/generate.go b/cmd/crossplane/composition/generate.go index dbf7c797..38129738 100644 --- a/cmd/crossplane/composition/generate.go +++ b/cmd/crossplane/composition/generate.go @@ -36,12 +36,9 @@ import ( pkgv1 "github.com/crossplane/crossplane/apis/v2/pkg/v1" "github.com/crossplane/cli/v2/apis/dev/v1alpha1" - "github.com/crossplane/cli/v2/internal/config" "github.com/crossplane/cli/v2/internal/dependency" "github.com/crossplane/cli/v2/internal/project/projectfile" - "github.com/crossplane/cli/v2/internal/schemas/generator" "github.com/crossplane/cli/v2/internal/terminal" - clixpkg "github.com/crossplane/cli/v2/internal/xpkg" _ "embed" ) @@ -73,49 +70,17 @@ func (c *generateCmd) Help() string { } // AfterApply sets up the project filesystem. -func (c *generateCmd) AfterApply(cfg *config.Config) error { - projFilePath, err := filepath.Abs(c.ProjectFile) - if err != nil { - return err - } - projDirPath := filepath.Dir(projFilePath) - c.projFS = afero.NewBasePathFs(afero.NewOsFs(), projDirPath) - - proj, err := projectfile.Parse(c.projFS, filepath.Base(c.ProjectFile)) +func (c *generateCmd) AfterApply() error { + proj, projFS, err := projectfile.Load(c.ProjectFile) if err != nil { return err } + c.projFS = projFS c.proj = proj c.apisFS = afero.NewBasePathFs(c.projFS, proj.Spec.Paths.APIs) - cacheDir := c.CacheDir - if cacheDir == "" { - cacheDir = dependency.DefaultCacheDir() - } - - client, err := clixpkg.NewClient( - clixpkg.NewRemoteFetcher(), - clixpkg.WithCacheDir(afero.NewOsFs(), cacheDir), - clixpkg.WithImageConfigs(proj.Spec.ImageConfigs), - ) - if err != nil { - return err - } - resolver := clixpkg.NewResolver(client) - - c.depManager = dependency.NewManager(proj, c.projFS, - dependency.WithProjectFile(filepath.Base(c.ProjectFile)), - dependency.WithSchemaGenerators(generator.Filter( - generator.AllLanguages( - generator.WithGoModelAccessors(cfg.Features.GenerateGoModelAccessors), - generator.WithGoRuntimeObjects(cfg.Features.GenerateGoRuntimeObjects), - ), - proj.Spec.Schemas.GetLanguages(), - )), - dependency.WithXpkgClient(client), - dependency.WithResolver(resolver), - ) - return nil + c.depManager, err = dependency.NewCachedManager(proj, c.projFS, c.ProjectFile, c.CacheDir) + return err } func (c *generateCmd) Run(sp terminal.SpinnerPrinter) error { diff --git a/cmd/crossplane/dependency/add.go b/cmd/crossplane/dependency/add.go index 6424f473..49623d08 100644 --- a/cmd/crossplane/dependency/add.go +++ b/cmd/crossplane/dependency/add.go @@ -18,22 +18,17 @@ package dependency import ( "context" - "path/filepath" "strings" "github.com/google/go-containerregistry/pkg/name" - "github.com/spf13/afero" "github.com/crossplane/crossplane-runtime/v2/pkg/errors" "github.com/crossplane/crossplane-runtime/v2/pkg/logging" "github.com/crossplane/cli/v2/apis/dev/v1alpha1" - "github.com/crossplane/cli/v2/internal/config" "github.com/crossplane/cli/v2/internal/dependency" "github.com/crossplane/cli/v2/internal/project/projectfile" - "github.com/crossplane/cli/v2/internal/schemas/generator" "github.com/crossplane/cli/v2/internal/terminal" - clixpkg "github.com/crossplane/cli/v2/internal/xpkg" _ "embed" ) @@ -58,49 +53,19 @@ func (c *addCmd) Help() string { } // Run executes the add command. -func (c *addCmd) Run(logger logging.Logger, sp terminal.SpinnerPrinter, cfg *config.Config) error { +func (c *addCmd) Run(logger logging.Logger, sp terminal.SpinnerPrinter) error { ctx := context.Background() - projFilePath, err := filepath.Abs(c.ProjectFile) + proj, projFS, err := projectfile.Load(c.ProjectFile) if err != nil { return err } - projDirPath := filepath.Dir(projFilePath) - projFS := afero.NewBasePathFs(afero.NewOsFs(), projDirPath) - proj, err := projectfile.Parse(projFS, filepath.Base(c.ProjectFile)) + m, err := dependency.NewCachedManager(proj, projFS, c.ProjectFile, c.CacheDir) if err != nil { return err } - cacheDir := c.CacheDir - if cacheDir == "" { - cacheDir = dependency.DefaultCacheDir() - } - - client, err := clixpkg.NewClient( - clixpkg.NewRemoteFetcher(), - clixpkg.WithCacheDir(afero.NewOsFs(), cacheDir), - clixpkg.WithImageConfigs(proj.Spec.ImageConfigs), - ) - if err != nil { - return err - } - resolver := clixpkg.NewResolver(client) - - m := dependency.NewManager(proj, projFS, - dependency.WithProjectFile(c.ProjectFile), - dependency.WithSchemaGenerators(generator.Filter( - generator.AllLanguages( - generator.WithGoModelAccessors(cfg.Features.GenerateGoModelAccessors), - generator.WithGoRuntimeObjects(cfg.Features.GenerateGoRuntimeObjects), - ), - proj.Spec.Schemas.GetLanguages(), - )), - dependency.WithXpkgClient(client), - dependency.WithResolver(resolver), - ) - dep, err := c.buildDependency() if err != nil { return err diff --git a/cmd/crossplane/dependency/cache.go b/cmd/crossplane/dependency/cache.go index 3cecc882..0c7b3f2f 100644 --- a/cmd/crossplane/dependency/cache.go +++ b/cmd/crossplane/dependency/cache.go @@ -19,7 +19,6 @@ package dependency import ( "context" "fmt" - "path/filepath" "github.com/alecthomas/kong" "github.com/spf13/afero" @@ -27,12 +26,10 @@ import ( "github.com/crossplane/crossplane-runtime/v2/pkg/logging" "github.com/crossplane/cli/v2/internal/async" - "github.com/crossplane/cli/v2/internal/config" "github.com/crossplane/cli/v2/internal/dependency" + "github.com/crossplane/cli/v2/internal/git" "github.com/crossplane/cli/v2/internal/project/projectfile" - "github.com/crossplane/cli/v2/internal/schemas/generator" "github.com/crossplane/cli/v2/internal/terminal" - clixpkg "github.com/crossplane/cli/v2/internal/xpkg" _ "embed" ) @@ -40,7 +37,8 @@ import ( //go:embed help/update-cache.md var updateHelp string -// updateCacheCmd updates the dependency cache by regenerating all schemas. +// updateCacheCmd refreshes the shared xpkg package cache and fetches the +// project's other dependencies. type updateCacheCmd struct { ProjectFile string `default:"${project_file}" help:"Path to project definition file." short:"f"` CacheDir string `env:"CROSSPLANE_XPKG_CACHE" help:"Directory for cached xpkg package contents." name:"cache-dir"` @@ -53,72 +51,36 @@ func (c *updateCacheCmd) Help() string { } // Run executes the update-cache command. -func (c *updateCacheCmd) Run(logger logging.Logger, sp terminal.SpinnerPrinter, cfg *config.Config) error { +func (c *updateCacheCmd) Run(logger logging.Logger, sp terminal.SpinnerPrinter) error { ctx := context.Background() - projFilePath, err := filepath.Abs(c.ProjectFile) + proj, projFS, err := projectfile.Load(c.ProjectFile) if err != nil { return err } - projDirPath := filepath.Dir(projFilePath) - projFS := afero.NewBasePathFs(afero.NewOsFs(), projDirPath) - proj, err := projectfile.Parse(projFS, filepath.Base(c.ProjectFile)) + m, err := dependency.NewCachedManager(proj, projFS, c.ProjectFile, c.CacheDir, dependency.WithGitAuthProvider(c.gitAuth())) if err != nil { return err } - cacheDir := c.CacheDir - if cacheDir == "" { - cacheDir = dependency.DefaultCacheDir() - } - - client, err := clixpkg.NewClient( - clixpkg.NewRemoteFetcher(), - clixpkg.WithCacheDir(afero.NewOsFs(), cacheDir), - clixpkg.WithImageConfigs(proj.Spec.ImageConfigs), - ) - if err != nil { - return err - } - resolver := clixpkg.NewResolver(client) - - opts := []dependency.ManagerOption{ - dependency.WithProjectFile(c.ProjectFile), - dependency.WithSchemaGenerators(generator.Filter( - generator.AllLanguages( - generator.WithGoModelAccessors(cfg.Features.GenerateGoModelAccessors), - generator.WithGoRuntimeObjects(cfg.Features.GenerateGoRuntimeObjects), - ), - proj.Spec.Schemas.GetLanguages(), - )), - dependency.WithXpkgClient(client), - dependency.WithResolver(resolver), - } - - if c.GitToken != "" { - opts = append(opts, dependency.WithGitAuthProvider(&gitAuthProvider{ - username: c.GitUsername, - token: c.GitToken, - })) - } - - m := dependency.NewManager(proj, projFS, opts...) - logger.Debug("Updating all dependencies") return sp.WrapAsyncWithSuccessSpinners(func(ch async.EventChannel) error { return m.RefreshAll(ctx, ch) }) } +// gitAuth returns the git authentication configured by the command's flags. +func (c *updateCacheCmd) gitAuth() git.AuthProvider { + return &git.HTTPSAuthProvider{Username: c.GitUsername, Password: c.GitToken} +} + //go:embed help/clean-cache.md var cleanHelp string -// cleanCacheCmd removes all generated schemas. +// cleanCacheCmd removes the shared xpkg package cache. type cleanCacheCmd struct { - ProjectFile string `default:"${project_file}" help:"Path to project definition file." short:"f"` - CacheDir string `env:"CROSSPLANE_XPKG_CACHE" help:"Directory for cached xpkg package contents." name:"cache-dir"` - KeepPackages bool `help:"Keep cached xpkg package contents; remove only generated schemas." name:"keep-packages"` + CacheDir string `env:"CROSSPLANE_XPKG_CACHE" help:"Directory for cached xpkg package contents." name:"cache-dir"` } func (c *cleanCacheCmd) Help() string { @@ -127,36 +89,14 @@ func (c *cleanCacheCmd) Help() string { // Run executes the clean-cache command. func (c *cleanCacheCmd) Run(k *kong.Context, _ logging.Logger) error { - projFilePath, err := filepath.Abs(c.ProjectFile) - if err != nil { - return err - } - projDirPath := filepath.Dir(projFilePath) - projFS := afero.NewBasePathFs(afero.NewOsFs(), projDirPath) - - proj, err := projectfile.Parse(projFS, filepath.Base(c.ProjectFile)) - if err != nil { - return err + cacheDir := c.CacheDir + if cacheDir == "" { + cacheDir = dependency.DefaultCacheDir() } - - m := dependency.NewManager(proj, projFS, - dependency.WithProjectFile(c.ProjectFile), - ) - - if err := m.Clean(); err != nil { + if err := dependency.CleanPackages(cacheDir, afero.NewOsFs()); err != nil { return err } - if !c.KeepPackages { - cacheDir := c.CacheDir - if cacheDir == "" { - cacheDir = dependency.DefaultCacheDir() - } - if err := dependency.CleanPackages(cacheDir, afero.NewOsFs()); err != nil { - return err - } - } - - fmt.Fprintln(k.Stdout, "Schema cache cleaned") //nolint:errcheck // TODO(adamwg): Clean up output. + fmt.Fprintln(k.Stdout, "Package cache cleaned") //nolint:errcheck // TODO(adamwg): Clean up output. return nil } diff --git a/cmd/crossplane/dependency/cache_test.go b/cmd/crossplane/dependency/cache_test.go new file mode 100644 index 00000000..dfec44e4 --- /dev/null +++ b/cmd/crossplane/dependency/cache_test.go @@ -0,0 +1,80 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dependency + +import ( + "testing" + + "github.com/alecthomas/kong" + "github.com/google/go-cmp/cmp" + + "github.com/crossplane/cli/v2/internal/git" +) + +func TestUpdateCacheGitAuth(t *testing.T) { + type args struct { + cmd []string + env map[string]string + } + type want struct { + auth git.AuthProvider + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "Defaults": { + reason: "Without flags the username defaults to x-access-token and there is no token.", + args: args{cmd: []string{"update-cache"}}, + want: want{auth: &git.HTTPSAuthProvider{Username: "x-access-token"}}, + }, + "Flags": { + reason: "The git flags configure the auth provider.", + args: args{cmd: []string{"update-cache", "--git-token", "t", "--git-username", "u"}}, + want: want{auth: &git.HTTPSAuthProvider{Username: "u", Password: "t"}}, + }, + "Env": { + reason: "The git environment variables configure the auth provider.", + args: args{ + cmd: []string{"update-cache"}, + env: map[string]string{"CROSSPLANE_GIT_TOKEN": "t", "CROSSPLANE_GIT_USERNAME": "u"}, + }, + want: want{auth: &git.HTTPSAuthProvider{Username: "u", Password: "t"}}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + var c Cmd + parser, err := kong.New(&c, kong.Vars{"project_file": "crossplane-project.yaml"}) + if err != nil { + t.Fatalf("kong.New(): %v", err) + } + for k, v := range tc.args.env { + t.Setenv(k, v) + } + if _, err := parser.Parse(tc.args.cmd); err != nil { + t.Fatalf("%s\nParse(%v): %v", tc.reason, tc.args.cmd, err) + } + if diff := cmp.Diff(tc.want.auth, c.UpdateCache.gitAuth()); diff != "" { + t.Errorf("%s\ngitAuth(): -want, +got:\n%s", tc.reason, diff) + } + }) + } +} diff --git a/cmd/crossplane/dependency/help/add.md b/cmd/crossplane/dependency/help/add.md index df9a1bfc..59fac478 100644 --- a/cmd/crossplane/dependency/help/add.md +++ b/cmd/crossplane/dependency/help/add.md @@ -1,6 +1,7 @@ The `dependency add` command adds a dependency to a Crossplane Project metadata -file and generates language bindings (schemas) for the dependency package's -CRDs. +file. It checks that the dependency can be fetched, and caches packages. It +doesn't generate language bindings (schemas); run `crossplane schema update` +afterwards to generate them for the dependency's CRDs. ## Dependency types @@ -21,37 +22,36 @@ Non-xpkg dependencies are always build-time dependencies. ## Examples -Retrieve the latest available semantic version of `provider-aws-eks`, generate -schemas for its CRDs, and add it to the project as a runtime dependency: +Retrieve the latest available semantic version of `provider-aws-eks` and add it +to the project as a runtime dependency: ```shell crossplane dependency add xpkg.crossplane.io/crossplane-contrib/provider-aws-eks ``` Retrieve the latest available version greater than `v1.1.0` of -`provider-gcp-storage`, generate schemas for its CRDs, and add it to the project -as a build-time only dependency: +`provider-gcp-storage` and add it to the project as a build-time only +dependency: ```shell crossplane dependency add --api-only 'xpkg.crossplane.io/crossplane-contrib/provider-gcp-storage:>v1.1.0' ``` -Generate schemas for the core resources from Kubernetes v1.33.0 and add it to -the project as a build-time dependency: +Add the core resources from Kubernetes v1.33.0 to the project as a build-time +dependency: ```shell crossplane dependency add k8s:v1.33.0 ``` -Generate schemas for a specific CRD from an HTTP URL and add it to the project -as a build-time dependency: +Add a specific CRD from an HTTP URL to the project as a build-time dependency: ```shell crossplane dependency add https://raw.githubusercontent.com/cert-manager/cert-manager/refs/heads/master/deploy/crds/cert-manager.io_certificaterequests.yaml ``` -Generate schemas for CRDs from a specific subdirectory of a git repository and -add it to the project as a build-time dependency: +Add CRDs from a specific subdirectory of a git repository to the project as a +build-time dependency: ```shell crossplane dependency add https://github.com/kubernetes-sigs/cluster-api \ diff --git a/cmd/crossplane/dependency/help/clean-cache.md b/cmd/crossplane/dependency/help/clean-cache.md index 614a838b..9deda82c 100644 --- a/cmd/crossplane/dependency/help/clean-cache.md +++ b/cmd/crossplane/dependency/help/clean-cache.md @@ -1,3 +1,7 @@ The `clean-cache` command removes all cached package images from the local cache -directory and removes generated schemas. This can help free up disk space, force -re-generation of schemas, or resolve issues with corrupted cache entries. +directory. This can help free up disk space or resolve issues with corrupted +cache entries. The cache is shared by all projects, so each project downloads +the packages it needs again the next time it needs them. + +The command doesn't remove generated schemas. Use `crossplane schema clean` for +that. diff --git a/cmd/crossplane/dependency/help/update-cache.md b/cmd/crossplane/dependency/help/update-cache.md index b016009a..304f0a2a 100644 --- a/cmd/crossplane/dependency/help/update-cache.md +++ b/cmd/crossplane/dependency/help/update-cache.md @@ -1,4 +1,14 @@ The `dependency update-cache` command updates the local dependency cache for the current project. It re-resolves semantic version constraints to specific -versions (fetching newer versions if available), caches all dependencies, and -re-generates language bindings (schemas) for them if needed. +versions (fetching newer versions if available) and fetches all dependencies: +xpkg packages and their transitive dependencies, git repositories, HTTP(S) +sources, and Kubernetes API definitions. xpkg packages are stored in the package +cache, which is shared by all projects. + +Git repositories are accessed over HTTPS. Use `--git-token` (or the +`CROSSPLANE_GIT_TOKEN` environment variable) to authenticate, and +`--git-username` (or `CROSSPLANE_GIT_USERNAME`, default `x-access-token`) to set +the username. + +The command doesn't generate schemas. Run `crossplane schema update` to generate +schemas for the cached packages. diff --git a/cmd/crossplane/function/generate.go b/cmd/crossplane/function/generate.go index 3d1fc20a..02056bd3 100644 --- a/cmd/crossplane/function/generate.go +++ b/cmd/crossplane/function/generate.go @@ -19,7 +19,6 @@ package function import ( "archive/tar" "bytes" - "context" "embed" "fmt" "io/fs" @@ -29,6 +28,7 @@ import ( "strings" "text/template" + "github.com/alecthomas/kong" "github.com/google/go-containerregistry/pkg/name" "github.com/spf13/afero" "github.com/spf13/afero/tarfs" @@ -39,13 +39,9 @@ import ( "github.com/crossplane/crossplane-runtime/v2/pkg/xpkg" v1alpha1 "github.com/crossplane/cli/v2/apis/dev/v1alpha1" - "github.com/crossplane/cli/v2/internal/config" "github.com/crossplane/cli/v2/internal/filesystem" "github.com/crossplane/cli/v2/internal/kcl" "github.com/crossplane/cli/v2/internal/project/projectfile" - "github.com/crossplane/cli/v2/internal/schemas/generator" - "github.com/crossplane/cli/v2/internal/schemas/manager" - "github.com/crossplane/cli/v2/internal/schemas/runner" "github.com/crossplane/cli/v2/internal/terminal" ) @@ -145,35 +141,11 @@ func functionSchemaLanguage(functionLang string) string { } // Run generates a function scaffold. -func (c *generateCmd) Run(sp terminal.SpinnerPrinter, cfg *config.Config) error { +func (c *generateCmd) Run(k *kong.Context, sp terminal.SpinnerPrinter) error { if err := c.validatePaths(); err != nil { return err } - ctx := context.Background() - apisFS := afero.NewBasePathFs(c.projFS, c.proj.Spec.Paths.APIs) - if c.proj.Spec.Paths.APIs == "/" { - apisFS = c.projFS - } - schemaMgr := manager.New( - c.schemasFS, - generator.Filter( - generator.AllLanguages( - generator.WithGoModelAccessors(cfg.Features.GenerateGoModelAccessors), - generator.WithGoRuntimeObjects(cfg.Features.GenerateGoRuntimeObjects), - ), - c.proj.Spec.Schemas.GetLanguages(), - ), - runner.NewRealSchemaRunner(runner.WithImageConfig(c.proj.Spec.ImageConfigs)), - ) - - if err := sp.WrapWithSuccessSpinner("Generating schemas", func() error { - _, err := schemaMgr.Generate(ctx, manager.NewFSSource(c.proj.Spec.Paths.APIs, apisFS)) - return err - }); err != nil { - return errors.Wrap(err, "failed to generate schemas") - } - type generatorFunc func(afero.Fs) error generators := map[string]generatorFunc{ "go": c.generateGoFiles, @@ -230,9 +202,26 @@ func (c *generateCmd) Run(sp terminal.SpinnerPrinter, cfg *config.Config) error } } + // The function imports the project's schemas. This command doesn't generate + // them, so that only `crossplane schema update` changes the schema lock. + schemaLang := functionSchemaLanguage(c.Language) + if populated, err := c.hasSchemas(schemaLang); err == nil && !populated { + fmt.Fprintf(k.Stdout, "No %s schemas found; run `crossplane schema update` so the function can import them.\n", schemaLang) //nolint:errcheck // Best-effort output. + } return nil } +// hasSchemas returns true if the project has generated schemas for the given +// language. +func (c *generateCmd) hasSchemas(lang string) (bool, error) { + exists, err := afero.DirExists(c.schemasFS, lang) + if err != nil || !exists { + return false, err + } + empty, err := afero.IsEmpty(c.schemasFS, lang) + return !empty, err +} + func (c *generateCmd) validatePaths() error { if c.PipelinePath != "" { exists, err := afero.Exists(c.projFS, c.PipelinePath) diff --git a/cmd/crossplane/function/generate_test.go b/cmd/crossplane/function/generate_test.go index 94e3f18c..e58ef7b4 100644 --- a/cmd/crossplane/function/generate_test.go +++ b/cmd/crossplane/function/generate_test.go @@ -22,6 +22,7 @@ import ( "strings" "testing" + "github.com/alecthomas/kong" "github.com/google/go-cmp/cmp" "github.com/spf13/afero" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -30,7 +31,6 @@ import ( apiextv1 "github.com/crossplane/crossplane/apis/v2/apiextensions/v1" v1alpha1 "github.com/crossplane/cli/v2/apis/dev/v1alpha1" - "github.com/crossplane/cli/v2/internal/config" "github.com/crossplane/cli/v2/internal/terminal" ) @@ -293,7 +293,6 @@ func TestRunErrors(t *testing.T) { name string language string seedFunctionsFS map[string][]byte - cfg *config.Config stage string // "afterApply" or "run" wantErrSubstring string }{ @@ -311,22 +310,6 @@ func TestRunErrors(t *testing.T) { stage: "run", wantErrSubstring: "already exists and is not empty", }, - "DirectoryNotEmptyWithGoRuntimeObjects": { - // Run reads the feature flag out of the config it's handed, so - // exercise the flag-on path too. Asserting on the generated - // artifacts isn't possible here: that needs the real schema - // runner, which runs a container. - name: "my-func", - language: "go-templating", - seedFunctionsFS: map[string][]byte{ - "my-func/existing.txt": []byte("data"), - }, - cfg: &config.Config{ - Features: config.Features{GenerateGoRuntimeObjects: true}, - }, - stage: "run", - wantErrSubstring: "already exists and is not empty", - }, } for name, tc := range cases { @@ -337,16 +320,12 @@ func TestRunErrors(t *testing.T) { functionsFS: seedFS(t, tc.seedFunctionsFS), projFS: afero.NewMemMapFs(), } - cfg := tc.cfg - if cfg == nil { - cfg = &config.Config{} - } var err error switch tc.stage { case "afterApply": err = c.AfterApply() case "run": - err = c.Run(terminal.NewSpinnerPrinter(io.Discard, false), cfg) + err = c.Run(nil, terminal.NewSpinnerPrinter(io.Discard, false)) } if err == nil { t.Fatalf("expected error containing %q, got nil", tc.wantErrSubstring) @@ -546,3 +525,71 @@ func TestAddStepToComposition(t *testing.T) { }) } } + +func TestRunSchemaHint(t *testing.T) { + type args struct { + language string + seedSchemas map[string][]byte + seedDirs []string + } + type want struct { + hint bool + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "MissingSchemaDir": { + reason: "The hint should be printed when the schema language directory doesn't exist.", + args: args{language: "go-templating"}, + want: want{hint: true}, + }, + "EmptySchemaDir": { + reason: "The hint should be printed when the schema language directory is empty.", + args: args{language: "go-templating", seedDirs: []string{"json"}}, + want: want{hint: true}, + }, + "PopulatedSchemaDir": { + reason: "The hint should not be printed when schemas exist.", + args: args{ + language: "go-templating", + seedSchemas: map[string][]byte{ + "json/index.schema.json": []byte("{}"), + }, + }, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + schemasFS := seedFS(t, tc.args.seedSchemas) + for _, d := range tc.args.seedDirs { + if err := schemasFS.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + + var out bytes.Buffer + k := &kong.Context{Kong: &kong.Kong{Stdout: &out}} + c := &generateCmd{ + Name: "my-func", + Language: tc.args.language, + proj: testProject(), + functionsFS: afero.NewMemMapFs(), + projFS: afero.NewMemMapFs(), + schemasFS: schemasFS, + fsPath: "functions/my-func", + } + if err := c.Run(k, terminal.NewSpinnerPrinter(io.Discard, false)); err != nil { + t.Fatalf("\n%s\nRun(...): unexpected error: %v", tc.reason, err) + } + + got := strings.Contains(out.String(), "No json schemas found; run `crossplane schema update`") + if diff := cmp.Diff(tc.want.hint, got); diff != "" { + t.Errorf("\n%s\nRun(...): -want hint printed, +got hint printed:\n%s\noutput: %q", tc.reason, diff, out.String()) + } + }) + } +} diff --git a/cmd/crossplane/main.go b/cmd/crossplane/main.go index bd2dd449..0b1aa88c 100644 --- a/cmd/crossplane/main.go +++ b/cmd/crossplane/main.go @@ -46,6 +46,7 @@ import ( "github.com/crossplane/cli/v2/cmd/crossplane/project" renderxr "github.com/crossplane/cli/v2/cmd/crossplane/render/xr" "github.com/crossplane/cli/v2/cmd/crossplane/resource" + "github.com/crossplane/cli/v2/cmd/crossplane/schema" "github.com/crossplane/cli/v2/cmd/crossplane/version" "github.com/crossplane/cli/v2/cmd/crossplane/xpkg" "github.com/crossplane/cli/v2/cmd/crossplane/xr" @@ -93,6 +94,7 @@ type cli struct { Operation operation.Cmd `cmd:"" help:"Work with Crossplane Operations." maturity:"alpha"` Project project.Cmd `cmd:"" help:"Work with control plane Projects." maturity:"beta"` Resource resource.Cmd `cmd:"" help:"Work with Crossplane resources." maturity:"beta"` + Schema schema.Cmd `cmd:"" help:"Manage the generated schemas of control plane Projects." maturity:"beta"` Version version.Cmd `cmd:"" help:"Print the client and server version information for the current context."` XPKG xpkg.Cmd `cmd:"" help:"Work with Crossplane packages."` XR xr.Cmd `cmd:"" help:"Work with Crossplane Composite Resources (XRs)." maturity:"alpha"` diff --git a/cmd/crossplane/project/build.go b/cmd/crossplane/project/build.go index 06222ae6..0171187d 100644 --- a/cmd/crossplane/project/build.go +++ b/cmd/crossplane/project/build.go @@ -36,11 +36,7 @@ import ( "github.com/crossplane/cli/v2/internal/project" "github.com/crossplane/cli/v2/internal/project/functions" "github.com/crossplane/cli/v2/internal/project/projectfile" - "github.com/crossplane/cli/v2/internal/schemas/generator" - "github.com/crossplane/cli/v2/internal/schemas/manager" - "github.com/crossplane/cli/v2/internal/schemas/runner" "github.com/crossplane/cli/v2/internal/terminal" - clixpkg "github.com/crossplane/cli/v2/internal/xpkg" _ "embed" ) @@ -97,39 +93,11 @@ func (c *buildCmd) Run(logger logging.Logger, sp terminal.SpinnerPrinter, cfg *c concurrency := max(1, c.MaxConcurrency) - schemasFS := afero.NewBasePathFs(c.projFS, c.proj.Spec.Paths.Schemas) - generators := generator.Filter( - generator.AllLanguages( - generator.WithGoModelAccessors(cfg.Features.GenerateGoModelAccessors), - generator.WithGoRuntimeObjects(cfg.Features.GenerateGoRuntimeObjects), - ), - c.proj.Spec.Schemas.GetLanguages(), - ) - schemaRunner := runner.NewRealSchemaRunner(runner.WithImageConfig(c.proj.Spec.ImageConfigs)) - schemaMgr := manager.New(schemasFS, generators, schemaRunner) - cacheDir := c.CacheDir - if cacheDir == "" { - cacheDir = dependency.DefaultCacheDir() - } - - client, err := clixpkg.NewClient( - clixpkg.NewRemoteFetcher(), - clixpkg.WithCacheDir(afero.NewOsFs(), cacheDir), - clixpkg.WithImageConfigs(c.proj.Spec.ImageConfigs), - ) + schemaMgr := project.NewSchemaManager(c.proj, c.projFS, cfg.Features) + depMgr, err := dependency.NewCachedManager(c.proj, c.projFS, c.ProjectFile, c.CacheDir) if err != nil { return err } - resolver := clixpkg.NewResolver(client) - - depMgr := dependency.NewManager(c.proj, c.projFS, - dependency.WithProjectFile(c.ProjectFile), - dependency.WithSchemaFS(schemasFS), - dependency.WithSchemaGenerators(generators), - dependency.WithSchemaRunner(schemaRunner), - dependency.WithXpkgClient(client), - dependency.WithResolver(resolver), - ) // The builder may decompress function runtime tarballs into this directory; // the built images read from it lazily, so we remove it only after the diff --git a/cmd/crossplane/project/help/build.md b/cmd/crossplane/project/help/build.md index b1de7f2e..c97e8cc3 100644 --- a/cmd/crossplane/project/help/build.md +++ b/cmd/crossplane/project/help/build.md @@ -14,8 +14,18 @@ with `--repository`. > building and pushing a project. The build reuses the dependency cache populated by `crossplane dependency add` -and `crossplane dependency update-cache`. Override the cache location with -`--cache-dir` or the `CROSSPLANE_XPKG_CACHE` environment variable. +and `crossplane schema update`. Override the cache location with `--cache-dir` +or the `CROSSPLANE_XPKG_CACHE` environment variable. + +If the project has no schema lock yet, the build runs a schema update first and +writes the lock. Otherwise the build never changes the project's generated +schemas or the schema lock, and never resolves versions again, so a new release +of a dependency can't change or break it. It checks that the schemas match the +lock, and fails if a source changed, the project's dependencies changed, or a +generated file was modified. Files that are missing are recreated from the exact +versions the lock pins, which works offline when the packages are cached. Run +`crossplane schema update` to accept changes and update the lock, or +`crossplane schema generate` to restore modified files. The CLI builds embedded functions onto a runtime base image from a registry, and caches those image layers on disk under `crossplane/base-images` in your user diff --git a/cmd/crossplane/project/run.go b/cmd/crossplane/project/run.go index 73ba0ad7..01658e85 100644 --- a/cmd/crossplane/project/run.go +++ b/cmd/crossplane/project/run.go @@ -48,11 +48,7 @@ import ( "github.com/crossplane/cli/v2/internal/project/controlplane" "github.com/crossplane/cli/v2/internal/project/functions" "github.com/crossplane/cli/v2/internal/project/projectfile" - "github.com/crossplane/cli/v2/internal/schemas/generator" - "github.com/crossplane/cli/v2/internal/schemas/manager" - "github.com/crossplane/cli/v2/internal/schemas/runner" "github.com/crossplane/cli/v2/internal/terminal" - clixpkg "github.com/crossplane/cli/v2/internal/xpkg" _ "embed" ) @@ -151,39 +147,11 @@ func (c *runCmd) Run(logger logging.Logger, sp terminal.SpinnerPrinter, cfg *con concurrency := max(1, c.MaxConcurrency) - schemasFS := afero.NewBasePathFs(c.projFS, c.proj.Spec.Paths.Schemas) - generators := generator.Filter( - generator.AllLanguages( - generator.WithGoModelAccessors(cfg.Features.GenerateGoModelAccessors), - generator.WithGoRuntimeObjects(cfg.Features.GenerateGoRuntimeObjects), - ), - c.proj.Spec.Schemas.GetLanguages(), - ) - schemaRunner := runner.NewRealSchemaRunner(runner.WithImageConfig(c.proj.Spec.ImageConfigs)) - schemaMgr := manager.New(schemasFS, generators, schemaRunner) - cacheDir := c.CacheDir - if cacheDir == "" { - cacheDir = dependency.DefaultCacheDir() - } - - client, err := clixpkg.NewClient( - clixpkg.NewRemoteFetcher(), - clixpkg.WithCacheDir(afero.NewOsFs(), cacheDir), - clixpkg.WithImageConfigs(c.proj.Spec.ImageConfigs), - ) + schemaMgr := project.NewSchemaManager(c.proj, c.projFS, cfg.Features) + depMgr, err := dependency.NewCachedManager(c.proj, c.projFS, c.ProjectFile, c.CacheDir) if err != nil { return err } - resolver := clixpkg.NewResolver(client) - - depMgr := dependency.NewManager(c.proj, c.projFS, - dependency.WithProjectFile(c.ProjectFile), - dependency.WithSchemaFS(schemasFS), - dependency.WithSchemaGenerators(generators), - dependency.WithSchemaRunner(schemaRunner), - dependency.WithXpkgClient(client), - dependency.WithResolver(resolver), - ) // The builder may decompress function runtime tarballs into this directory; // the built images read from it lazily, so we remove it only after they have diff --git a/cmd/crossplane/render/op/cmd.go b/cmd/crossplane/render/op/cmd.go index 065eb6a8..64fe118f 100644 --- a/cmd/crossplane/render/op/cmd.go +++ b/cmd/crossplane/render/op/cmd.go @@ -45,9 +45,6 @@ import ( "github.com/crossplane/cli/v2/internal/project" "github.com/crossplane/cli/v2/internal/project/functions" "github.com/crossplane/cli/v2/internal/project/projectfile" - "github.com/crossplane/cli/v2/internal/schemas/generator" - "github.com/crossplane/cli/v2/internal/schemas/manager" - "github.com/crossplane/cli/v2/internal/schemas/runner" "github.com/crossplane/cli/v2/internal/terminal" clixpkg "github.com/crossplane/cli/v2/internal/xpkg" @@ -360,16 +357,8 @@ func (c *Cmd) loadFunctions(ctx context.Context, log logging.Logger, sp terminal } resolver := clixpkg.NewResolver(xpkgClient) - // Built here rather than alongside the schema manager below so the - // dependency manager generates dependency schemas the same way. - generators := generator.AllLanguages( - generator.WithGoModelAccessors(cfg.Features.GenerateGoModelAccessors), - generator.WithGoRuntimeObjects(cfg.Features.GenerateGoRuntimeObjects), - ) - depMgr := dependency.NewManager(proj, projFS, dependency.WithProjectFile(filepath.Base(projFilePath)), - dependency.WithSchemaGenerators(generators), dependency.WithXpkgClient(xpkgClient), dependency.WithResolver(resolver), ) @@ -384,9 +373,7 @@ func (c *Cmd) loadFunctions(ctx context.Context, log logging.Logger, sp terminal } if err := sp.WrapAsyncWithSuccessSpinners(func(ch async.EventChannel) error { - schemasFS := afero.NewBasePathFs(projFS, proj.Spec.Paths.Schemas) - schemaRunner := runner.NewRealSchemaRunner(runner.WithImageConfig(proj.Spec.ImageConfigs)) - schemaMgr := manager.New(schemasFS, generators, schemaRunner) + schemaMgr := project.NewSchemaManager(proj, projFS, cfg.Features) // The builder may decompress function runtime tarballs into this // directory; the built images read from it lazily, so we remove it only diff --git a/cmd/crossplane/render/xr/cmd.go b/cmd/crossplane/render/xr/cmd.go index 10cd4332..df52559f 100644 --- a/cmd/crossplane/render/xr/cmd.go +++ b/cmd/crossplane/render/xr/cmd.go @@ -52,9 +52,6 @@ import ( "github.com/crossplane/cli/v2/internal/project" "github.com/crossplane/cli/v2/internal/project/functions" "github.com/crossplane/cli/v2/internal/project/projectfile" - "github.com/crossplane/cli/v2/internal/schemas/generator" - "github.com/crossplane/cli/v2/internal/schemas/manager" - "github.com/crossplane/cli/v2/internal/schemas/runner" "github.com/crossplane/cli/v2/internal/terminal" clixpkg "github.com/crossplane/cli/v2/internal/xpkg" xrpkg "github.com/crossplane/cli/v2/pkg/xr" @@ -474,14 +471,8 @@ func (c *Cmd) loadFunctionsFromProject(ctx context.Context, log logging.Logger, return nil, err } - generators := generator.AllLanguages( - generator.WithGoModelAccessors(cfg.Features.GenerateGoModelAccessors), - generator.WithGoRuntimeObjects(cfg.Features.GenerateGoRuntimeObjects), - ) - depMgr := dependency.NewManager(proj, projFS, dependency.WithProjectFile(projFileName), - dependency.WithSchemaGenerators(generators), dependency.WithXpkgClient(xpkgClient), dependency.WithResolver(resolver), ) @@ -496,9 +487,7 @@ func (c *Cmd) loadFunctionsFromProject(ctx context.Context, log logging.Logger, } if err := sp.WrapAsyncWithSuccessSpinners(func(ch async.EventChannel) error { - schemasFS := afero.NewBasePathFs(projFS, proj.Spec.Paths.Schemas) - schemaRunner := runner.NewRealSchemaRunner(runner.WithImageConfig(proj.Spec.ImageConfigs)) - schemaMgr := manager.New(schemasFS, generators, schemaRunner) + schemaMgr := project.NewSchemaManager(proj, projFS, cfg.Features) tempDir, err := os.MkdirTemp("", "crossplane-build-") if err != nil { diff --git a/cmd/crossplane/schema/clean.go b/cmd/crossplane/schema/clean.go new file mode 100644 index 00000000..525cd2cf --- /dev/null +++ b/cmd/crossplane/schema/clean.go @@ -0,0 +1,53 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package schema + +import ( + "fmt" + + "github.com/alecthomas/kong" + + "github.com/crossplane/cli/v2/internal/config" + + _ "embed" +) + +//go:embed help/clean.md +var cleanHelp string + +// cleanCmd removes generated schemas and the schema lock. +type cleanCmd struct { + ProjectFlags +} + +func (c *cleanCmd) Help() string { + return cleanHelp +} + +// Run executes the clean command. +func (c *cleanCmd) Run(k *kong.Context, cfg *config.Config) error { + e, err := newEnv(c.ProjectFile, cfg, nil) + if err != nil { + return err + } + if err := e.schemas.Clean(); err != nil { + return err + } + + fmt.Fprintln(k.Stdout, "Schemas cleaned") //nolint:errcheck // Best-effort output. + return nil +} diff --git a/cmd/crossplane/schema/env.go b/cmd/crossplane/schema/env.go new file mode 100644 index 00000000..cd212b6d --- /dev/null +++ b/cmd/crossplane/schema/env.go @@ -0,0 +1,77 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package schema + +import ( + "github.com/spf13/afero" + + "github.com/crossplane/cli/v2/apis/dev/v1alpha1" + "github.com/crossplane/cli/v2/internal/config" + "github.com/crossplane/cli/v2/internal/dependency" + "github.com/crossplane/cli/v2/internal/git" + "github.com/crossplane/cli/v2/internal/project" + "github.com/crossplane/cli/v2/internal/project/projectfile" + "github.com/crossplane/cli/v2/internal/schemas/manager" +) + +// ProjectFlags are the flags every schema command needs to find the project. +type ProjectFlags struct { + ProjectFile string `default:"${project_file}" help:"Path to project definition file." short:"f"` +} + +// SourceFlags are the flags commands need to fetch the project's dependencies. +type SourceFlags struct { + CacheDir string `env:"CROSSPLANE_XPKG_CACHE" help:"Directory for cached xpkg package contents." name:"cache-dir"` + GitToken string `env:"CROSSPLANE_GIT_TOKEN" help:"Token for git HTTPS authentication."` + GitUsername string `default:"x-access-token" env:"CROSSPLANE_GIT_USERNAME" help:"Username for git HTTPS authentication."` +} + +// env is everything a schema command works with: the project, its schema +// manager, and, for commands that fetch dependencies, its dependency manager. +type env struct { + proj *v1alpha1.Project + projFS afero.Fs + schemas *manager.Manager + deps *dependency.Manager +} + +// newEnv loads the project. The dependency manager is only set up if src is +// non-nil, so commands that work on the schema directory alone need no +// registry access. +func newEnv(projectFile string, cfg *config.Config, src *SourceFlags) (*env, error) { + proj, projFS, err := projectfile.Load(projectFile) + if err != nil { + return nil, err + } + + e := &env{ + proj: proj, + projFS: projFS, + schemas: project.NewSchemaManager(proj, projFS, cfg.Features), + } + if src == nil { + return e, nil + } + + e.deps, err = dependency.NewCachedManager(proj, projFS, projectFile, src.CacheDir, + dependency.WithGitAuthProvider(&git.HTTPSAuthProvider{Username: src.GitUsername, Password: src.GitToken}), + ) + if err != nil { + return nil, err + } + return e, nil +} diff --git a/cmd/crossplane/schema/generate.go b/cmd/crossplane/schema/generate.go new file mode 100644 index 00000000..2a0625aa --- /dev/null +++ b/cmd/crossplane/schema/generate.go @@ -0,0 +1,73 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package schema + +import ( + "context" + "fmt" + + "github.com/alecthomas/kong" + + "github.com/crossplane/crossplane-runtime/v2/pkg/errors" + "github.com/crossplane/crossplane-runtime/v2/pkg/logging" + + "github.com/crossplane/cli/v2/internal/config" + "github.com/crossplane/cli/v2/internal/project" + "github.com/crossplane/cli/v2/internal/terminal" + + _ "embed" +) + +//go:embed help/generate.md +var generateHelp string + +// generateCmd recreates schemas from the schema lock without changing it. +type generateCmd struct { + ProjectFlags + SourceFlags +} + +func (c *generateCmd) Help() string { + return generateHelp +} + +// Run executes the generate command. +func (c *generateCmd) Run(k *kong.Context, logger logging.Logger, sp terminal.SpinnerPrinter, cfg *config.Config) error { + ctx := context.Background() + + e, err := newEnv(c.ProjectFile, cfg, &c.SourceFlags) + if err != nil { + return err + } + + hasLock, err := e.schemas.HasLock() + if err != nil { + return err + } + if !hasLock { + return errors.New("no schema lock; run `crossplane schema update`") + } + + logger.Debug("Generating schemas from the schema lock") + if err := sp.WrapWithSuccessSpinner("Generating schemas from the schema lock", func() error { + return project.VerifySchemas(ctx, e.schemas, e.deps, e.proj, e.projFS, true) + }); err != nil { + return err + } + fmt.Fprintln(k.Stdout, "Schemas match the schema lock") //nolint:errcheck // Best-effort output. + return nil +} diff --git a/cmd/crossplane/schema/help/clean.md b/cmd/crossplane/schema/help/clean.md new file mode 100644 index 00000000..e320fb85 --- /dev/null +++ b/cmd/crossplane/schema/help/clean.md @@ -0,0 +1,3 @@ +The `schema clean` command removes all generated schemas and the schema lock. +Cached packages are kept; use `crossplane dependency clean-cache` to remove them +too. Run `crossplane schema update` to generate the schemas again. diff --git a/cmd/crossplane/schema/help/generate.md b/cmd/crossplane/schema/help/generate.md new file mode 100644 index 00000000..f277eef6 --- /dev/null +++ b/cmd/crossplane/schema/help/generate.md @@ -0,0 +1,8 @@ +The `schema generate` command recreates schemas from the schema lock. It +restores generated files that are missing or were modified, using the exact +package digests and git commits the lock pins, and never changes the lock. It +never resolves versions again, and works offline when the packages are cached. + +The command fails if there is no schema lock, or if a source or the project's +dependencies changed since the lock was written. Run `crossplane schema update` +to create the lock or accept the change. diff --git a/cmd/crossplane/schema/help/status.md b/cmd/crossplane/schema/help/status.md new file mode 100644 index 00000000..6a677570 --- /dev/null +++ b/cmd/crossplane/schema/help/status.md @@ -0,0 +1,6 @@ +The `schema status` command lists generated schema files that are missing or +differ from the schema lock. It only reads the schemas directory, so it works +offline, and it exits with an error if any file differs. + +It doesn't check whether sources changed since the lock was written. Run +`crossplane schema update` for that. diff --git a/cmd/crossplane/schema/help/update.md b/cmd/crossplane/schema/help/update.md new file mode 100644 index 00000000..e20ef5f6 --- /dev/null +++ b/cmd/crossplane/schema/help/update.md @@ -0,0 +1,22 @@ +The `schema update` command generates schemas for the current project's APIs +(XRDs) and dependencies, and records what it generated in the schema lock +(`.lock.json` in the schemas directory). It is the command to run during +development, after changing an XRD or the project's dependencies. + +Sources whose content did not change are skipped. Schemas of dependencies that +are no longer used are removed. Set `spec.schemas.apiGroups` in the project file +to generate schemas only for matching API groups. + +Versions already in the schema lock are preferred, as long as they still satisfy +the version constraints of your dependencies. Use `--refresh` to ignore the +locked versions, resolve to the newest versions that are allowed, and pull them +again. + +When several packages require the same dependency, a dependency you declare in +the project always wins. If its version violates a constraint of another +package, the command prints a warning and uses it anyway. Otherwise it picks the +highest version that satisfies every constraint, and fails if there is none. + +`crossplane project build` only runs an update if there is no schema lock. After +that it never changes the lock, and only recreates missing schemas or fails if +they no longer match the lock. diff --git a/cmd/crossplane/dependency/auth.go b/cmd/crossplane/schema/schema.go similarity index 50% rename from cmd/crossplane/dependency/auth.go rename to cmd/crossplane/schema/schema.go index 2a3ecca8..6cb78d67 100644 --- a/cmd/crossplane/dependency/auth.go +++ b/cmd/crossplane/schema/schema.go @@ -14,24 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package dependency +// Package schema contains commands for managing the generated schemas of a +// project. +package schema -import ( - "github.com/go-git/go-git/v5/plumbing/transport" - "github.com/go-git/go-git/v5/plumbing/transport/http" -) - -type gitAuthProvider struct { - username string - token string -} - -func (g *gitAuthProvider) GetAuthMethod() (transport.AuthMethod, error) { - if g.token != "" { - return &http.BasicAuth{ - Username: g.username, - Password: g.token, - }, nil - } - return nil, nil +// Cmd contains commands for managing schemas. +type Cmd struct { + Update updateCmd `cmd:"" help:"Update schemas and the schema lock from the project's APIs and dependencies."` + Generate generateCmd `cmd:"" help:"Recreate missing and modified schemas from the schema lock."` + Status statusCmd `cmd:"" help:"Show generated schemas that differ from the schema lock."` + Clean cleanCmd `cmd:"" help:"Remove generated schemas and the schema lock."` } diff --git a/cmd/crossplane/schema/schema_test.go b/cmd/crossplane/schema/schema_test.go new file mode 100644 index 00000000..7e1f53a4 --- /dev/null +++ b/cmd/crossplane/schema/schema_test.go @@ -0,0 +1,208 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package schema + +import ( + "bytes" + "maps" + "os" + "path/filepath" + "reflect" + "slices" + "strings" + "testing" + + "github.com/alecthomas/kong" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/crossplane/crossplane-runtime/v2/pkg/logging" + + "github.com/crossplane/cli/v2/internal/config" + "github.com/crossplane/cli/v2/internal/terminal" +) + +const projectYAML = `apiVersion: dev.crossplane.io/v1alpha1 +kind: Project +metadata: + name: test +spec: + repository: registry.example.com/test + schemas: + languages: [json] +` + +// newParser returns a kong parser for the schema commands that binds what the +// commands' Run methods ask for. +func newParser(t *testing.T, stdout *bytes.Buffer) *kong.Kong { + t.Helper() + var c Cmd + parser, err := kong.New(&c, + kong.Vars{"project_file": "crossplane-project.yaml"}, + kong.Bind(&config.Config{}), + kong.BindTo(logging.NewNopLogger(), (*logging.Logger)(nil)), + kong.BindTo(terminal.NewSpinnerPrinter(stdout, false), (*terminal.SpinnerPrinter)(nil)), + kong.Writers(stdout, stdout), + ) + if err != nil { + t.Fatalf("kong.New(): %v", err) + } + return parser +} + +func TestParse(t *testing.T) { + t.Parallel() + + type args struct { + cmd []string + } + type want struct { + command string + refresh bool + err error + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "Update": {reason: "update has no required arguments.", args: args{cmd: []string{"update"}}, want: want{command: "update"}}, + "UpdateRefresh": {reason: "--refresh asks for fresh resolution.", args: args{cmd: []string{"update", "--refresh"}}, want: want{command: "update", refresh: true}}, + "Generate": {reason: "generate takes the same source flags as update.", args: args{cmd: []string{"generate", "--cache-dir", "/tmp/x"}}, want: want{command: "generate"}}, + "Status": {reason: "status is offline, so it takes only the project file.", args: args{cmd: []string{"status", "-f", "p.yaml"}}, want: want{command: "status"}}, + "StatusNoCacheFlag": {reason: "status must not accept source flags.", args: args{cmd: []string{"status", "--cache-dir", "/tmp/x"}}, want: want{err: cmpopts.AnyError}}, + "Clean": {reason: "clean is offline too.", args: args{cmd: []string{"clean"}}, want: want{command: "clean"}}, + "RefreshOnlyUpdate": {reason: "--refresh belongs to update only.", args: args{cmd: []string{"generate", "--refresh"}}, want: want{err: cmpopts.AnyError}}, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + ctx, err := newParser(t, &bytes.Buffer{}).Parse(tc.args.cmd) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nParse(%v): -want err, +got err:\n%s", tc.reason, tc.args.cmd, diff) + } + if err != nil { + return + } + got := want{command: ctx.Command()} + if u, ok := reflect.TypeAssert[*updateCmd](ctx.Selected().Target.Addr()); ok { + got.refresh = u.Refresh + } + if diff := cmp.Diff(tc.want, got, cmp.AllowUnexported(want{})); diff != "" { + t.Errorf("%s\nParse(%v): -want, +got:\n%s", tc.reason, tc.args.cmd, diff) + } + }) + } +} + +func TestOfflineCommands(t *testing.T) { + t.Parallel() + + type args struct { + cmd []string + // files are written under the project directory before running. + files map[string]string + // cache is whether the command takes a package cache directory. + cache bool + } + type want struct { + stdout string + err error + // errMsg is a substring the error message must contain. + errMsg string + // schemaFiles is whether the schemas directory contains files afterwards. + schemaFiles bool + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "StatusWithoutLock": { + reason: "A project that never generated schemas has no drift.", + args: args{cmd: []string{"status"}}, + want: want{stdout: "Schemas match the schema lock"}, + }, + "StatusReportsDriftedFile": { + reason: "A file recorded in the lock but gone from disk is reported, and the command fails.", + args: args{cmd: []string{"status"}, files: map[string]string{ + "schemas/.lock.json": `{"lockVersion":1,"sources":{"fs://apis":{"sourceHash":"h","flags":{"goRuntimeObjects":false},"files":[{"path":"json/a.json","sha256":"x"}]}}}`, + }}, + want: want{stdout: "fs://apis: json/a.json (missing)", err: cmpopts.AnyError, errMsg: "1 generated files differ", schemaFiles: true}, + }, + "GenerateWithoutLock": { + reason: "Generate only works from a lock, and says how to create one.", + args: args{cmd: []string{"generate"}, cache: true}, + want: want{err: cmpopts.AnyError, errMsg: "no schema lock; run `crossplane schema update`"}, + }, + "CleanRemovesSchemas": { + reason: "Clean removes the generated files and the lock.", + args: args{cmd: []string{"clean"}, files: map[string]string{"schemas/.lock.json": "{}", "schemas/json/a.json": "{}"}}, + want: want{stdout: "Schemas cleaned"}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + files := map[string]string{"crossplane-project.yaml": projectYAML} + maps.Copy(files, tc.args.files) + for p, c := range files { + full := filepath.Join(dir, p) + if err := os.MkdirAll(filepath.Dir(full), 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(c), 0o600); err != nil { + t.Fatal(err) + } + } + + cmd := append(slices.Clone(tc.args.cmd), "-f", filepath.Join(dir, "crossplane-project.yaml")) + if tc.args.cache { + cmd = append(cmd, "--cache-dir", filepath.Join(dir, "cache")) + } + + var stdout bytes.Buffer + parser := newParser(t, &stdout) + ctx, err := parser.Parse(cmd) + if err != nil { + t.Fatalf("%s\nParse: %v", tc.reason, err) + } + err = ctx.Run() + + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("%s\nRun(): -want err, +got err:\n%s", tc.reason, diff) + } + if tc.want.errMsg != "" && (err == nil || !strings.Contains(err.Error(), tc.want.errMsg)) { + t.Errorf("%s\nRun(): err = %v, want error containing %q", tc.reason, err, tc.want.errMsg) + } + if !strings.Contains(stdout.String(), tc.want.stdout) { + t.Errorf("%s\nstdout = %q, want it to contain %q", tc.reason, stdout.String(), tc.want.stdout) + } + entries, _ := os.ReadDir(filepath.Join(dir, "schemas")) + if diff := cmp.Diff(tc.want.schemaFiles, len(entries) > 0); diff != "" { + t.Errorf("%s\nschema files left behind (-want +got):\n%s", tc.reason, diff) + } + }) + } +} diff --git a/cmd/crossplane/schema/status.go b/cmd/crossplane/schema/status.go new file mode 100644 index 00000000..aacadede --- /dev/null +++ b/cmd/crossplane/schema/status.go @@ -0,0 +1,76 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package schema + +import ( + "context" + "fmt" + + "github.com/alecthomas/kong" + + "github.com/crossplane/crossplane-runtime/v2/pkg/errors" + + "github.com/crossplane/cli/v2/internal/config" + "github.com/crossplane/cli/v2/internal/schemas/manager" + + _ "embed" +) + +//go:embed help/status.md +var statusHelp string + +// statusCmd reports generated schemas that differ from the schema lock. +type statusCmd struct { + ProjectFlags +} + +func (c *statusCmd) Help() string { + return statusHelp +} + +// Run executes the status command. +func (c *statusCmd) Run(k *kong.Context, cfg *config.Config) error { + e, err := newEnv(c.ProjectFile, cfg, nil) + if err != nil { + return err + } + + statuses, err := e.schemas.Status(context.Background()) + if err != nil { + return err + } + + for _, s := range statuses { + for _, d := range s.Drift { + fmt.Fprintf(k.Stdout, "%s: %s (%s)\n", s.Key, d.Path, d.Kind) //nolint:errcheck // Best-effort output. + } + } + if drifted := countDrift(statuses); drifted > 0 { + return errors.Errorf("%d generated files differ from the schema lock; run `crossplane schema generate` to restore them", drifted) + } + + fmt.Fprintln(k.Stdout, "Schemas match the schema lock") //nolint:errcheck // Best-effort output. + return nil +} + +func countDrift(statuses []manager.SourceStatus) int { + n := 0 + for _, s := range statuses { + n += len(s.Drift) + } + return n +} diff --git a/cmd/crossplane/schema/update.go b/cmd/crossplane/schema/update.go new file mode 100644 index 00000000..5c9c1a5a --- /dev/null +++ b/cmd/crossplane/schema/update.go @@ -0,0 +1,84 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package schema + +import ( + "context" + "fmt" + + "github.com/alecthomas/kong" + + "github.com/crossplane/crossplane-runtime/v2/pkg/logging" + + "github.com/crossplane/cli/v2/internal/async" + "github.com/crossplane/cli/v2/internal/config" + "github.com/crossplane/cli/v2/internal/project" + "github.com/crossplane/cli/v2/internal/terminal" + + _ "embed" +) + +//go:embed help/update.md +var updateHelp string + +// updateCmd generates schemas for the project's APIs and dependencies and +// writes the schema lock. +type updateCmd struct { + ProjectFlags + SourceFlags + + Refresh bool `help:"Ignore the versions in the schema lock, resolve to the newest allowed versions and pull them again." name:"refresh"` +} + +func (c *updateCmd) Help() string { + return updateHelp +} + +// Run executes the update command. +func (c *updateCmd) Run(k *kong.Context, logger logging.Logger, sp terminal.SpinnerPrinter, cfg *config.Config) error { + ctx := context.Background() + e, err := newEnv(c.ProjectFile, cfg, &c.SourceFlags) + if err != nil { + return err + } + + logger.Debug("Updating schemas", "refresh", c.Refresh) + var res *project.UpdateResult + if err := sp.WrapAsyncWithSuccessSpinners(func(ch async.EventChannel) error { + var err error + res, err = project.UpdateSchemas(ctx, e.schemas, e.deps, e.proj, e.projFS, ch, c.Refresh) + return err + }); err != nil { + return err + } + for _, w := range res.Warnings { + fmt.Fprintf(k.Stderr, "Warning: %s\n", w) //nolint:errcheck // Best-effort output. + } + for _, key := range res.Removed { + fmt.Fprintf(k.Stdout, "Removed unused schemas of %s\n", key) //nolint:errcheck // Best-effort output. + } + + // Add never repairs files, so tell the user if some are still off. + statuses, err := e.schemas.Status(ctx) + if err != nil { + return err + } + if drifted := countDrift(statuses); drifted > 0 { + fmt.Fprintf(k.Stdout, "%d generated files differ from the schema lock; run `crossplane schema generate` to restore them\n", drifted) //nolint:errcheck // Best-effort output. + } + return nil +} diff --git a/internal/dependency/manager.go b/internal/dependency/manager.go index 6e34c251..912f6d58 100644 --- a/internal/dependency/manager.go +++ b/internal/dependency/manager.go @@ -14,16 +14,25 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package dependency manages schema generation for Crossplane project -// dependencies. +// Package dependency manages the dependencies of Crossplane projects: it records +// them in the project file, fetches their packages into the shared package +// cache, and finds the schema sources they contribute. Generating schemas from +// those sources is the schema manager's job. package dependency import ( "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" "fmt" + "maps" + "path/filepath" + "slices" "strings" "sync" + "github.com/Masterminds/semver/v3" "github.com/google/go-containerregistry/pkg/name" conregv1 "github.com/google/go-containerregistry/pkg/v1" "github.com/spf13/afero" @@ -40,19 +49,16 @@ import ( "github.com/crossplane/cli/v2/internal/async" "github.com/crossplane/cli/v2/internal/git" "github.com/crossplane/cli/v2/internal/project/projectfile" - "github.com/crossplane/cli/v2/internal/schemas/generator" smanager "github.com/crossplane/cli/v2/internal/schemas/manager" - "github.com/crossplane/cli/v2/internal/schemas/runner" clixpkg "github.com/crossplane/cli/v2/internal/xpkg" ) // Manager manages dependencies for a Crossplane project, including fetching -// packages, extracting CRDs, and generating schemas. +// packages and listing the schema sources they provide. type Manager struct { proj *v1alpha1.Project projFS afero.Fs projFile string - schemas *smanager.Manager gitCloner git.Cloner gitAuthProvider git.AuthProvider @@ -61,24 +67,16 @@ type Manager struct { resolver *clixpkg.Resolver updateMutex sync.Mutex - - // visited tracks package refs already processed during this invocation, - // deduplicating shared transitive dependencies and breaking cycles. - visited map[string]struct{} - visitedMu sync.Mutex } // ManagerOption configures the dependency manager. type ManagerOption func(*managerOptions) type managerOptions struct { - projFile string - schemaFS afero.Fs - schemaGenerators []generator.Interface - schemaRunner runner.SchemaRunner - gitAuthProvider git.AuthProvider - client runtimexpkg.Client - resolver *clixpkg.Resolver + projFile string + gitAuthProvider git.AuthProvider + client runtimexpkg.Client + resolver *clixpkg.Resolver } // WithProjectFile sets the path to the project file. @@ -88,27 +86,6 @@ func WithProjectFile(path string) ManagerOption { } } -// WithSchemaFS sets the filesystem to use for schemas. -func WithSchemaFS(fs afero.Fs) ManagerOption { - return func(opts *managerOptions) { - opts.schemaFS = fs - } -} - -// WithSchemaRunner sets the runner to use when generating schemas. -func WithSchemaRunner(r runner.SchemaRunner) ManagerOption { - return func(opts *managerOptions) { - opts.schemaRunner = r - } -} - -// WithSchemaGenerators sets the schema generators to call. -func WithSchemaGenerators(gs []generator.Interface) ManagerOption { - return func(opts *managerOptions) { - opts.schemaGenerators = gs - } -} - // WithGitAuthProvider sets the auth provider for git operations. func WithGitAuthProvider(p git.AuthProvider) ManagerOption { return func(opts *managerOptions) { @@ -132,18 +109,10 @@ func WithResolver(r *clixpkg.Resolver) ManagerOption { } } -// NewManager returns an initialized dependency manager. The default schema -// generators are configured with every generator option off, so a caller that -// generates schemas must pass WithSchemaGenerators to honor the feature flags -// in the user's config. +// NewManager returns an initialized dependency manager. func NewManager(proj *v1alpha1.Project, projFS afero.Fs, opts ...ManagerOption) *Manager { options := &managerOptions{ - projFile: clixpkg.ProjectFile, - schemaFS: afero.NewBasePathFs(projFS, proj.Spec.Paths.Schemas), - schemaGenerators: generator.AllLanguages(), - schemaRunner: runner.NewRealSchemaRunner( - runner.WithImageConfig(proj.Spec.ImageConfigs), - ), + projFile: clixpkg.ProjectFile, gitAuthProvider: &git.HTTPSAuthProvider{}, } @@ -151,25 +120,41 @@ func NewManager(proj *v1alpha1.Project, projFS afero.Fs, opts ...ManagerOption) opt(options) } - schemas := smanager.New( - options.schemaFS, - options.schemaGenerators, - options.schemaRunner, - ) - return &Manager{ proj: proj, projFS: projFS, projFile: options.projFile, - schemas: schemas, gitCloner: &git.DefaultCloner{}, gitAuthProvider: options.gitAuthProvider, client: options.client, resolver: options.resolver, - visited: make(map[string]struct{}), } } +// NewCachedManager returns a dependency manager for a project that fetches +// packages through the xpkg cache in cacheDir, or in DefaultCacheDir if it is +// empty. projFile is the project file's path; only its base name is used, +// because projFS is rooted at the project directory. +func NewCachedManager(proj *v1alpha1.Project, projFS afero.Fs, projFile, cacheDir string, opts ...ManagerOption) (*Manager, error) { + if cacheDir == "" { + cacheDir = DefaultCacheDir() + } + client, err := clixpkg.NewClient( + clixpkg.NewRemoteFetcher(), + clixpkg.WithCacheDir(afero.NewOsFs(), cacheDir), + clixpkg.WithImageConfigs(proj.Spec.ImageConfigs), + ) + if err != nil { + return nil, err + } + opts = append([]ManagerOption{ + WithProjectFile(filepath.Base(projFile)), + WithXpkgClient(client), + WithResolver(clixpkg.NewResolver(client)), + }, opts...) + return NewManager(proj, projFS, opts...), nil +} + // ResolveRef resolves a version constraint in an OCI ref to a concrete tag. If // the ref has no tag, has an exact semver version, or is not a valid // constraint, it is returned unchanged. @@ -178,30 +163,400 @@ func (m *Manager) ResolveRef(ref string) (name.Reference, error) { return resolved, err } -// AddPackage adds a package to the dependency manager. If refresh is set, the -// package's ref will be re-resolved regardless of whether it is cached. -// Schemas are also generated recursively for the dependencies declared in the -// package's metadata. -func (m *Manager) AddPackage(ctx context.Context, ref string, refresh bool) (*schema.GroupVersionKind, error) { - return m.addPackage(ctx, ref, refresh) +// maxResolveRounds caps the rounds of the fixed point in Resolve, so that +// requirements that keep changing each other's selections fail instead of +// looping. +const maxResolveRounds = 50 + +// ResolveOptions configures Resolve. +type ResolveOptions struct { + // Refresh resolves version constraints against the registry and pulls + // packages again, ignoring Locked. + Refresh bool + + // Locked holds the pins of the schema lock, keyed by source key. Unless + // Refresh is set, a locked version is kept as long as it still satisfies + // the requirements, and a locked git commit is cloned instead of the ref. + Locked map[string]smanager.LockedSource +} + +// A Warning reports a requirement that the resolved dependencies do not meet. +type Warning struct { + // Package is the repository of the direct dependency. + Package string + + // Version is the version the direct dependency resolved to. + Version string + + // Requirer is the package that requires a different version. + Requirer string + + // Constraint is the version constraint that Version violates. + Constraint string +} + +// String returns the warning as a message. +func (w Warning) String() string { + return fmt.Sprintf("dependency %s resolved to %s, which does not satisfy %s required by %s", w.Package, w.Version, w.Constraint, w.Requirer) } -func (m *Manager) addPackage(ctx context.Context, ref string, refresh bool) (*schema.GroupVersionKind, error) { - // Record the ref unconditionally: top-level calls must always fully - // process the package so callers get its GVK, while transitive walks - // consult the visited set before recursing. - m.claim(ref) +// A requirement is a version of a package that a requirer asks for. The +// project is the requirer of its direct dependencies, and has an empty name. +type requirement struct { + requirer string + version string +} - resolvedRef, version, err := m.resolver.Resolve(ctx, ref) +func (r requirement) String() string { + requirer, version := r.requirer, r.version + if requirer == "" { + requirer = "the project" + } + if version == "" { + version = "any version" + } + return fmt.Sprintf("%s requires %s", requirer, version) +} + +// A selection is the package version chosen for a repository. +type selection struct { + ref string + version string + pkg *runtimexpkg.Package + gvk *schema.GroupVersionKind + crdFS afero.Fs + + // requires are the versions of the packages that this package depends on, + // by repository. + requires map[string]string +} + +// A resolution is the outcome of resolving dependencies. +type resolution struct { + sources []smanager.Source + warnings []Warning + + // gvks are the runtime GVKs of the resolved packages, by repository. + gvks map[string]*schema.GroupVersionKind +} + +// A resolver finds the package versions that satisfy the requirements of a set +// of dependencies. It is single use. +type resolver struct { + m *Manager + opts ResolveOptions + + // direct are the versions the project asks for, by repository. + direct map[string]string + + selected map[string]*selection + + // decided and fetched remember earlier work for the rest of the + // resolution. They are only written between rounds. + decided map[string]string + fetched map[string]*selection +} + +// Resolve returns the schema sources of every dependency in the project, +// direct and transitive, sorted by key. Packages are fetched, from the cache if +// present, to find their dependencies. +// +// The versions are chosen deterministically. A direct dependency's version wins +// over what other packages require, and a Warning is returned for each +// requirement it violates. Otherwise the highest version that satisfies every +// requirement wins. Unless opts.Refresh is set, the locked version is +// preferred when it still satisfies them. +// +// Callers that reconcile the schema lock must add the project's own sources, +// such as its XRDs, to the result. +func (m *Manager) Resolve(ctx context.Context, opts ResolveOptions) ([]smanager.Source, []Warning, error) { + res, err := m.resolve(ctx, m.proj.Spec.Dependencies, opts) if err != nil { - return nil, errors.Wrapf(err, "cannot resolve %s", ref) + return nil, nil, err } + return res.sources, res.warnings, nil +} + +func (m *Manager) resolve(ctx context.Context, deps []v1alpha1.Dependency, opts ResolveOptions) (*resolution, error) { + r := &resolver{ + m: m, + opts: opts, + direct: make(map[string]string), + selected: make(map[string]*selection), + decided: make(map[string]string), + fetched: make(map[string]*selection), + } + + sources := make(map[string]smanager.Source) + for _, dep := range deps { + if dep.Type == v1alpha1.DependencyTypeXpkg { + if dep.Xpkg == nil { + return nil, errors.New("xpkg dependency has no package reference") + } + r.direct[dep.Xpkg.Package] = dep.Xpkg.Version + continue + } + + src, err := m.source(dep) + if err != nil { + return nil, err + } + // Clone the locked commit instead of whatever the ref points to now. + if pin, ok := opts.Locked[src.Key()]; ok && !opts.Refresh && dep.Git != nil && git.CheckSHA1Hash(pin.Pin) { + if src, err = m.pinnedSource(dep, pin.Pin); err != nil { + return nil, err + } + } + if _, ok := sources[src.Key()]; !ok { + sources[src.Key()] = src + } + } + + if err := r.run(ctx); err != nil { + return nil, err + } + + res := &resolution{ + warnings: r.warnings(), + gvks: make(map[string]*schema.GroupVersionKind), + } + for repo, sel := range r.selected { + res.gvks[repo] = sel.gvk + src := smanager.NewXpkgSource(sel.pkg.Source, sel.version, sel.pkg.Digest, sel.crdFS) + sources[src.Key()] = src + } + for _, k := range slices.Sorted(maps.Keys(sources)) { + res.sources = append(res.sources, sources[k]) + } + return res, nil +} + +// source returns the schema source of a dependency that is not a package. +func (m *Manager) source(dep v1alpha1.Dependency) (smanager.Source, error) { + switch { + case dep.Git != nil: + return smanager.NewGitSource(dep, m.gitCloner, m.gitAuthProvider), nil + case dep.HTTP != nil: + return smanager.NewHTTPSource(dep), nil + case dep.K8s != nil: + return smanager.NewK8sSource(dep), nil + default: + return nil, errors.New("dependency has no source configured") + } +} + +// pinnedSource returns the schema source of a dependency that is not a +// package. A git dependency is cloned at pin instead of its ref. +func (m *Manager) pinnedSource(dep v1alpha1.Dependency, pin string) (smanager.Source, error) { + if dep.Git != nil { + return smanager.NewPinnedGitSource(dep, pin, m.gitCloner, m.gitAuthProvider), nil + } + return m.source(dep) +} + +// run selects package versions until a round changes nothing. Each round bases +// its choices on the previous round's selections only, so the order in which +// packages are fetched does not matter. +func (r *resolver) run(ctx context.Context) error { + for range maxResolveRounds { + reqs := r.requirements() + repos := slices.Sorted(maps.Keys(reqs)) + + type result struct { + decision string + sel *selection + } + results := make([]result, len(repos)) + + eg, egCtx := errgroup.WithContext(ctx) + for i, repo := range repos { + eg.Go(func() error { + ref, err := r.decide(egCtx, repo, reqs[repo]) + if err != nil { + return err + } + results[i].decision = ref + if cur := r.selected[repo]; cur != nil && cur.ref == ref { + return nil + } + if sel, ok := r.fetched[ref]; ok { + results[i].sel = sel + return nil + } + results[i].sel, err = r.fetch(egCtx, ref) + return err + }) + } + if err := eg.Wait(); err != nil { + return err + } + + // Apply the results in a fixed order. + changed := false + for i, repo := range repos { + r.decided[decisionKey(repo, reqs[repo])] = results[i].decision + if results[i].sel == nil { + continue + } + r.fetched[results[i].sel.ref] = results[i].sel + r.selected[repo] = results[i].sel + changed = true + } + if !changed { + return nil + } + } + return errors.New("dependency resolution does not converge") +} + +// requirements returns what is required of each package that is still needed, +// sorted by requirer. A package is needed if the project depends on it, or if +// a needed package that has been selected depends on it. Selections of packages +// that are no longer needed are dropped. +func (r *resolver) requirements() map[string][]requirement { + reqs := make(map[string][]requirement) + needed := make(map[string]bool) + + queue := slices.Sorted(maps.Keys(r.direct)) + for _, repo := range queue { + needed[repo] = true + reqs[repo] = append(reqs[repo], requirement{version: r.direct[repo]}) + } + for len(queue) > 0 { + repo := queue[0] + queue = queue[1:] + + sel := r.selected[repo] + if sel == nil { + continue + } + for _, dep := range slices.Sorted(maps.Keys(sel.requires)) { + reqs[dep] = append(reqs[dep], requirement{requirer: repo, version: sel.requires[dep]}) + if !needed[dep] { + needed[dep] = true + queue = append(queue, dep) + } + } + } + + for repo := range r.selected { + if !needed[repo] { + delete(r.selected, repo) + } + } + for _, rs := range reqs { + slices.SortFunc(rs, func(a, b requirement) int { + return strings.Compare(a.requirer, b.requirer) + }) + } + return reqs +} +func decisionKey(repo string, reqs []requirement) string { + parts := make([]string, 0, len(reqs)+1) + parts = append(parts, repo) + for _, req := range reqs { + parts = append(parts, req.requirer+"="+req.version) + } + return strings.Join(parts, "\x00") +} + +// decide returns the reference of the version of repo that satisfies reqs. +func (r *resolver) decide(ctx context.Context, repo string, reqs []requirement) (string, error) { + if ref, ok := r.decided[decisionKey(repo, reqs)]; ok { + return ref, nil + } + + // A direct dependency wins over everything else. + if reqs[0].requirer == "" { + reqs = reqs[:1] + } + + var digests, tags, constraints []string + for _, req := range reqs { + switch _, err := semver.NewConstraint(req.version); { + case req.version == "": + case isDigest(req.version): + digests = append(digests, req.version) + case err != nil: + tags = append(tags, req.version) + default: + constraints = append(constraints, req.version) + } + } + + ref, err := r.choose(ctx, repo, digests, tags, constraints) + if err != nil { + descs := make([]string, len(reqs)) + for i, req := range reqs { + descs[i] = req.String() + } + return "", errors.Wrapf(err, "cannot resolve %s (%s)", repo, strings.Join(descs, ", ")) + } + return ref, nil +} + +func isDigest(version string) bool { + _, err := conregv1.NewHash(version) + return err == nil +} + +func (r *resolver) choose(ctx context.Context, repo string, digests, tags, constraints []string) (string, error) { + switch { + case len(digests) > 0: + // Packages that agree on the digest agree on the version. + if d := slices.Compact(slices.Sorted(slices.Values(digests))); len(d) > 1 { + return "", errors.Errorf("conflicting digests %s", strings.Join(d, ", ")) + } + dgst, err := name.NewDigest(xpkgRef(repo, digests[0]), name.StrictValidation) + if err != nil { + return "", err + } + return dgst.String(), nil + + case len(tags) > 0: + if len(constraints) > 0 || len(slices.Compact(slices.Sorted(slices.Values(tags)))) > 1 { + return "", errors.Errorf("incompatible versions %s", strings.Join(append(tags, constraints...), ", ")) + } + tag, err := name.NewTag(xpkgRef(repo, tags[0]), name.StrictValidation) + if err != nil { + return "", err + } + return tag.String(), nil + } + + if len(constraints) == 0 { + constraints = []string{">=v0.0.0"} + } + + // Stay on the locked version if it is still allowed, to not depend on the + // registry. + if locked, ok := r.opts.Locked["xpkg://"+repo]; ok && !r.opts.Refresh && locked.Version != "" { + if v, err := semver.NewVersion(locked.Version); err == nil { + if c, err := semver.NewConstraint(strings.Join(constraints, ", ")); err == nil && c.Check(v) { + n, err := name.NewRepository(repo, name.StrictValidation) + if err != nil { + return "", err + } + return n.Tag(locked.Version).String(), nil + } + } + } + + resolved, _, err := r.m.resolver.ResolveConstraints(ctx, repo, constraints) + if err != nil { + return "", err + } + return resolved.String(), nil +} + +// fetch gets the package at ref and records what it requires. +func (r *resolver) fetch(ctx context.Context, ref string) (*selection, error) { pullPolicy := corev1.PullIfNotPresent - if refresh { + if r.opts.Refresh { pullPolicy = corev1.PullAlways } - pkg, err := m.client.Get(ctx, resolvedRef.String(), runtimexpkg.WithPullPolicy(pullPolicy)) + pkg, err := r.m.client.Get(ctx, ref, runtimexpkg.WithPullPolicy(pullPolicy)) if err != nil { return nil, errors.Wrapf(err, "failed to fetch %s", ref) } @@ -215,52 +570,45 @@ func (m *Manager) addPackage(ctx context.Context, ref string, refresh bool) (*sc return nil, errors.Wrapf(err, "cannot extract CRDs from %s", ref) } - // Use the resolved version so constraint and exact-version inputs - // collapse to one schema-lock entry. Digest-pinned refs (no resolved - // version) intentionally use a digest-form ID and remain separate. - id := pkg.Source + "@" + pkg.Digest - if version != "" { - id = pkg.Source + ":" + version - } - src := smanager.NewXpkgSource(id, pkg.Digest, crdFS) - if err := m.schemas.Add(ctx, src); err != nil { - return nil, err + // The tag is not part of the source key, so constraint, exact-version and + // digest-pinned refs to one package share a schema-lock entry. + var version string + if tag, err := name.NewTag(ref, name.StrictValidation); err == nil { + version = tag.TagStr() } - if err := m.addTransitiveDeps(ctx, pkg, refresh); err != nil { - return nil, err + + requires := make(map[string]string) + for _, dep := range pkg.GetDependencies() { + if depRepo := dependencyRepo(dep); depRepo != "" { + requires[depRepo] = dep.Version + } } - return gvk, nil + return &selection{ref: ref, version: version, pkg: pkg, gvk: gvk, crdFS: crdFS, requires: requires}, nil } -// addTransitiveDeps generates schemas for the dependencies declared in the -// package's metadata. Transitive dependencies are not added to the project -// file. -func (m *Manager) addTransitiveDeps(ctx context.Context, pkg *runtimexpkg.Package, refresh bool) error { - for _, dep := range pkg.GetDependencies() { - repo := dependencyRepo(dep) - if repo == "" { +// warnings returns the requirements that direct dependencies override. +func (r *resolver) warnings() []Warning { + var warnings []Warning + reqs := r.requirements() + for _, repo := range slices.Sorted(maps.Keys(r.direct)) { + sel := r.selected[repo] + if sel == nil || sel.version == "" { continue } - ref := xpkgRef(repo, dep.Version) - if !m.claim(ref) { - continue // Already processed (or in progress) this invocation. + v, err := semver.NewVersion(sel.version) + if err != nil { + continue } - if _, err := m.addPackage(ctx, ref, refresh); err != nil { - return errors.Wrapf(err, "cannot add transitive dependency %s of %s", ref, pkg.Source) + for _, req := range reqs[repo] { + if req.requirer == "" { + continue + } + if c, err := semver.NewConstraint(req.version); err == nil && !c.Check(v) { + warnings = append(warnings, Warning{Package: repo, Version: sel.version, Requirer: req.requirer, Constraint: req.version}) + } } } - return nil -} - -// claim records ref as processed and reports whether it was newly claimed. -func (m *Manager) claim(ref string) bool { - m.visitedMu.Lock() - defer m.visitedMu.Unlock() - if _, ok := m.visited[ref]; ok { - return false - } - m.visited[ref] = struct{}{} - return true + return warnings } // dependencyRepo returns the OCI repository of a package metadata dependency, @@ -310,20 +658,29 @@ func runtimeGVKForPackage(pkg *runtimexpkg.Package) (*schema.GroupVersionKind, e }, nil } -// AddDependency adds a dependency, generates schemas for it, and persists the -// dependency to the project file. +// AddDependency checks that a dependency can be fetched, caches it if it is a +// package, and persists it to the project file. It does not generate schemas; +// `crossplane schema update` does. func (m *Manager) AddDependency(ctx context.Context, dep *v1alpha1.Dependency) error { - gvk, err := m.addDependencyNoWrite(ctx, dep, false) + // Fetching the sources proves that the dependency exists, so a typo in a + // URL or version is caught here and not at the next schema update. Nothing + // is generated. + res, err := m.resolve(ctx, []v1alpha1.Dependency{*dep}, ResolveOptions{}) if err != nil { return err } + if err := hashAll(ctx, res.sources); err != nil { + return err + } // For xpkg dependencies, fill in the apiVersion and kind. We don't care // about these for development purposes, but they are required to install an // xpkg dependency into a cluster at runtime. - if dep.Type == v1alpha1.DependencyTypeXpkg && dep.Xpkg != nil && gvk != nil { - dep.Xpkg.APIVersion = gvk.GroupVersion().String() - dep.Xpkg.Kind = gvk.Kind + if dep.Type == v1alpha1.DependencyTypeXpkg && dep.Xpkg != nil { + if gvk := res.gvks[dep.Xpkg.Package]; gvk != nil { + dep.Xpkg.APIVersion = gvk.GroupVersion().String() + dep.Xpkg.Kind = gvk.Kind + } } m.updateMutex.Lock() @@ -335,60 +692,89 @@ func (m *Manager) AddDependency(ctx context.Context, dep *v1alpha1.Dependency) e }) } -// AddAll adds all dependencies configured in the project. If ch is non-nil, -// events will be sent for each dependency as it is processed. -func (m *Manager) AddAll(ctx context.Context, ch async.EventChannel) error { - return m.addAll(ctx, ch, false) -} - -// RefreshAll re-resolves every dependency's version constraint against the -// registry. Used by `dependency update-cache`. -func (m *Manager) RefreshAll(ctx context.Context, ch async.EventChannel) error { - return m.addAll(ctx, ch, true) -} - -func (m *Manager) addAll(ctx context.Context, ch async.EventChannel, refresh bool) error { +// hashAll hashes the sources, which fetches their content. +func hashAll(ctx context.Context, sources []smanager.Source) error { eg, egCtx := errgroup.WithContext(ctx) - - for i := range m.proj.Spec.Dependencies { - dep := &m.proj.Spec.Dependencies[i] - desc := "Updating dependency " + GetSourceDescription(*dep) + for _, s := range sources { eg.Go(func() error { - ch.SendEvent(desc, async.EventStatusStarted) - if _, err := m.addDependencyNoWrite(egCtx, dep, refresh); err != nil { - ch.SendEvent(desc, async.EventStatusFailure) - return err - } - ch.SendEvent(desc, async.EventStatusSuccess) - return nil + _, err := s.Hash(egCtx) + return errors.Wrapf(err, "cannot fetch %s", s.Key()) }) } - return eg.Wait() } -func (m *Manager) addDependencyNoWrite(ctx context.Context, dep *v1alpha1.Dependency, refresh bool) (*schema.GroupVersionKind, error) { - switch { - case dep.Type == v1alpha1.DependencyTypeXpkg: - if dep.Xpkg == nil { - return nil, errors.New("xpkg dependency has no package reference") +// RefreshAll re-resolves every dependency's version constraint against the +// registry and refreshes the package cache. It also fetches the git, http and +// Kubernetes dependencies. Used by `dependency update-cache`. +func (m *Manager) RefreshAll(ctx context.Context, ch async.EventChannel) error { + descs := make([]string, len(m.proj.Spec.Dependencies)) + for i, dep := range m.proj.Spec.Dependencies { + descs[i] = "Updating dependency " + GetSourceDescription(dep) + ch.SendEvent(descs[i], async.EventStatusStarted) + } + + err := func() error { + sources, _, err := m.Resolve(ctx, ResolveOptions{Refresh: true}) + if err != nil { + return err } + return hashAll(ctx, sources) + }() - return m.addPackage(ctx, xpkgRef(dep.Xpkg.Package, dep.Xpkg.Version), refresh) - case dep.Git != nil: - return nil, m.schemas.Add(ctx, smanager.NewGitSource(*dep, m.gitCloner, m.gitAuthProvider)) - case dep.HTTP != nil: - return nil, m.schemas.Add(ctx, smanager.NewHTTPSource(*dep)) - case dep.K8s != nil: - return nil, m.schemas.Add(ctx, smanager.NewK8sSource(*dep)) - default: - return nil, errors.New("dependency has no source configured") + status := async.EventStatusSuccess + if err != nil { + status = async.EventStatusFailure + } + for _, desc := range descs { + ch.SendEvent(desc, status) } + return err } -// Clean removes all generated schemas. -func (m *Manager) Clean() error { - return m.projFS.RemoveAll(m.proj.Spec.Paths.Schemas) +// FetchPinned returns the source locked under key, fetched at the pin recorded +// in the lock instead of being resolved again. A package is taken from the +// cache if it is there. +func (m *Manager) FetchPinned(ctx context.Context, key string, entry smanager.LockedSource) (smanager.Source, error) { + if strings.HasPrefix(key, "xpkg://") { + pkg, err := m.client.Get(ctx, entry.Pin, runtimexpkg.WithPullPolicy(corev1.PullIfNotPresent)) + if err != nil { + return nil, errors.Wrapf(err, "failed to fetch %s", entry.Pin) + } + crdFS, err := clixpkg.CRDFilesystem(pkg.Package) + if err != nil { + return nil, errors.Wrapf(err, "cannot extract CRDs from %s", entry.Pin) + } + return smanager.NewXpkgSource(pkg.Source, entry.Version, pkg.Digest, crdFS), nil + } + + for _, dep := range m.proj.Spec.Dependencies { + if dep.Type == v1alpha1.DependencyTypeXpkg { + continue + } + src, err := m.source(dep) + if err != nil { + return nil, err + } + if src.Key() != key { + continue + } + + if dep.Git != nil { + return m.pinnedSource(dep, entry.Pin) + } + if dep.HTTP != nil { + hash, err := src.Hash(ctx) + if err != nil { + return nil, errors.Wrapf(err, "cannot fetch %s", key) + } + if hash != entry.Pin { + return nil, errors.Errorf("%s changed: its hash is %s, but the lock has %s", key, hash, entry.Pin) + } + } + return src, nil + } + return nil, errors.Errorf("no dependency in the project provides %s", key) } // CleanPackages removes the per-user xpkg cache directory at root. @@ -399,6 +785,22 @@ func CleanPackages(root string, fs afero.Fs) error { return nil } +// DependenciesHash returns a hash of the project's dependencies. It does not +// depend on their order, and changes when any dependency changes. +func DependenciesHash(proj *v1alpha1.Project) (string, error) { + deps := slices.Clone(proj.Spec.Dependencies) + slices.SortStableFunc(deps, func(a, b v1alpha1.Dependency) int { + return strings.Compare(GetSourceDescription(a), GetSourceDescription(b)) + }) + + bs, err := json.Marshal(deps) + if err != nil { + return "", errors.Wrap(err, "cannot marshal dependencies") + } + sum := sha256.Sum256(bs) + return hex.EncodeToString(sum[:]), nil +} + // GetSourceDescription returns a human-readable description of a dependency. func GetSourceDescription(dep v1alpha1.Dependency) string { switch { diff --git a/internal/dependency/manager_test.go b/internal/dependency/manager_test.go index 0d2339d5..21f95e2d 100644 --- a/internal/dependency/manager_test.go +++ b/internal/dependency/manager_test.go @@ -18,16 +18,23 @@ package dependency import ( "context" - "encoding/json" "fmt" "io" - "maps" + "math/rand/v2" + "net/http" + "net/http/httptest" + "path/filepath" "slices" "strings" "sync" "testing" + "time" + "github.com/go-git/go-billy/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/storage" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/spf13/afero" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/yaml" @@ -37,8 +44,8 @@ import ( "github.com/crossplane/crossplane-runtime/v2/pkg/xpkg/parser" "github.com/crossplane/cli/v2/apis/dev/v1alpha1" - "github.com/crossplane/cli/v2/internal/async" - "github.com/crossplane/cli/v2/internal/schemas/generator" + "github.com/crossplane/cli/v2/internal/git" + smanager "github.com/crossplane/cli/v2/internal/schemas/manager" clixpkg "github.com/crossplane/cli/v2/internal/xpkg" ) @@ -186,11 +193,22 @@ type fakeClient struct { tags []string tagsByRepo map[string][]string + // jitter delays every call by a random time, to shuffle the order in which + // concurrent fetches finish. + jitter bool + mu sync.Mutex gets map[string]int } +func (f *fakeClient) delay() { + if f.jitter { + time.Sleep(time.Duration(rand.IntN(300)) * time.Microsecond) + } +} + func (f *fakeClient) Get(_ context.Context, ref string, _ ...runtimexpkg.GetOption) (*runtimexpkg.Package, error) { + f.delay() f.mu.Lock() if f.gets == nil { f.gets = make(map[string]int) @@ -206,6 +224,7 @@ func (f *fakeClient) Get(_ context.Context, ref string, _ ...runtimexpkg.GetOpti } func (f *fakeClient) ListVersions(_ context.Context, repo string, _ ...runtimexpkg.GetOption) ([]string, error) { + f.delay() if tags, ok := f.tagsByRepo[repo]; ok { return tags, nil } @@ -228,22 +247,34 @@ func makePackageWithBody(t *testing.T, source, digest, version, body string) *ru } } -func newTestManager(t *testing.T, fc *fakeClient, generators ...generator.Interface) (*Manager, afero.Fs) { +func newTestManager(t *testing.T, fc *fakeClient) *Manager { t.Helper() - schemaFS := afero.NewMemMapFs() - m := NewManager( + return NewManager( &v1alpha1.Project{ Spec: v1alpha1.ProjectSpec{ Paths: &v1alpha1.ProjectPaths{Schemas: "schemas"}, }, }, afero.NewMemMapFs(), - WithSchemaFS(schemaFS), - WithSchemaGenerators(generators), WithXpkgClient(fc), WithResolver(clixpkg.NewResolver(fc)), ) - return m, schemaFS +} + +// sourceKeys returns the keys of the sources. +func sourceKeys(sources []smanager.Source) []string { + keys := make([]string, 0, len(sources)) + for _, src := range sources { + keys = append(keys, src.Key()) + } + return keys +} + +// refDep builds an xpkg dependency from an OCI ref, whose version may be a tag, +// a constraint or a digest. +func refDep(ref string) *v1alpha1.Dependency { + repo := refRepo(ref) + return xpkgDep(repo, strings.TrimLeft(strings.TrimPrefix(ref, repo), ":@")) } // xpkgDep builds an xpkg dependency the same way cmd/crossplane/dependency/add.go @@ -530,8 +561,6 @@ func TestManager_AddDependency(t *testing.T) { m := NewManager(proj, projFS, WithProjectFile("crossplane-project.yaml"), - WithSchemaFS(afero.NewMemMapFs()), - WithSchemaGenerators([]generator.Interface{}), WithXpkgClient(fc), WithResolver(clixpkg.NewResolver(fc)), ) @@ -567,104 +596,100 @@ func TestManager_AddDependency(t *testing.T) { } } -func TestManager_AddPackage(t *testing.T) { +func TestManager_Resolve(t *testing.T) { const ( cfgXRDPkg = "xpkg.crossplane.io/crossplane-contrib/configuration-xrd" cfgXRDTag = "v0.1.0" ) - tests := map[string]struct { + type args struct { ref string tags []string fetchAt string // body is the package YAML served by the fake client; empty // defaults to configurationPackageYAML. body string - // generators are the schema generators wired into the manager; - // empty/nil means no schema files are actually rendered. - generators []generator.Interface - wantKey string - // wantSchemaGlob, when set, must match at least one file in - // schemaFS after AddPackage. - wantSchemaGlob string + } + type want struct { + // key is the only schema source the package should yield. + key string + // crds is whether the source's resources contain CRD files. + crds bool + err error + } + + cases := map[string]struct { + reason string + args args + want want }{ "ConstraintCollapsesToResolvedVersion": { - ref: "pkg.example/foo:>=v0.0.0", - tags: []string{"v0.5.2"}, - fetchAt: "pkg.example/foo:v0.5.2", - wantKey: "xpkg://pkg.example/foo:v0.5.2", + reason: "A semver constraint resolves to a tag, but the source key never contains a version.", + args: args{ref: "pkg.example/foo:>=v0.0.0", tags: []string{"v0.5.2"}, fetchAt: "pkg.example/foo:v0.5.2"}, + want: want{key: "xpkg://pkg.example/foo", crds: true}, }, "ExactVersionMatchesResolved": { - ref: "pkg.example/foo:v0.5.2", - tags: []string{"v0.5.2"}, - fetchAt: "pkg.example/foo:v0.5.2", - wantKey: "xpkg://pkg.example/foo:v0.5.2", - }, - "DigestRefUsesDigestForm": { - ref: "pkg.example/foo@sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03", - fetchAt: "pkg.example/foo@sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03", - wantKey: "xpkg://pkg.example/foo@sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03", - }, - // ConfigurationWithXRDGeneratesSchemas verifies that adding a - // Configuration dependency that bundles XRDs (rather than raw - // CRDs) generates real schema output, not just an empty - // successful pass. Before internal/xpkg.CRDFilesystem learned to - // convert XRDs to their derived CRD form, this produced a lock - // entry but zero schema content. - "ConfigurationWithXRDGeneratesSchemas": { - ref: cfgXRDPkg + ":" + cfgXRDTag, - tags: []string{cfgXRDTag}, - fetchAt: cfgXRDPkg + ":" + cfgXRDTag, - body: configurationWithXRDPackageYAML, - generators: generator.Filter(generator.AllLanguages(), []string{v1alpha1.SchemaLanguageJSON}), - wantKey: "xpkg://" + cfgXRDPkg + ":" + cfgXRDTag, - wantSchemaGlob: "json/*.schema.json", - }, - } - - for name, tc := range tests { + reason: "An exact tag yields the same key as a constraint on the same package.", + args: args{ref: "pkg.example/foo:v0.5.2", tags: []string{"v0.5.2"}, fetchAt: "pkg.example/foo:v0.5.2"}, + want: want{key: "xpkg://pkg.example/foo", crds: true}, + }, + "DigestRefSharesKeyWithTags": { + reason: "A digest-pinned ref yields the same key, so one lock entry tracks the package whatever the ref form.", + args: args{ + ref: "pkg.example/foo@sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03", + fetchAt: "pkg.example/foo@sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03", + }, + want: want{key: "xpkg://pkg.example/foo", crds: true}, + }, + // A Configuration that bundles XRDs (rather than raw CRDs) must yield + // real CRD resources, not an empty source. Before + // internal/xpkg.CRDFilesystem learned to convert XRDs to their derived + // CRD form, this produced a source with zero schema content. + "ConfigurationWithXRDYieldsCRDs": { + reason: "XRDs bundled in a Configuration are converted to CRDs for schema generation.", + args: args{ + ref: cfgXRDPkg + ":" + cfgXRDTag, + tags: []string{cfgXRDTag}, + fetchAt: cfgXRDPkg + ":" + cfgXRDTag, + body: configurationWithXRDPackageYAML, + }, + want: want{key: "xpkg://" + cfgXRDPkg, crds: true}, + }, + } + + for name, tc := range cases { t.Run(name, func(t *testing.T) { - body := tc.body + body := tc.args.body if body == "" { body = configurationPackageYAML } fc := &fakeClient{ packages: map[string]*runtimexpkg.Package{ - tc.fetchAt: makePackageWithBody(t, refRepo(tc.ref), "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03", "", body), + tc.args.fetchAt: makePackageWithBody(t, refRepo(tc.args.ref), "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03", "", body), }, - tags: tc.tags, + tags: tc.args.tags, } - m, schemaFS := newTestManager(t, fc, tc.generators...) + m := newTestManager(t, fc) - if _, err := m.AddPackage(context.Background(), tc.ref, false); err != nil { - t.Fatalf("AddPackage: %v", err) + m.proj.Spec.Dependencies = []v1alpha1.Dependency{*refDep(tc.args.ref)} + sources, _, err := m.Resolve(context.Background(), ResolveOptions{}) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nResolve(...): -want err, +got err:\n%s", tc.reason, diff) } - bs, err := afero.ReadFile(schemaFS, ".lock.json") - if err != nil { - t.Fatalf("read lock: %v", err) - } - var got struct { - Packages map[string]string `json:"packages"` + if diff := cmp.Diff([]string{tc.want.key}, sourceKeys(sources)); diff != "" { + t.Errorf("%s\nResolve(...): -want source keys, +got source keys:\n%s", tc.reason, diff) } - if err := json.Unmarshal(bs, &got); err != nil { - t.Fatalf("unmarshal lock: %v", err) + resources, err := sources[0].Resources(context.Background()) + if err != nil { + t.Fatal(err) } - if _, ok := got.Packages[tc.wantKey]; !ok { - t.Errorf("lock has no entry for %q; keys = %v", tc.wantKey, slices.Collect(maps.Keys(got.Packages))) + files, err := afero.Glob(resources, "*.yaml") + if err != nil { + t.Fatal(err) } - if len(got.Packages) != 1 { - t.Errorf("lock packages = %d, want 1; got %v", len(got.Packages), got.Packages) - } - - if tc.wantSchemaGlob != "" { - files, err := afero.Glob(schemaFS, tc.wantSchemaGlob) - if err != nil { - t.Fatalf("glob generated schemas: %v", err) - } - if len(files) == 0 { - t.Errorf("no files matched %q; no schemas were generated", tc.wantSchemaGlob) - } + if diff := cmp.Diff(tc.want.crds, len(files) > 0); diff != "" { + t.Errorf("%s\nResolve(...): -want CRD files, +got CRD files:\n%s\nfiles: %v", tc.reason, diff, files) } }) } @@ -682,25 +707,7 @@ func refRepo(ref string) string { return ref } -// readLockKeys returns the package keys recorded in the schema lock file. -func readLockKeys(t *testing.T, schemaFS afero.Fs) []string { - t.Helper() - bs, err := afero.ReadFile(schemaFS, ".lock.json") - if err != nil { - t.Fatalf("read lock: %v", err) - } - var lock struct { - Packages map[string]string `json:"packages"` - } - if err := json.Unmarshal(bs, &lock); err != nil { - t.Fatalf("unmarshal lock: %v", err) - } - keys := slices.Collect(maps.Keys(lock.Packages)) - slices.Sort(keys) - return keys -} - -func TestManager_AddPackage_TransitiveDeps(t *testing.T) { +func TestManager_Resolve_TransitiveDeps(t *testing.T) { const ( provA = "xpkg.example/prov-a" provB = "xpkg.example/prov-b" @@ -713,165 +720,211 @@ func TestManager_AddPackage_TransitiveDeps(t *testing.T) { body string } - tests := map[string]struct { - packages []pkgFixture - tagsByRepo map[string][]string - addRefs []string - wantLockKeys []string - wantErr string - wantGets map[string]int + type args struct { + packages []pkgFixture + tagsByRepo map[string][]string + addRefs []string + } + type want struct { + // keys are the source keys of the lock. + keys []string + // gets are the expected fetch counts, by ref. + gets map[string]int + err error + // errMsg is a substring the error must contain: it must name the + // package that could not be fetched. + errMsg string + } + + cases := map[string]struct { + reason string + args args + want want }{ "TransitiveDepGeneratesSchemas": { - packages: []pkgFixture{ - {ref: provA + ":v0.1.0", body: fmt.Sprintf(providerDependsOnPackageYAML, family, "v1.0.0")}, - {ref: family + ":v1.0.0", body: providerPackageYAML}, - }, - tagsByRepo: map[string][]string{ - provA: {"v0.1.0"}, - family: {"v1.0.0"}, + reason: "A dependency of a package is resolved and gets its own source, so schemas are generated for it too.", + args: args{ + packages: []pkgFixture{ + {ref: provA + ":v0.1.0", body: fmt.Sprintf(providerDependsOnPackageYAML, family, "v1.0.0")}, + {ref: family + ":v1.0.0", body: providerPackageYAML}, + }, + tagsByRepo: map[string][]string{ + provA: {"v0.1.0"}, + family: {"v1.0.0"}, + }, + addRefs: []string{provA + ":v0.1.0"}, }, - addRefs: []string{provA + ":v0.1.0"}, - wantLockKeys: []string{ - "xpkg://" + family + ":v1.0.0", - "xpkg://" + provA + ":v0.1.0", + want: want{ + keys: []string{ + "xpkg://" + family, + "xpkg://" + provA, + }, }, }, "TransitiveDepWithConstraint": { - packages: []pkgFixture{ - {ref: provA + ":v0.1.0", body: fmt.Sprintf(providerDependsOnPackageYAML, family, ">=v1.0.0")}, - {ref: family + ":v1.2.3", body: providerPackageYAML}, - }, - tagsByRepo: map[string][]string{ - provA: {"v0.1.0"}, - family: {"v1.2.3"}, + reason: "A version constraint on a dependency of a package resolves to the matching tag.", + args: args{ + packages: []pkgFixture{ + {ref: provA + ":v0.1.0", body: fmt.Sprintf(providerDependsOnPackageYAML, family, ">=v1.0.0")}, + {ref: family + ":v1.2.3", body: providerPackageYAML}, + }, + tagsByRepo: map[string][]string{ + provA: {"v0.1.0"}, + family: {"v1.2.3"}, + }, + addRefs: []string{provA + ":v0.1.0"}, }, - addRefs: []string{provA + ":v0.1.0"}, - wantLockKeys: []string{ - "xpkg://" + family + ":v1.2.3", - "xpkg://" + provA + ":v0.1.0", + want: want{ + keys: []string{ + "xpkg://" + family, + "xpkg://" + provA, + }, }, }, "TransitiveDepByDigest": { - packages: []pkgFixture{ - {ref: provA + ":v0.1.0", body: fmt.Sprintf(providerDependsOnPackageYAML, family, digestVal)}, - {ref: family + "@" + digestVal, body: providerPackageYAML}, - }, - tagsByRepo: map[string][]string{ - provA: {"v0.1.0"}, + reason: "A dependency of a package that is pinned by digest is fetched by that digest.", + args: args{ + packages: []pkgFixture{ + {ref: provA + ":v0.1.0", body: fmt.Sprintf(providerDependsOnPackageYAML, family, digestVal)}, + {ref: family + "@" + digestVal, body: providerPackageYAML}, + }, + tagsByRepo: map[string][]string{ + provA: {"v0.1.0"}, + }, + addRefs: []string{provA + ":v0.1.0"}, }, - addRefs: []string{provA + ":v0.1.0"}, - wantLockKeys: []string{ - "xpkg://" + family + "@" + digestVal, - "xpkg://" + provA + ":v0.1.0", + want: want{ + keys: []string{ + "xpkg://" + family, + "xpkg://" + provA, + }, }, }, "DeprecatedProviderField": { - packages: []pkgFixture{ - {ref: provA + ":v0.1.0", body: fmt.Sprintf(providerDependsOnProviderYAML, family, "v1.0.0")}, - {ref: family + ":v1.0.0", body: providerPackageYAML}, - }, - tagsByRepo: map[string][]string{ - provA: {"v0.1.0"}, - family: {"v1.0.0"}, + reason: "A dependency declared through the deprecated provider field is resolved like any other.", + args: args{ + packages: []pkgFixture{ + {ref: provA + ":v0.1.0", body: fmt.Sprintf(providerDependsOnProviderYAML, family, "v1.0.0")}, + {ref: family + ":v1.0.0", body: providerPackageYAML}, + }, + tagsByRepo: map[string][]string{ + provA: {"v0.1.0"}, + family: {"v1.0.0"}, + }, + addRefs: []string{provA + ":v0.1.0"}, }, - addRefs: []string{provA + ":v0.1.0"}, - wantLockKeys: []string{ - "xpkg://" + family + ":v1.0.0", - "xpkg://" + provA + ":v0.1.0", + want: want{ + keys: []string{ + "xpkg://" + family, + "xpkg://" + provA, + }, }, }, "DiamondSharedDepFetchedOnce": { - packages: []pkgFixture{ - {ref: provA + ":v0.1.0", body: fmt.Sprintf(providerDependsOnPackageYAML, family, "v1.0.0")}, - {ref: provB + ":v0.2.0", body: fmt.Sprintf(providerDependsOnPackageYAML, family, "v1.0.0")}, - {ref: family + ":v1.0.0", body: providerPackageYAML}, - }, - tagsByRepo: map[string][]string{ - provA: {"v0.1.0"}, - provB: {"v0.2.0"}, - family: {"v1.0.0"}, - }, - addRefs: []string{provA + ":v0.1.0", provB + ":v0.2.0"}, - wantLockKeys: []string{ - "xpkg://" + family + ":v1.0.0", - "xpkg://" + provA + ":v0.1.0", - "xpkg://" + provB + ":v0.2.0", + reason: "A dependency shared by two packages is fetched only once.", + args: args{ + packages: []pkgFixture{ + {ref: provA + ":v0.1.0", body: fmt.Sprintf(providerDependsOnPackageYAML, family, "v1.0.0")}, + {ref: provB + ":v0.2.0", body: fmt.Sprintf(providerDependsOnPackageYAML, family, "v1.0.0")}, + {ref: family + ":v1.0.0", body: providerPackageYAML}, + }, + tagsByRepo: map[string][]string{ + provA: {"v0.1.0"}, + provB: {"v0.2.0"}, + family: {"v1.0.0"}, + }, + addRefs: []string{provA + ":v0.1.0", provB + ":v0.2.0"}, }, - wantGets: map[string]int{ - family + ":v1.0.0": 1, + want: want{ + keys: []string{ + "xpkg://" + family, + "xpkg://" + provA, + "xpkg://" + provB, + }, + gets: map[string]int{ + family + ":v1.0.0": 1, + }, }, }, "CycleTerminates": { - packages: []pkgFixture{ - {ref: provA + ":v0.1.0", body: fmt.Sprintf(providerDependsOnPackageYAML, provB, "v0.2.0")}, - {ref: provB + ":v0.2.0", body: fmt.Sprintf(providerDependsOnPackageYAML, provA, "v0.1.0")}, - }, - tagsByRepo: map[string][]string{ - provA: {"v0.1.0"}, - provB: {"v0.2.0"}, + reason: "Packages that depend on each other do not make resolution loop forever.", + args: args{ + packages: []pkgFixture{ + {ref: provA + ":v0.1.0", body: fmt.Sprintf(providerDependsOnPackageYAML, provB, "v0.2.0")}, + {ref: provB + ":v0.2.0", body: fmt.Sprintf(providerDependsOnPackageYAML, provA, "v0.1.0")}, + }, + tagsByRepo: map[string][]string{ + provA: {"v0.1.0"}, + provB: {"v0.2.0"}, + }, + addRefs: []string{provA + ":v0.1.0"}, }, - addRefs: []string{provA + ":v0.1.0"}, - wantLockKeys: []string{ - "xpkg://" + provA + ":v0.1.0", - "xpkg://" + provB + ":v0.2.0", + want: want{ + keys: []string{ + "xpkg://" + provA, + "xpkg://" + provB, + }, }, }, "TransitiveFetchFailure": { - packages: []pkgFixture{ - {ref: provA + ":v0.1.0", body: fmt.Sprintf(providerDependsOnPackageYAML, family, "v9.9.9")}, + reason: "A dependency of a package that cannot be fetched is an error naming that dependency.", + args: args{ + packages: []pkgFixture{ + {ref: provA + ":v0.1.0", body: fmt.Sprintf(providerDependsOnPackageYAML, family, "v9.9.9")}, + }, + tagsByRepo: map[string][]string{ + provA: {"v0.1.0"}, + family: {}, + }, + addRefs: []string{provA + ":v0.1.0"}, }, - tagsByRepo: map[string][]string{ - provA: {"v0.1.0"}, - family: {}, + want: want{ + err: cmpopts.AnyError, errMsg: family, }, - addRefs: []string{provA + ":v0.1.0"}, - wantErr: family + ":v9.9.9", }, } - for name, tc := range tests { + for name, tc := range cases { t.Run(name, func(t *testing.T) { fc := &fakeClient{ packages: map[string]*runtimexpkg.Package{}, - tagsByRepo: tc.tagsByRepo, + tagsByRepo: tc.args.tagsByRepo, } - for _, p := range tc.packages { + for _, p := range tc.args.packages { fc.packages[p.ref] = makePackageWithBody(t, refRepo(p.ref), "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03", "", p.body) } - m, schemaFS := newTestManager(t, fc) + m := newTestManager(t, fc) - var err error - for _, ref := range tc.addRefs { - if _, err = m.AddPackage(context.Background(), ref, false); err != nil { - break - } + for _, ref := range tc.args.addRefs { + m.proj.Spec.Dependencies = append(m.proj.Spec.Dependencies, *refDep(ref)) } - if tc.wantErr != "" { - if err == nil || !strings.Contains(err.Error(), tc.wantErr) { - t.Fatalf("expected error containing %q, got %v", tc.wantErr, err) + sources, _, err := m.Resolve(context.Background(), ResolveOptions{}) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nResolve(...): -want err, +got err:\n%s", tc.reason, diff) + } + if tc.want.err != nil { + if !strings.Contains(err.Error(), tc.want.errMsg) { + t.Errorf("%s\nResolve(...): error %q does not contain %q", tc.reason, err, tc.want.errMsg) } return } - if err != nil { - t.Fatalf("AddPackage: %v", err) - } - want := slices.Clone(tc.wantLockKeys) + want := slices.Clone(tc.want.keys) slices.Sort(want) - if diff := cmp.Diff(want, readLockKeys(t, schemaFS)); diff != "" { - t.Errorf("lock keys (-want +got):\n%s", diff) + if diff := cmp.Diff(want, sourceKeys(sources)); diff != "" { + t.Errorf("%s\nResolve(...): -want source keys, +got source keys:\n%s", tc.reason, diff) } - for ref, wantCount := range tc.wantGets { - if got := fc.getCount(ref); got != wantCount { - t.Errorf("fetch count for %s = %d, want %d", ref, got, wantCount) + for ref, wantCount := range tc.want.gets { + if diff := cmp.Diff(wantCount, fc.getCount(ref)); diff != "" { + t.Errorf("%s\nResolve(...): -want fetch count for %s, +got fetch count:\n%s", tc.reason, ref, diff) } } }) } } -func TestManager_AddAll_SharedTransitiveDep(t *testing.T) { +func TestManager_Resolve_SharedTransitiveDep(t *testing.T) { const ( provA = "xpkg.example/prov-a" provB = "xpkg.example/prov-b" @@ -879,52 +932,78 @@ func TestManager_AddAll_SharedTransitiveDep(t *testing.T) { digest = "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03" ) - fc := &fakeClient{ - packages: map[string]*runtimexpkg.Package{ - provA + ":v0.1.0": makePackageWithBody(t, provA, digest, "", fmt.Sprintf(providerDependsOnPackageYAML, family, "v1.0.0")), - provB + ":v0.2.0": makePackageWithBody(t, provB, digest, "", fmt.Sprintf(providerDependsOnPackageYAML, family, "v1.0.0")), - family + ":v1.0.0": makePackageWithBody(t, family, digest, "", providerPackageYAML), - }, - tagsByRepo: map[string][]string{ - provA: {"v0.1.0"}, - provB: {"v0.2.0"}, - family: {"v1.0.0"}, - }, + type args struct { + packages map[string]*runtimexpkg.Package + tagsByRepo map[string][]string + deps []v1alpha1.Dependency + } + type want struct { + // gets are the expected fetch counts, by ref. + gets map[string]int + keys []string + err error } - schemaFS := afero.NewMemMapFs() - m := NewManager( - &v1alpha1.Project{ - Spec: v1alpha1.ProjectSpec{ - Dependencies: []v1alpha1.Dependency{ - *xpkgDep(provA, "v0.1.0"), - *xpkgDep(provB, "v0.2.0"), + cases := map[string]struct { + reason string + args args + want want + }{ + "TwoPackagesShareDependency": { + reason: "A dependency needed by several packages is fetched once, however many need it, and listed once.", + args: args{ + packages: map[string]*runtimexpkg.Package{ + provA + ":v0.1.0": makePackageWithBody(t, provA, digest, "", fmt.Sprintf(providerDependsOnPackageYAML, family, "v1.0.0")), + provB + ":v0.2.0": makePackageWithBody(t, provB, digest, "", fmt.Sprintf(providerDependsOnPackageYAML, family, "v1.0.0")), + family + ":v1.0.0": makePackageWithBody(t, family, digest, "", providerPackageYAML), }, - Paths: &v1alpha1.ProjectPaths{Schemas: "schemas"}, + tagsByRepo: map[string][]string{ + provA: {"v0.1.0"}, + provB: {"v0.2.0"}, + family: {"v1.0.0"}, + }, + deps: []v1alpha1.Dependency{*xpkgDep(provA, "v0.1.0"), *xpkgDep(provB, "v0.2.0")}, + }, + want: want{ + gets: map[string]int{family + ":v1.0.0": 1}, + keys: []string{"xpkg://" + family, "xpkg://" + provA, "xpkg://" + provB}, }, }, - afero.NewMemMapFs(), - WithSchemaFS(schemaFS), - WithSchemaGenerators([]generator.Interface{}), - WithXpkgClient(fc), - WithResolver(clixpkg.NewResolver(fc)), - ) - - var ch async.EventChannel // nil channel; SendEvent is a no-op. - if err := m.AddAll(context.Background(), ch); err != nil { - t.Fatalf("AddAll: %v", err) } - want := []string{ - "xpkg://" + family + ":v1.0.0", - "xpkg://" + provA + ":v0.1.0", - "xpkg://" + provB + ":v0.2.0", - } - if diff := cmp.Diff(want, readLockKeys(t, schemaFS)); diff != "" { - t.Errorf("lock keys (-want +got):\n%s", diff) - } - if got := fc.getCount(family + ":v1.0.0"); got != 1 { - t.Errorf("fetch count for %s = %d, want 1", family+":v1.0.0", got) + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + fc := &fakeClient{ + packages: tc.args.packages, + tagsByRepo: tc.args.tagsByRepo, + } + + m := NewManager( + &v1alpha1.Project{ + Spec: v1alpha1.ProjectSpec{ + Dependencies: tc.args.deps, + Paths: &v1alpha1.ProjectPaths{Schemas: "schemas"}, + }, + }, + afero.NewMemMapFs(), + WithXpkgClient(fc), + WithResolver(clixpkg.NewResolver(fc)), + ) + + sources, _, err := m.Resolve(context.Background(), ResolveOptions{}) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nResolve(...): -want err, +got err:\n%s", tc.reason, diff) + } + + for ref, wantCount := range tc.want.gets { + if diff := cmp.Diff(wantCount, fc.getCount(ref)); diff != "" { + t.Errorf("%s\nResolve(...): -want fetch count for %s, +got fetch count:\n%s", tc.reason, ref, diff) + } + } + if diff := cmp.Diff(tc.want.keys, sourceKeys(sources)); diff != "" { + t.Errorf("%s\nResolve(...): -want source keys, +got source keys:\n%s", tc.reason, diff) + } + }) } } @@ -935,18 +1014,121 @@ func TestManager_AddDependency_TransitiveNotPersisted(t *testing.T) { digest = "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03" ) - fc := &fakeClient{ - packages: map[string]*runtimexpkg.Package{ - provA + ":v0.1.0": makePackageWithBody(t, provA, digest, "", fmt.Sprintf(providerDependsOnPackageYAML, family, "v1.0.0")), - family + ":v1.0.0": makePackageWithBody(t, family, digest, "", providerPackageYAML), - }, - tagsByRepo: map[string][]string{ - provA: {"v0.1.0"}, - family: {"v1.0.0"}, + type args struct { + packages map[string]*runtimexpkg.Package + tagsByRepo map[string][]string + dep *v1alpha1.Dependency + } + type want struct { + // gets are the expected fetch counts, by ref. + gets map[string]int + // deps are the dependencies persisted to the project, in memory and on + // disk. + deps []v1alpha1.Dependency + err error + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "OnlyDeclaredPersisted": { + reason: "The transitive package is fetched into the cache, but only the declared dependency is persisted to the project.", + args: args{ + packages: map[string]*runtimexpkg.Package{ + provA + ":v0.1.0": makePackageWithBody(t, provA, digest, "", fmt.Sprintf(providerDependsOnPackageYAML, family, "v1.0.0")), + family + ":v1.0.0": makePackageWithBody(t, family, digest, "", providerPackageYAML), + }, + tagsByRepo: map[string][]string{ + provA: {"v0.1.0"}, + family: {"v1.0.0"}, + }, + dep: xpkgDep(provA, "v0.1.0"), + }, + want: want{ + gets: map[string]int{family + ":v1.0.0": 1}, + deps: []v1alpha1.Dependency{{ + Type: v1alpha1.DependencyTypeXpkg, + Xpkg: &v1alpha1.XpkgDependency{ + APIVersion: "pkg.crossplane.io/v1", + Kind: "Provider", + Package: provA, + Version: "v0.1.0", + }, + }}, + }, }, } - projFS := afero.NewMemMapFs() + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + fc := &fakeClient{ + packages: tc.args.packages, + tagsByRepo: tc.args.tagsByRepo, + } + + projFS := afero.NewMemMapFs() + proj := &v1alpha1.Project{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "dev.crossplane.io/v1alpha1", + Kind: "Project", + }, + ObjectMeta: metav1.ObjectMeta{Name: "test-project"}, + Spec: v1alpha1.ProjectSpec{ + Repository: "xpkg.crossplane.io/test/test", + Paths: &v1alpha1.ProjectPaths{Schemas: "schemas"}, + }, + } + bs, err := yaml.Marshal(proj) + if err != nil { + t.Fatal(err) + } + if err := afero.WriteFile(projFS, "crossplane-project.yaml", bs, 0o644); err != nil { + t.Fatal(err) + } + + m := NewManager(proj, projFS, + WithProjectFile("crossplane-project.yaml"), + WithXpkgClient(fc), + WithResolver(clixpkg.NewResolver(fc)), + ) + + err = m.AddDependency(context.Background(), tc.args.dep) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nAddDependency(...): -want err, +got err:\n%s", tc.reason, diff) + } + + for ref, wantCount := range tc.want.gets { + if diff := cmp.Diff(wantCount, fc.getCount(ref)); diff != "" { + t.Errorf("%s\nAddDependency(...): -want fetch count for %s, +got fetch count:\n%s", tc.reason, ref, diff) + } + } + + if diff := cmp.Diff(tc.want.deps, proj.Spec.Dependencies); diff != "" { + t.Errorf("%s\nAddDependency(...): -want project deps, +got project deps:\n%s", tc.reason, diff) + } + gotBytes, err := afero.ReadFile(projFS, "crossplane-project.yaml") + if err != nil { + t.Fatal(err) + } + var gotProj v1alpha1.Project + if err := yaml.Unmarshal(gotBytes, &gotProj); err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(tc.want.deps, gotProj.Spec.Dependencies); diff != "" { + t.Errorf("%s\nAddDependency(...): -want on-disk project deps, +got on-disk project deps:\n%s", tc.reason, diff) + } + }) + } +} + +const testDigest = "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03" + +// newProjectManager returns a manager for a project with the given +// dependencies, with its project file on a memory filesystem. +func newProjectManager(t *testing.T, fc *fakeClient, deps []v1alpha1.Dependency) (*Manager, afero.Fs) { + t.Helper() proj := &v1alpha1.Project{ TypeMeta: metav1.TypeMeta{ APIVersion: "dev.crossplane.io/v1alpha1", @@ -954,62 +1136,869 @@ func TestManager_AddDependency_TransitiveNotPersisted(t *testing.T) { }, ObjectMeta: metav1.ObjectMeta{Name: "test-project"}, Spec: v1alpha1.ProjectSpec{ - Repository: "xpkg.crossplane.io/test/test", - Paths: &v1alpha1.ProjectPaths{Schemas: "schemas"}, + Repository: "xpkg.crossplane.io/test/test", + Dependencies: deps, + Paths: &v1alpha1.ProjectPaths{Schemas: "schemas"}, }, } bs, err := yaml.Marshal(proj) if err != nil { t.Fatal(err) } + projFS := afero.NewMemMapFs() if err := afero.WriteFile(projFS, "crossplane-project.yaml", bs, 0o644); err != nil { t.Fatal(err) } - - schemaFS := afero.NewMemMapFs() m := NewManager(proj, projFS, WithProjectFile("crossplane-project.yaml"), - WithSchemaFS(schemaFS), - WithSchemaGenerators([]generator.Interface{}), WithXpkgClient(fc), WithResolver(clixpkg.NewResolver(fc)), ) + return m, projFS +} + +// providerDep is an xpkg dependency as stored in a project file, which requires +// the package's apiVersion and kind. +func providerDep(pkg string) *v1alpha1.Dependency { + dep := xpkgDep(pkg, "v0.1.0") + dep.Xpkg.APIVersion = "pkg.crossplane.io/v1" + dep.Xpkg.Kind = "Provider" + return dep +} - if err := m.AddDependency(context.Background(), xpkgDep(provA, "v0.1.0")); err != nil { - t.Fatalf("AddDependency: %v", err) +// familyFixture is a registry with the given providers, which all depend on a +// shared provider package. +func familyFixture(t *testing.T, provs ...string) *fakeClient { + t.Helper() + const family = "xpkg.example/family" + fc := &fakeClient{ + packages: map[string]*runtimexpkg.Package{family + ":v1.0.0": makePackageWithBody(t, family, testDigest, "", providerPackageYAML)}, + tagsByRepo: map[string][]string{family: {"v1.0.0"}}, + } + for _, p := range provs { + fc.packages[p+":v0.1.0"] = makePackageWithBody(t, p, testDigest, "", fmt.Sprintf(providerDependsOnPackageYAML, family, "v1.0.0")) + fc.tagsByRepo[p] = []string{"v0.1.0"} } + return fc +} + +func TestManager_Resolve_Sources(t *testing.T) { + const ( + provA = "xpkg.example/prov-a" + provB = "xpkg.example/prov-b" + family = "xpkg.example/family" + ) - // Schemas were generated for both the declared and the transitive package. - wantKeys := []string{ - "xpkg://" + family + ":v1.0.0", - "xpkg://" + provA + ":v0.1.0", + type args struct { + deps []v1alpha1.Dependency + // provs are the providers available in the fake registry. + provs []string } - if diff := cmp.Diff(wantKeys, readLockKeys(t, schemaFS)); diff != "" { - t.Errorf("lock keys (-want +got):\n%s", diff) + type want struct { + keys []string + err error } - // Only the declared dependency is persisted to the project file. - wantDeps := []v1alpha1.Dependency{{ - Type: v1alpha1.DependencyTypeXpkg, - Xpkg: &v1alpha1.XpkgDependency{ - APIVersion: "pkg.crossplane.io/v1", - Kind: "Provider", - Package: provA, - Version: "v0.1.0", + cases := map[string]struct { + reason string + args args + want want + }{ + "NoDependencies": { + reason: "A project without dependencies has no sources.", + }, + "IncludesTransitiveDependencies": { + reason: "Sources of packages the dependency depends on are live too, otherwise reconciling would delete them.", + args: args{deps: []v1alpha1.Dependency{*xpkgDep(provA, "v0.1.0")}, provs: []string{provA}}, + want: want{keys: []string{"xpkg://" + family, "xpkg://" + provA}}, + }, + "SharedTransitiveListedOnce": { + reason: "Two providers sharing a dependency list it once.", + args: args{deps: []v1alpha1.Dependency{*xpkgDep(provA, "v0.1.0"), *xpkgDep(provB, "v0.1.0")}, provs: []string{provA, provB}}, + want: want{keys: []string{"xpkg://" + family, "xpkg://" + provA, "xpkg://" + provB}}, + }, + "AllSourceTypes": { + reason: "k8s, http and git dependencies are listed without being fetched.", + args: args{deps: []v1alpha1.Dependency{ + {Type: v1alpha1.DependencyTypeK8s, K8s: &v1alpha1.K8sDependency{Version: "1.33.0"}}, + {Type: v1alpha1.DependencyTypeCRD, HTTP: &v1alpha1.HTTPDependency{URL: "https://example.com/crd.yaml"}}, + {Type: v1alpha1.DependencyTypeCRD, Git: &v1alpha1.GitDependency{Repository: "https://example.com/r.git", Path: "crds"}}, + }}, + want: want{keys: []string{"git://https://example.com/r.git/crds", "http://https://example.com/crd.yaml", "k8s://1.33.0"}}, + }, + "UnavailablePackage": { + reason: "A dependency that can't be fetched is an error, not silently dropped from the live set.", + args: args{deps: []v1alpha1.Dependency{*xpkgDep("xpkg.example/missing", "v0.1.0")}}, + want: want{err: cmpopts.AnyError}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + fc := familyFixture(t, tc.args.provs...) + m, _ := newProjectManager(t, fc, tc.args.deps) + + sources, _, err := m.Resolve(context.Background(), ResolveOptions{}) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nResolve(...): -want err, +got err:\n%s", tc.reason, diff) + } + got := make([]string, 0, len(sources)) + for _, s := range sources { + got = append(got, s.Key()) + } + if diff := cmp.Diff(tc.want.keys, got, cmpopts.EquateEmpty()); diff != "" { + t.Errorf("%s\nResolve(...): -want, +got:\n%s", tc.reason, diff) + } + }) + } +} + +func TestManager_ResolveAndRefreshAll(t *testing.T) { + const ( + provA = "xpkg.example/prov-a" + family = "xpkg.example/family" + ) + + type args struct { + refresh bool + } + type want struct { + // fetched are the refs that must have been fetched into the cache. + fetched []string + err error + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "Resolve": { + reason: "Resolve caches the declared package and its transitive dependency.", + want: want{fetched: []string{provA + ":v0.1.0", family + ":v1.0.0"}}, + }, + "RefreshAll": { + reason: "RefreshAll fetches the same packages, asking the cache to refresh them.", + args: args{refresh: true}, + want: want{fetched: []string{provA + ":v0.1.0", family + ":v1.0.0"}}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + fc := familyFixture(t, provA) + m, _ := newProjectManager(t, fc, []v1alpha1.Dependency{*providerDep(provA)}) + + var err error + if tc.args.refresh { + err = m.RefreshAll(context.Background(), nil) + } else { + _, _, err = m.Resolve(context.Background(), ResolveOptions{}) + } + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nResolve/RefreshAll(...): -want err, +got err:\n%s", tc.reason, diff) + } + + for _, ref := range tc.want.fetched { + if fc.getCount(ref) == 0 { + t.Errorf("%s\nResolve/RefreshAll(...): %s was not fetched", tc.reason, ref) + } + } + }) + } +} + +func TestDependenciesHash(t *testing.T) { + t.Parallel() + + xpkg := func(pkg, version string) v1alpha1.Dependency { + return v1alpha1.Dependency{Type: v1alpha1.DependencyTypeXpkg, Xpkg: &v1alpha1.XpkgDependency{Package: pkg, Version: version}} + } + k8s := v1alpha1.Dependency{Type: v1alpha1.DependencyTypeK8s, K8s: &v1alpha1.K8sDependency{Version: "v1.33.0"}} + proj := func(deps ...v1alpha1.Dependency) *v1alpha1.Project { + return &v1alpha1.Project{Spec: v1alpha1.ProjectSpec{Dependencies: deps}} + } + + a, b := xpkg("xpkg.example.com/a", "v1.0.0"), xpkg("xpkg.example.com/b", "v1.0.0") + baseProj := proj(a, b, k8s) + + type args struct { + // base is the project whose hash is the baseline. + base *v1alpha1.Project + proj *v1alpha1.Project + } + type want struct { + // same is whether proj hashes the same as base. + same bool + err error + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "Reordered": { + reason: "The order of the dependencies does not change the hash.", + args: args{base: baseProj, proj: proj(k8s, b, a)}, + want: want{same: true}, + }, + "ChangedVersion": { + reason: "Changing a dependency changes the hash.", + args: args{base: baseProj, proj: proj(a, xpkg("xpkg.example.com/b", "v2.0.0"), k8s)}, }, - }} - if diff := cmp.Diff(wantDeps, proj.Spec.Dependencies); diff != "" { - t.Errorf("project deps (-want +got):\n%s", diff) + "RemovedDependency": { + reason: "Removing a dependency changes the hash.", + args: args{base: baseProj, proj: proj(a, k8s)}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + base, err := DependenciesHash(tc.args.base) + if err != nil { + t.Fatal(err) + } + got, err := DependenciesHash(tc.args.proj) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nDependenciesHash(...): -want err, +got err:\n%s", tc.reason, diff) + } + if diff := cmp.Diff(tc.want.same, got == base); diff != "" { + t.Errorf("%s\nDependenciesHash(...): -want same as base, +got same as base:\n%s", tc.reason, diff) + } + }) } - gotBytes, err := afero.ReadFile(projFS, "crossplane-project.yaml") +} + +// regPkg is a package in a fake registry. +type regPkg struct { + repo string + version string // empty if the package is only available by digest. + digest string + body string +} + +// digestN returns a valid digest that is different for each n. +func digestN(n int) string { + return fmt.Sprintf("sha256:%064x", n) +} + +// registry returns a fake client serving the packages. +func registry(t *testing.T, pkgs ...regPkg) *fakeClient { + t.Helper() + fc := &fakeClient{ + packages: make(map[string]*runtimexpkg.Package), + tagsByRepo: make(map[string][]string), + } + for _, p := range pkgs { + ref := p.repo + "@" + p.digest + if p.version != "" { + ref = p.repo + ":" + p.version + fc.tagsByRepo[p.repo] = append(fc.tagsByRepo[p.repo], p.version) + } + fc.packages[ref] = makePackageWithBody(t, p.repo, p.digest, p.version, p.body) + } + return fc +} + +// dependsOn returns the body of a provider that depends on repo at version. +func dependsOn(repo, version string) string { + return fmt.Sprintf(providerDependsOnPackageYAML, repo, version) +} + +// pinsOf returns the pin of each source, by key. +func pinsOf(t *testing.T, sources []smanager.Source) map[string]string { + t.Helper() + pins := make(map[string]string) + for _, src := range sources { + pin, err := src.(smanager.Pinned).Pin(context.Background()) + if err != nil { + t.Fatal(err) + } + pins[src.Key()] = pin + } + return pins +} + +// versionsOf returns the resolved version of each source, by key. +func versionsOf(sources []smanager.Source) map[string]string { + versions := make(map[string]string) + for _, src := range sources { + versions[src.Key()] = src.(smanager.Pinned).Version() + } + return versions +} + +func TestManager_Resolve_Versions(t *testing.T) { + const ( + provA = "xpkg.example/prov-a" + provB = "xpkg.example/prov-b" + provC = "xpkg.example/prov-c" + family = "xpkg.example/family" + libOld = "xpkg.example/lib-old" + libNew = "xpkg.example/lib-new" + ) + + familyVersions := func(versions ...string) []regPkg { + pkgs := make([]regPkg, 0, len(versions)) + for i, v := range versions { + pkgs = append(pkgs, regPkg{repo: family, version: v, digest: digestN(100 + i), body: providerPackageYAML}) + } + return pkgs + } + + type args struct { + pkgs []regPkg + deps []v1alpha1.Dependency + opts ResolveOptions + } + type want struct { + // versions are the resolved versions, by repository. Repositories that + // are not listed must not be resolved at all. + versions map[string]string + warnings []Warning + err error + // errMsgs are substrings the error must contain, for cases where the + // message (e.g. naming the conflicting package) is the behaviour. + errMsgs []string + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "DirectBeatsTransitive": { + reason: "A direct dependency wins over what a package requires, and the violated requirement is reported.", + args: args{ + pkgs: append(familyVersions("v1.0.0", "v2.0.0"), + regPkg{repo: provA, version: "v0.1.0", digest: digestN(1), body: dependsOn(family, ">=v2.0.0")}, + ), + deps: []v1alpha1.Dependency{*xpkgDep(provA, "v0.1.0"), *xpkgDep(family, "v1.0.0")}, + }, + want: want{ + versions: map[string]string{provA: "v0.1.0", family: "v1.0.0"}, + warnings: []Warning{{Package: family, Version: "v1.0.0", Requirer: provA, Constraint: ">=v2.0.0"}}, + }, + }, + "DirectSatisfyingTransitive": { + reason: "A direct dependency that meets the requirements of the packages does not cause a warning.", + args: args{ + pkgs: append(familyVersions("v1.0.0", "v2.0.0"), + regPkg{repo: provA, version: "v0.1.0", digest: digestN(1), body: dependsOn(family, ">=v1.0.0")}, + ), + deps: []v1alpha1.Dependency{*xpkgDep(provA, "v0.1.0"), *xpkgDep(family, "v1.0.0")}, + }, + want: want{versions: map[string]string{provA: "v0.1.0", family: "v1.0.0"}}, + }, + "HighestSatisfyingBothRequirers": { + reason: "Two packages that require different ranges get the highest version that satisfies both.", + args: args{ + pkgs: append(familyVersions("v1.0.0", "v1.1.0", "v1.2.0", "v2.0.0"), + regPkg{repo: provA, version: "v0.1.0", digest: digestN(1), body: dependsOn(family, ">=v1.0.0")}, + regPkg{repo: provB, version: "v0.1.0", digest: digestN(2), body: dependsOn(family, "=v2.0.0")}, + regPkg{repo: provB, version: "v0.1.0", digest: digestN(2), body: dependsOn(family, "=v2.0.0", provB, "=v1.0.0")}, + ), + deps: []v1alpha1.Dependency{*xpkgDep(provA, "v0.1.0")}, + opts: ResolveOptions{Locked: map[string]smanager.LockedSource{ + "xpkg://" + family: {Version: "v1.0.0"}, + }}, + }, + want: want{versions: map[string]string{provA: "v0.1.0", family: "v1.0.0"}}, + }, + "LockedVersionReplacedByRefresh": { + reason: "With Refresh, the lock is ignored and the highest version wins.", + args: args{ + pkgs: append(familyVersions("v1.0.0", "v1.1.0"), + regPkg{repo: provA, version: "v0.1.0", digest: digestN(1), body: dependsOn(family, ">=v1.0.0")}, + ), + deps: []v1alpha1.Dependency{*xpkgDep(provA, "v0.1.0")}, + opts: ResolveOptions{Refresh: true, Locked: map[string]smanager.LockedSource{ + "xpkg://" + family: {Version: "v1.0.0"}, + }}, + }, + want: want{versions: map[string]string{provA: "v0.1.0", family: "v1.1.0"}}, + }, + "LockedVersionNoLongerAllowed": { + reason: "A locked version that violates the requirements is replaced.", + args: args{ + pkgs: append(familyVersions("v1.0.0", "v1.1.0"), + regPkg{repo: provA, version: "v0.1.0", digest: digestN(1), body: dependsOn(family, ">=v1.1.0")}, + ), + deps: []v1alpha1.Dependency{*xpkgDep(provA, "v0.1.0")}, + opts: ResolveOptions{Locked: map[string]smanager.LockedSource{ + "xpkg://" + family: {Version: "v1.0.0"}, + }}, + }, + want: want{versions: map[string]string{provA: "v0.1.0", family: "v1.1.0"}}, + }, + "ParentVersionChangeDropsTransitive": { + reason: "When a package is moved to another version, the packages only the old version needed are dropped.", + args: args{ + // provA is first chosen at v0.2.0 because of provB. Once provC + // is found, which pins it below v0.2.0, it moves to v0.1.0. + pkgs: []regPkg{ + {repo: provA, version: "v0.1.0", digest: digestN(1), body: dependsOn(libOld, "v1.0.0")}, + {repo: provA, version: "v0.2.0", digest: digestN(2), body: dependsOn(libNew, "v1.0.0")}, + {repo: libOld, version: "v1.0.0", digest: digestN(3), body: providerPackageYAML}, + {repo: libNew, version: "v1.0.0", digest: digestN(4), body: providerPackageYAML}, + {repo: provB, version: "v0.1.0", digest: digestN(5), body: dependsOn(provA, ">=v0.1.0")}, + {repo: provC, version: "v0.1.0", digest: digestN(6), body: dependsOn(provA, "=v2.0.0")}, + {repo: provA, version: "v2.0.0", digest: digestN(2), body: dependsOn(provB, "=v2.0.0")}, + {repo: provC, version: "v0.1.0", digest: digestN(5), body: dependsOn(provA, ">=v1.0.0")}, + }, + deps: []v1alpha1.Dependency{*xpkgDep(provC, "v0.1.0")}, + }, + want: want{err: cmpopts.AnyError, errMsgs: []string{"does not converge"}}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + m, _ := newProjectManager(t, registry(t, tc.args.pkgs...), tc.args.deps) + + sources, warnings, err := m.Resolve(context.Background(), tc.args.opts) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nResolve(...): -want err, +got err:\n%s", tc.reason, diff) + } + if tc.want.err != nil { + for _, s := range tc.want.errMsgs { + if !strings.Contains(err.Error(), s) { + t.Errorf("%s\nResolve(...): error %q does not contain %q", tc.reason, err, s) + } + } + return + } + + got := make(map[string]string) + for key, v := range versionsOf(sources) { + got[strings.TrimPrefix(key, "xpkg://")] = v + } + if diff := cmp.Diff(tc.want.versions, got); diff != "" { + t.Errorf("%s\nResolve(...): -want versions, +got versions:\n%s", tc.reason, diff) + } + if diff := cmp.Diff(tc.want.warnings, warnings, cmpopts.EquateEmpty()); diff != "" { + t.Errorf("%s\nResolve(...): -want warnings, +got warnings:\n%s", tc.reason, diff) + } + }) + } +} + +// xpkgDepRepo is a package that depends on a package that the project does not. +const xpkgDepRepo = "xpkg.example/prov-d" + +func TestManager_Resolve_Deterministic(t *testing.T) { + const ( + provA = "xpkg.example/prov-a" + provB = "xpkg.example/prov-b" + provC = "xpkg.example/prov-c" + family = "xpkg.example/family" + ) + + pkgs := make([]regPkg, 0, 8) + pkgs = append(pkgs, + regPkg{repo: provA, version: "v0.1.0", digest: digestN(1), body: dependsOn(family, ">=v1.0.0")}, + regPkg{repo: provB, version: "v0.1.0", digest: digestN(2), body: dependsOn(family, "=v0.1.0")}, + ) + for i, v := range []string{"v1.0.0", "v1.1.0", "v1.2.0", "v1.3.0", "v2.0.0"} { + pkgs = append(pkgs, regPkg{repo: family, version: v, digest: digestN(100 + i), body: providerPackageYAML}) + } + + type args struct { + pkgs []regPkg + deps []v1alpha1.Dependency + // runs is how many times to resolve against a registry with jittery + // latency. + runs int + } + type want struct { + // pins are the pins that every run must produce, by source key. + pins map[string]string + err error + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "ConcurrentFetchOrder": { + reason: "Resolving yields the same pins however fetches interleave, and picks the highest version all requirements allow.", + args: args{ + pkgs: pkgs, + deps: []v1alpha1.Dependency{*xpkgDep(provC, "v0.1.0"), *xpkgDep(provB, "v0.1.0")}, + runs: 20, + }, + want: want{pins: map[string]string{ + "xpkg://" + provA: provA + "@" + digestN(1), + "xpkg://" + provB: provB + "@" + digestN(2), + "xpkg://" + provC: provC + "@" + digestN(3), + "xpkg://" + family: family + "@" + digestN(102), + }}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + for i := range tc.args.runs { + fc := registry(t, tc.args.pkgs...) + fc.jitter = true + m, _ := newProjectManager(t, fc, tc.args.deps) + + sources, _, err := m.Resolve(context.Background(), ResolveOptions{}) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nResolve(...) run %d: -want err, +got err:\n%s", tc.reason, i, diff) + } + if diff := cmp.Diff(tc.want.pins, pinsOf(t, sources)); diff != "" { + t.Fatalf("%s\nResolve(...) run %d: -want pins, +got pins:\n%s", tc.reason, i, diff) + } + } + }) + } +} + +// fakeCloner is a git.Cloner that records the ref it was asked to clone. +type fakeCloner struct { + refs []string +} + +func (c *fakeCloner) CloneRepository(_ storage.Storer, fs billy.Filesystem, _ git.AuthProvider, opts git.CloneOptions) (*plumbing.Reference, error) { + c.refs = append(c.refs, opts.RefName) + f, err := fs.Create(opts.Path + "/crd.yaml") + if err != nil { + return nil, err + } + if _, err := f.Write([]byte(configurationPackageYAML)); err != nil { + return nil, err + } + if err := f.Close(); err != nil { + return nil, err + } + return plumbing.NewHashReference("refs/heads/main", plumbing.NewHash(strings.Repeat("a", 40))), nil +} + +func TestManager_Resolve_LockedGit(t *testing.T) { + sha := strings.Repeat("b", 40) + dep := v1alpha1.Dependency{ + Type: v1alpha1.DependencyTypeCRD, + Git: &v1alpha1.GitDependency{Repository: "https://example.com/r.git", Ref: "main", Path: "crds"}, + } + locked := map[string]smanager.LockedSource{"git://https://example.com/r.git/crds": {Pin: sha}} + + type args struct { + opts ResolveOptions + } + type want struct { + // ref is the single ref the cloner must be asked for. + ref string + err error + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "Locked": { + reason: "The locked commit is cloned instead of the branch.", + args: args{opts: ResolveOptions{Locked: locked}}, + want: want{ref: sha}, + }, + "Refresh": { + reason: "With Refresh, the branch is cloned.", + args: args{opts: ResolveOptions{Refresh: true, Locked: locked}}, + want: want{ref: "refs/heads/main"}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + m, _ := newProjectManager(t, registry(t), []v1alpha1.Dependency{dep}) + cloner := &fakeCloner{} + m.gitCloner = cloner + + sources, _, err := m.Resolve(context.Background(), tc.args.opts) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nResolve(...): -want err, +got err:\n%s", tc.reason, diff) + } + if _, err := sources[0].Hash(context.Background()); err != nil { + t.Fatal(err) + } + if diff := cmp.Diff([]string{tc.want.ref}, cloner.refs); diff != "" { + t.Errorf("%s\nResolve(...): -want cloned refs, +got cloned refs:\n%s", tc.reason, diff) + } + if diff := cmp.Diff(dep.Git.Ref, sources[0].(smanager.Pinned).Version()); diff != "" { + t.Errorf("%s\nResolve(...): -want version, +got version:\n%s", tc.reason, diff) + } + }) + } +} + +func TestManager_FetchPinned(t *testing.T) { + const prov = "xpkg.example/prov-a" + + sha := strings.Repeat("c", 40) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(configurationPackageYAML)) + })) + t.Cleanup(srv.Close) + + httpDep := v1alpha1.Dependency{Type: v1alpha1.DependencyTypeCRD, HTTP: &v1alpha1.HTTPDependency{URL: srv.URL + "/crd.yaml"}} + gitDep := v1alpha1.Dependency{ + Type: v1alpha1.DependencyTypeCRD, + Git: &v1alpha1.GitDependency{Repository: "https://example.com/r.git", Ref: "main", Path: "crds"}, + } + + // The pin of the http source is the hash of its content. + m, _ := newProjectManager(t, registry(t), []v1alpha1.Dependency{httpDep}) + httpSrc, err := m.source(httpDep) if err != nil { t.Fatal(err) } - var gotProj v1alpha1.Project - if err := yaml.Unmarshal(gotBytes, &gotProj); err != nil { + httpHash, err := httpSrc.Hash(context.Background()) + if err != nil { t.Fatal(err) } - if diff := cmp.Diff(wantDeps, gotProj.Spec.Dependencies); diff != "" { - t.Errorf("on-disk project deps (-want +got):\n%s", diff) + + type args struct { + deps []v1alpha1.Dependency + key string + entry smanager.LockedSource + } + type want struct { + // pin is the pin of the returned source. + pin string + err error + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "XpkgByDigest": { + reason: "A package is fetched by the digest in the lock, not by a tag.", + args: args{ + key: "xpkg://" + prov, + entry: smanager.LockedSource{Pin: prov + "@" + digestN(1), Version: "v0.1.0"}, + }, + want: want{pin: prov + "@" + digestN(1)}, + }, + "XpkgMissing": { + reason: "A package that cannot be fetched is an error.", + args: args{ + key: "xpkg://" + prov, + entry: smanager.LockedSource{Pin: prov + "@" + digestN(2)}, + }, + want: want{err: cmpopts.AnyError}, + }, + "GitAtSHA": { + reason: "A git dependency is cloned at the locked commit.", + args: args{ + deps: []v1alpha1.Dependency{gitDep}, + key: "git://https://example.com/r.git/crds", + entry: smanager.LockedSource{Pin: sha}, + }, + want: want{pin: sha}, + }, + "HTTPHashMatches": { + reason: "An http dependency whose content is unchanged is returned.", + args: args{ + deps: []v1alpha1.Dependency{httpDep}, + key: "http://" + srv.URL + "/crd.yaml", + entry: smanager.LockedSource{Pin: httpHash}, + }, + want: want{pin: httpHash}, + }, + "HTTPHashMismatch": { + reason: "An http dependency whose content changed is an error.", + args: args{ + deps: []v1alpha1.Dependency{httpDep}, + key: "http://" + srv.URL + "/crd.yaml", + entry: smanager.LockedSource{Pin: "not-the-hash"}, + }, + want: want{err: cmpopts.AnyError}, + }, + "NotInProject": { + reason: "A source that no dependency provides is an error.", + args: args{ + deps: []v1alpha1.Dependency{httpDep}, + key: "git://https://example.com/other.git", + entry: smanager.LockedSource{Pin: sha}, + }, + want: want{err: cmpopts.AnyError}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + fc := registry(t, regPkg{repo: prov, version: "v0.1.0", digest: digestN(1), body: providerPackageYAML}) + fc.packages[prov+"@"+digestN(1)] = fc.packages[prov+":v0.1.0"] + m, _ := newProjectManager(t, fc, tc.args.deps) + cloner := &fakeCloner{} + m.gitCloner = cloner + + src, err := m.FetchPinned(context.Background(), tc.args.key, tc.args.entry) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nFetchPinned(...): -want err, +got err:\n%s", tc.reason, diff) + } + if tc.want.err != nil { + return + } + if diff := cmp.Diff(tc.args.key, src.Key()); diff != "" { + t.Errorf("%s\nFetchPinned(...): -want key, +got key:\n%s", tc.reason, diff) + } + + // A git source reports the commit it cloned; clone it first. + if _, err := src.Resources(context.Background()); err != nil { + t.Fatal(err) + } + pin, err := src.(smanager.Pinned).Pin(context.Background()) + if err != nil { + t.Fatal(err) + } + if strings.HasPrefix(tc.args.key, "git://") { + // The fake cloner reports its own commit; what matters is that the + // pin was cloned, and that the version is still the branch. + if diff := cmp.Diff([]string{tc.want.pin}, cloner.refs); diff != "" { + t.Errorf("%s\nFetchPinned(...): -want cloned refs, +got cloned refs:\n%s", tc.reason, diff) + } + if diff := cmp.Diff(gitDep.Git.Ref, src.(smanager.Pinned).Version()); diff != "" { + t.Errorf("%s\nFetchPinned(...): -want version, +got version:\n%s", tc.reason, diff) + } + return + } + if diff := cmp.Diff(tc.want.pin, pin); diff != "" { + t.Errorf("%s\nFetchPinned(...): -want pin, +got pin:\n%s", tc.reason, diff) + } + }) + } +} + +func TestNewCachedManager(t *testing.T) { + // t.Setenv makes these tests unsuitable for t.Parallel. + explicit := filepath.Join(t.TempDir(), "explicit") + + type args struct { + cacheDir string + projFile string + } + type want struct { + // projFile is the project file name the manager uses. + projFile string + // cacheDir returns the directory that must have been created. + cacheDir func() string + err error + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "ExplicitCacheDir": { + reason: "An explicit cache directory should be created and used instead of the default.", + args: args{cacheDir: explicit, projFile: "crossplane-project.yaml"}, + want: want{projFile: "crossplane-project.yaml", cacheDir: func() string { return explicit }}, + }, + "DefaultCacheDir": { + reason: "An empty cache directory should fall back to DefaultCacheDir.", + args: args{projFile: "crossplane-project.yaml"}, + want: want{projFile: "crossplane-project.yaml", cacheDir: DefaultCacheDir}, + }, + "ProjectFileBaseName": { + reason: "Only the base name of the project file should be used, because the project filesystem is rooted at the project directory.", + args: args{cacheDir: explicit, projFile: "/some/dir/my-project.yaml"}, + want: want{projFile: "my-project.yaml", cacheDir: func() string { return explicit }}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + // Point the user cache dir at a temp dir, so DefaultCacheDir + // doesn't touch the real cache. + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_CACHE_HOME", filepath.Join(home, ".cache")) + + m, err := NewCachedManager(&v1alpha1.Project{}, afero.NewMemMapFs(), tc.args.projFile, tc.args.cacheDir) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nNewCachedManager(...): -want err, +got err:\n%s", tc.reason, diff) + } + + if diff := cmp.Diff(tc.want.projFile, m.projFile); diff != "" { + t.Errorf("%s\nNewCachedManager(...): -want projFile, +got projFile:\n%s", tc.reason, diff) + } + + dir := tc.want.cacheDir() + if ok, err := afero.DirExists(afero.NewOsFs(), dir); err != nil || !ok { + t.Errorf("%s\nNewCachedManager(...): cache dir %q not created (err: %v)", tc.reason, dir, err) + } + }) } } diff --git a/internal/project/build.go b/internal/project/build.go index 81de2ca7..de7d7380 100644 --- a/internal/project/build.go +++ b/internal/project/build.go @@ -86,8 +86,10 @@ func BuildWithMaxConcurrency(n uint) BuilderOption { } } -// BuildWithSchemaManager sets the schema manager that will be used to generate -// language-specific schemas from XRDs before building functions. +// BuildWithSchemaManager sets the schema manager that will be used to verify +// that the project's generated schemas match the schema lock before building +// functions. Verification recreates missing schema files but never changes the +// lock. func BuildWithSchemaManager(m *manager.Manager) BuilderOption { return func(b *Builder) { b.schemaManager = m @@ -95,8 +97,7 @@ func BuildWithSchemaManager(m *manager.Manager) BuilderOption { } // BuildWithDependencyManager sets the dependency manager that will be used to -// ensure schemas are present for the project's declared dependencies before -// building functions. +// find the project's declared dependencies when verifying schemas. func BuildWithDependencyManager(m *dependency.Manager) BuilderOption { return func(b *Builder) { b.dependencyManager = m @@ -172,6 +173,26 @@ type Builder struct { baseImageCacheDir string } +// verifySchemas verifies the schemas of the project's APIs and dependencies +// against the schema lock. If there is no lock yet, it runs a schema update to +// create one. +func (b *Builder) verifySchemas(ctx context.Context, project *devv1alpha1.Project, projectFS afero.Fs, ch async.EventChannel) error { + ok, err := b.schemaManager.HasLock() + if err != nil { + return err + } + if !ok { + ch.SendEvent("No schema lock; running schema update", async.EventStatusStarted) + if _, err := UpdateSchemas(ctx, b.schemaManager, b.dependencyManager, project, projectFS, ch, false); err != nil { + ch.SendEvent("No schema lock; running schema update", async.EventStatusFailure) + return err + } + ch.SendEvent("No schema lock; running schema update", async.EventStatusSuccess) + return nil + } + return VerifySchemas(ctx, b.schemaManager, b.dependencyManager, project, projectFS, false) +} + // Build builds a project into a set of packages. It returns a map containing // images that were built from the project. The returned map will always include // one image with the ConfigurationTag, which is the configuration package built @@ -214,15 +235,14 @@ func (b *Builder) Build(ctx context.Context, project *devv1alpha1.Project, proje return nil, errors.Wrap(err, "failed to resolve functions") } - apisSource := projectFS - apiExcludes := []string{ - project.Spec.Paths.Examples, - project.Spec.Paths.Functions, - project.Spec.Paths.Operations, - } - if project.Spec.Paths.APIs != "/" { - apisSource = afero.NewBasePathFs(projectFS, project.Spec.Paths.APIs) - apiExcludes = []string{} + apisSource := apisFS(project, projectFS) + apiExcludes := []string{} + if project.Spec.Paths.APIs == "/" { + apiExcludes = []string{ + project.Spec.Paths.Examples, + project.Spec.Paths.Functions, + project.Spec.Paths.Operations, + } } // Not all projects have operations; ignore them if not present. @@ -262,23 +282,17 @@ func (b *Builder) Build(ctx context.Context, project *devv1alpha1.Project, proje } o.eventCh.SendEvent("Collecting resources", async.EventStatusSuccess) - // Generate schemas for declared dependencies. The dependency manager - // short-circuits sources whose recorded version still matches, so this is - // cheap on the steady-state path. - if b.dependencyManager != nil { - if err := b.dependencyManager.AddAll(ctx, o.eventCh); err != nil { - return nil, errors.Wrap(err, "failed to generate dependency schemas") - } - } - - // Generate language-specific schemas from XRDs. + // Check that the generated schemas match the schema lock, recreating files + // that are missing. Unless there is no lock yet, a build never changes the + // lock or the schemas' content, so it is reproducible; `crossplane schema + // update` does that. if b.schemaManager != nil { - o.eventCh.SendEvent("Generating schemas", async.EventStatusStarted) - if _, err := b.schemaManager.Generate(ctx, manager.NewFSSource(project.Spec.Paths.APIs, apisSource)); err != nil { - o.eventCh.SendEvent("Generating schemas", async.EventStatusFailure) - return nil, errors.Wrap(err, "failed to generate schemas") + o.eventCh.SendEvent("Verifying schemas", async.EventStatusStarted) + if err := b.verifySchemas(ctx, project, projectFS, o.eventCh); err != nil { + o.eventCh.SendEvent("Verifying schemas", async.EventStatusFailure) + return nil, errors.Wrap(err, "schemas are out of date") } - o.eventCh.SendEvent("Generating schemas", async.EventStatusSuccess) + o.eventCh.SendEvent("Verifying schemas", async.EventStatusSuccess) } // Build the resolved functions. diff --git a/internal/project/build_test.go b/internal/project/build_test.go index a027bb28..d6943224 100644 --- a/internal/project/build_test.go +++ b/internal/project/build_test.go @@ -18,8 +18,11 @@ package project import ( "compress/gzip" + "context" "fmt" "io" + "net/http" + "net/http/httptest" "os" "path/filepath" "strings" @@ -35,10 +38,15 @@ import ( "github.com/spf13/afero" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/crossplane/crossplane-runtime/v2/pkg/errors" "github.com/crossplane/crossplane-runtime/v2/pkg/xpkg" devv1alpha1 "github.com/crossplane/cli/v2/apis/dev/v1alpha1" + "github.com/crossplane/cli/v2/internal/dependency" "github.com/crossplane/cli/v2/internal/project/functions" + "github.com/crossplane/cli/v2/internal/schemas/generator" + "github.com/crossplane/cli/v2/internal/schemas/manager" + "github.com/crossplane/cli/v2/internal/schemas/runner" ) // xrdYAML returns an XRD manifest for a resource with the given group/kind. @@ -297,6 +305,318 @@ func TestBuilderDependsOn(t *testing.T) { } } +// fakeSchemaGenerator generates one fixed file per source, whatever the source +// contains, so tests can tell generated files from the lock's point of view. +type fakeSchemaGenerator struct{} + +func (fakeSchemaGenerator) Language() string { return "fake" } + +func (fakeSchemaGenerator) GenerateFromCRD(_ context.Context, _ afero.Fs, _ runner.SchemaRunner) (afero.Fs, error) { + out := afero.NewMemMapFs() + return out, afero.WriteFile(out, "schema.txt", []byte("generated"), 0o644) +} + +func (fakeSchemaGenerator) GenerateFromOpenAPI(_ context.Context, _ afero.Fs, _ runner.SchemaRunner) (afero.Fs, error) { + return nil, nil +} + +// failingClient is an xpkg client that fails the test on any call, to prove a +// build doesn't touch the registry. +type failingClient struct { + t *testing.T +} + +func (c failingClient) Get(_ context.Context, ref string, _ ...xpkg.GetOption) (*xpkg.Package, error) { + c.t.Errorf("unexpected Get(%q)", ref) + return nil, errors.New("unexpected registry access") +} + +func (c failingClient) ListVersions(_ context.Context, repo string, _ ...xpkg.GetOption) ([]string, error) { + c.t.Errorf("unexpected ListVersions(%q)", repo) + return nil, errors.New("unexpected registry access") +} + +// perFileGenerator generates one file per input file, named after it, so tests +// can tell the files of different sources apart. +type perFileGenerator struct{} + +func (perFileGenerator) Language() string { return "fake" } + +func (perFileGenerator) GenerateFromCRD(_ context.Context, from afero.Fs, _ runner.SchemaRunner) (afero.Fs, error) { + out := afero.NewMemMapFs() + err := afero.Walk(from, ".", func(p string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() { + return err + } + return afero.WriteFile(out, filepath.Base(p)+".txt", []byte("generated "+filepath.Base(p)), 0o644) + }) + return out, err +} + +func (perFileGenerator) GenerateFromOpenAPI(_ context.Context, _ afero.Fs, _ runner.SchemaRunner) (afero.Fs, error) { + return nil, nil +} + +func TestBuilderBuild_VerifiesSchemas(t *testing.T) { + t.Parallel() + + const schemaFile = "fake/schema.txt" + + type args struct { + // lock seeds the schema lock, as `crossplane schema update` would, before + // the build. + lock bool + // change modifies the project or the schemas between update and build. + change func(t *testing.T, proj *devv1alpha1.Project, projFS, schemaFS afero.Fs) + } + type want struct { + err error + // errMsg is a substring of the error message, which is the behavior + // under test: it tells the user which command to run. + errMsg string + // lockWritten is whether the build creates the lock. Otherwise it must + // leave the lock alone. + lockWritten bool + // schema is the content of the generated file afterwards, "" if absent. + schema string + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "MatchesLock": { + reason: "A build whose schemas match the lock succeeds and leaves them alone.", + args: args{lock: true}, + want: want{schema: "generated"}, + }, + "MissingFileIsRecreated": { + reason: "A deleted generated file is restored, because that doesn't change what is locked.", + args: args{lock: true, change: func(t *testing.T, _ *devv1alpha1.Project, _, schemaFS afero.Fs) { + t.Helper() + if err := schemaFS.Remove(schemaFile); err != nil { + t.Fatal(err) + } + }}, + want: want{schema: "generated"}, + }, + "NoLock": { + reason: "Without a lock the build runs a schema update, which writes the lock and the schemas.", + want: want{schema: "generated", lockWritten: true}, + }, + "DependenciesChangedSinceLock": { + reason: "A dependency added after the lock was written fails the build without resolving it, because a build never touches the registry.", + args: args{lock: true, change: func(t *testing.T, proj *devv1alpha1.Project, _, _ afero.Fs) { + t.Helper() + proj.Spec.Dependencies = append(proj.Spec.Dependencies, *xpkgDependency("xpkg.example/provider", ">=v1")) + }}, + want: want{err: cmpopts.AnyError, errMsg: "crossplane schema update", schema: "generated"}, + }, + "XRDChangedSinceLock": { + reason: "An XRD edited after the lock was written makes the lock stale; the build must not silently regenerate.", + args: args{lock: true, change: func(t *testing.T, _ *devv1alpha1.Project, projFS, _ afero.Fs) { + t.Helper() + if err := afero.WriteFile(projFS, "apis/db.yaml", []byte(xrdYAML("acme.example.com", "xdatabases", "xdatabase", "XDatabase")+"# changed\n"), 0o644); err != nil { + t.Fatal(err) + } + }}, + want: want{err: cmpopts.AnyError, errMsg: "crossplane schema update", schema: "generated"}, + }, + "ModifiedFile": { + reason: "A hand-edited generated file is not overwritten by a build.", + args: args{lock: true, change: func(t *testing.T, _ *devv1alpha1.Project, _, schemaFS afero.Fs) { + t.Helper() + if err := afero.WriteFile(schemaFS, schemaFile, []byte("edited"), 0o644); err != nil { + t.Fatal(err) + } + }}, + want: want{err: cmpopts.AnyError, errMsg: "crossplane schema generate", schema: "edited"}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + projFS := afero.NewMemMapFs() + writeProject(t, projFS, + map[string]string{ + "db.yaml": xrdYAML("acme.example.com", "xdatabases", "xdatabase", "XDatabase"), + "db-comp.yaml": compositionYAML("xdatabase-comp", "acme.example.com", "XDatabase"), + }, nil) + proj := &devv1alpha1.Project{ + ObjectMeta: metav1.ObjectMeta{Name: "test-project"}, + Spec: devv1alpha1.ProjectSpec{Repository: "xpkg.crossplane.io/example/test"}, + } + proj.Default() + + schemaFS := afero.NewMemMapFs() + schemas := manager.New(schemaFS, []generator.Interface{fakeSchemaGenerator{}}, nil) + if tc.args.lock { + if err := schemas.Add(t.Context(), APISchemaSource(proj, projFS)); err != nil { + t.Fatal(err) + } + depsHash, err := dependency.DependenciesHash(proj) + if err != nil { + t.Fatal(err) + } + if err := schemas.SetDependenciesHash(depsHash); err != nil { + t.Fatal(err) + } + } + if tc.args.change != nil { + tc.args.change(t, proj, projFS, schemaFS) + } + lockBefore, _ := afero.ReadFile(schemaFS, ".lock.json") + + b := NewBuilder( + BuildWithFunctionIdentifier(functions.FakeIdentifier), + BuildWithSchemaManager(schemas), + BuildWithDependencyManager(dependency.NewManager(proj, projFS, dependency.WithXpkgClient(failingClient{t: t}))), + ) + _, err := b.Build(t.Context(), proj, projFS) + + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("%s\nBuild(...): -want err, +got err:\n%s", tc.reason, diff) + } + if tc.want.errMsg != "" && (err == nil || !strings.Contains(err.Error(), tc.want.errMsg)) { + t.Errorf("%s\nBuild(...): err = %v, want error containing %q", tc.reason, err, tc.want.errMsg) + } + got, _ := afero.ReadFile(schemaFS, schemaFile) + if diff := cmp.Diff(tc.want.schema, string(got)); diff != "" { + t.Errorf("%s\ngenerated file (-want +got):\n%s", tc.reason, diff) + } + lockAfter, _ := afero.ReadFile(schemaFS, ".lock.json") + if tc.want.lockWritten { + if len(lockBefore) != 0 || len(lockAfter) == 0 { + t.Errorf("%s\nBuild should have written the lock; before %d bytes, after %d bytes", tc.reason, len(lockBefore), len(lockAfter)) + } + return + } + if diff := cmp.Diff(string(lockBefore), string(lockAfter)); diff != "" { + t.Errorf("%s\nBuild must never change the lock (-before +after):\n%s", tc.reason, diff) + } + }) + } +} + +func TestBuilderBuild_RecreatesDependencySchemas(t *testing.T) { + t.Parallel() + + const crd = `apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: widgets.example.org +spec: + group: example.org + names: + kind: Widget + plural: widgets + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object +` + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, crd) + })) + t.Cleanup(srv.Close) + + type args struct { + // remove is the generated file deleted after the update. + remove string + } + type want struct { + // files are the generated files that exist after the build. + files []string + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "MissingDependencyFile": { + reason: "A deleted dependency file is recreated from the pin in the lock, without resolving the dependency again.", + args: args{remove: "fake/crd.yaml.txt"}, + want: want{files: []string{"fake/crd.yaml.txt", "fake/db.yaml.txt"}}, + }, + "MissingLocalFile": { + reason: "A deleted file of the project's own APIs is recreated from the project, without fetching any dependency.", + args: args{remove: "fake/db.yaml.txt"}, + want: want{files: []string{"fake/crd.yaml.txt", "fake/db.yaml.txt"}}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + projFS := afero.NewMemMapFs() + writeProject(t, projFS, + map[string]string{ + "db.yaml": xrdYAML("acme.example.com", "xdatabases", "xdatabase", "XDatabase"), + "db-comp.yaml": compositionYAML("xdatabase-comp", "acme.example.com", "XDatabase"), + }, nil) + proj := &devv1alpha1.Project{ + ObjectMeta: metav1.ObjectMeta{Name: "test-project"}, + Spec: devv1alpha1.ProjectSpec{ + Repository: "xpkg.crossplane.io/example/test", + Dependencies: []devv1alpha1.Dependency{{ + Type: devv1alpha1.DependencyTypeCRD, + HTTP: &devv1alpha1.HTTPDependency{URL: srv.URL + "/crd.yaml"}, + }}, + }, + } + proj.Default() + + schemaFS := afero.NewMemMapFs() + schemas := manager.New(schemaFS, []generator.Interface{perFileGenerator{}}, nil) + deps := dependency.NewManager(proj, projFS, dependency.WithXpkgClient(failingClient{t: t})) + + if _, err := UpdateSchemas(t.Context(), schemas, deps, proj, projFS, nil, false); err != nil { + t.Fatalf("%s\nUpdateSchemas(...): %v", tc.reason, err) + } + if err := schemaFS.Remove(tc.args.remove); err != nil { + t.Fatal(err) + } + + b := NewBuilder( + BuildWithFunctionIdentifier(functions.FakeIdentifier), + BuildWithSchemaManager(schemas), + BuildWithDependencyManager(deps), + ) + if _, err := b.Build(t.Context(), proj, projFS); err != nil { + t.Fatalf("%s\nBuild(...): %v", tc.reason, err) + } + + var got []string + for _, f := range []string{"fake/crd.yaml.txt", "fake/db.yaml.txt"} { + if ok, _ := afero.Exists(schemaFS, f); ok { + got = append(got, f) + } + } + if diff := cmp.Diff(tc.want.files, got); diff != "" { + t.Errorf("%s\ngenerated files (-want +got):\n%s", tc.reason, diff) + } + }) + } +} + +// xpkgDependency returns an xpkg dependency on the given package and version. +func xpkgDependency(pkg, version string) *devv1alpha1.Dependency { + return &devv1alpha1.Dependency{ + Type: devv1alpha1.DependencyTypeXpkg, + Xpkg: &devv1alpha1.XpkgDependency{APIVersion: "pkg.crossplane.io/v1", Kind: "Provider", Package: pkg, Version: version}, + } +} + func constructTag(repo, tag string) (name.Tag, error) { return name.NewTag(fmt.Sprintf("%s:%s", repo, tag)) } diff --git a/internal/project/projectfile/projectfile.go b/internal/project/projectfile/projectfile.go index b79fc5a3..a93df132 100644 --- a/internal/project/projectfile/projectfile.go +++ b/internal/project/projectfile/projectfile.go @@ -18,6 +18,8 @@ limitations under the License. package projectfile import ( + "path/filepath" + "github.com/spf13/afero" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/yaml" @@ -51,6 +53,21 @@ func IsProjectFile(fs afero.Fs, filePath string) (bool, error) { return tm.APIVersion == APIVersion && tm.Kind == Kind, nil } +// Load parses the project file at the given OS path. It returns the project, +// with defaults applied, and a filesystem rooted at the project directory. +func Load(path string) (*v1alpha1.Project, afero.Fs, error) { + abs, err := filepath.Abs(path) + if err != nil { + return nil, nil, err + } + projFS := afero.NewBasePathFs(afero.NewOsFs(), filepath.Dir(abs)) + proj, err := Parse(projFS, filepath.Base(abs)) + if err != nil { + return nil, nil, err + } + return proj, projFS, nil +} + // Parse parses and validates the project file, returning a Project with // defaults applied. func Parse(projFS afero.Fs, projFilePath string) (*v1alpha1.Project, error) { diff --git a/internal/project/projectfile/projectfile_test.go b/internal/project/projectfile/projectfile_test.go index 431051d4..9f17eeb6 100644 --- a/internal/project/projectfile/projectfile_test.go +++ b/internal/project/projectfile/projectfile_test.go @@ -18,9 +18,12 @@ package projectfile import ( "os" + "path/filepath" + "strings" "testing" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/spf13/afero" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/yaml" @@ -370,3 +373,100 @@ func TestUpdate(t *testing.T) { t.Errorf("on-disk contents (-want +got):\n%s", diff) } } + +func TestLoad(t *testing.T) { + // t.Chdir makes these tests unsuitable for t.Parallel. + const validProject = `apiVersion: dev.crossplane.io/v1alpha1 +kind: Project +metadata: + name: test +spec: + repository: xpkg.crossplane.io/foo/bar +` + + type args struct { + files map[string]string // relative to the temp dir + chdir string // relative to the temp dir + path string // absolute if it starts with "@"; the temp dir is prepended + } + type want struct { + name string + rooted string // file that must exist in the returned fs + err error + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "AbsolutePath": { + reason: "An absolute path should load the project and root the fs at its directory.", + args: args{ + files: map[string]string{"proj/crossplane-project.yaml": validProject}, + path: "@proj/crossplane-project.yaml", + }, + want: want{name: "test", rooted: "crossplane-project.yaml"}, + }, + "RelativePath": { + reason: "A relative path should resolve against the working directory, and the fs should be rooted at the project directory.", + args: args{ + files: map[string]string{"proj/crossplane-project.yaml": validProject, "proj/other.txt": ""}, + chdir: ".", + path: "proj/crossplane-project.yaml", + }, + want: want{name: "test", rooted: "other.txt"}, + }, + "MissingFile": { + reason: "A missing project file should return an error.", + args: args{path: "@crossplane-project.yaml"}, + want: want{err: cmpopts.AnyError}, + }, + "InvalidProject": { + reason: "A file that isn't a valid project should return an error.", + args: args{ + files: map[string]string{"crossplane-project.yaml": `apiVersion: foo.example.com/v1 +kind: Other +`}, + path: "@crossplane-project.yaml", + }, + want: want{err: cmpopts.AnyError}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + dir := t.TempDir() + for p, content := range tc.args.files { + full := filepath.Join(dir, p) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + if tc.args.chdir != "" { + t.Chdir(filepath.Join(dir, tc.args.chdir)) + } + path := tc.args.path + if strings.HasPrefix(path, "@") { + path = filepath.Join(dir, path[1:]) + } + + proj, projFS, err := Load(path) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("\n%s\nLoad(...): -want err, +got err:\n%s", tc.reason, diff) + } + if err != nil { + return + } + if diff := cmp.Diff(tc.want.name, proj.Name); diff != "" { + t.Errorf("\n%s\nLoad(...): -want name, +got name:\n%s", tc.reason, diff) + } + if ok, err := afero.Exists(projFS, tc.want.rooted); err != nil || !ok { + t.Errorf("\n%s\nLoad(...): fs not rooted at project dir, %q missing (err: %v)", tc.reason, tc.want.rooted, err) + } + }) + } +} diff --git a/internal/project/schemas.go b/internal/project/schemas.go new file mode 100644 index 00000000..796d41b2 --- /dev/null +++ b/internal/project/schemas.go @@ -0,0 +1,162 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package project + +import ( + "context" + + "github.com/spf13/afero" + "golang.org/x/sync/errgroup" + + "github.com/crossplane/crossplane-runtime/v2/pkg/errors" + + "github.com/crossplane/cli/v2/apis/dev/v1alpha1" + "github.com/crossplane/cli/v2/internal/async" + "github.com/crossplane/cli/v2/internal/config" + "github.com/crossplane/cli/v2/internal/dependency" + "github.com/crossplane/cli/v2/internal/schemas/generator" + "github.com/crossplane/cli/v2/internal/schemas/manager" + "github.com/crossplane/cli/v2/internal/schemas/runner" +) + +// NewSchemaManager returns the schema manager for a project. It writes to the +// project's schemas path and generates the languages the project asks for. +func NewSchemaManager(proj *v1alpha1.Project, projFS afero.Fs, features config.Features) *manager.Manager { + generators := generator.Filter( + generator.AllLanguages( + generator.WithGoModelAccessors(features.GenerateGoModelAccessors), + generator.WithGoRuntimeObjects(features.GenerateGoRuntimeObjects), + ), + proj.Spec.Schemas.GetLanguages(), + ) + return manager.New( + afero.NewBasePathFs(projFS, proj.Spec.Paths.Schemas), + generators, + runner.NewRealSchemaRunner(runner.WithImageConfig(proj.Spec.ImageConfigs)), + manager.WithFlags(manager.FlagsFor(proj.Spec.Schemas, features.GenerateGoRuntimeObjects)), + ) +} + +// apisFS returns the filesystem holding the project's own APIs. +func apisFS(proj *v1alpha1.Project, projFS afero.Fs) afero.Fs { + if proj.Spec.Paths.APIs == "/" { + return projFS + } + return afero.NewBasePathFs(projFS, proj.Spec.Paths.APIs) +} + +// APISchemaSource returns the schema source for the project's own APIs, i.e. +// the XRDs under the project's APIs path. +func APISchemaSource(proj *v1alpha1.Project, projFS afero.Fs) manager.Source { + return manager.NewFSSource(proj.Spec.Paths.APIs, apisFS(proj, projFS)) +} + +// UpdateResult describes what UpdateSchemas found and removed. +type UpdateResult struct { + // Warnings are the requirements that the resolved dependencies do not meet. + Warnings []dependency.Warning + // Removed are the keys of locked sources that are not used any more. + Removed []string +} + +// UpdateSchemas resolves the project's dependencies, generates schemas for them +// and the project's own APIs, and records the result in the schema lock. It is +// the only place where versions are resolved. Unless refresh is set, versions +// in the lock are kept while they still satisfy the dependencies' constraints. +// An event for each source is sent to ch. +func UpdateSchemas(ctx context.Context, sm *manager.Manager, dm *dependency.Manager, proj *v1alpha1.Project, projFS afero.Fs, ch async.EventChannel, refresh bool) (*UpdateResult, error) { + if dm == nil { + return nil, errors.New("dependency manager is not configured") + } + + locked, err := sm.Locked() + if err != nil { + return nil, err + } + + ch.SendEvent("Resolving dependencies", async.EventStatusStarted) + sources, warnings, err := dm.Resolve(ctx, dependency.ResolveOptions{Refresh: refresh, Locked: locked}) + if err != nil { + ch.SendEvent("Resolving dependencies", async.EventStatusFailure) + return nil, err + } + ch.SendEvent("Resolving dependencies", async.EventStatusSuccess) + sources = append(sources, APISchemaSource(proj, projFS)) + + if err := addAll(ctx, sm, ch, sources); err != nil { + return nil, err + } + + // Anything in the lock that is not a live source any more is left over from + // a removed dependency or XRD. + res, err := sm.Reconcile(ctx, sources) + if err != nil { + return nil, err + } + + depsHash, err := dependency.DependenciesHash(proj) + if err != nil { + return nil, err + } + if err := sm.SetDependenciesHash(depsHash); err != nil { + return nil, err + } + return &UpdateResult{Warnings: warnings, Removed: res.Sources}, nil +} + +// addAll generates schemas for the given sources concurrently, sending an event +// for each to ch. +func addAll(ctx context.Context, sm *manager.Manager, ch async.EventChannel, sources []manager.Source) error { + eg, egCtx := errgroup.WithContext(ctx) + for _, src := range sources { + desc := "Generating schemas for " + src.Key() + eg.Go(func() error { + ch.SendEvent(desc, async.EventStatusStarted) + if err := sm.Add(egCtx, src); err != nil { + ch.SendEvent(desc, async.EventStatusFailure) + return errors.Wrapf(err, "cannot generate schemas for %s", src.Key()) + } + ch.SendEvent(desc, async.EventStatusSuccess) + return nil + }) + } + return eg.Wait() +} + +// VerifySchemas checks that the generated schemas match the schema lock, +// recreating files that are missing from the exact pins the lock records. It +// never resolves anything. If repair is set, files that were modified are +// restored too. dm is only used to recreate files, and may be nil. +func VerifySchemas(ctx context.Context, sm *manager.Manager, dm *dependency.Manager, proj *v1alpha1.Project, projFS afero.Fs, repair bool) error { + depsHash, err := dependency.DependenciesHash(proj) + if err != nil { + return err + } + + fetch := func(context.Context, string, manager.LockedSource) (manager.Source, error) { + return nil, errors.New("no dependency manager to fetch the locked sources") + } + if dm != nil { + fetch = dm.FetchPinned + } + + local := []manager.Source{APISchemaSource(proj, projFS)} + if repair { + return sm.Generate(ctx, local, depsHash, fetch) + } + return sm.Verify(ctx, local, depsHash, fetch) +} diff --git a/internal/schemas/manager/filter.go b/internal/schemas/manager/filter.go new file mode 100644 index 00000000..23af4b57 --- /dev/null +++ b/internal/schemas/manager/filter.go @@ -0,0 +1,159 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package manager + +import ( + "bufio" + "bytes" + "io" + "io/fs" + "path" + "path/filepath" + "strings" + + "github.com/spf13/afero" + utilyaml "k8s.io/apimachinery/pkg/util/yaml" + "sigs.k8s.io/yaml" + + "github.com/crossplane/crossplane-runtime/v2/pkg/errors" +) + +// openAPIGroupPrefix starts the file name of a Kubernetes OpenAPI spec for a +// named API group, e.g. apis__apps__v1_openapi.json. Specs for the core group +// (api__v1_openapi.json) and files with other names are never filtered. +const openAPIGroupPrefix = "apis__" + +// filterAPIGroups returns a filesystem containing only the resources of from +// whose API group matches one of the globs. With no globs it returns from +// unchanged. CRD and XRD files are filtered per YAML document, so a file that +// bundles several groups keeps only the matching ones. +func filterAPIGroups(from afero.Fs, sourceType SourceType, globs []string) (afero.Fs, error) { + if len(globs) == 0 || from == nil { + return from, nil + } + + out := afero.NewMemMapFs() + err := afero.Walk(from, ".", func(p string, info fs.FileInfo, err error) error { + if err != nil || info.IsDir() { + return err + } + content, err := afero.ReadFile(from, p) + if err != nil { + return err + } + + switch { + case sourceType == SourceTypeOpenAPI: + group, ok := openAPIGroup(filepath.Base(p)) + if ok && !matchesAny(globs, group) { + return nil + } + case isYAML(p): + content, err = filterYAMLDocuments(content, globs) + if err != nil { + return errors.Wrapf(err, "cannot filter %s", p) + } + if content == nil { + return nil + } + } + + if err := out.MkdirAll(filepath.Dir(p), 0o750); err != nil { + return err + } + return afero.WriteFile(out, p, content, 0o644) + }) + return out, err +} + +// filterYAMLDocuments keeps the documents of a YAML stream that are not +// resources with an API group, or whose group matches. It returns nil if no +// document is left, and the input unchanged if none was dropped. +func filterYAMLDocuments(content []byte, globs []string) ([]byte, error) { + var kept [][]byte + dropped := false + err := eachYAMLDocument(content, func(doc []byte) error { + // Documents that don't parse or have no spec.group aren't API + // resources. Keep them and leave complaining to the generators. + var res struct { + Spec struct { + Group string `json:"group"` + } `json:"spec"` + } + if err := yaml.Unmarshal(doc, &res); err == nil && res.Spec.Group != "" && !matchesAny(globs, res.Spec.Group) { + dropped = true + return nil + } + kept = append(kept, doc) + return nil + }) + if err != nil { + return nil, err + } + + switch { + case !dropped: + return content, nil + case len(kept) == 0: + return nil, nil + default: + return bytes.Join(kept, []byte("---\n")), nil + } +} + +// eachYAMLDocument calls fn with every document of a YAML stream, in order. It +// stops at the first error. +func eachYAMLDocument(content []byte, fn func(doc []byte) error) error { + reader := utilyaml.NewYAMLReader(bufio.NewReader(bytes.NewReader(content))) + for { + doc, err := reader.Read() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + if err := fn(doc); err != nil { + return err + } + } +} + +// openAPIGroup returns the API group encoded in an OpenAPI spec file name, and +// whether the name has one. +func openAPIGroup(name string) (string, bool) { + rest, ok := strings.CutPrefix(name, openAPIGroupPrefix) + if !ok { + return "", false + } + group, _, _ := strings.Cut(rest, "__") + return group, true +} + +func matchesAny(globs []string, group string) bool { + for _, g := range globs { + if ok, err := path.Match(g, group); err == nil && ok { + return true + } + } + return false +} + +func isYAML(p string) bool { + ext := strings.ToLower(filepath.Ext(p)) + return ext == ".yaml" || ext == ".yml" +} diff --git a/internal/schemas/manager/filter_test.go b/internal/schemas/manager/filter_test.go new file mode 100644 index 00000000..57302465 --- /dev/null +++ b/internal/schemas/manager/filter_test.go @@ -0,0 +1,190 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package manager + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/spf13/afero" +) + +const ( + storageXRD = `apiVersion: apiextensions.crossplane.io/v2 +kind: CompositeResourceDefinition +spec: + group: storage.example.org +` + networkXRD = `apiVersion: apiextensions.crossplane.io/v2 +kind: CompositeResourceDefinition +spec: + group: network.example.org +` +) + +func TestFilterAPIGroups(t *testing.T) { + t.Parallel() + + type args struct { + files map[string]string + sourceType SourceType + globs []string + } + type want struct { + files map[string]string + err error + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "NoGlobsKeepsEverything": { + reason: "Without globs nothing is filtered.", + args: args{ + files: map[string]string{"a.yaml": storageXRD, "b.yaml": networkXRD}, + globs: nil, + }, + want: want{files: map[string]string{"a.yaml": storageXRD, "b.yaml": networkXRD}}, + }, + "MatchingFilesAreKept": { + reason: "A file whose group matches a glob is kept byte for byte; others are dropped.", + args: args{ + files: map[string]string{"a.yaml": storageXRD, "b.yaml": networkXRD}, + sourceType: SourceTypeCRD, + globs: []string{"storage.*"}, + }, + want: want{files: map[string]string{"a.yaml": storageXRD}}, + }, + "AnyGlobMatches": { + reason: "Globs are alternatives.", + args: args{ + files: map[string]string{"a.yaml": storageXRD, "b.yaml": networkXRD}, + sourceType: SourceTypeCRD, + globs: []string{"nomatch", "*.example.org"}, + }, + want: want{files: map[string]string{"a.yaml": storageXRD, "b.yaml": networkXRD}}, + }, + "MultiDocumentFileIsSplit": { + reason: "A file bundling several groups keeps only the matching documents.", + args: args{ + files: map[string]string{"all.yaml": storageXRD + "---\n" + networkXRD}, + sourceType: SourceTypeCRD, + globs: []string{"network.example.org"}, + }, + want: want{files: map[string]string{"all.yaml": networkXRD}}, + }, + "NonResourceDocumentsAreKept": { + reason: "Files without a spec.group, such as a kustomization, are not API resources and are not filtered.", + args: args{ + files: map[string]string{"kustomization.yaml": "resources: []\n", "a.yaml": networkXRD}, + sourceType: SourceTypeCRD, + globs: []string{"storage.*"}, + }, + want: want{files: map[string]string{"kustomization.yaml": "resources: []\n"}}, + }, + "OpenAPIFilteredByFileNameGroup": { + reason: "Kubernetes OpenAPI specs are filtered by the group in the file name; the core group spec is always kept.", + args: args{ + files: map[string]string{ + "api__v1_openapi.json": "{}", + "apis__apps__v1_openapi.json": "{}", + "apis__networking.k8s.io__v1_openapi.json": "{}", + }, + sourceType: SourceTypeOpenAPI, + globs: []string{"apps"}, + }, + want: want{files: map[string]string{ + "api__v1_openapi.json": "{}", + "apis__apps__v1_openapi.json": "{}", + }}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + from := afero.NewMemMapFs() + for p, c := range tc.args.files { + if err := afero.WriteFile(from, p, []byte(c), 0o600); err != nil { + t.Fatal(err) + } + } + + got, err := filterAPIGroups(from, tc.args.sourceType, tc.args.globs) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nfilterAPIGroups(...): -want err, +got err:\n%s", tc.reason, diff) + } + if diff := cmp.Diff(tc.want.files, readFiles(t, got)); diff != "" { + t.Errorf("%s\nfilterAPIGroups(...): -want, +got:\n%s", tc.reason, diff) + } + }) + } +} + +func TestEachYAMLDocument(t *testing.T) { + t.Parallel() + + type args struct { + content string + } + type want struct { + docs []string + err error + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "Empty": { + reason: "An empty stream has no documents.", + args: args{content: ""}, + want: want{}, + }, + "Single": { + reason: "A single document is passed as is.", + args: args{content: "a: 1\n"}, + want: want{docs: []string{"a: 1\n"}}, + }, + "Multiple": { + reason: "Documents are passed in order, without separators.", + args: args{content: "a: 1\n---\nb: 2\n"}, + want: want{docs: []string{"a: 1\n", "b: 2\n"}}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + var got want + err := eachYAMLDocument([]byte(tc.args.content), func(doc []byte) error { + got.docs = append(got.docs, string(doc)) + return nil + }) + got.err = err + if diff := cmp.Diff(tc.want, got, cmp.AllowUnexported(want{}), cmpopts.EquateErrors()); diff != "" { + t.Errorf("%s\neachYAMLDocument(...): -want, +got:\n%s", tc.reason, diff) + } + }) + } +} diff --git a/internal/schemas/manager/lock.go b/internal/schemas/manager/lock.go index e4387d97..7fb6102b 100644 --- a/internal/schemas/manager/lock.go +++ b/internal/schemas/manager/lock.go @@ -16,16 +16,93 @@ limitations under the License. package manager -const lockFileName = ".lock.json" +import ( + "crypto/sha256" + "encoding/hex" + "slices" +) -// lock tracks the versions of sources whose schemas are present in the -// manager. It is persisted to the manager's filesystem. +const ( + lockFileName = ".lock.json" + + // lockVersion is the current lock file format. A lock with any other + // version is ignored and overwritten on the next change. + lockVersion = 1 +) + +// Flags are the generation options, beyond languages, that change the content +// of the generated files. They are recorded per source so that changing one +// regenerates the schemas it affects. +type Flags struct { + GoRuntimeObjects bool `json:"goRuntimeObjects"` + // APIGroups are the globs resources are filtered by, sorted. Empty means + // no filtering. + APIGroups []string `json:"apiGroups,omitempty"` +} + +func (f Flags) equal(o Flags) bool { + return f.GoRuntimeObjects == o.GoRuntimeObjects && slices.Equal(f.APIGroups, o.APIGroups) +} + +// lock tracks the sources whose schemas are present in the manager and the +// files generated from each. It is persisted to the manager's filesystem. type lock struct { - Packages map[string]string `json:"packages"` + LockVersion int `json:"lockVersion"` + // DependenciesHash is the hash of the project's dependencies the lock was + // last updated for. It lets Verify notice changed dependencies without the + // network. + DependenciesHash string `json:"dependenciesHash,omitempty"` + Sources map[string]lockSource `json:"sources"` +} + +// lockSource is the lock entry of a single source. +type lockSource struct { + SourceHash string `json:"sourceHash"` + // Pin is an immutable reference to fetch the source's content by, if the + // source has one. + Pin string `json:"pin,omitempty"` + // Version is the resolved tag or ref, shown to people. It is optional. + Version string `json:"version,omitempty"` + Flags Flags `json:"flags"` + // Languages are the sorted languages the source was generated for. They + // are recorded because a language may produce no files for a source. + Languages []string `json:"languages"` + Files []lockFile `json:"files"` +} + +// LockedSource is the read-only view of a source's lock entry. +type LockedSource struct { + Pin string + Version string + SourceHash string +} + +// lockFile is a generated file, identified by its slash-separated path +// relative to the managed directory, and the hash of its content. +type lockFile struct { + Path string `json:"path"` + SHA256 string `json:"sha256"` } func newLock() *lock { return &lock{ - Packages: make(map[string]string), + LockVersion: lockVersion, + Sources: make(map[string]lockSource), + } +} + +// owned returns the set of file paths recorded by any source. +func (l *lock) owned() map[string]struct{} { + owned := make(map[string]struct{}) + for _, src := range l.Sources { + for _, f := range src.Files { + owned[f.Path] = struct{}{} + } } + return owned +} + +func hashContent(content []byte) string { + sum := sha256.Sum256(content) + return hex.EncodeToString(sum[:]) } diff --git a/internal/schemas/manager/manager.go b/internal/schemas/manager/manager.go index 3fe60e7c..d824efdc 100644 --- a/internal/schemas/manager/manager.go +++ b/internal/schemas/manager/manager.go @@ -21,7 +21,10 @@ import ( "context" "encoding/json" "io/fs" + "path" "path/filepath" + "slices" + "sort" "sync" "github.com/invopop/jsonschema" @@ -31,54 +34,450 @@ import ( "github.com/crossplane/crossplane-runtime/v2/pkg/errors" devv1alpha1 "github.com/crossplane/cli/v2/apis/dev/v1alpha1" - "github.com/crossplane/cli/v2/internal/filesystem" "github.com/crossplane/cli/v2/internal/schemas/generator" "github.com/crossplane/cli/v2/internal/schemas/runner" ) -// Manager is a schema manager. It manages a directory of schemas, generating -// new schemas only when necessary. +// Manager is a schema manager. It manages a directory of schemas and a lock +// that records, per source, the content hash it was generated from and the +// files that came out of it. Schemas are generated only when necessary. type Manager struct { fs afero.Fs generators []generator.Interface runner runner.SchemaRunner + flags Flags + // lockMu guards the lock file and the generated files it describes. lockMu sync.RWMutex } +// Option configures a Manager. +type Option func(*Manager) + +// WithFlags sets the generation flags recorded in the lock. Sources locked +// with different flags are regenerated. +func WithFlags(f Flags) Option { + return func(m *Manager) { + // Sort a copy so the same globs in another order don't regenerate. + f.APIGroups = slices.Sorted(slices.Values(f.APIGroups)) + m.flags = f + } +} + +// FlagsFor returns the generation flags for a project's schema configuration. +// It is safe to call with nil schemas. +func FlagsFor(schemas *devv1alpha1.ProjectSchemas, goRuntimeObjects bool) Flags { + return Flags{ + GoRuntimeObjects: goRuntimeObjects, + APIGroups: schemas.GetAPIGroups(), + } +} + +// DriftKind is the way a generated file differs from the lock. +type DriftKind string + +const ( + // DriftMissing means the file is recorded in the lock but not on disk. + DriftMissing DriftKind = "missing" + // DriftModified means the file on disk differs from the recorded hash. + DriftModified DriftKind = "modified" +) + +// DriftedFile is a generated file that differs from the lock. +type DriftedFile struct { + Path string + Kind DriftKind +} + +// ReconcileResult describes what Reconcile removed. +type ReconcileResult struct { + // Sources are the keys of lock entries that were removed. + Sources []string + // Files are the generated files that were deleted with them. + Files []string +} + +// SourceStatus is the on-disk state of one locked source. +type SourceStatus struct { + Key string + Drift []DriftedFile +} + // Add ensures schemas for resources in the given source are present in the -// managed directory. +// managed directory and recorded in the lock. The source is regenerated if its +// hash, the generation flags or the set of languages changed. Files that +// merely drifted from the lock are left alone; see Manager.Generate. func (m *Manager) Add(ctx context.Context, source Source) error { - version, err := source.Version(ctx) + hash, err := source.Hash(ctx) if err != nil { return err } - existing, err := m.currentVersion(source.ID()) + m.lockMu.RLock() + l, err := m.getLock() + m.lockMu.RUnlock() if err != nil { return err } - if existing == version { + + if entry, ok := l.Sources[source.Key()]; ok && m.upToDate(source.Key(), entry, hash) { return nil } - _, err = m.Generate(ctx, source) - return err + files, err := m.render(ctx, source) + if err != nil { + return err + } + return m.commit(ctx, source, hash, files) +} + +// HasLock returns true if a usable schema lock exists. A lock written by an +// earlier version is not usable. +func (m *Manager) HasLock() (bool, error) { + m.lockMu.RLock() + defer m.lockMu.RUnlock() + + _, ok, err := m.readLock() + return ok, err +} + +// Locked returns the lock entry of every locked source, by source key. +func (m *Manager) Locked() (map[string]LockedSource, error) { + m.lockMu.RLock() + defer m.lockMu.RUnlock() + + l, err := m.getLock() + if err != nil { + return nil, err + } + + locked := make(map[string]LockedSource, len(l.Sources)) + for key, src := range l.Sources { + locked[key] = LockedSource{Pin: src.Pin, Version: src.Version, SourceHash: src.SourceHash} + } + return locked, nil +} + +// SetDependenciesHash records the hash of the project's dependencies in the +// lock, leaving its sources untouched. +func (m *Manager) SetDependenciesHash(hash string) error { + m.lockMu.Lock() + defer m.lockMu.Unlock() + + l, err := m.getLock() + if err != nil { + return err + } + l.DependenciesHash = hash + return m.updateLock(l) +} + +// Reconcile removes every locked source that is not in liveSources, together +// with its generated files. It must be called with all live sources, not a +// subset, or sources that are still in use are removed. +func (m *Manager) Reconcile(_ context.Context, liveSources []Source) (*ReconcileResult, error) { + live := make(map[string]struct{}, len(liveSources)) + for _, s := range liveSources { + live[s.Key()] = struct{}{} + } + + m.lockMu.Lock() + defer m.lockMu.Unlock() + + l, err := m.getLock() + if err != nil { + return nil, err + } + + res := &ReconcileResult{} + var orphaned []lockFile + for key, entry := range l.Sources { + if _, ok := live[key]; ok { + continue + } + res.Sources = append(res.Sources, key) + orphaned = append(orphaned, entry.Files...) + delete(l.Sources, key) + } + sort.Strings(res.Sources) + if len(res.Sources) == 0 { + return res, nil + } + + res.Files, err = m.removeUnowned(l, orphaned) + if err != nil { + return nil, err + } + if err := m.rebuildIndexes(); err != nil { + return nil, err + } + return res, m.updateLock(l) +} + +// Status reports, for every locked source, which generated files differ from +// the lock. It only reads the schema directory; it never resolves sources. +func (m *Manager) Status(_ context.Context) ([]SourceStatus, error) { + m.lockMu.RLock() + defer m.lockMu.RUnlock() + + l, err := m.getLock() + if err != nil { + return nil, err + } + + statuses := make([]SourceStatus, 0, len(l.Sources)) + for key, entry := range l.Sources { + drift, err := m.drift(entry) + if err != nil { + return nil, err + } + statuses = append(statuses, SourceStatus{Key: key, Drift: drift}) + } + sort.Slice(statuses, func(i, j int) bool { return statuses[i].Key < statuses[j].Key }) + return statuses, nil +} + +// Fetcher returns the locked source key again, fetched at the pin its lock +// entry records instead of being resolved anew. +type Fetcher func(ctx context.Context, key string, entry LockedSource) (Source, error) + +// Verify checks that the lock describes the project and recreates generated +// files that are missing. It fails if the dependencies, a local source, the +// generation flags or the languages changed since the lock was written, or if a +// generated file was modified. It never changes the lock. +// +// The local sources are those of the project itself, which are always at hand. +// depsHash identifies the project's dependencies. Their sources are only +// fetched, by fetch, when one of their files has to be recreated. +func (m *Manager) Verify(ctx context.Context, local []Source, depsHash string, fetch Fetcher) error { + return m.restore(ctx, local, depsHash, fetch, false) +} + +// Generate recreates missing and modified generated files from the lock. The +// project must still match the lock; it fails otherwise and never changes the +// lock. +func (m *Manager) Generate(ctx context.Context, local []Source, depsHash string, fetch Fetcher) error { + return m.restore(ctx, local, depsHash, fetch, true) +} + +// Clean removes all generated schemas and the lock. +func (m *Manager) Clean() error { + m.lockMu.Lock() + defer m.lockMu.Unlock() + + entries, err := afero.ReadDir(m.fs, ".") + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return errors.Wrap(err, "failed to list schema directory") + } + for _, e := range entries { + if err := m.fs.RemoveAll(e.Name()); err != nil { + return errors.Wrapf(err, "failed to remove %s", e.Name()) + } + } + return nil +} + +func (m *Manager) restore(ctx context.Context, local []Source, depsHash string, fetch Fetcher, repairModified bool) error { + m.lockMu.Lock() + defer m.lockMu.Unlock() + + l, err := m.getLock() + if err != nil { + return err + } + if l.DependenciesHash != depsHash { + return errors.New("dependencies changed since the schema lock was written; run `crossplane schema update`") + } + + locals := make(map[string]Source, len(local)) + for _, s := range local { + locals[s.Key()] = s + } + if problems := m.stateProblems(ctx, l, locals); len(problems) > 0 { + return errors.Join(problems...) + } + + var problems []error + rebuild := false + for _, key := range sortedKeys(l.Sources) { + entry := l.Sources[key] + drift, err := m.drift(entry) + if err != nil { + return err + } + + var repair []string + for _, d := range drift { + if d.Kind == DriftModified && !repairModified { + problems = append(problems, errors.Errorf("generated file %s was modified; run `crossplane schema generate` to restore it", d.Path)) + continue + } + repair = append(repair, d.Path) + } + if len(repair) == 0 { + continue + } + + rebuild = true + p, err := m.restoreSource(ctx, key, entry, locals[key], fetch, repair) + if err != nil { + return err + } + problems = append(problems, p...) + } + + if rebuild { + if err := m.rebuildIndexes(); err != nil { + return err + } + } + return errors.Join(problems...) +} + +// stateProblems reports the reasons the lock does not describe the project: +// local sources that are not locked or changed, and locked sources generated +// with other flags or languages than the manager has now. It doesn't need the +// network. +func (m *Manager) stateProblems(ctx context.Context, l *lock, locals map[string]Source) []error { + var problems []error + for _, key := range sortedKeys(locals) { + entry, ok := l.Sources[key] + if !ok { + problems = append(problems, errors.Errorf("source %s is not in the schema lock; run `crossplane schema update`", key)) + continue + } + hash, err := locals[key].Hash(ctx) + if err != nil { + problems = append(problems, errors.Wrapf(err, "cannot hash source %s", key)) + continue + } + if hash != entry.SourceHash { + problems = append(problems, errors.Errorf("source %s changed since the schema lock was written; run `crossplane schema update`", key)) + } + } + for _, key := range sortedKeys(l.Sources) { + entry := l.Sources[key] + if !m.upToDate(key, entry, entry.SourceHash) { + problems = append(problems, errors.Errorf("source %s was generated with other flags or languages than configured; run `crossplane schema update`", key)) + } + } + return problems } -// Generate generates and returns schemas using the manager's generators, and -// adds them to the manager. Unlike Add, Generate will always generate schemas, -// regardless of whether they're already present in the manager. -func (m *Manager) Generate(ctx context.Context, source Source) (map[string]afero.Fs, error) { - version, err := source.Version(ctx) +// restoreSource recreates the given files of one locked source, fetching the +// source at its pin unless it is local. It returns the reasons it could not +// restore something. An error means the attempt itself failed. +func (m *Manager) restoreSource(ctx context.Context, key string, entry lockSource, src Source, fetch Fetcher, paths []string) ([]error, error) { + if src == nil { + if fetch == nil { + return nil, errors.Errorf("cannot recreate schemas of %s: no way to fetch it", key) + } + var err error + src, err = fetch(ctx, key, LockedSource{Pin: entry.Pin, Version: entry.Version, SourceHash: entry.SourceHash}) + if err != nil { + return nil, errors.Wrapf(err, "cannot fetch %s", key) + } + hash, err := src.Hash(ctx) + if err != nil { + return nil, errors.Wrapf(err, "cannot hash %s", key) + } + if hash != entry.SourceHash { + return []error{errors.Errorf("source %s fetched at its pin does not match the schema lock; run `crossplane schema update`", key)}, nil + } + } + return m.recreate(ctx, entry, src, paths) +} + +// recreate renders the source again and writes the given files, provided they +// come out exactly as the lock recorded them. +func (m *Manager) recreate(ctx context.Context, entry lockSource, src Source, paths []string) ([]error, error) { + files, err := m.render(ctx, src) if err != nil { return nil, err } + locked := make(map[string]string, len(entry.Files)) + for _, f := range entry.Files { + locked[f.Path] = f.SHA256 + } + + var problems []error + for _, p := range paths { + content, ok := files[p] + if !ok || hashContent(content) != locked[p] { + problems = append(problems, errors.Errorf("cannot recreate %s exactly as locked; run `crossplane schema update`", p)) + continue + } + if err := m.writeFile(p, content); err != nil { + return problems, err + } + } + return problems, nil +} + +// upToDate returns true if the lock entry was generated from the given hash +// with the flags that apply to the source and for exactly the manager's +// languages. +func (m *Manager) upToDate(key string, entry lockSource, hash string) bool { + return entry.SourceHash == hash && + entry.Flags.equal(m.flagsFor(key)) && + slices.Equal(entry.Languages, m.languages()) +} +// languages returns the sorted languages the manager generates. +func (m *Manager) languages() []string { + langs := make([]string, 0, len(m.generators)) + for _, gen := range m.generators { + langs = append(langs, gen.Language()) + } + slices.Sort(langs) + return slices.Compact(langs) +} + +// flagsFor returns the generation flags that apply to the source with the +// given key. API group filters never apply to local sources, so they are +// cleared for them. +func (m *Manager) flagsFor(key string) Flags { + f := m.flags + if isLocalKey(key) { + f.APIGroups = nil + } + return f +} + +// drift compares the generated files of a lock entry with the disk. +func (m *Manager) drift(entry lockSource) ([]DriftedFile, error) { + var drift []DriftedFile + for _, f := range entry.Files { + content, err := afero.ReadFile(m.fs, f.Path) + switch { + case errors.Is(err, fs.ErrNotExist): + drift = append(drift, DriftedFile{Path: f.Path, Kind: DriftMissing}) + case err != nil: + return nil, errors.Wrapf(err, "failed to read %s", f.Path) + case hashContent(content) != f.SHA256: + drift = append(drift, DriftedFile{Path: f.Path, Kind: DriftModified}) + } + } + return drift, nil +} + +// render runs the generators on the source and returns the generated files +// keyed by their slash-separated path relative to the managed directory, i.e. +// "/". +func (m *Manager) render(ctx context.Context, source Source) (map[string][]byte, error) { fromFS, err := source.Resources(ctx) if err != nil { return nil, err } + if !isLocalKey(source.Key()) { + fromFS, err = filterAPIGroups(fromFS, source.Type(), m.flags.APIGroups) + if err != nil { + return nil, errors.Wrapf(err, "failed to filter %s by API group", source.Key()) + } + } var schemasMu sync.Mutex schemas := make(map[string]afero.Fs) @@ -114,35 +513,143 @@ func (m *Manager) Generate(ctx context.Context, source Source) (map[string]afero return nil, err } - // Copy generated schemas into our schema repository. Generators produce - // output into models/ — we strip that prefix by copying from models/ into - // the language directory. + // Generators produce output into models/; we strip that prefix so files + // land directly in the language directory. + files := make(map[string][]byte) for lang, genFS := range schemas { - langFS := afero.NewBasePathFs(m.fs, lang) + root := genFS + if fi, err := genFS.Stat("models"); err == nil && fi.IsDir() { + root = afero.NewBasePathFs(genFS, "models") + } + err := afero.Walk(root, ".", func(p string, info fs.FileInfo, err error) error { + if err != nil || info.IsDir() { + return err + } + content, err := afero.ReadFile(root, p) + if err != nil { + return err + } + files[path.Join(lang, filepath.ToSlash(p))] = content + return nil + }) + if err != nil { + return nil, errors.Wrapf(err, "failed to read %s schemas", lang) + } + } + return files, nil +} - // Try to copy from models/ subdirectory first (generators put output there). - modelsFS := afero.NewBasePathFs(genFS, "models") - hasModels := false - if fi, err := modelsFS.Stat("."); err == nil && fi.IsDir() { - hasModels = true +// commit writes the files of a source and records them in the lock. Files the +// source owned before and no longer produces are deleted, unless another +// source also records them. +func (m *Manager) commit(ctx context.Context, source Source, hash string, files map[string][]byte) error { + key := source.Key() + + // Resolve the pin before taking the lock; it may need the network. + var pin, version string + if p, ok := source.(Pinned); ok { + var err error + if pin, err = p.Pin(ctx); err != nil { + return errors.Wrapf(err, "cannot pin source %s", key) } + version = p.Version() + } - if hasModels { - if err := filesystem.CopyFilesBetweenFs(modelsFS, langFS); err != nil { - return nil, err - } - } else { - if err := filesystem.CopyFilesBetweenFs(genFS, langFS); err != nil { - return nil, err + m.lockMu.Lock() + defer m.lockMu.Unlock() + + l, err := m.getLock() + if err != nil { + return err + } + + entry := lockSource{SourceHash: hash, Pin: pin, Version: version, Flags: m.flagsFor(key), Languages: m.languages(), Files: make([]lockFile, 0, len(files))} + for p, content := range files { + if err := m.writeFile(p, content); err != nil { + return err + } + entry.Files = append(entry.Files, lockFile{Path: p, SHA256: hashContent(content)}) + } + sort.Slice(entry.Files, func(i, j int) bool { return entry.Files[i].Path < entry.Files[j].Path }) + + var stale []lockFile + for _, f := range l.Sources[key].Files { + if _, ok := files[f.Path]; !ok { + stale = append(stale, f) + } + } + l.Sources[key] = entry + if _, err := m.removeUnowned(l, stale); err != nil { + return err + } + + if err := m.rebuildIndexes(); err != nil { + return err + } + return m.updateLock(l) +} + +// removeUnowned deletes the given files unless a source in the lock still +// records them, and returns the paths it deleted. +func (m *Manager) removeUnowned(l *lock, files []lockFile) ([]string, error) { + owned := l.owned() + var removed []string + for _, f := range files { + if _, ok := owned[f.Path]; ok { + continue + } + if err := m.fs.Remove(f.Path); err != nil { + if errors.Is(err, fs.ErrNotExist) { + continue // Already gone. } + return removed, errors.Wrapf(err, "failed to remove %s", f.Path) } + removed = append(removed, f.Path) + m.removeEmptyParents(f.Path) + } + sort.Strings(removed) + return removed, nil +} - if err := postProcessForLanguage(lang, langFS); err != nil { - return nil, err +// removeEmptyParents removes the directories above the file that became empty. +func (m *Manager) removeEmptyParents(filePath string) { + for dir := path.Dir(filePath); dir != "." && dir != "/"; dir = path.Dir(dir) { + entries, err := afero.ReadDir(m.fs, dir) + if err != nil || len(entries) > 0 { + return + } + if err := m.fs.Remove(dir); err != nil { + return + } + } +} + +func (m *Manager) writeFile(filePath string, content []byte) error { + if err := m.fs.MkdirAll(path.Dir(filePath), 0o750); err != nil { + return errors.Wrapf(err, "failed to create directory for %s", filePath) + } + return errors.Wrapf(afero.WriteFile(m.fs, filePath, content, 0o644), "failed to write %s", filePath) +} + +// rebuildIndexes refreshes per-language files that are derived from all +// generated files, such as the JSON index schema. +func (m *Manager) rebuildIndexes() error { + for _, gen := range m.generators { + lang := gen.Language() + if err := postProcessForLanguage(lang, afero.NewBasePathFs(m.fs, lang)); err != nil { + return err } } + return nil +} - return schemas, m.updateVersion(source.ID(), version) +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys } func postProcessForLanguage(language string, langFS afero.Fs) error { @@ -165,6 +672,13 @@ func jsonBuildIndexSchema(langFS afero.Fs) error { } metaFile := "index.schema.json" + if len(schemas) == 0 { + // Nothing to index; don't leave a stale or empty index behind. + if err := langFS.Remove(metaFile); err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + return nil + } var metaSchema jsonschema.Schema for _, schema := range schemas { if schema == metaFile { @@ -182,48 +696,40 @@ func jsonBuildIndexSchema(langFS afero.Fs) error { return afero.WriteFile(langFS, metaFile, bs, 0o644) } -func (m *Manager) currentVersion(id string) (string, error) { - m.lockMu.RLock() - defer m.lockMu.RUnlock() - - l, err := m.getLock() +func (m *Manager) getLock() (*lock, error) { + l, ok, err := m.readLock() if err != nil { - return "", err + return nil, err } - - return l.Packages[id], nil -} - -func (m *Manager) updateVersion(id, version string) error { - m.lockMu.Lock() - defer m.lockMu.Unlock() - - l, err := m.getLock() - if err != nil { - return err + if !ok { + return newLock(), nil } - - l.Packages[id] = version - - return m.updateLock(l) + return l, nil } -func (m *Manager) getLock() (*lock, error) { +// readLock reads the lock file. It reports false if there is no usable lock: +// the file is missing, or was written by an earlier version. +func (m *Manager) readLock() (*lock, bool, error) { lf, err := m.fs.Open(lockFileName) if err != nil { if errors.Is(err, fs.ErrNotExist) { - return newLock(), nil + return nil, false, nil } - return nil, err + return nil, false, err } defer func() { _ = lf.Close() }() var l lock if err := json.NewDecoder(lf).Decode(&l); err != nil { - return nil, err + return nil, false, err + } + // Locks written by earlier versions are not migrated. Starting from an + // empty lock regenerates every source once and rewrites the file. + if l.LockVersion != lockVersion || l.Sources == nil { + return nil, false, nil } - return &l, nil + return &l, true, nil } func (m *Manager) updateLock(l *lock) error { @@ -247,10 +753,14 @@ func (m *Manager) updateLock(l *lock) error { } // New returns an initialized manager. -func New(fs afero.Fs, gens []generator.Interface, r runner.SchemaRunner) *Manager { - return &Manager{ +func New(fs afero.Fs, gens []generator.Interface, r runner.SchemaRunner, opts ...Option) *Manager { + m := &Manager{ fs: fs, generators: gens, runner: r, } + for _, opt := range opts { + opt(m) + } + return m } diff --git a/internal/schemas/manager/manager_test.go b/internal/schemas/manager/manager_test.go index 62e1b18c..5bdad286 100644 --- a/internal/schemas/manager/manager_test.go +++ b/internal/schemas/manager/manager_test.go @@ -19,199 +19,941 @@ package manager import ( "context" "io/fs" + "strings" "testing" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/spf13/afero" + devv1alpha1 "github.com/crossplane/cli/v2/apis/dev/v1alpha1" "github.com/crossplane/cli/v2/internal/schemas/generator" "github.com/crossplane/cli/v2/internal/schemas/runner" ) +// lf builds the expected lock entry for a generated file. +func lf(path, content string) lockFile { + return lockFile{Path: path, SHA256: hashContent([]byte(content))} +} + func TestManager_Add(t *testing.T) { t.Parallel() - tcs := map[string]struct { - lock *lock - gen generator.Interface - src Source + // run is one Add call. Every run gets a fresh manager over the same + // filesystem, like consecutive invocations of the CLI. + type run struct { + gens []generator.Interface + flags Flags + src *mockSource + // tamper changes the managed directory before the call. + tamper func(t *testing.T, fs afero.Fs) + } + type args struct { + // lockJSON is written to the lock file before the first run. + lockJSON string + runs []run + } + type want struct { + lock *lock + files map[string]string + err error + } + + cr := func(files map[string]string) []generator.Interface { + return []generator.Interface{&mockGenerator{lang: "mock", files: files}} + } + src := func(hash string) *mockSource { return &mockSource{key: "xpkg://pkg", hash: hash} } - expectedLock *lock - expectedFiles map[string]string - expectErr bool + cases := map[string]struct { + reason string + args args + want want }{ - // Version already matches: skip generation. - "AlreadyGenerated": { - lock: &lock{ - Packages: map[string]string{ - "xpkg.upbound.io/my-org/my-pkg": "v1.0.0", - }, + "EmptyLock": { + reason: "A source that is not locked is generated and recorded with its hash and files.", + args: args{runs: []run{ + {gens: cr(map[string]string{"a.json": "A"}), src: src("h1")}, + }}, + want: want{ + lock: &lock{LockVersion: 1, Sources: map[string]lockSource{ + "xpkg://pkg": {SourceHash: "h1", Languages: []string{"mock"}, Files: []lockFile{lf("mock/a.json", "A")}}, + }}, + files: map[string]string{"mock/a.json": "A"}, }, - gen: &mockGenerator{ - files: map[string]string{ - "should-not-exist": "does not get created", - }, + }, + "UnchangedHashIsNoOp": { + reason: "A source whose hash matches the lock is not regenerated, even if the generator would now produce different files.", + args: args{runs: []run{ + {gens: cr(map[string]string{"a.json": "A"}), src: src("h1")}, + {gens: cr(map[string]string{"a.json": "CHANGED"}), src: src("h1")}, + }}, + want: want{ + lock: &lock{LockVersion: 1, Sources: map[string]lockSource{ + "xpkg://pkg": {SourceHash: "h1", Languages: []string{"mock"}, Files: []lockFile{lf("mock/a.json", "A")}}, + }}, + files: map[string]string{"mock/a.json": "A"}, }, - src: &mockSource{ - id: "xpkg.upbound.io/my-org/my-pkg", - version: "v1.0.0", + }, + "ChangedHashRegeneratesAndPrunes": { + reason: "A new hash regenerates the source, and files it no longer produces are removed and reported.", + args: args{runs: []run{ + {gens: cr(map[string]string{"a.json": "A", "b.json": "B"}), src: src("h1")}, + {gens: cr(map[string]string{"a.json": "A2"}), src: src("h2")}, + }}, + want: want{ + lock: &lock{LockVersion: 1, Sources: map[string]lockSource{ + "xpkg://pkg": {SourceHash: "h2", Languages: []string{"mock"}, Files: []lockFile{lf("mock/a.json", "A2")}}, + }}, + files: map[string]string{"mock/a.json": "A2"}, }, - expectedLock: &lock{ - Packages: map[string]string{ - "xpkg.upbound.io/my-org/my-pkg": "v1.0.0", - }, + }, + "ChangedFlagsRegenerate": { + reason: "Flags are part of what a source was generated with, so changing one regenerates it.", + args: args{runs: []run{ + {gens: cr(map[string]string{"a.json": "A"}), src: src("h1")}, + {gens: cr(map[string]string{"a.json": "A2"}), src: src("h1"), flags: Flags{GoRuntimeObjects: true}}, + }}, + want: want{ + lock: &lock{LockVersion: 1, Sources: map[string]lockSource{ + "xpkg://pkg": {SourceHash: "h1", Flags: Flags{GoRuntimeObjects: true}, Languages: []string{"mock"}, Files: []lockFile{lf("mock/a.json", "A2")}}, + }}, + files: map[string]string{"mock/a.json": "A2"}, }, }, - // No lock at all: generate and record version. - "EmptyLock": { - gen: &mockGenerator{ - files: map[string]string{ - "should-exist": "does get created", - }, + "ChangedAPIGroupsRegenerate": { + reason: "Changing the API group filter changes what a source generates, so it is regenerated and recorded sorted.", + args: args{runs: []run{ + {gens: cr(map[string]string{"a.json": "A", "b.json": "B"}), src: src("h1")}, + {gens: cr(map[string]string{"a.json": "A"}), src: src("h1"), flags: Flags{APIGroups: []string{"b.*", "a.*"}}}, + }}, + want: want{ + lock: &lock{LockVersion: 1, Sources: map[string]lockSource{ + "xpkg://pkg": {SourceHash: "h1", Flags: Flags{APIGroups: []string{"a.*", "b.*"}}, Languages: []string{"mock"}, Files: []lockFile{lf("mock/a.json", "A")}}, + }}, + files: map[string]string{"mock/a.json": "A"}, }, - src: &mockSource{ - id: "xpkg.upbound.io/my-org/my-pkg", - version: "v1.0.0", + }, + "APIGroupOrderIsIgnored": { + reason: "The same globs in another order are the same filter and must not regenerate.", + args: args{runs: []run{ + {gens: cr(map[string]string{"a.json": "A"}), src: src("h1"), flags: Flags{APIGroups: []string{"a.*", "b.*"}}}, + {gens: cr(map[string]string{"a.json": "CHANGED"}), src: src("h1"), flags: Flags{APIGroups: []string{"b.*", "a.*"}}}, + }}, + want: want{ + lock: &lock{LockVersion: 1, Sources: map[string]lockSource{ + "xpkg://pkg": {SourceHash: "h1", Flags: Flags{APIGroups: []string{"a.*", "b.*"}}, Languages: []string{"mock"}, Files: []lockFile{lf("mock/a.json", "A")}}, + }}, + files: map[string]string{"mock/a.json": "A"}, }, - expectedLock: &lock{ - Packages: map[string]string{ - "xpkg.upbound.io/my-org/my-pkg": "v1.0.0", + }, + "AddedLanguageRegenerates": { + reason: "A language that has no files for the source yet is detected from the file paths and generated.", + args: args{runs: []run{ + {gens: cr(map[string]string{"a.json": "A"}), src: src("h1")}, + { + gens: []generator.Interface{ + &mockGenerator{lang: "mock", files: map[string]string{"a.json": "A"}}, + &mockGenerator{lang: "other", files: map[string]string{"a.txt": "T"}}, + }, + src: src("h1"), }, - }, - expectedFiles: map[string]string{ - "mock/should-exist": "does get created", + }}, + want: want{ + lock: &lock{LockVersion: 1, Sources: map[string]lockSource{ + "xpkg://pkg": {SourceHash: "h1", Languages: []string{"mock", "other"}, Files: []lockFile{lf("mock/a.json", "A"), lf("other/a.txt", "T")}}, + }}, + files: map[string]string{"mock/a.json": "A", "other/a.txt": "T"}, }, }, - // Version changed: regenerate and update lock. - "VersionUpdated": { - lock: &lock{ - Packages: map[string]string{ - "xpkg.upbound.io/my-org/my-pkg": "v1.0.0", + "RemovedLanguageRegeneratesAndPrunes": { + reason: "A language that is no longer generated regenerates the source, and its files are removed.", + args: args{runs: []run{ + { + gens: []generator.Interface{ + &mockGenerator{lang: "mock", files: map[string]string{"a.json": "A"}}, + &mockGenerator{lang: "other", files: map[string]string{"a.txt": "T"}}, + }, + src: src("h1"), }, + {gens: cr(map[string]string{"a.json": "A"}), src: src("h1")}, + }}, + want: want{ + lock: &lock{LockVersion: 1, Sources: map[string]lockSource{ + "xpkg://pkg": {SourceHash: "h1", Languages: []string{"mock"}, Files: []lockFile{lf("mock/a.json", "A")}}, + }}, + files: map[string]string{"mock/a.json": "A"}, }, - gen: &mockGenerator{ - files: map[string]string{ - "should-exist": "does get created", + }, + "DriftIsReportedNotResolved": { + reason: "Add reports edited and deleted files but leaves them alone; repairing is the job of Generate.", + args: args{runs: []run{ + {gens: cr(map[string]string{"a.json": "A", "b.json": "B"}), src: src("h1")}, + { + gens: cr(map[string]string{"a.json": "A", "b.json": "B"}), + src: src("h1"), + tamper: func(t *testing.T, fs afero.Fs) { + t.Helper() + if err := afero.WriteFile(fs, "mock/a.json", []byte("edited"), 0o600); err != nil { + t.Fatal(err) + } + if err := fs.Remove("mock/b.json"); err != nil { + t.Fatal(err) + } + }, }, + }}, + want: want{ + lock: &lock{LockVersion: 1, Sources: map[string]lockSource{ + "xpkg://pkg": {SourceHash: "h1", Languages: []string{"mock"}, Files: []lockFile{lf("mock/a.json", "A"), lf("mock/b.json", "B")}}, + }}, + files: map[string]string{"mock/a.json": "edited"}, }, - src: &mockSource{ - id: "xpkg.upbound.io/my-org/my-pkg", - version: "v1.1.0", + }, + "OldLockIsOverwritten": { + reason: "A lock without lockVersion 1 is ignored and replaced.", + args: args{ + lockJSON: `{"packages":{"xpkg://pkg":"h1"}}`, + runs: []run{{gens: cr(map[string]string{"a.json": "A"}), src: src("h1")}}, }, - expectedLock: &lock{ - Packages: map[string]string{ - "xpkg.upbound.io/my-org/my-pkg": "v1.1.0", - }, + want: want{ + lock: &lock{LockVersion: 1, Sources: map[string]lockSource{ + "xpkg://pkg": {SourceHash: "h1", Languages: []string{"mock"}, Files: []lockFile{lf("mock/a.json", "A")}}, + }}, + files: map[string]string{"mock/a.json": "A"}, }, - expectedFiles: map[string]string{ - "mock/should-exist": "does get created", + }, + "LanguageWithoutOutputIsRecorded": { + reason: "A language that produces no files for a source is still recorded, so the source is up to date on the next Add.", + args: args{runs: []run{ + {gens: []generator.Interface{&mockGenerator{lang: "mock", files: map[string]string{"a.json": "A"}}, &mockGenerator{lang: "empty"}}, src: src("h1")}, + {gens: []generator.Interface{&mockGenerator{lang: "mock", files: map[string]string{"a.json": "CHANGED"}}, &mockGenerator{lang: "empty"}}, src: src("h1")}, + }}, + want: want{ + lock: &lock{LockVersion: 1, Sources: map[string]lockSource{ + "xpkg://pkg": {SourceHash: "h1", Languages: []string{"empty", "mock"}, Files: []lockFile{lf("mock/a.json", "A")}}, + }}, + files: map[string]string{"mock/a.json": "A"}, + }, + }, + "SourceWithoutOutputIsUpToDate": { + reason: "A source that generates nothing is recorded once and not regenerated on every Add.", + args: args{runs: []run{ + {gens: cr(nil), src: src("h1")}, + {gens: cr(nil), src: src("h1")}, + }}, + want: want{ + lock: &lock{LockVersion: 1, Sources: map[string]lockSource{ + "xpkg://pkg": {SourceHash: "h1", Languages: []string{"mock"}, Files: []lockFile{}}, + }}, }, }, } - for name, tc := range tcs { + for name, tc := range cases { t.Run(name, func(t *testing.T) { t.Parallel() - testFS := afero.NewMemMapFs() - - m := New(testFS, []generator.Interface{tc.gen}, nil) - if tc.lock != nil { - if err := m.updateLock(tc.lock); err != nil { + memFS := afero.NewMemMapFs() + if tc.args.lockJSON != "" { + if err := afero.WriteFile(memFS, lockFileName, []byte(tc.args.lockJSON), 0o600); err != nil { t.Fatal(err) } } - err := m.Add(t.Context(), tc.src) - if tc.expectErr { - if err == nil { - t.Fatal("expected error, got nil") + for _, r := range tc.args.runs { + if r.tamper != nil { + r.tamper(t, memFS) + } + err := New(memFS, r.gens, nil, WithFlags(r.flags)).Add(t.Context(), r.src) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nAdd(...): -want err, +got err:\n%s", tc.reason, diff) } - return } + + gotLock, err := New(memFS, nil, nil).getLock() if err != nil { t.Fatal(err) } + if diff := cmp.Diff(tc.want.lock, gotLock); diff != "" { + t.Errorf("%s\nlock: -want, +got:\n%s", tc.reason, diff) + } + if diff := cmp.Diff(tc.want.files, readFiles(t, memFS)); diff != "" { + t.Errorf("%s\nfiles: -want, +got:\n%s", tc.reason, diff) + } + }) + } +} + +func TestManager_Reconcile(t *testing.T) { + t.Parallel() + + type args struct { + // added are the sources added before reconciling, each with one file. + added map[string]string // source key -> file name + live []string + } + type want struct { + result *ReconcileResult + keys []string + files map[string]string + err error + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "AllLive": { + reason: "Nothing is removed while every locked source is live.", + args: args{added: map[string]string{"a": "a.json", "b": "b.json"}, live: []string{"a", "b"}}, + want: want{ + result: &ReconcileResult{}, + keys: []string{"a", "b"}, + files: map[string]string{"mock/a.json": "a.json", "mock/b.json": "b.json"}, + }, + }, + "OrphanRemoved": { + reason: "A locked source that is no longer live loses its lock entry and files.", + args: args{added: map[string]string{"a": "a.json", "b": "b.json"}, live: []string{"a"}}, + want: want{ + result: &ReconcileResult{Sources: []string{"b"}, Files: []string{"mock/b.json"}}, + keys: []string{"a"}, + files: map[string]string{"mock/a.json": "a.json"}, + }, + }, + "SharedFileKept": { + reason: "A file that another live source also records must survive its orphaned co-owner.", + args: args{added: map[string]string{"a": "shared.json", "b": "shared.json"}, live: []string{"a"}}, + want: want{ + result: &ReconcileResult{Sources: []string{"b"}}, + keys: []string{"a"}, + files: map[string]string{"mock/shared.json": "shared.json"}, + }, + }, + "NoLiveSources": { + reason: "With no live sources everything is removed.", + args: args{added: map[string]string{"a": "a.json"}}, + want: want{ + result: &ReconcileResult{Sources: []string{"a"}, Files: []string{"mock/a.json"}}, + keys: []string{}, + }, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() - _ = afero.Walk(testFS, ".", func(path string, info fs.FileInfo, err error) error { - if err != nil { + memFS := afero.NewMemMapFs() + live := make([]Source, 0, len(tc.args.live)) + for key, file := range tc.args.added { + m := New(memFS, []generator.Interface{&mockGenerator{lang: "mock", files: map[string]string{file: file}}}, nil) + if err := m.Add(t.Context(), &mockSource{key: key, hash: "h"}); err != nil { t.Fatal(err) } - if info.Name() == lockFileName { - return nil + } + for _, key := range tc.args.live { + live = append(live, &mockSource{key: key}) + } + + got, err := New(memFS, nil, nil).Reconcile(t.Context(), live) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nReconcile(...): -want err, +got err:\n%s", tc.reason, diff) + } + if diff := cmp.Diff(tc.want.result, got); diff != "" { + t.Errorf("%s\nReconcile(...): -want, +got:\n%s", tc.reason, diff) + } + l, _ := New(memFS, nil, nil).getLock() + if diff := cmp.Diff(tc.want.keys, sortedKeys(l.Sources)); diff != "" { + t.Errorf("%s\nlocked sources: -want, +got:\n%s", tc.reason, diff) + } + if diff := cmp.Diff(tc.want.files, readFiles(t, memFS)); diff != "" { + t.Errorf("%s\nfiles: -want, +got:\n%s", tc.reason, diff) + } + }) + } +} + +func TestManager_Status(t *testing.T) { + t.Parallel() + + type args struct { + tamper func(t *testing.T, fs afero.Fs) + } + type want struct { + statuses []SourceStatus + err error + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "Clean": { + reason: "Untouched files report no drift.", + args: args{tamper: func(*testing.T, afero.Fs) {}}, + want: want{statuses: []SourceStatus{{Key: "k"}}}, + }, + "EditedAndMissing": { + reason: "Edited and deleted files are named with their kind.", + args: args{tamper: func(t *testing.T, fs afero.Fs) { + t.Helper() + _ = afero.WriteFile(fs, "mock/a.json", []byte("edited"), 0o600) + _ = fs.Remove("mock/b.json") + }}, + want: want{statuses: []SourceStatus{{Key: "k", Drift: []DriftedFile{ + {Path: "mock/a.json", Kind: DriftModified}, + {Path: "mock/b.json", Kind: DriftMissing}, + }}}}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + memFS := afero.NewMemMapFs() + m := New(memFS, []generator.Interface{&mockGenerator{lang: "mock", files: map[string]string{"a.json": "A", "b.json": "B"}}}, nil) + if err := m.Add(t.Context(), &mockSource{key: "k", hash: "h"}); err != nil { + t.Fatal(err) + } + tc.args.tamper(t, memFS) + + got, err := m.Status(t.Context()) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nStatus(...): -want err, +got err:\n%s", tc.reason, diff) + } + if diff := cmp.Diff(tc.want.statuses, got); diff != "" { + t.Errorf("%s\nStatus(...): -want, +got:\n%s", tc.reason, diff) + } + }) + } +} + +func TestManager_VerifyAndGenerate(t *testing.T) { + t.Parallel() + + // The lock has a local source "fs://api" (mock/l.json, hash "h-api") and a + // dependency "xpkg://dep" (mock/d.json, hash "h-dep", pinned), written for + // dependencies "deps1". + type args struct { + // generate selects Generate (repairs modified files) over Verify. + generate bool + tamper func(t *testing.T, fs afero.Fs) + flags Flags + // extraLanguage adds a generator the lock was not written with. + extraLanguage bool + depsHash string + // localHash is the hash of the local source passed in. Empty means the + // local source is not passed at all. + localHash string + // localKey is the key of the local source. It defaults to the locked one. + localKey string + // fetchHash is the hash of the source the fetcher returns. + fetchHash string + } + type want struct { + // err is the expected error; cmpopts.AnyError when only its presence + // and message matter. + err error + // errMsg is a substring of the expected error message. + errMsg string + // fetched are the keys the fetcher was called with. + fetched []string + files map[string]string + } + + remove := func(path string) func(*testing.T, afero.Fs) { + return func(t *testing.T, fs afero.Fs) { + t.Helper() + _ = fs.Remove(path) + } + } + edit := func(path string) func(*testing.T, afero.Fs) { + return func(t *testing.T, fs afero.Fs) { + t.Helper() + _ = afero.WriteFile(fs, path, []byte("edited"), 0o600) + } + } + none := func(*testing.T, afero.Fs) {} + clean := map[string]string{"mock/l.json": "L", "mock/d.json": "D"} + with := func(path, content string) map[string]string { + files := map[string]string{"mock/l.json": "L", "mock/d.json": "D"} + if content == "" { + delete(files, path) + } else { + files[path] = content + } + return files + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "VerifyCleanProject": { + reason: "Nothing to do when disk and lock agree, and nothing is fetched.", + args: args{tamper: none, depsHash: "deps1", localHash: "h-api"}, + want: want{files: clean}, + }, + "VerifyRecreatesMissingLocalFile": { + reason: "A missing file of a local source is recreated from the local source, without fetching.", + args: args{tamper: remove("mock/l.json"), depsHash: "deps1", localHash: "h-api"}, + want: want{files: clean}, + }, + "VerifyFetchesMissingDependencyFile": { + reason: "A missing file of a dependency is recreated from the source fetched at its pin.", + args: args{tamper: remove("mock/d.json"), depsHash: "deps1", localHash: "h-api", fetchHash: "h-dep"}, + want: want{fetched: []string{"xpkg://dep"}, files: clean}, + }, + "VerifyRejectsFetchedSourceWithOtherHash": { + reason: "A source fetched at its pin whose content differs from the lock must not be used.", + args: args{tamper: remove("mock/d.json"), depsHash: "deps1", localHash: "h-api", fetchHash: "other"}, + want: want{err: cmpopts.AnyError, errMsg: "does not match the schema lock", fetched: []string{"xpkg://dep"}, files: with("mock/d.json", "")}, + }, + "VerifyRejectsModifiedFile": { + reason: "Build must not silently overwrite a hand-edited generated file, and needs no fetch to say so.", + args: args{tamper: edit("mock/d.json"), depsHash: "deps1", localHash: "h-api"}, + want: want{err: cmpopts.AnyError, errMsg: "crossplane schema generate", files: with("mock/d.json", "edited")}, + }, + "VerifyRejectsChangedDependencies": { + reason: "Other dependencies than the ones locked mean the lock is stale; this is known without the network.", + args: args{tamper: remove("mock/d.json"), depsHash: "deps2", localHash: "h-api"}, + want: want{err: cmpopts.AnyError, errMsg: "dependencies changed", files: with("mock/d.json", "")}, + }, + "VerifyRejectsChangedFlags": { + reason: "Sources generated with other flags need an update first.", + args: args{tamper: none, depsHash: "deps1", localHash: "h-api", flags: Flags{GoRuntimeObjects: true}}, + want: want{err: cmpopts.AnyError, errMsg: "other flags or languages", files: clean}, + }, + "VerifyRejectsChangedAPIGroups": { + reason: "Changing the API group filter needs an update first.", + args: args{tamper: none, depsHash: "deps1", localHash: "h-api", flags: Flags{APIGroups: []string{"a.*"}}}, + want: want{err: cmpopts.AnyError, errMsg: "other flags or languages", files: clean}, + }, + "VerifyRejectsChangedLanguages": { + reason: "A language the lock was not written with needs an update first.", + args: args{tamper: none, depsHash: "deps1", localHash: "h-api", extraLanguage: true}, + want: want{err: cmpopts.AnyError, errMsg: "other flags or languages", files: clean}, + }, + "VerifyRejectsChangedLocalSource": { + reason: "A local source hash that differs from the lock means the lock is stale.", + args: args{tamper: remove("mock/l.json"), depsHash: "deps1", localHash: "other"}, + want: want{err: cmpopts.AnyError, errMsg: "source fs://api changed", files: with("mock/l.json", "")}, + }, + "VerifyRejectsUnlockedLocalSource": { + reason: "A local source with no lock entry needs an update first.", + args: args{tamper: none, depsHash: "deps1", localHash: "h-new", localKey: "fs://new"}, + want: want{err: cmpopts.AnyError, errMsg: "source fs://new is not in the schema lock", files: clean}, + }, + "GenerateRepairsModifiedLocalFile": { + reason: "`schema generate` is the explicit repair for edited files.", + args: args{generate: true, tamper: edit("mock/l.json"), depsHash: "deps1", localHash: "h-api"}, + want: want{files: clean}, + }, + "GenerateRepairsModifiedDependencyFile": { + reason: "`schema generate` restores an edited dependency file from the source fetched at its pin.", + args: args{generate: true, tamper: edit("mock/d.json"), depsHash: "deps1", localHash: "h-api", fetchHash: "h-dep"}, + want: want{fetched: []string{"xpkg://dep"}, files: clean}, + }, + "GenerateRejectsChangedDependencies": { + reason: "Generate never re-resolves what the lock pinned; changed dependencies fail.", + args: args{generate: true, tamper: edit("mock/d.json"), depsHash: "deps2", localHash: "h-api"}, + want: want{err: cmpopts.AnyError, errMsg: "dependencies changed", files: with("mock/d.json", "edited")}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + memFS := afero.NewMemMapFs() + for key, src := range map[string]*pinnedSource{ + "l": {mockSource: &mockSource{key: "fs://api", hash: "h-api"}}, + "d": {mockSource: &mockSource{key: "xpkg://dep", hash: "h-dep"}, pin: "dep@sha256:x", version: "v1"}, + } { + file := key + ".json" + m := New(memFS, []generator.Interface{&mockGenerator{lang: "mock", files: map[string]string{file: strings.ToUpper(key)}}}, nil) + if err := m.Add(t.Context(), src); err != nil { + t.Fatal(err) } - if info.IsDir() { - return nil + } + if err := New(memFS, nil, nil).SetDependenciesHash("deps1"); err != nil { + t.Fatal(err) + } + lockBefore, _ := afero.ReadFile(memFS, lockFileName) + tc.args.tamper(t, memFS) + + // The generator renders every file, but only the ones the lock records + // for the source being restored are written. + gens := []generator.Interface{&mockGenerator{lang: "mock", files: map[string]string{"l.json": "L", "d.json": "D"}}} + if tc.args.extraLanguage { + gens = append(gens, &mockGenerator{lang: "other", files: map[string]string{"a.txt": "T"}}) + } + m := New(memFS, gens, nil, WithFlags(tc.args.flags)) + + var local []Source + if tc.args.localHash != "" { + key := tc.args.localKey + if key == "" { + key = "fs://api" } + local = append(local, &mockSource{key: key, hash: tc.args.localHash}) + } + var fetched []string + fetch := func(_ context.Context, key string, entry LockedSource) (Source, error) { + fetched = append(fetched, key) + if entry.Pin != "dep@sha256:x" || entry.Version != "v1" { + t.Errorf("%s\nfetch(...): entry = %+v, want the locked pin and version", tc.reason, entry) + } + return &mockSource{key: key, hash: tc.args.fetchHash}, nil + } + + var err error + if tc.args.generate { + err = m.Generate(t.Context(), local, tc.args.depsHash, fetch) + } else { + err = m.Verify(t.Context(), local, tc.args.depsHash, fetch) + } + + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("%s\nVerify/Generate(...): -want err, +got err:\n%s", tc.reason, diff) + } + if err != nil && !strings.Contains(err.Error(), tc.want.errMsg) { + t.Errorf("%s\nVerify/Generate(...): err = %q, want it to contain %q", tc.reason, err, tc.want.errMsg) + } + if diff := cmp.Diff(tc.want.fetched, fetched); diff != "" { + t.Errorf("%s\nfetched: -want, +got:\n%s", tc.reason, diff) + } + if diff := cmp.Diff(tc.want.files, readFiles(t, memFS)); diff != "" { + t.Errorf("%s\nfiles: -want, +got:\n%s", tc.reason, diff) + } + lockAfter, _ := afero.ReadFile(memFS, lockFileName) + if diff := cmp.Diff(string(lockBefore), string(lockAfter)); diff != "" { + t.Errorf("%s\nlock must not change: -before, +after:\n%s", tc.reason, diff) + } + }) + } +} + +const localCRD = `apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: widgets.local.example.org +spec: + group: local.example.org +` + +// echoGenerator generates a file with the content of each input file, so tests +// can see what the manager handed to the generators. +type echoGenerator struct{} + +func (echoGenerator) Language() string { return "echo" } - want, ok := tc.expectedFiles[path] - if !ok { - t.Errorf("unexpected file %q generated", path) +func (echoGenerator) GenerateFromCRD(_ context.Context, from afero.Fs, _ runner.SchemaRunner) (afero.Fs, error) { + out := afero.NewMemMapFs() + err := afero.Walk(from, ".", func(p string, info fs.FileInfo, err error) error { + if err != nil || info.IsDir() { + return err + } + content, err := afero.ReadFile(from, p) + if err != nil { + return err + } + return afero.WriteFile(out, p, content, 0o600) + }) + return out, err +} + +func (echoGenerator) GenerateFromOpenAPI(_ context.Context, _ afero.Fs, _ runner.SchemaRunner) (afero.Fs, error) { + return nil, nil +} + +func TestManager_LocalSourcesIgnoreAPIGroups(t *testing.T) { + t.Parallel() + + const depCRD = "kind: CustomResourceDefinition\nspec:\n group: dep.example.org\n" + local := func() Source { + fs := afero.NewMemMapFs() + _ = afero.WriteFile(fs, "widget.yaml", []byte(localCRD), 0o600) + return NewFSSource("api", fs) + } + dep := func() Source { + fs := afero.NewMemMapFs() + _ = afero.WriteFile(fs, "dep.yaml", []byte(depCRD), 0o600) + return &mockSource{key: "xpkg://dep", hash: "h-dep", fs: fs} + } + gens := []generator.Interface{echoGenerator{}} + groups := func(g ...string) []Option { return []Option{WithFlags(Flags{APIGroups: g})} } + + localHash, _ := local().Hash(t.Context()) + localLock := lockSource{SourceHash: localHash, Languages: []string{"echo"}, Files: []lockFile{lf("echo/widget.yaml", localCRD)}} + + type args struct { + // rounds are the API group globs of consecutive Add rounds. Each round + // adds the local source and the dependency. + rounds [][]string + // verify makes the test call Verify with verifyGlobs after the last round. + verify bool + verifyGlobs []string + } + type want struct { + files map[string]string + lock map[string]lockSource + err error + } + + // An unrelated glob filters the dependency's CRD away but not the local one. + afterFirst := want{ + files: map[string]string{"echo/widget.yaml": localCRD}, + lock: map[string]lockSource{ + "fs://api": localLock, + "xpkg://dep": {SourceHash: "h-dep", Flags: Flags{APIGroups: []string{"other.*"}}, Languages: []string{"echo"}, Files: []lockFile{}}, + }, + } + // Changing the globs regenerates the dependency but leaves the local source + // alone. + afterSecond := want{ + files: map[string]string{"echo/widget.yaml": localCRD, "echo/dep.yaml": depCRD}, + lock: map[string]lockSource{ + "fs://api": localLock, + "xpkg://dep": {SourceHash: "h-dep", Flags: Flags{APIGroups: []string{"dep.*"}}, Languages: []string{"echo"}, Files: []lockFile{lf("echo/dep.yaml", depCRD)}}, + }, + } + withErr := func(w want, err error) want { + w.err = err + return w + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "LocalSourceIgnoresAPIGroups": { + reason: "A local source must ignore apiGroups, a dependency must not.", + args: args{rounds: [][]string{{"other.*"}}}, + want: afterFirst, + }, + "ChangedAPIGroupsRegenerateOnlyDependency": { + reason: "Changed apiGroups must regenerate only the dependency.", + args: args{rounds: [][]string{{"other.*"}, {"dep.*"}}}, + want: afterSecond, + }, + // Verify accepts the local source after its globs changed, but not the + // dependency. + "VerifyUnchangedGlobs": { + reason: "The lock matches the globs.", + args: args{rounds: [][]string{{"other.*"}, {"dep.*"}}, verify: true, verifyGlobs: []string{"dep.*"}}, + want: afterSecond, + }, + "VerifyChangedGlobs": { + reason: "Changed globs make the dependency stale.", + args: args{rounds: [][]string{{"other.*"}, {"dep.*"}}, verify: true, verifyGlobs: []string{"x.*"}}, + want: withErr(afterSecond, cmpopts.AnyError), + }, + "VerifyChangedGlobsLocals": { + reason: "Changed globs make the dependency stale but never the local source.", + args: args{rounds: [][]string{{"other.*"}, {"dep.*"}}, verify: true, verifyGlobs: []string{"other.*"}}, + want: withErr(afterSecond, cmpopts.AnyError), + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + memFS := afero.NewMemMapFs() + for _, globs := range tc.args.rounds { + m := New(memFS, gens, nil, groups(globs...)...) + for _, s := range []Source{local(), dep()} { + if err := m.Add(t.Context(), s); err != nil { + t.Fatal(err) + } } + } - got, err := afero.ReadFile(testFS, path) - if err != nil { + var err error + if tc.args.verify { + if err := New(memFS, nil, nil).SetDependenciesHash("deps"); err != nil { t.Fatal(err) } - if diff := cmp.Diff(want, string(got)); diff != "" { - t.Errorf("file %q content (-want +got):\n%s", path, diff) - } - - return nil - }) + err = New(memFS, gens, nil, groups(tc.args.verifyGlobs...)...).Verify(t.Context(), []Source{local()}, "deps", nil) + } + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("%s\nVerify(...): -want err, +got err:\n%s", tc.reason, diff) + } + if err != nil && strings.Contains(err.Error(), "fs://api") { + t.Errorf("%s\nVerify(...): err = %v, must not mention the local source", tc.reason, err) + } - gotLock, err := m.getLock() - if err != nil { - t.Fatal(err) + l, lerr := New(memFS, nil, nil).getLock() + if lerr != nil { + t.Fatal(lerr) + } + if diff := cmp.Diff(tc.want.files, readFiles(t, memFS)); diff != "" { + t.Errorf("%s\nfiles: -want, +got:\n%s", tc.reason, diff) } - if diff := cmp.Diff(tc.expectedLock, gotLock); diff != "" { - t.Errorf("lock (-want +got):\n%s", diff) + if diff := cmp.Diff(tc.want.lock, l.Sources); diff != "" { + t.Errorf("%s\nlock sources: -want, +got:\n%s", tc.reason, diff) } }) } } -func TestUpdateLockIsIndented(t *testing.T) { - // The lock file must be written with one entry per line (indented) so it is - // human-readable and avoids spurious merge conflicts. See issue #168. - fs := afero.NewMemMapFs() - m := New(fs, nil, nil) +func TestManager_Clean(t *testing.T) { + t.Parallel() - l := newLock() - l.Packages["xpkg://pkg.example/bar:v1.0.0"] = "sha256:bbb" - l.Packages["xpkg://pkg.example/foo:v0.5.2"] = "sha256:aaa" + type args struct{ add bool } + type want struct { + files map[string]string + err error + } - if err := m.updateLock(l); err != nil { - t.Fatalf("updateLock: %v", err) + cases := map[string]struct { + reason string + args args + want want + }{ + "RemovesSchemasAndLock": { + reason: "Clean wipes generated files and the lock.", + args: args{add: true}, + }, + "EmptyDirectory": { + reason: "Cleaning a project without schemas is not an error.", + }, } - got, err := afero.ReadFile(fs, lockFileName) - if err != nil { - t.Fatalf("read lock: %v", err) + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + memFS := afero.NewMemMapFs() + m := New(memFS, []generator.Interface{&mockGenerator{lang: "mock", files: map[string]string{"a.json": "A"}}}, nil) + if tc.args.add { + if err := m.Add(t.Context(), &mockSource{key: "k", hash: "h"}); err != nil { + t.Fatal(err) + } + } + if diff := cmp.Diff(tc.want.err, m.Clean(), cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nClean(): -want err, +got err:\n%s", tc.reason, diff) + } + if diff := cmp.Diff(tc.want.files, readFiles(t, memFS)); diff != "" { + t.Errorf("%s\nfiles: -want, +got:\n%s", tc.reason, diff) + } + }) + } +} + +func TestUpdateLockIsIndented(t *testing.T) { + type args struct { + sources map[string]lockSource + } + type want struct { + content string + err error } - // Map keys are marshalled in sorted order, so this golden is deterministic. - want := `{ - "packages": { - "xpkg://pkg.example/bar:v1.0.0": "sha256:bbb", - "xpkg://pkg.example/foo:v0.5.2": "sha256:aaa" + cases := map[string]struct { + reason string + args args + want want + }{ + "OneEntryPerLine": { + // See issue #168. + reason: "The lock file must be written with one entry per line (indented) so it is human-readable and avoids spurious merge conflicts.", + args: args{sources: map[string]lockSource{ + "xpkg://pkg.example/foo": { + SourceHash: "sha256:aaa", + Languages: []string{"go"}, + Files: []lockFile{{Path: "go/foo.go", SHA256: "bbb"}}, + }, + }}, + want: want{content: `{ + "lockVersion": 1, + "sources": { + "xpkg://pkg.example/foo": { + "sourceHash": "sha256:aaa", + "flags": { + "goRuntimeObjects": false + }, + "languages": [ + "go" + ], + "files": [ + { + "path": "go/foo.go", + "sha256": "bbb" + } + ] + } } } -` - if diff := cmp.Diff(want, string(got)); diff != "" { - t.Errorf("lock file is not indented as expected (-want +got):\n%s", diff) +`}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + fs := afero.NewMemMapFs() + m := New(fs, nil, nil) + + l := newLock() + l.Sources = tc.args.sources + + err := m.updateLock(l) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nupdateLock(...): -want err, +got err:\n%s", tc.reason, diff) + } + + got, err := afero.ReadFile(fs, lockFileName) + if err != nil { + t.Fatalf("read lock: %v", err) + } + if diff := cmp.Diff(tc.want.content, string(got)); diff != "" { + t.Errorf("%s\nupdateLock(...): lock file: -want, +got:\n%s", tc.reason, diff) + } + }) + } +} + +// readFiles returns all files except the lock, keyed by path. +func readFiles(t *testing.T, f afero.Fs) map[string]string { + t.Helper() + var files map[string]string + err := afero.Walk(f, ".", func(path string, info fs.FileInfo, err error) error { + if err != nil || info.IsDir() || info.Name() == lockFileName || info.Name() == "index.schema.json" { + return err + } + content, err := afero.ReadFile(f, path) + if files == nil { + files = map[string]string{} + } + files[path] = string(content) + return err + }) + if err != nil { + t.Fatal(err) } + return files } type mockGenerator struct { + lang string files map[string]string } func (g *mockGenerator) Language() string { - return "mock" + return g.lang } func (g *mockGenerator) GenerateFromCRD(_ context.Context, _ afero.Fs, _ runner.SchemaRunner) (afero.Fs, error) { + if len(g.files) == 0 { + return nil, nil + } fs := afero.NewMemMapFs() for path, contents := range g.files { if err := afero.WriteFile(fs, path, []byte(contents), 0o600); err != nil { @@ -226,22 +968,347 @@ func (g *mockGenerator) GenerateFromOpenAPI(_ context.Context, _ afero.Fs, _ run } type mockSource struct { - id string - version string + key string + hash string + fs afero.Fs } -func (s *mockSource) ID() string { - return s.id +func (s *mockSource) Key() string { + return s.key } -func (s *mockSource) Version(_ context.Context) (string, error) { - return s.version, nil +func (s *mockSource) Hash(_ context.Context) (string, error) { + return s.hash, nil } func (s *mockSource) Resources(_ context.Context) (afero.Fs, error) { - return nil, nil + return s.fs, nil } func (s *mockSource) Type() SourceType { return SourceTypeCRD } + +// pinnedSource is a mockSource that also implements Pinned. +type pinnedSource struct { + *mockSource + pin string + version string +} + +func (s *pinnedSource) Pin(_ context.Context) (string, error) { + return s.pin, nil +} + +func (s *pinnedSource) Version() string { + return s.version +} + +func TestManager_AddRecordsPin(t *testing.T) { + t.Parallel() + + base := func() *mockSource { return &mockSource{key: "xpkg://pkg", hash: "h1"} } + + type args struct { + src Source + } + type want struct { + locked map[string]LockedSource + err error + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "PinnedSource": { + reason: "The pin and version of a source that implements Pinned are recorded in the lock.", + args: args{src: &pinnedSource{mockSource: base(), pin: "pkg@sha256:abc", version: "v1"}}, + want: want{locked: map[string]LockedSource{"xpkg://pkg": {Pin: "pkg@sha256:abc", Version: "v1", SourceHash: "h1"}}}, + }, + "UnpinnedSource": { + reason: "A source that does not implement Pinned records neither.", + args: args{src: NewFSSource("apis", afero.NewMemMapFs())}, + want: want{locked: map[string]LockedSource{"fs://apis": {SourceHash: mustFSHash(t)}}}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + m := New(afero.NewMemMapFs(), []generator.Interface{&mockGenerator{lang: "mock", files: map[string]string{"a.json": "A"}}}, nil) + if diff := cmp.Diff(tc.want.err, m.Add(t.Context(), tc.args.src), cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nAdd(...): -want err, +got err:\n%s", tc.reason, diff) + } + got, err := m.Locked() + if err != nil { + t.Fatalf("%s\nLocked(): %v", tc.reason, err) + } + if diff := cmp.Diff(tc.want.locked, got); diff != "" { + t.Errorf("%s\nLocked(): -want, +got:\n%s", tc.reason, diff) + } + }) + } +} + +// mustFSHash returns the hash of an empty filesystem source. +func mustFSHash(t *testing.T) string { + t.Helper() + h, err := NewFSSource("apis", afero.NewMemMapFs()).Hash(t.Context()) + if err != nil { + t.Fatal(err) + } + return h +} + +func TestManager_SetDependenciesHash(t *testing.T) { + t.Parallel() + + type args struct { + // src is added before the hash is set. + src Source + hash string + } + type want struct { + lock *lock + err error + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "HashIsRecordedNextToSources": { + reason: "Setting the dependencies hash keeps the locked sources and records the hash in the lock.", + args: args{ + src: &pinnedSource{mockSource: &mockSource{key: "xpkg://pkg", hash: "h1"}, pin: "pkg@sha256:abc", version: "v1"}, + hash: "deps", + }, + want: want{lock: &lock{ + LockVersion: 1, + DependenciesHash: "deps", + Sources: map[string]lockSource{ + "xpkg://pkg": {SourceHash: "h1", Pin: "pkg@sha256:abc", Version: "v1", Languages: []string{"mock"}, Files: []lockFile{lf("mock/a.json", "A")}}, + }, + }}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + memFS := afero.NewMemMapFs() + gens := []generator.Interface{&mockGenerator{lang: "mock", files: map[string]string{"a.json": "A"}}} + + if err := New(memFS, gens, nil).Add(t.Context(), tc.args.src); err != nil { + t.Fatal(err) + } + err := New(memFS, nil, nil).SetDependenciesHash(tc.args.hash) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nSetDependenciesHash(...): -want err, +got err:\n%s", tc.reason, diff) + } + + got, err := New(memFS, nil, nil).getLock() + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(tc.want.lock, got); diff != "" { + t.Errorf("%s\nSetDependenciesHash(...): -want lock, +got:\n%s", tc.reason, diff) + } + }) + } +} + +func TestManager_OldLockVersionIsIgnored(t *testing.T) { + t.Parallel() + + type args struct { + lockJSON string + } + type want struct { + locked map[string]LockedSource + err error + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "UnversionedLock": { + reason: "A lock without a lockVersion should be ignored.", + args: args{lockJSON: `{"sources":{"xpkg://pkg":{"sourceHash":"h1","flags":{"goRuntimeObjects":false},"files":[]}}}`}, + want: want{locked: map[string]LockedSource{}}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + memFS := afero.NewMemMapFs() + if err := afero.WriteFile(memFS, lockFileName, []byte(tc.args.lockJSON), 0o600); err != nil { + t.Fatal(err) + } + + got, err := New(memFS, nil, nil).Locked() + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nLocked(): -want err, +got err:\n%s", tc.reason, diff) + } + if diff := cmp.Diff(tc.want.locked, got); diff != "" { + t.Errorf("%s\nLocked(): -want, +got:\n%s", tc.reason, diff) + } + }) + } +} + +func TestFlagsFor(t *testing.T) { + t.Parallel() + + type args struct { + schemas *devv1alpha1.ProjectSchemas + goRuntimeObjects bool + } + type want struct { + flags Flags + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "NilSchemas": { + reason: "A project without a schemas section has no filter.", + want: want{flags: Flags{}}, + }, + "APIGroups": { + reason: "The project's apiGroups become part of the flags recorded in the lock.", + args: args{schemas: &devv1alpha1.ProjectSchemas{APIGroups: []string{"*.m.upbound.io"}}, goRuntimeObjects: true}, + want: want{flags: Flags{GoRuntimeObjects: true, APIGroups: []string{"*.m.upbound.io"}}}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + if diff := cmp.Diff(tc.want.flags, FlagsFor(tc.args.schemas, tc.args.goRuntimeObjects)); diff != "" { + t.Errorf("%s\nFlagsFor(...): -want, +got:\n%s", tc.reason, diff) + } + }) + } +} + +func TestManager_HasLock(t *testing.T) { + t.Parallel() + + type args struct { + lockJSON string + } + type want struct { + has bool + err error + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "NoFile": { + reason: "A project that never ran an update has no lock.", + }, + "CurrentLock": { + reason: "A lock in the current format is usable.", + args: args{lockJSON: `{"lockVersion":1,"sources":{}}`}, + want: want{has: true}, + }, + "OldLock": { + reason: "A lock from an earlier version is ignored, so it counts as no lock.", + args: args{lockJSON: `{"sources":{}}`}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + memFS := afero.NewMemMapFs() + if tc.args.lockJSON != "" { + if err := afero.WriteFile(memFS, lockFileName, []byte(tc.args.lockJSON), 0o600); err != nil { + t.Fatal(err) + } + } + got, err := New(memFS, nil, nil).HasLock() + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nHasLock(): -want err, +got err:\n%s", tc.reason, diff) + } + if diff := cmp.Diff(tc.want.has, got); diff != "" { + t.Errorf("%s\nHasLock(): -want, +got:\n%s", tc.reason, diff) + } + }) + } +} + +func TestManager_removeUnowned(t *testing.T) { + t.Parallel() + + type args struct { + readOnly bool + exists bool + } + type want struct { + removed []string + err error + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "AlreadyGone": { + reason: "A file that is already gone is not an error.", + }, + "Removed": { + reason: "An unowned file is removed and reported.", + args: args{exists: true}, + want: want{removed: []string{"mock/a.json"}}, + }, + "CannotRemove": { + reason: "Any other failure to remove a file is returned.", + args: args{exists: true, readOnly: true}, + want: want{err: cmpopts.AnyError}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + memFS := afero.NewMemMapFs() + if tc.args.exists { + if err := afero.WriteFile(memFS, "mock/a.json", []byte("A"), 0o600); err != nil { + t.Fatal(err) + } + } + f := memFS + if tc.args.readOnly { + f = afero.NewReadOnlyFs(memFS) + } + + got, err := New(f, nil, nil).removeUnowned(newLock(), []lockFile{lf("mock/a.json", "A")}) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nremoveUnowned(...): -want err, +got err:\n%s", tc.reason, diff) + } + if diff := cmp.Diff(tc.want.removed, got); diff != "" { + t.Errorf("%s\nremoveUnowned(...): -want, +got:\n%s", tc.reason, diff) + } + }) + } +} diff --git a/internal/schemas/manager/source.go b/internal/schemas/manager/source.go index feed85a7..798ab409 100644 --- a/internal/schemas/manager/source.go +++ b/internal/schemas/manager/source.go @@ -17,6 +17,7 @@ limitations under the License. package manager import ( + "bytes" "context" "crypto/sha256" "fmt" @@ -28,6 +29,7 @@ import ( "path/filepath" "slices" "strings" + "sync" "time" "github.com/Masterminds/semver/v3" @@ -36,6 +38,7 @@ import ( "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5/storage/memory" "github.com/spf13/afero" + "sigs.k8s.io/yaml" "github.com/crossplane/crossplane-runtime/v2/pkg/errors" @@ -56,10 +59,13 @@ const ( // Source is a source of resources for which schemas can be generated. type Source interface { - // ID returns a unique identifier for this source. - ID() string - // Version returns a revision identifier for this source. - Version(ctx context.Context) (string, error) + // Key returns a stable identifier for this source. It never encodes a + // version, so a new revision of a source replaces its lock entry instead of + // orphaning it. + Key() string + // Hash returns the authoritative content hash of this source. It is what + // gets recorded in the lock as the source hash. + Hash(ctx context.Context) (string, error) // Resources returns a filesystem containing resources for which schemas // need to be generated. Resources(ctx context.Context) (afero.Fs, error) @@ -67,6 +73,17 @@ type Source interface { Type() SourceType } +// Pinned is implemented by sources that can be fetched again by an immutable +// reference. The manager records it in the lock when it adds the source. +type Pinned interface { + // Pin returns the immutable reference of the source's content, for example + // a digest or a commit. + Pin(ctx context.Context) (string, error) + // Version returns the human-meaningful version the pin was resolved from, + // such as a tag or ref. It may be empty. + Version() string +} + // calculateFilesystemHash calculates a SHA256 hash of the filesystem contents. func calculateFilesystemHash(filesystem afero.Fs, sourceType SourceType) (string, error) { h := sha256.New() @@ -112,28 +129,120 @@ func calculateFilesystemHash(filesystem afero.Fs, sourceType SourceType) (string return fmt.Sprintf("%x", h.Sum(nil)), nil } -// fsSource is a resource source backed by a filesystem. +// fsKeyPrefix prefixes the keys of filesystem sources, which are the project's +// own local APIs. +const fsKeyPrefix = "fs://" + +// isLocalKey returns true if the key belongs to a local source. It is for lock +// entries, whose sources may not be at hand. +func isLocalKey(key string) bool { + return strings.HasPrefix(key, fsKeyPrefix) +} + +// isDefinitionKind returns true for the kinds of the documents that define a +// local API. +func isDefinitionKind(kind string) bool { + return kind == "CompositeResourceDefinition" || kind == "CustomResourceDefinition" +} + +// fsDocument is one API definition document of a filesystem source. +type fsDocument struct { + path string + doc []byte +} + +// fsSource is a resource source backed by a filesystem. Only its XRD and CRD +// documents count: they are what Resources returns and what Hash covers, so +// the hash and the generator input always agree. type fsSource struct { id string fs afero.Fs + + once sync.Once + docs []fsDocument + err error } -func (f *fsSource) ID() string { - return "fs://" + f.id +func (f *fsSource) Key() string { + return fsKeyPrefix + f.id } -func (f *fsSource) Version(_ context.Context) (string, error) { - return calculateFilesystemHash(f.fs, SourceTypeCRD) +func (f *fsSource) Hash(_ context.Context) (string, error) { + docs, err := f.definitions() + if err != nil { + return "", err + } + + // Paths are part of the hash so renaming a file or splitting it into two + // changes it. + h := sha256.New() + for _, d := range docs { + _, _ = h.Write([]byte(d.path)) + _, _ = h.Write([]byte{0}) + _, _ = h.Write(d.doc) + _, _ = h.Write([]byte{0}) + } + return fmt.Sprintf("%x", h.Sum(nil)), nil } func (f *fsSource) Resources(_ context.Context) (afero.Fs, error) { - return f.fs, nil + docs, err := f.definitions() + if err != nil { + return nil, err + } + + byPath := make(map[string][][]byte) + var paths []string + for _, d := range docs { + if _, ok := byPath[d.path]; !ok { + paths = append(paths, d.path) + } + byPath[d.path] = append(byPath[d.path], d.doc) + } + + out := afero.NewMemMapFs() + for _, p := range paths { + if err := out.MkdirAll(filepath.Dir(p), 0o750); err != nil { + return nil, err + } + if err := afero.WriteFile(out, p, bytes.Join(byPath[p], []byte("---\n")), 0o644); err != nil { + return nil, err + } + } + return out, nil } func (f *fsSource) Type() SourceType { return SourceTypeCRD } +// definitions returns the XRD and CRD documents of the filesystem in walk +// order. It reads the filesystem once. +func (f *fsSource) definitions() ([]fsDocument, error) { + f.once.Do(func() { + f.err = afero.Walk(f.fs, ".", func(p string, info fs.FileInfo, err error) error { + if err != nil || info.IsDir() || !isYAML(p) { + return err + } + content, err := afero.ReadFile(f.fs, p) + if err != nil { + return err + } + return eachYAMLDocument(content, func(doc []byte) error { + // Documents that don't parse aren't definitions. + var res struct { + Kind string `json:"kind"` + } + if err := yaml.Unmarshal(doc, &res); err == nil && isDefinitionKind(res.Kind) { + f.docs = append(f.docs, fsDocument{path: p, doc: doc}) + } + return nil + }) + }) + }) + return f.docs, f.err +} + // NewFSSource returns a new filesystem-backed resource source. The id should // be a stable, location-independent identifier (e.g. a project-relative path) // since it is persisted in the schema manager's lockfile. @@ -144,17 +253,18 @@ func NewFSSource(id string, fs afero.Fs) Source { // xpkgSource is a source backed by extracted CRDs from an xpkg. Unlike the // up CLI, this always generates client-side (never uses pre-packaged schemas). type xpkgSource struct { - id string - version string - crdFS afero.Fs + key string + ref string + digest string + crdFS afero.Fs } -func (s *xpkgSource) ID() string { - return "xpkg://" + s.id +func (s *xpkgSource) Key() string { + return "xpkg://" + s.key } -func (s *xpkgSource) Version(_ context.Context) (string, error) { - return s.version, nil +func (s *xpkgSource) Hash(_ context.Context) (string, error) { + return s.digest, nil } func (s *xpkgSource) Resources(_ context.Context) (afero.Fs, error) { @@ -165,13 +275,24 @@ func (s *xpkgSource) Type() SourceType { return SourceTypeCRD } -// NewXpkgSource returns a new xpkg-backed resource source. The crdFS should -// contain extracted CRD YAML files from the package. -func NewXpkgSource(id, version string, crdFS afero.Fs) Source { +func (s *xpkgSource) Pin(_ context.Context) (string, error) { + return s.key + "@" + s.digest, nil +} + +func (s *xpkgSource) Version() string { + return s.ref +} + +// NewXpkgSource returns a new xpkg-backed resource source. The key identifies +// the package without its version (e.g. its repository), the ref is the +// resolved tag or version, and the digest is the resolved package digest. The +// crdFS should contain extracted CRD YAML files from the package. +func NewXpkgSource(key, ref, digest string, crdFS afero.Fs) Source { return &xpkgSource{ - id: id, - version: version, - crdFS: crdFS, + key: key, + ref: ref, + digest: digest, + crdFS: crdFS, } } @@ -185,9 +306,12 @@ type gitSource struct { sourceType SourceType fs afero.Fs commitSHA string + // cloneRef is what to clone when it differs from the dependency's ref, i.e. + // the commit a lock pinned. Empty means clone gitRef.Ref. + cloneRef string } -func (g *gitSource) ID() string { +func (g *gitSource) Key() string { id := fmt.Sprintf("git://%s", g.gitRef.Repository) if g.gitRef.Path != "" { id = fmt.Sprintf("%s/%s", id, g.gitRef.Path) @@ -195,7 +319,7 @@ func (g *gitSource) ID() string { return id } -func (g *gitSource) Version(ctx context.Context) (string, error) { +func (g *gitSource) Hash(ctx context.Context) (string, error) { if g.commitSHA != "" { return g.commitSHA, nil } @@ -211,7 +335,11 @@ func (g *gitSource) Resources(_ context.Context) (afero.Fs, error) { return g.fs, nil } - ref := g.normalizeRef(g.gitRef.Ref) + cloneRef := g.cloneRef + if cloneRef == "" { + cloneRef = g.gitRef.Ref + } + ref := g.normalizeRef(cloneRef) var memFS billy.Filesystem var lastErr error @@ -265,6 +393,14 @@ func (g *gitSource) Type() SourceType { return g.sourceType } +func (g *gitSource) Pin(ctx context.Context) (string, error) { + return g.Hash(ctx) +} + +func (g *gitSource) Version() string { + return g.gitRef.Ref +} + func (g *gitSource) normalizeRef(ref string) string { if ref == "" { return "refs/heads/main" @@ -300,6 +436,14 @@ func (g *gitSource) verifyClone(fs billy.Filesystem, path string) error { // NewGitSource returns a new git-backed resource source. func NewGitSource(dep v1alpha1.Dependency, cloner git.Cloner, authProvider git.AuthProvider) Source { + return NewPinnedGitSource(dep, "", cloner, authProvider) +} + +// NewPinnedGitSource returns a git-backed resource source that clones pin, a +// commit, instead of the dependency's ref. Its Version is still the ref, so a +// lock written from it keeps the branch or tag the pin was resolved from. An +// empty pin clones the ref. +func NewPinnedGitSource(dep v1alpha1.Dependency, pin string, cloner git.Cloner, authProvider git.AuthProvider) Source { sourceType := SourceTypeCRD if dep.Type == v1alpha1.DependencyTypeK8s { sourceType = SourceTypeOpenAPI @@ -310,6 +454,7 @@ func NewGitSource(dep v1alpha1.Dependency, cloner git.Cloner, authProvider git.A cloner: cloner, authProvider: authProvider, sourceType: sourceType, + cloneRef: pin, } } @@ -326,11 +471,11 @@ type httpSource struct { fs afero.Fs } -func (h *httpSource) ID() string { +func (h *httpSource) Key() string { return fmt.Sprintf("http://%s", h.httpRef.URL) } -func (h *httpSource) Version(ctx context.Context) (string, error) { +func (h *httpSource) Hash(ctx context.Context) (string, error) { if h.fs != nil { return h.calculateHash() } @@ -395,6 +540,14 @@ func (h *httpSource) Type() SourceType { return h.sourceType } +func (h *httpSource) Pin(ctx context.Context) (string, error) { + return h.Hash(ctx) +} + +func (h *httpSource) Version() string { + return "" +} + func (h *httpSource) calculateHash() (string, error) { return calculateFilesystemHash(h.fs, h.sourceType) } @@ -440,12 +593,18 @@ type k8sOpenAPISource struct { fs afero.Fs } -func (k *k8sOpenAPISource) ID() string { +func (k *k8sOpenAPISource) Key() string { return fmt.Sprintf("k8s://%s", k.k8sRef.Version) } -func (k *k8sOpenAPISource) Version(_ context.Context) (string, error) { - return k.k8sRef.Version, nil +// Hash hashes the downloaded OpenAPI specs rather than the release string, so +// a changed spec is noticed even when the version is not. +func (k *k8sOpenAPISource) Hash(ctx context.Context) (string, error) { + fromFS, err := k.Resources(ctx) + if err != nil { + return "", err + } + return calculateFilesystemHash(fromFS, SourceTypeOpenAPI) } func (k *k8sOpenAPISource) Resources(ctx context.Context) (afero.Fs, error) { @@ -520,6 +679,14 @@ func (k *k8sOpenAPISource) Type() SourceType { return SourceTypeOpenAPI } +func (k *k8sOpenAPISource) Pin(_ context.Context) (string, error) { + return k.k8sRef.Version, nil +} + +func (k *k8sOpenAPISource) Version() string { + return k.k8sRef.Version +} + // NewK8sSource returns a source for Kubernetes built-in APIs. func NewK8sSource(dep v1alpha1.Dependency) Source { return &k8sOpenAPISource{ diff --git a/internal/schemas/manager/source_test.go b/internal/schemas/manager/source_test.go index 387cfe75..15947a2a 100644 --- a/internal/schemas/manager/source_test.go +++ b/internal/schemas/manager/source_test.go @@ -17,6 +17,8 @@ limitations under the License. package manager import ( + "errors" + "os" "testing" "github.com/go-git/go-billy/v5" @@ -25,6 +27,7 @@ import ( "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/storage" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/spf13/afero" "github.com/crossplane/cli/v2/apis/dev/v1alpha1" @@ -118,7 +121,7 @@ func TestGitSourceVersion(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() - gotVersion, err := tc.source.Version(t.Context()) + gotVersion, err := tc.source.Hash(t.Context()) if tc.wantError { if err == nil { @@ -137,6 +140,67 @@ func TestGitSourceVersion(t *testing.T) { } } +func TestNewPinnedGitSource(t *testing.T) { + t.Parallel() + + const sha = "abc123def456789012345678901234567890abcd" + + type args struct { + pin string + } + type want struct { + cloned string + version string + err error + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "Pinned": { + reason: "A pinned source clones the commit but still reports the branch as its version.", + args: args{pin: sha}, + want: want{cloned: sha, version: "main"}, + }, + "Unpinned": { + reason: "Without a pin the dependency's ref is cloned.", + want: want{cloned: "refs/heads/main", version: "main"}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + cloner := &recordingCloner{mockCloner: mockCloner{ref: plumbing.NewHashReference("refs/heads/main", plumbing.NewHash(sha))}} + dep := v1alpha1.Dependency{Git: &v1alpha1.GitDependency{Repository: "https://example.com/r.git", Ref: "main"}} + src := NewPinnedGitSource(dep, tc.args.pin, cloner, &mockAuthProvider{}) + + _, err := src.Hash(t.Context()) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nHash(...): -want err, +got err:\n%s", tc.reason, diff) + } + got := want{cloned: cloner.refName, version: src.(Pinned).Version()} + if diff := cmp.Diff(tc.want, got, cmp.AllowUnexported(want{}), cmpopts.EquateErrors()); diff != "" { + t.Errorf("%s\nNewPinnedGitSource(...): -want, +got:\n%s", tc.reason, diff) + } + }) + } +} + +// recordingCloner remembers the ref it was asked to clone. +type recordingCloner struct { + mockCloner + refName string +} + +func (r *recordingCloner) CloneRepository(st storage.Storer, fs billy.Filesystem, auth git.AuthProvider, opts git.CloneOptions) (*plumbing.Reference, error) { + r.refName = opts.RefName + return r.mockCloner.CloneRepository(st, fs, auth, opts) +} + func TestGitSourceResources(t *testing.T) { t.Parallel() @@ -259,7 +323,7 @@ func TestGitSourceID(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() - gotID := tc.source.ID() + gotID := tc.source.Key() if diff := cmp.Diff(tc.wantID, gotID); diff != "" { t.Errorf("(-want +got):\n%s", diff) } @@ -680,3 +744,271 @@ func TestCalculateFilesystemHashDifferentContent(t *testing.T) { }) } } + +func TestSourceVersionAndHash(t *testing.T) { + t.Parallel() + + openAPIFS := afero.NewMemMapFs() + _ = afero.WriteFile(openAPIFS, "api__v1_openapi.json", []byte(`{"a":1}`), 0o600) + + type args struct { + source Source + } + type want struct { + key string + version string + hash string + err error + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "XpkgHashIsDigestNotTag": { + reason: "The lock must pin the resolved digest; the tag is only the version shown to people.", + args: args{source: NewXpkgSource("xpkg.upbound.io/acme/provider", "v1.2.3", "sha256:abc", nil)}, + want: want{key: "xpkg://xpkg.upbound.io/acme/provider", version: "v1.2.3", hash: "sha256:abc"}, + }, + "K8sHashIsContentNotRelease": { + reason: "A k8s source is hashed by its fetched OpenAPI bytes so a changed spec is noticed without a version bump.", + args: args{source: &k8sOpenAPISource{k8sRef: &v1alpha1.K8sDependency{Version: "1.33.0"}, fs: openAPIFS}}, + want: want{ + key: "k8s://1.33.0", + version: "1.33.0", + hash: "015abd7f5cc57a2dd94b7590f04ad8084273905ee33ec5cebeae62276a97f862", + }, + }, + "GitRefIsBranchHashIsCommit": { + reason: "A git source's version is the configured branch or tag; its hash is the commit.", + args: args{source: &gitSource{ + gitRef: &v1alpha1.GitDependency{Repository: "https://example.com/r.git", Ref: "main"}, + commitSHA: "deadbeef", + }}, + want: want{key: "git://https://example.com/r.git", version: "main", hash: "deadbeef"}, + }, + "PinnedGitKeepsBranchAsVersion": { + reason: "A git source cloned from a locked commit still reports the branch it was resolved from, so the lock keeps it.", + args: args{source: &gitSource{ + gitRef: &v1alpha1.GitDependency{Repository: "https://example.com/r.git", Ref: "main"}, + cloneRef: "abc123def456789012345678901234567890abcd", + commitSHA: "abc123def456789012345678901234567890abcd", + }}, + want: want{key: "git://https://example.com/r.git", version: "main", hash: "abc123def456789012345678901234567890abcd"}, + }, + "FSHasNoVersion": { + reason: "A filesystem source has no version.", + args: args{source: NewFSSource("apis", afero.NewMemMapFs())}, + want: want{ + key: "fs://apis", + hash: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + }, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + gotVersion := "" + if p, ok := tc.args.source.(Pinned); ok { + gotVersion = p.Version() + } + gotHash, err := tc.args.source.Hash(t.Context()) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nHash(...): -want err, +got err:\n%s", tc.reason, diff) + } + got := want{key: tc.args.source.Key(), version: gotVersion, hash: gotHash} + if diff := cmp.Diff(tc.want, got, cmp.AllowUnexported(want{}), cmpopts.EquateErrors()); diff != "" { + t.Errorf("%s\nKey(), Version(), Hash(...): -want, +got:\n%s", tc.reason, diff) + } + }) + } +} + +func TestFSSourceHash(t *testing.T) { + t.Parallel() + + const ( + xrd = "apiVersion: apiextensions.crossplane.io/v2\nkind: CompositeResourceDefinition\nmetadata:\n name: a\n" + xrdEdited = "apiVersion: apiextensions.crossplane.io/v2\nkind: CompositeResourceDefinition\nmetadata:\n name: b\n" + crd = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: c\n" + other = "apiVersion: apiextensions.crossplane.io/v1\nkind: Composition\nmetadata:\n name: x\n" + ) + + type args struct { + a map[string]string + b map[string]string + } + type want struct { + same bool + err error + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "AddingCompositionIsIgnored": { + reason: "A Composition is not an API definition, so adding one must not change the hash.", + args: args{ + a: map[string]string{"xrd.yaml": xrd}, + b: map[string]string{"xrd.yaml": xrd, "composition.yaml": other}, + }, + want: want{same: true}, + }, + "EditingExampleAndWorkflowIsIgnored": { + reason: "Examples, workflows and function YAML are not API definitions.", + args: args{ + a: map[string]string{"xrd.yaml": xrd, "examples/xr.yaml": "kind: XR\nspec: {a: 1}\n", ".github/ci.yaml": "name: ci\n", "fn/fn.yaml": "kind: Function\n"}, + b: map[string]string{"xrd.yaml": xrd, "examples/xr.yaml": "kind: XR\nspec: {a: 2}\n", ".github/ci.yaml": "name: ci2\n", "fn/fn.yaml": "kind: Function\nx: y\n"}, + }, + want: want{same: true}, + }, + "NonYAMLIsIgnored": { + reason: "Only YAML files are read.", + args: args{ + a: map[string]string{"xrd.yaml": xrd}, + b: map[string]string{"xrd.yaml": xrd, "README.md": xrd}, + }, + want: want{same: true}, + }, + "EditingXRDChangesHash": { + reason: "Editing an XRD must change the hash.", + args: args{ + a: map[string]string{"xrd.yaml": xrd}, + b: map[string]string{"xrd.yaml": xrdEdited}, + }, + want: want{same: false}, + }, + "CRDCountsAnyGroup": { + reason: "A CRD counts regardless of its apiVersion group.", + args: args{ + a: map[string]string{"crd.yaml": crd}, + b: map[string]string{}, + }, + want: want{same: false}, + }, + "RenamingFileChangesHash": { + reason: "Renaming a file with an XRD must change the hash.", + args: args{ + a: map[string]string{"a.yaml": xrd}, + b: map[string]string{"b.yaml": xrd}, + }, + want: want{same: false}, + }, + "SplittingFileChangesHash": { + reason: "Splitting one multi-document file into two files must change the hash.", + args: args{ + a: map[string]string{"all.yaml": xrd + "---\n" + crd}, + b: map[string]string{"xrd.yaml": xrd, "crd.yaml": crd}, + }, + want: want{same: false}, + }, + "SameContentSameHash": { + reason: "The hash is deterministic.", + args: args{ + a: map[string]string{"b.yaml": crd, "a.yaml": xrd}, + b: map[string]string{"a.yaml": xrd, "b.yaml": crd}, + }, + want: want{same: true}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + hash := func(files map[string]string) (string, error) { + fs := afero.NewMemMapFs() + for p, c := range files { + _ = afero.WriteFile(fs, p, []byte(c), 0o600) + } + return NewFSSource("apis", fs).Hash(t.Context()) + } + + ha, errA := hash(tc.args.a) + hb, errB := hash(tc.args.b) + err := errors.Join(errA, errB) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nHash(...): -want err, +got err:\n%s", tc.reason, diff) + } + got := want{same: ha == hb} + if diff := cmp.Diff(tc.want, got, cmp.AllowUnexported(want{}), cmpopts.EquateErrors()); diff != "" { + t.Errorf("%s\nHash(...): -want, +got:\n%s", tc.reason, diff) + } + }) + } +} + +func TestFSSourceResources(t *testing.T) { + t.Parallel() + + const ( + xrd = "apiVersion: apiextensions.crossplane.io/v2\nkind: CompositeResourceDefinition\nmetadata:\n name: a\n" + crd = "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: c\n" + other = "apiVersion: apiextensions.crossplane.io/v1\nkind: Composition\nmetadata:\n name: x\n" + ) + + type args struct { + files map[string]string + } + type want struct { + files map[string]string + err error + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "DropsOtherDocuments": { + reason: "Only XRDs and CRDs are returned; files without any are omitted.", + args: args{files: map[string]string{ + "xrd.yaml": xrd, + "composition.yaml": other, + "README.md": "hi", + ".github/ci.yaml": "name: ci\n", + }}, + want: want{files: map[string]string{"xrd.yaml": xrd}}, + }, + "KeepsMultiDocumentFiles": { + reason: "Definitions of a multi-document file stay together, other documents are dropped.", + args: args{files: map[string]string{ + "sub/all.yml": xrd + "---\n" + other + "---\n" + crd, + }}, + want: want{files: map[string]string{"sub/all.yml": xrd + "---\n" + crd}}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + fs := afero.NewMemMapFs() + for p, c := range tc.args.files { + _ = afero.WriteFile(fs, p, []byte(c), 0o600) + } + out, err := NewFSSource("apis", fs).Resources(t.Context()) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Fatalf("%s\nResources(...): -want err, +got err:\n%s", tc.reason, diff) + } + + got := want{files: map[string]string{}} + _ = afero.Walk(out, ".", func(p string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() { + return err + } + b, _ := afero.ReadFile(out, p) + got.files[p] = string(b) + return nil + }) + if diff := cmp.Diff(tc.want, got, cmp.AllowUnexported(want{})); diff != "" { + t.Errorf("%s\nResources(...): -want, +got:\n%s", tc.reason, diff) + } + }) + } +} diff --git a/internal/xpkg/resolver.go b/internal/xpkg/resolver.go index 0dbf666d..76e761f4 100644 --- a/internal/xpkg/resolver.go +++ b/internal/xpkg/resolver.go @@ -93,6 +93,37 @@ func (r *Resolver) Resolve(ctx context.Context, ref string) (name.Reference, str return repo.Tag(v), v, nil } +// ResolveConstraints returns the highest version of repo that satisfies every +// constraint, as a tag reference together with the version. The constraints are +// combined with a logical AND, so a constraint using "||" should be passed +// on its own. +func (r *Resolver) ResolveConstraints(ctx context.Context, repo string, constraints []string) (name.Reference, string, error) { + if len(constraints) == 0 { + return nil, "", errors.Errorf("no version constraints for %s", repo) + } + + n, err := name.NewRepository(repo, name.StrictValidation) + if err != nil { + return nil, "", errors.Wrapf(err, "invalid repository %s", repo) + } + + sc, err := semver.NewConstraint(strings.Join(constraints, ", ")) + if err != nil { + return nil, "", errors.Wrapf(err, "invalid version constraints %q", constraints) + } + + tags, err := r.client.ListVersions(ctx, n.Name()) + if err != nil { + return nil, "", errors.Wrapf(err, "cannot list versions for %s", n) + } + v := highestSatisfying(tags, sc) + if v == "" { + return nil, "", errors.Errorf("cannot find version to satisfy constraint %s", sc) + } + + return n.Tag(v), v, nil +} + // highestSatisfying returns the original-form string of the highest // version in tags that satisfies c, or "" if none matches. func highestSatisfying(tags []string, c *semver.Constraints) string { diff --git a/internal/xpkg/resolver_test.go b/internal/xpkg/resolver_test.go index bccb90b8..6bff55a1 100644 --- a/internal/xpkg/resolver_test.go +++ b/internal/xpkg/resolver_test.go @@ -24,9 +24,12 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" "github.com/google/go-containerregistry/pkg/name" + "github.com/crossplane/crossplane-runtime/v2/pkg/errors" "github.com/crossplane/crossplane-runtime/v2/pkg/xpkg" ) +var errBoom = errors.New("boom") + // fakeClient is a fake xpkg.Client whose ListVersions returns a fixed // list of tags and whose Get is unused by the resolver. type fakeClient struct { @@ -149,3 +152,80 @@ func TestResolver_Resolve(t *testing.T) { }) } } + +func TestResolver_ResolveConstraints(t *testing.T) { + tags := []string{"v1.0.0", "v1.1.0", "v1.2.0", "v2.0.0", "latest"} + + type args struct { + repo string + constraints []string + listErr error + } + + type want struct { + ref name.Reference + version string + err error + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "SingleConstraint": { + reason: "A single constraint resolves to the highest matching version.", + args: args{repo: "pkg.example/foo", constraints: []string{">=v1.0.0, =v1.0.0", "=v2.0.0", "=v1.0.0"}, listErr: errBoom}, + want: want{err: cmpopts.AnyError}, + }, + } + + for tname, tc := range cases { + t.Run(tname, func(t *testing.T) { + r := NewResolver(&fakeClient{tags: tags, listErr: tc.args.listErr}) + gotRef, gotVer, err := r.ResolveConstraints(context.Background(), tc.args.repo, tc.args.constraints) + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("%s\nResolveConstraints(...), -want error, +got error:\n%s", tc.reason, diff) + } + + ignoreUnexported := cmpopts.IgnoreUnexported(name.Registry{}, name.Repository{}, name.Tag{}, name.Digest{}) + if diff := cmp.Diff(tc.want.ref, gotRef, ignoreUnexported); diff != "" { + t.Errorf("%s\nResolveConstraints(...), -want ref +got ref:\n%s", tc.reason, diff) + } + if diff := cmp.Diff(tc.want.version, gotVer); diff != "" { + t.Errorf("%s\nResolveConstraints(...), -want version +got version:\n%s", tc.reason, diff) + } + }) + } +} From 00ea4c04e16952b718acb78b30f402fe6b7911df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20N=C3=A4f?= Date: Tue, 20 Oct 2026 08:10:29 +0200 Subject: [PATCH 2/2] fix: Streamline git authentication on all places dealing with schemas/dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cyrill Näf --- cmd/crossplane/dependency/add.go | 7 +- cmd/crossplane/dependency/add_test.go | 80 +++++++++++++++++++++++ cmd/crossplane/dependency/cache.go | 6 +- cmd/crossplane/dependency/help/add.md | 14 ++++ cmd/crossplane/project/build.go | 7 +- cmd/crossplane/project/build_test.go | 94 +++++++++++++++++++++++++++ cmd/crossplane/project/help/build.md | 5 ++ cmd/crossplane/project/run.go | 7 +- cmd/crossplane/render/op/cmd.go | 3 + cmd/crossplane/render/xr/cmd.go | 3 + cmd/crossplane/schema/env.go | 8 +-- internal/git/flags.go | 30 +++++++++ internal/git/flags_test.go | 79 ++++++++++++++++++++++ 13 files changed, 333 insertions(+), 10 deletions(-) create mode 100644 cmd/crossplane/dependency/add_test.go create mode 100644 cmd/crossplane/project/build_test.go create mode 100644 internal/git/flags.go create mode 100644 internal/git/flags_test.go diff --git a/cmd/crossplane/dependency/add.go b/cmd/crossplane/dependency/add.go index 49623d08..5b03e34b 100644 --- a/cmd/crossplane/dependency/add.go +++ b/cmd/crossplane/dependency/add.go @@ -27,6 +27,7 @@ import ( "github.com/crossplane/cli/v2/apis/dev/v1alpha1" "github.com/crossplane/cli/v2/internal/dependency" + "github.com/crossplane/cli/v2/internal/git" "github.com/crossplane/cli/v2/internal/project/projectfile" "github.com/crossplane/cli/v2/internal/terminal" @@ -38,6 +39,8 @@ var addHelp string // addCmd adds a dependency to the current project. type addCmd struct { + git.HTTPSAuthFlags `embed:""` + Package string `arg:"" help:"Package to add (xpkg OCI reference, k8s:, git repository URL, or HTTP(S) URL)."` ProjectFile string `default:"${project_file}" help:"Path to project definition file." short:"f"` CacheDir string `env:"CROSSPLANE_XPKG_CACHE" help:"Directory for cached xpkg package contents." name:"cache-dir"` @@ -61,7 +64,9 @@ func (c *addCmd) Run(logger logging.Logger, sp terminal.SpinnerPrinter) error { return err } - m, err := dependency.NewCachedManager(proj, projFS, c.ProjectFile, c.CacheDir) + m, err := dependency.NewCachedManager(proj, projFS, c.ProjectFile, c.CacheDir, + dependency.WithGitAuthProvider(c.AuthProvider()), + ) if err != nil { return err } diff --git a/cmd/crossplane/dependency/add_test.go b/cmd/crossplane/dependency/add_test.go new file mode 100644 index 00000000..4e2f993f --- /dev/null +++ b/cmd/crossplane/dependency/add_test.go @@ -0,0 +1,80 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dependency + +import ( + "testing" + + "github.com/alecthomas/kong" + "github.com/google/go-cmp/cmp" + + "github.com/crossplane/cli/v2/internal/git" +) + +func TestAddGitAuth(t *testing.T) { + type args struct { + cmd []string + env map[string]string + } + type want struct { + auth git.AuthProvider + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "Defaults": { + reason: "Without flags the username defaults to x-access-token and there is no token.", + args: args{cmd: []string{"add", "pkg"}}, + want: want{auth: &git.HTTPSAuthProvider{Username: "x-access-token"}}, + }, + "Flags": { + reason: "The git flags configure the auth provider.", + args: args{cmd: []string{"add", "pkg", "--git-token", "t", "--git-username", "u"}}, + want: want{auth: &git.HTTPSAuthProvider{Username: "u", Password: "t"}}, + }, + "Env": { + reason: "The git environment variables configure the auth provider.", + args: args{ + cmd: []string{"add", "pkg"}, + env: map[string]string{"CROSSPLANE_GIT_TOKEN": "t", "CROSSPLANE_GIT_USERNAME": "u"}, + }, + want: want{auth: &git.HTTPSAuthProvider{Username: "u", Password: "t"}}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + var c Cmd + parser, err := kong.New(&c, kong.Vars{"project_file": "crossplane-project.yaml"}) + if err != nil { + t.Fatalf("kong.New(): %v", err) + } + for k, v := range tc.args.env { + t.Setenv(k, v) + } + if _, err := parser.Parse(tc.args.cmd); err != nil { + t.Fatalf("%s\nParse(%v): %v", tc.reason, tc.args.cmd, err) + } + if diff := cmp.Diff(tc.want.auth, c.Add.AuthProvider()); diff != "" { + t.Errorf("%s\nAuthProvider(): -want, +got:\n%s", tc.reason, diff) + } + }) + } +} diff --git a/cmd/crossplane/dependency/cache.go b/cmd/crossplane/dependency/cache.go index 0c7b3f2f..b6eec198 100644 --- a/cmd/crossplane/dependency/cache.go +++ b/cmd/crossplane/dependency/cache.go @@ -40,10 +40,10 @@ var updateHelp string // updateCacheCmd refreshes the shared xpkg package cache and fetches the // project's other dependencies. type updateCacheCmd struct { + git.HTTPSAuthFlags `embed:""` + ProjectFile string `default:"${project_file}" help:"Path to project definition file." short:"f"` CacheDir string `env:"CROSSPLANE_XPKG_CACHE" help:"Directory for cached xpkg package contents." name:"cache-dir"` - GitToken string `env:"CROSSPLANE_GIT_TOKEN" help:"Token for git HTTPS authentication."` - GitUsername string `default:"x-access-token" env:"CROSSPLANE_GIT_USERNAME" help:"Username for git HTTPS authentication."` } func (c *updateCacheCmd) Help() string { @@ -72,7 +72,7 @@ func (c *updateCacheCmd) Run(logger logging.Logger, sp terminal.SpinnerPrinter) // gitAuth returns the git authentication configured by the command's flags. func (c *updateCacheCmd) gitAuth() git.AuthProvider { - return &git.HTTPSAuthProvider{Username: c.GitUsername, Password: c.GitToken} + return c.AuthProvider() } //go:embed help/clean-cache.md diff --git a/cmd/crossplane/dependency/help/add.md b/cmd/crossplane/dependency/help/add.md index 59fac478..85f76b49 100644 --- a/cmd/crossplane/dependency/help/add.md +++ b/cmd/crossplane/dependency/help/add.md @@ -20,6 +20,11 @@ dependencies. Use the `--api-only` flag to add a build-time xpkg dependency. Non-xpkg dependencies are always build-time dependencies. +Git repositories are accessed over HTTPS. Use `--git-token` (or the +`CROSSPLANE_GIT_TOKEN` environment variable) to authenticate, and +`--git-username` (or `CROSSPLANE_GIT_USERNAME`, default `x-access-token`) to set +the username. + ## Examples Retrieve the latest available semantic version of `provider-aws-eks` and add it @@ -57,3 +62,12 @@ build-time dependency: crossplane dependency add https://github.com/kubernetes-sigs/cluster-api \ --git-ref=release-1.11 --git-path=config/crd/bases ``` + +Add CRDs from a private git repository, authenticating with a token from the +environment: + +```shell +export CROSSPLANE_GIT_TOKEN= +crossplane dependency add https:////.git \ + --git-ref=main --git-path=config/crd/bases +``` diff --git a/cmd/crossplane/project/build.go b/cmd/crossplane/project/build.go index 0171187d..fabac478 100644 --- a/cmd/crossplane/project/build.go +++ b/cmd/crossplane/project/build.go @@ -33,6 +33,7 @@ import ( "github.com/crossplane/cli/v2/internal/async" "github.com/crossplane/cli/v2/internal/config" "github.com/crossplane/cli/v2/internal/dependency" + "github.com/crossplane/cli/v2/internal/git" "github.com/crossplane/cli/v2/internal/project" "github.com/crossplane/cli/v2/internal/project/functions" "github.com/crossplane/cli/v2/internal/project/projectfile" @@ -46,6 +47,8 @@ var buildHelp string // buildCmd builds a project into Crossplane packages. type buildCmd struct { + git.HTTPSAuthFlags `embed:""` + ProjectFile string `default:"${project_file}" help:"Path to project definition." short:"f"` Repository string `help:"Override the repository in the project file." optional:""` OutputDir string `default:"_output" help:"Output directory for packages." short:"o"` @@ -94,7 +97,9 @@ func (c *buildCmd) Run(logger logging.Logger, sp terminal.SpinnerPrinter, cfg *c concurrency := max(1, c.MaxConcurrency) schemaMgr := project.NewSchemaManager(c.proj, c.projFS, cfg.Features) - depMgr, err := dependency.NewCachedManager(c.proj, c.projFS, c.ProjectFile, c.CacheDir) + depMgr, err := dependency.NewCachedManager(c.proj, c.projFS, c.ProjectFile, c.CacheDir, + dependency.WithGitAuthProvider(c.AuthProvider()), + ) if err != nil { return err } diff --git a/cmd/crossplane/project/build_test.go b/cmd/crossplane/project/build_test.go new file mode 100644 index 00000000..d67e7c00 --- /dev/null +++ b/cmd/crossplane/project/build_test.go @@ -0,0 +1,94 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/alecthomas/kong" + "github.com/google/go-cmp/cmp" + + "github.com/crossplane/cli/v2/internal/git" +) + +const projectYAML = `apiVersion: dev.crossplane.io/v1alpha1 +kind: Project +metadata: + name: test-project +spec: + repository: xpkg.crossplane.io/test/project +` + +func TestBuildGitAuth(t *testing.T) { + type args struct { + cmd []string + env map[string]string + } + type want struct { + auth git.AuthProvider + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "Defaults": { + reason: "Without flags the username defaults to x-access-token and there is no token.", + args: args{cmd: []string{"build"}}, + want: want{auth: &git.HTTPSAuthProvider{Username: "x-access-token"}}, + }, + "Flags": { + reason: "The git flags configure the auth provider.", + args: args{cmd: []string{"build", "--git-token", "t", "--git-username", "u"}}, + want: want{auth: &git.HTTPSAuthProvider{Username: "u", Password: "t"}}, + }, + "Env": { + reason: "The git environment variables configure the auth provider.", + args: args{ + cmd: []string{"build"}, + env: map[string]string{"CROSSPLANE_GIT_TOKEN": "t", "CROSSPLANE_GIT_USERNAME": "u"}, + }, + want: want{auth: &git.HTTPSAuthProvider{Username: "u", Password: "t"}}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + projFile := filepath.Join(t.TempDir(), "crossplane-project.yaml") + if err := os.WriteFile(projFile, []byte(projectYAML), 0o600); err != nil { + t.Fatal(err) + } + var c Cmd + parser, err := kong.New(&c, kong.Vars{"project_file": projFile}) + if err != nil { + t.Fatalf("kong.New(): %v", err) + } + for k, v := range tc.args.env { + t.Setenv(k, v) + } + if _, err := parser.Parse(tc.args.cmd); err != nil { + t.Fatalf("%s\nParse(%v): %v", tc.reason, tc.args.cmd, err) + } + if diff := cmp.Diff(tc.want.auth, c.Build.AuthProvider()); diff != "" { + t.Errorf("%s\nAuthProvider(): -want, +got:\n%s", tc.reason, diff) + } + }) + } +} diff --git a/cmd/crossplane/project/help/build.md b/cmd/crossplane/project/help/build.md index c97e8cc3..60ead057 100644 --- a/cmd/crossplane/project/help/build.md +++ b/cmd/crossplane/project/help/build.md @@ -27,6 +27,11 @@ versions the lock pins, which works offline when the packages are cached. Run `crossplane schema update` to accept changes and update the lock, or `crossplane schema generate` to restore modified files. +Creating the lock or recreating missing schema files may clone git +dependencies. The build authenticates with the same `--git-token` and +`--git-username` flags, or the `CROSSPLANE_GIT_TOKEN` and +`CROSSPLANE_GIT_USERNAME` environment variables, as `dependency add`. + The CLI builds embedded functions onto a runtime base image from a registry, and caches those image layers on disk under `crossplane/base-images` in your user cache directory. A build that needs a layer already in the cache reads it diff --git a/cmd/crossplane/project/run.go b/cmd/crossplane/project/run.go index 01658e85..31e0e39f 100644 --- a/cmd/crossplane/project/run.go +++ b/cmd/crossplane/project/run.go @@ -44,6 +44,7 @@ import ( "github.com/crossplane/cli/v2/internal/async" "github.com/crossplane/cli/v2/internal/config" "github.com/crossplane/cli/v2/internal/dependency" + "github.com/crossplane/cli/v2/internal/git" "github.com/crossplane/cli/v2/internal/project" "github.com/crossplane/cli/v2/internal/project/controlplane" "github.com/crossplane/cli/v2/internal/project/functions" @@ -58,6 +59,8 @@ var runHelp string // runCmd builds a project and runs it in a local dev control plane. type runCmd struct { + git.HTTPSAuthFlags `embed:""` + ProjectFile string `default:"${project_file}" help:"Path to project definition." short:"f"` Repository string `help:"Override the repository." optional:""` MaxConcurrency uint `default:"8" help:"Max concurrent builds."` @@ -148,7 +151,9 @@ func (c *runCmd) Run(logger logging.Logger, sp terminal.SpinnerPrinter, cfg *con concurrency := max(1, c.MaxConcurrency) schemaMgr := project.NewSchemaManager(c.proj, c.projFS, cfg.Features) - depMgr, err := dependency.NewCachedManager(c.proj, c.projFS, c.ProjectFile, c.CacheDir) + depMgr, err := dependency.NewCachedManager(c.proj, c.projFS, c.ProjectFile, c.CacheDir, + dependency.WithGitAuthProvider(c.AuthProvider()), + ) if err != nil { return err } diff --git a/cmd/crossplane/render/op/cmd.go b/cmd/crossplane/render/op/cmd.go index 64fe118f..e94ad709 100644 --- a/cmd/crossplane/render/op/cmd.go +++ b/cmd/crossplane/render/op/cmd.go @@ -42,6 +42,7 @@ import ( "github.com/crossplane/cli/v2/internal/async" "github.com/crossplane/cli/v2/internal/config" "github.com/crossplane/cli/v2/internal/dependency" + "github.com/crossplane/cli/v2/internal/git" "github.com/crossplane/cli/v2/internal/project" "github.com/crossplane/cli/v2/internal/project/functions" "github.com/crossplane/cli/v2/internal/project/projectfile" @@ -56,6 +57,7 @@ var helpDetail string // Cmd arguments and flags for alpha render op subcommand. type Cmd struct { + git.HTTPSAuthFlags `embed:""` render.EngineFlags `prefix:""` // Arguments. @@ -361,6 +363,7 @@ func (c *Cmd) loadFunctions(ctx context.Context, log logging.Logger, sp terminal dependency.WithProjectFile(filepath.Base(projFilePath)), dependency.WithXpkgClient(xpkgClient), dependency.WithResolver(resolver), + dependency.WithGitAuthProvider(c.AuthProvider()), ) var fns []pkgv1.Function diff --git a/cmd/crossplane/render/xr/cmd.go b/cmd/crossplane/render/xr/cmd.go index df52559f..556cffe9 100644 --- a/cmd/crossplane/render/xr/cmd.go +++ b/cmd/crossplane/render/xr/cmd.go @@ -49,6 +49,7 @@ import ( "github.com/crossplane/cli/v2/internal/async" "github.com/crossplane/cli/v2/internal/config" "github.com/crossplane/cli/v2/internal/dependency" + "github.com/crossplane/cli/v2/internal/git" "github.com/crossplane/cli/v2/internal/project" "github.com/crossplane/cli/v2/internal/project/functions" "github.com/crossplane/cli/v2/internal/project/projectfile" @@ -64,6 +65,7 @@ var helpDetail string // Cmd arguments and flags for the `render xr` subcommand. type Cmd struct { + git.HTTPSAuthFlags `embed:""` render.EngineFlags `prefix:""` // Arguments. @@ -475,6 +477,7 @@ func (c *Cmd) loadFunctionsFromProject(ctx context.Context, log logging.Logger, dependency.WithProjectFile(projFileName), dependency.WithXpkgClient(xpkgClient), dependency.WithResolver(resolver), + dependency.WithGitAuthProvider(c.AuthProvider()), ) var fns []pkgv1.Function diff --git a/cmd/crossplane/schema/env.go b/cmd/crossplane/schema/env.go index cd212b6d..38d85a8a 100644 --- a/cmd/crossplane/schema/env.go +++ b/cmd/crossplane/schema/env.go @@ -35,9 +35,9 @@ type ProjectFlags struct { // SourceFlags are the flags commands need to fetch the project's dependencies. type SourceFlags struct { - CacheDir string `env:"CROSSPLANE_XPKG_CACHE" help:"Directory for cached xpkg package contents." name:"cache-dir"` - GitToken string `env:"CROSSPLANE_GIT_TOKEN" help:"Token for git HTTPS authentication."` - GitUsername string `default:"x-access-token" env:"CROSSPLANE_GIT_USERNAME" help:"Username for git HTTPS authentication."` + git.HTTPSAuthFlags `embed:""` + + CacheDir string `env:"CROSSPLANE_XPKG_CACHE" help:"Directory for cached xpkg package contents." name:"cache-dir"` } // env is everything a schema command works with: the project, its schema @@ -68,7 +68,7 @@ func newEnv(projectFile string, cfg *config.Config, src *SourceFlags) (*env, err } e.deps, err = dependency.NewCachedManager(proj, projFS, projectFile, src.CacheDir, - dependency.WithGitAuthProvider(&git.HTTPSAuthProvider{Username: src.GitUsername, Password: src.GitToken}), + dependency.WithGitAuthProvider(src.AuthProvider()), ) if err != nil { return nil, err diff --git a/internal/git/flags.go b/internal/git/flags.go new file mode 100644 index 00000000..5bbbad1a --- /dev/null +++ b/internal/git/flags.go @@ -0,0 +1,30 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +// HTTPSAuthFlags are the flags for git HTTPS authentication. Embed it into a +// command's Kong flag struct with the `embed:""` tag, then pass AuthProvider to +// the dependency manager. +type HTTPSAuthFlags struct { + GitToken string `env:"CROSSPLANE_GIT_TOKEN" help:"Token for git HTTPS authentication."` + GitUsername string `default:"x-access-token" env:"CROSSPLANE_GIT_USERNAME" help:"Username for git HTTPS authentication."` +} + +// AuthProvider returns the auth provider configured by the flags. +func (f HTTPSAuthFlags) AuthProvider() AuthProvider { + return &HTTPSAuthProvider{Username: f.GitUsername, Password: f.GitToken} +} diff --git a/internal/git/flags_test.go b/internal/git/flags_test.go new file mode 100644 index 00000000..1f589219 --- /dev/null +++ b/internal/git/flags_test.go @@ -0,0 +1,79 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +import ( + "testing" + + "github.com/alecthomas/kong" + "github.com/google/go-cmp/cmp" +) + +func TestHTTPSAuthFlags(t *testing.T) { + type args struct { + cmd []string + env map[string]string + } + type want struct { + auth AuthProvider + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "Defaults": { + reason: "Without flags the username defaults to x-access-token and there is no token.", + args: args{}, + want: want{auth: &HTTPSAuthProvider{Username: "x-access-token"}}, + }, + "Flags": { + reason: "The git flags configure the auth provider.", + args: args{cmd: []string{"--git-token", "t", "--git-username", "u"}}, + want: want{auth: &HTTPSAuthProvider{Username: "u", Password: "t"}}, + }, + "Env": { + reason: "The git environment variables configure the auth provider.", + args: args{ + env: map[string]string{"CROSSPLANE_GIT_TOKEN": "t", "CROSSPLANE_GIT_USERNAME": "u"}, + }, + want: want{auth: &HTTPSAuthProvider{Username: "u", Password: "t"}}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + var c struct { + HTTPSAuthFlags `embed:""` + } + parser, err := kong.New(&c) + if err != nil { + t.Fatalf("kong.New(): %v", err) + } + for k, v := range tc.args.env { + t.Setenv(k, v) + } + if _, err := parser.Parse(tc.args.cmd); err != nil { + t.Fatalf("%s\nParse(%v): %v", tc.reason, tc.args.cmd, err) + } + if diff := cmp.Diff(tc.want.auth, c.AuthProvider()); diff != "" { + t.Errorf("%s\nAuthProvider(): -want, +got:\n%s", tc.reason, diff) + } + }) + } +}