From bf65abce389d58fbe4e8f35c90e4c43438eed6d7 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Wed, 8 Oct 2025 22:20:33 -0600 Subject: [PATCH 01/12] Avoid @// --- example/person/BUILD.bazel | 10 +++++----- example/place/BUILD.bazel | 10 +++++----- example/routeguide/BUILD.bazel | 10 +++++----- example/thing/BUILD.bazel | 10 +++++----- google/protobuf/BUILD.bazel | 10 +++++----- rules/go/README.md | 4 ++-- rules/golden_filegroup.bzl | 25 ++++++++++++++++++++----- rules/proto_compile_gencopy.bzl | 2 ++ rules_proto_config.yaml | 10 +++++----- 9 files changed, 54 insertions(+), 37 deletions(-) diff --git a/example/person/BUILD.bazel b/example/person/BUILD.bazel index 61a5b20a1..f78ef8a17 100644 --- a/example/person/BUILD.bazel +++ b/example/person/BUILD.bazel @@ -151,11 +151,11 @@ proto_ts_library( srcs = ["person.ts"], visibility = ["//visibility:public"], deps = [ + "//:node_modules/@nestjs/microservices", + "//:node_modules/@types/node", + "//:node_modules/long", + "//:node_modules/protobufjs", + "//:node_modules/rxjs", "//example/place:place_ts_proto", - "@//:node_modules/@nestjs/microservices", - "@//:node_modules/@types/node", - "@//:node_modules/long", - "@//:node_modules/protobufjs", - "@//:node_modules/rxjs", ], ) diff --git a/example/place/BUILD.bazel b/example/place/BUILD.bazel index b06408baf..fe7c3ebfc 100644 --- a/example/place/BUILD.bazel +++ b/example/place/BUILD.bazel @@ -151,11 +151,11 @@ proto_ts_library( srcs = ["place.ts"], visibility = ["//visibility:public"], deps = [ + "//:node_modules/@nestjs/microservices", + "//:node_modules/@types/node", + "//:node_modules/long", + "//:node_modules/protobufjs", + "//:node_modules/rxjs", "//example/thing:thing_ts_proto", - "@//:node_modules/@nestjs/microservices", - "@//:node_modules/@types/node", - "@//:node_modules/long", - "@//:node_modules/protobufjs", - "@//:node_modules/rxjs", ], ) diff --git a/example/routeguide/BUILD.bazel b/example/routeguide/BUILD.bazel index e31dee27d..2078d05a3 100644 --- a/example/routeguide/BUILD.bazel +++ b/example/routeguide/BUILD.bazel @@ -252,10 +252,10 @@ proto_ts_library( srcs = ["routeguide.ts"], visibility = ["//visibility:public"], deps = [ - "@//:node_modules/@nestjs/microservices", - "@//:node_modules/@types/node", - "@//:node_modules/long", - "@//:node_modules/protobufjs", - "@//:node_modules/rxjs", + "//:node_modules/@nestjs/microservices", + "//:node_modules/@types/node", + "//:node_modules/long", + "//:node_modules/protobufjs", + "//:node_modules/rxjs", ], ) diff --git a/example/thing/BUILD.bazel b/example/thing/BUILD.bazel index c08fd89b1..cc7da8a2a 100644 --- a/example/thing/BUILD.bazel +++ b/example/thing/BUILD.bazel @@ -151,11 +151,11 @@ proto_ts_library( srcs = ["thing.ts"], visibility = ["//visibility:public"], deps = [ + "//:node_modules/@nestjs/microservices", + "//:node_modules/@types/node", + "//:node_modules/long", + "//:node_modules/protobufjs", + "//:node_modules/rxjs", "//google/protobuf:timestamppb_ts_proto", - "@//:node_modules/@nestjs/microservices", - "@//:node_modules/@types/node", - "@//:node_modules/long", - "@//:node_modules/protobufjs", - "@//:node_modules/rxjs", ], ) diff --git a/google/protobuf/BUILD.bazel b/google/protobuf/BUILD.bazel index f315f7003..db878c667 100644 --- a/google/protobuf/BUILD.bazel +++ b/google/protobuf/BUILD.bazel @@ -27,10 +27,10 @@ proto_ts_library( srcs = ["timestamp.ts"], visibility = ["//visibility:public"], deps = [ - "@//:node_modules/@nestjs/microservices", - "@//:node_modules/@types/node", - "@//:node_modules/long", - "@//:node_modules/protobufjs", - "@//:node_modules/rxjs", + "//:node_modules/@nestjs/microservices", + "//:node_modules/@types/node", + "//:node_modules/long", + "//:node_modules/protobufjs", + "//:node_modules/rxjs", ], ) diff --git a/rules/go/README.md b/rules/go/README.md index f42be3ddb..42dda9c8e 100644 --- a/rules/go/README.md +++ b/rules/go/README.md @@ -15,7 +15,7 @@ One solution is to manually copy all the needed `.proto` files from the different repos into the default workspace and generate protos from there. This is a common solution but can be troublesome to maintain. -The solution described here (`bazel run @//:proto_go_modules`) has the following +The solution described here (`bazel run //:proto_go_modules`) has the following two side effects: 1. creates a "vendored" file tree in `./local` @@ -149,7 +149,7 @@ proto_repository.archive( ], build_file_proto_mode = "file", build_file_generation = "clean", - cfgs = ["@//:rules_proto_config.yaml"], + cfgs = ["//:rules_proto_config.yaml"], deleted_files = [ "google/protobuf/*test*.proto", "google/protobuf/*unittest*.proto", diff --git a/rules/golden_filegroup.bzl b/rules/golden_filegroup.bzl index fba19c7f5..8f6abfa58 100644 --- a/rules/golden_filegroup.bzl +++ b/rules/golden_filegroup.bzl @@ -19,16 +19,21 @@ load("@build_stack_rules_proto//rules:providers.bzl", "ProtoCompileInfo") def _files_impl(ctx): dep = ctx.attr.dep[DefaultInfo] + outputs = dep.files.to_list() + output_files_by_rel_path = {"/".join([ctx.label.package, f.basename]): f for f in outputs} + return ProtoCompileInfo( label = ctx.attr.dep.label, - outputs = dep.files.to_list(), - output_files_by_rel_path = {}, + outputs = outputs, + output_files_by_rel_path = output_files_by_rel_path, ) _files = rule( doc = """Provider Adapter from DefaultInfo to ProtoCompileInfo.""", implementation = _files_impl, - attrs = {"dep": attr.label(providers = [DefaultInfo])}, + attrs = { + "dep": attr.label(providers = [DefaultInfo]), + }, ) def golden_filegroup( @@ -55,9 +60,19 @@ def golden_filegroup( tags = kwargs.pop("tags", []) srcs = kwargs.pop("srcs", []) goldens = [src + extension for src in srcs] - native.filegroup(name = name, srcs = srcs, tags = tags, **kwargs) - _files(name = name_sources, dep = name, tags = tags) + native.filegroup( + name = name, + srcs = srcs, + tags = tags, + **kwargs + ) + + _files( + name = name_sources, + dep = name, + tags = tags, + ) proto_compile_gencopy_test( name = name_test, diff --git a/rules/proto_compile_gencopy.bzl b/rules/proto_compile_gencopy.bzl index ae53f6ce9..0290280b8 100644 --- a/rules/proto_compile_gencopy.bzl +++ b/rules/proto_compile_gencopy.bzl @@ -22,6 +22,8 @@ def _proto_compile_gencopy_run_impl(ctx): source_files.append(rel) generated_files.append(generated_file.short_path) + if len(source_files) == 0: + fail("source files cannot be empty") config.packageConfigs.append( struct( targetLabel = str(info.label), diff --git a/rules_proto_config.yaml b/rules_proto_config.yaml index 21c4f14db..1769f57be 100644 --- a/rules_proto_config.yaml +++ b/rules_proto_config.yaml @@ -208,11 +208,11 @@ rules: visibility: - //visibility:public deps: - - "@//:node_modules/@nestjs/microservices" - - "@//:node_modules/@types/node" - - "@//:node_modules/long" - - "@//:node_modules/protobufjs" - - "@//:node_modules/rxjs" + - "//:node_modules/@nestjs/microservices" + - "//:node_modules/@types/node" + - "//:node_modules/long" + - "//:node_modules/protobufjs" + - "//:node_modules/rxjs" languages: # CLOSURE From 8e0b4fc400a106502b092b104b9d7828865d3e39 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Sun, 23 Nov 2025 10:31:04 -0700 Subject: [PATCH 02/12] Initial symbol extension --- WORKSPACE | 0 cmd/gazelle/BUILD.bazel | 1 + cmd/gazelle/langs.go | 2 + language/symbol/BUILD.bazel | 18 + language/symbol/symbol.go | 351 ++++++++++++++++++ rules/private/proto_repository_tools.bzl | 4 +- rules/private/proto_repository_tools_srcs.bzl | 2 + .../bazel-gazelle/config/BUILD.bazel | 41 ++ .../bazelbuild/bazel-gazelle/flag/BUILD.bazel | 24 ++ .../bazel-gazelle/internal/module/BUILD.bazel | 43 +++ .../internal/version/BUILD.bazel | 31 ++ .../bazel-gazelle/internal/wspace/BUILD.bazel | 32 ++ .../bazel-gazelle/label/BUILD.bazel | 35 ++ .../bazel-gazelle/language/BUILD.bazel | 42 +++ .../bazel-gazelle/language/go/BUILD.bazel | 162 ++++++++ .../bazel-gazelle/language/proto/BUILD.bazel | 127 +++++++ .../bazel-gazelle/merger/BUILD.bazel | 52 +++ .../bazel-gazelle/pathtools/BUILD.bazel | 32 ++ .../bazelbuild/bazel-gazelle/repo/BUILD.bazel | 54 +++ .../bazel-gazelle/resolve/BUILD.bazel | 47 +++ .../bazelbuild/bazel-gazelle/rule/BUILD.bazel | 67 ++++ .../bazel-gazelle/testtools/BUILD.bazel | 34 ++ .../bazelbuild/bazel-gazelle/walk/BUILD.bazel | 58 +++ .../bazelbuild/buildtools/build/BUILD.bazel | 64 ++++ .../bazelbuild/buildtools/labels/BUILD.bazel | 20 + .../bazelbuild/buildtools/tables/BUILD.bazel | 25 ++ .../rules_go/go/runfiles/BUILD.bazel | 61 +++ .../rules_go/go/tools/bazel/BUILD.bazel | 34 ++ 28 files changed, 1461 insertions(+), 2 deletions(-) create mode 100644 WORKSPACE create mode 100644 language/symbol/BUILD.bazel create mode 100644 language/symbol/symbol.go create mode 100644 vendor/github.com/bazelbuild/bazel-gazelle/config/BUILD.bazel create mode 100644 vendor/github.com/bazelbuild/bazel-gazelle/flag/BUILD.bazel create mode 100644 vendor/github.com/bazelbuild/bazel-gazelle/internal/module/BUILD.bazel create mode 100644 vendor/github.com/bazelbuild/bazel-gazelle/internal/version/BUILD.bazel create mode 100644 vendor/github.com/bazelbuild/bazel-gazelle/internal/wspace/BUILD.bazel create mode 100644 vendor/github.com/bazelbuild/bazel-gazelle/label/BUILD.bazel create mode 100644 vendor/github.com/bazelbuild/bazel-gazelle/language/BUILD.bazel create mode 100644 vendor/github.com/bazelbuild/bazel-gazelle/language/go/BUILD.bazel create mode 100644 vendor/github.com/bazelbuild/bazel-gazelle/language/proto/BUILD.bazel create mode 100644 vendor/github.com/bazelbuild/bazel-gazelle/merger/BUILD.bazel create mode 100644 vendor/github.com/bazelbuild/bazel-gazelle/pathtools/BUILD.bazel create mode 100644 vendor/github.com/bazelbuild/bazel-gazelle/repo/BUILD.bazel create mode 100644 vendor/github.com/bazelbuild/bazel-gazelle/resolve/BUILD.bazel create mode 100644 vendor/github.com/bazelbuild/bazel-gazelle/rule/BUILD.bazel create mode 100644 vendor/github.com/bazelbuild/bazel-gazelle/testtools/BUILD.bazel create mode 100644 vendor/github.com/bazelbuild/bazel-gazelle/walk/BUILD.bazel create mode 100644 vendor/github.com/bazelbuild/buildtools/build/BUILD.bazel create mode 100644 vendor/github.com/bazelbuild/buildtools/labels/BUILD.bazel create mode 100644 vendor/github.com/bazelbuild/buildtools/tables/BUILD.bazel create mode 100644 vendor/github.com/bazelbuild/rules_go/go/runfiles/BUILD.bazel create mode 100644 vendor/github.com/bazelbuild/rules_go/go/tools/bazel/BUILD.bazel diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/gazelle/BUILD.bazel b/cmd/gazelle/BUILD.bazel index 038ba6469..b0a060f2d 100644 --- a/cmd/gazelle/BUILD.bazel +++ b/cmd/gazelle/BUILD.bazel @@ -29,6 +29,7 @@ go_library( "//cmd/gazelle/internal/wspace", "//language/proto_go_modules", "//language/protobuf", + "//language/symbol", "@bazel_gazelle//config:go_default_library", "@bazel_gazelle//flag:go_default_library", "@bazel_gazelle//label:go_default_library", diff --git a/cmd/gazelle/langs.go b/cmd/gazelle/langs.go index 021652958..7202e0a08 100644 --- a/cmd/gazelle/langs.go +++ b/cmd/gazelle/langs.go @@ -21,6 +21,7 @@ import ( "github.com/bazelbuild/bazel-gazelle/language/proto" "github.com/stackb/rules_proto/language/proto_go_modules" "github.com/stackb/rules_proto/language/protobuf" + "github.com/stackb/rules_proto/language/symbol" ) var languages = []language.Language{ @@ -28,4 +29,5 @@ var languages = []language.Language{ protobuf.NewLanguage(), golang.NewLanguage(), proto_go_modules.NewLanguage(), + symbol.NewLanguage(), } diff --git a/language/symbol/BUILD.bazel b/language/symbol/BUILD.bazel new file mode 100644 index 000000000..bd1ec100a --- /dev/null +++ b/language/symbol/BUILD.bazel @@ -0,0 +1,18 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "symbol", + srcs = ["symbol.go"], + importpath = "github.com/stackb/rules_proto/language/symbol", + visibility = ["//visibility:public"], + deps = [ + "@bazel_gazelle//config:go_default_library", + "@bazel_gazelle//label:go_default_library", + "@bazel_gazelle//language:go_default_library", + "@bazel_gazelle//pathtools:go_default_library", + "@bazel_gazelle//repo:go_default_library", + "@bazel_gazelle//resolve:go_default_library", + "@bazel_gazelle//rule:go_default_library", + "@com_github_bazelbuild_buildtools//build:go_default_library", + ], +) diff --git a/language/symbol/symbol.go b/language/symbol/symbol.go new file mode 100644 index 000000000..addf2b215 --- /dev/null +++ b/language/symbol/symbol.go @@ -0,0 +1,351 @@ +/* Copyright 2020 The Bazel Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package symbol generates a `symbol_library` target for every `.bzl` file in +// each package. At the root of the module, a single symbol_library is +// populated with deps that include all other symbol_libraries. +// +// The original code for this gazelle extension started from +// https://github.com/bazelbuild/bazel-skylib/blob/main/gazelle/bzl/gazelle.go. +package symbol + +import ( + "flag" + "fmt" + "log" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/bazelbuild/bazel-gazelle/config" + "github.com/bazelbuild/bazel-gazelle/label" + "github.com/bazelbuild/bazel-gazelle/language" + "github.com/bazelbuild/bazel-gazelle/pathtools" + "github.com/bazelbuild/bazel-gazelle/repo" + "github.com/bazelbuild/bazel-gazelle/resolve" + "github.com/bazelbuild/bazel-gazelle/rule" + "github.com/bazelbuild/buildtools/build" +) + +const ( + languageName = "symbol" + symbolLibraryKind = "symbol_library" + fileType = ".bzl" +) + +var ignoreSuffix = suffixes{ + "_tests.bzl", + "_test.bzl", +} + +var kinds = map[string]rule.KindInfo{ + symbolLibraryKind: { + NonEmptyAttrs: map[string]bool{"srcs": true, "deps": true}, + MergeableAttrs: map[string]bool{"srcs": true}, + }, +} + +type suffixes []string + +func (s suffixes) Matches(test string) bool { + for _, v := range s { + if strings.HasSuffix(test, v) { + return true + } + } + return false +} + +type symbolLang struct { + enabled bool +} + +// NewLanguage is called by Gazelle to install this language extension in a +// binary. +func NewLanguage() language.Language { + return &symbolLang{} +} + +// Name returns the name of the language. This should be a prefix of the kinds +// of rules generated by the language, e.g., "go" for the Go extension since it +// generates "go_library" rules. +func (*symbolLang) Name() string { return languageName } + +// The following methods are implemented to satisfy the +// https://pkg.go.dev/github.com/bazelbuild/bazel-gazelle/resolve?tab=doc#Resolver +// interface, but are otherwise unused. +func (l *symbolLang) RegisterFlags(fs *flag.FlagSet, cmd string, c *config.Config) { + fs.BoolVar(&l.enabled, "symbol_language_enabled", false, "whather this extension is turned on") +} +func (*symbolLang) CheckFlags(fs *flag.FlagSet, c *config.Config) error { return nil } +func (*symbolLang) KnownDirectives() []string { return nil } +func (*symbolLang) Configure(c *config.Config, rel string, f *rule.File) {} + +// Kinds returns a map of maps rule names (kinds) and information on how to +// match and merge attributes that may be found in rules of those kinds. All +// kinds of rules generated for this language may be found here. +func (*symbolLang) Kinds() map[string]rule.KindInfo { + return kinds +} + +// Loads returns .bzl files and symbols they define. Every rule generated by +// GenerateRules, now or in the past, should be loadable from one of these +// files. +func (*symbolLang) Loads() []rule.LoadInfo { + return []rule.LoadInfo{{ + Name: "@build_stack_rules_proto//rules:symbol_library.bzl", + Symbols: []string{symbolLibraryKind}, + }} +} + +// Fix repairs deprecated usage of language-specific rules in f. This is called +// before the file is indexed. Unless c.ShouldFix is true, fixes that delete or +// rename rules should not be performed. +func (*symbolLang) Fix(c *config.Config, f *rule.File) {} + +// Imports returns a list of ImportSpecs that can be used to import the rule r. +// This is used to populate RuleIndex. +// +// If nil is returned, the rule will not be indexed. If any non-nil slice is +// returned, including an empty slice, the rule will be indexed. +func (b *symbolLang) Imports(c *config.Config, r *rule.Rule, f *rule.File) []resolve.ImportSpec { + srcs := r.AttrStrings("srcs") + imports := make([]resolve.ImportSpec, 0, len(srcs)) + + for _, src := range srcs { + spec := resolve.ImportSpec{ + // Lang is the language in which the import string appears (this should + // match Resolver.Name). + Lang: languageName, + // Imp is an import string for the library. + Imp: fmt.Sprintf("//%s:%s", f.Pkg, src), + } + + imports = append(imports, spec) + } + + return imports +} + +// Embeds returns a list of labels of rules that the given rule embeds. If a +// rule is embedded by another importable rule of the same language, only the +// embedding rule will be indexed. The embedding rule will inherit the imports +// of the embedded rule. Since SkyLark doesn't support embedding this should +// always return nil. +func (*symbolLang) Embeds(r *rule.Rule, from label.Label) []label.Label { return nil } + +// Resolve translates imported libraries for a given rule into Bazel +// dependencies. Information about imported libraries is returned for each rule +// generated by language.GenerateRules in language.GenerateResult.Imports. +// Resolve generates a "deps" attribute (or the appropriate language-specific +// equivalent) for each import according to language-specific rules and +// heuristics. +func (*symbolLang) Resolve(c *config.Config, ix *resolve.RuleIndex, rc *repo.RemoteCache, r *rule.Rule, importsRaw interface{}, from label.Label) { + imports := importsRaw.([]string) + + r.DelAttr("deps") + + if len(imports) == 0 { + return + } + + deps := make([]string, 0, len(imports)) + for _, imp := range imports { + impLabel, err := label.Parse(imp) + if err != nil { + log.Printf("%s: import of %q is invalid: %v", from.String(), imp, err) + continue + } + + // the index only contains absolute labels, not relative + impLabel = impLabel.Abs(from.Repo, from.Pkg) + + if impLabel.Repo == "bazel_tools" { + // The @bazel_tools repo is tricky because it is a part of the + // "shipped with bazel" core library for interacting with the + // outside world. This means that it can not depend on skylib. + // Fortunately there is a fairly simple workaround for this, which + // is that you can add those bzl files as `deps` entries. + deps = append(deps, imp) + continue + } + + if impLabel.Repo != "" || !c.IndexLibraries { + // This is a dependency that is external to the current repo, or + // indexing is disabled so take a guess at what the target name + // should be. + deps = append(deps, strings.TrimSuffix(imp, fileType)) + continue + } + + res := resolve.ImportSpec{ + Lang: languageName, + Imp: impLabel.String(), + } + matches := ix.FindRulesByImportWithConfig(c, res, languageName) + if len(matches) == 0 { + log.Printf("%s: %q (%s) was not found in dependency index. Skipping. This may result in an incomplete deps section and require manual BUILD file intervention.\n", from.String(), imp, impLabel.String()) + } + + for _, m := range matches { + depLabel := m.Label + depLabel = depLabel.Rel(from.Repo, from.Pkg) + deps = append(deps, depLabel.String()) + } + } + + sort.Strings(deps) + if len(deps) > 0 { + r.SetAttr("deps", deps) + } +} + +// GenerateRules extracts build metadata from source files in a directory. +// GenerateRules is called in each directory where an update is requested in +// depth-first post-order. +// +// args contains the arguments for GenerateRules. This is passed as a struct to +// avoid breaking implementations in the future when new fields are added. +// +// A GenerateResult struct is returned. Optional fields may be added to this +// type in the future. +// +// Any non-fatal errors this function encounters should be logged using +// log.Print. +func (l *symbolLang) GenerateRules(args language.GenerateArgs) language.GenerateResult { + if !l.enabled { + return language.GenerateResult{} + } + + var rules []*rule.Rule + var imports []any + for _, f := range append(args.RegularFiles, args.GenFiles...) { + if !isBzlSourceFile(f) { + continue + } + r, loads := makeSymbolLibraryRule(args, f) + rules = append(rules, r) + imports = append(imports, loads) + } + + return language.GenerateResult{ + Gen: rules, + Imports: imports, + Empty: generateEmpty(args), + } +} + +func makeSymbolLibraryRule(args language.GenerateArgs, f string) (*rule.Rule, []string) { + name := strings.TrimSuffix(f, fileType) + r := rule.NewRule(symbolLibraryKind, name) + + r.SetAttr("srcs", []string{f}) + + shouldSetVisibility := args.File == nil || !args.File.HasDefaultVisibility() + if shouldSetVisibility { + vis := checkInternalVisibility(args.Rel, "//visibility:public") + r.SetAttr("visibility", []string{vis}) + } + + fullPath := filepath.Join(args.Dir, f) + loads, err := getBzlFileLoads(fullPath) + + if err != nil { + log.Printf("%s: contains syntax errors: %v", fullPath, err) + // Don't `continue` since it is reasonable to create a target even + // without deps. + } + + return r, loads +} + +func getBzlFileLoads(path string) ([]string, error) { + f, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("os.ReadFile(%q) error: %v", path, err) + } + ast, err := build.ParseBuild(path, f) + if err != nil { + return nil, fmt.Errorf("build.Parse(%q) error: %v", f, err) + } + + var loads []string + build.WalkOnce(ast, func(expr *build.Expr) { + n := *expr + if l, ok := n.(*build.LoadStmt); ok { + loads = append(loads, l.Module.Value) + } + }) + sort.Strings(loads) + + return loads, nil +} + +func isBzlSourceFile(f string) bool { + return strings.HasSuffix(f, fileType) && !ignoreSuffix.Matches(f) +} + +// generateEmpty generates the list of rules that don't need to exist in the +// BUILD file any more. For each symbol_library rule in args.File that only has +// srcs that aren't in args.RegularFiles or args.GenFiles, add a symbol_library +// with no srcs or deps. That will let Gazelle delete symbol_library rules after +// the corresponding .bzl files are deleted. +func generateEmpty(args language.GenerateArgs) []*rule.Rule { + var ret []*rule.Rule + if args.File == nil { + return ret + } + for _, r := range args.File.Rules { + if r.Kind() != symbolLibraryKind { + continue + } + name := r.AttrString("name") + + exists := make(map[string]bool) + for _, f := range args.RegularFiles { + exists[f] = true + } + for _, f := range args.GenFiles { + exists[f] = true + } + for _, r := range args.File.Rules { + srcsExist := false + for _, f := range r.AttrStrings("srcs") { + if exists[f] { + srcsExist = true + break + } + } + if !srcsExist { + ret = append(ret, rule.NewRule(symbolLibraryKind, name)) + } + } + } + return ret +} + +// checkInternalVisibility overrides the given visibility if the package is +// internal. +func checkInternalVisibility(rel, visibility string) string { + if i := pathtools.Index(rel, "internal"); i > 0 { + visibility = fmt.Sprintf("//%s:__subpackages__", rel[:i-1]) + } else if i := pathtools.Index(rel, "private"); i > 0 { + visibility = fmt.Sprintf("//%s:__subpackages__", rel[:i-1]) + } else if pathtools.HasPrefix(rel, "internal") || pathtools.HasPrefix(rel, "private") { + visibility = "//:__subpackages__" + } + return visibility +} diff --git a/rules/private/proto_repository_tools.bzl b/rules/private/proto_repository_tools.bzl index 2f7bbfc6d..fef1520c1 100644 --- a/rules/private/proto_repository_tools.bzl +++ b/rules/private/proto_repository_tools.bzl @@ -68,11 +68,11 @@ def _proto_repository_tools_impl(ctx): ctx.path(ctx.attr._list_repository_tools_srcs), "-dir=src/github.com/stackb/rules_proto", # Run it under 'check' to assert file is up-to-date - "-check=rules/private/proto_repository_tools_srcs.bzl", + # "-check=rules/private/proto_repository_tools_srcs.bzl", # Run it under 'skip' to not check (only for internal testing) # "-skip=rules/private/proto_repository_tools_srcs.bzl", # Run it under 'generate' to recreate the list - # "-generate=rules/private/proto_repository_tools_srcs.bzl", + "-generate=rules/private/proto_repository_tools_srcs.bzl", ], environment = env, ) diff --git a/rules/private/proto_repository_tools_srcs.bzl b/rules/private/proto_repository_tools_srcs.bzl index 9810725aa..062f2bcb1 100644 --- a/rules/private/proto_repository_tools_srcs.bzl +++ b/rules/private/proto_repository_tools_srcs.bzl @@ -43,6 +43,8 @@ PROTO_REPOSITORY_TOOLS_SRCS = [ "@build_stack_rules_proto//language/proto_go_modules:rule.go", "@build_stack_rules_proto//language/protobuf:BUILD.bazel", "@build_stack_rules_proto//language/protobuf:protobuf.go", + "@build_stack_rules_proto//language/symbol:BUILD.bazel", + "@build_stack_rules_proto//language/symbol:symbol.go", "@build_stack_rules_proto//pkg:BUILD.bazel", "@build_stack_rules_proto//pkg/goldentest:BUILD.bazel", "@build_stack_rules_proto//pkg/goldentest:cases.go", diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/config/BUILD.bazel b/vendor/github.com/bazelbuild/bazel-gazelle/config/BUILD.bazel new file mode 100644 index 000000000..2d5718174 --- /dev/null +++ b/vendor/github.com/bazelbuild/bazel-gazelle/config/BUILD.bazel @@ -0,0 +1,41 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "config", + srcs = [ + "config.go", + "constants.go", + ], + importpath = "github.com/bazelbuild/bazel-gazelle/config", + visibility = ["//visibility:public"], + deps = [ + "//internal/module", + "//internal/wspace", + "//rule", + ], +) + +go_test( + name = "config_test", + srcs = ["config_test.go"], + embed = [":config"], + deps = ["//rule"], +) + +filegroup( + name = "all_files", + testonly = True, + srcs = [ + "BUILD.bazel", + "config.go", + "config_test.go", + "constants.go", + ], + visibility = ["//visibility:public"], +) + +alias( + name = "go_default_library", + actual = ":config", + visibility = ["//visibility:public"], +) diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/flag/BUILD.bazel b/vendor/github.com/bazelbuild/bazel-gazelle/flag/BUILD.bazel new file mode 100644 index 000000000..6820783c6 --- /dev/null +++ b/vendor/github.com/bazelbuild/bazel-gazelle/flag/BUILD.bazel @@ -0,0 +1,24 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "flag", + srcs = ["flag.go"], + importpath = "github.com/bazelbuild/bazel-gazelle/flag", + visibility = ["//visibility:public"], +) + +filegroup( + name = "all_files", + testonly = True, + srcs = [ + "BUILD.bazel", + "flag.go", + ], + visibility = ["//visibility:public"], +) + +alias( + name = "go_default_library", + actual = ":flag", + visibility = ["//visibility:public"], +) diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/internal/module/BUILD.bazel b/vendor/github.com/bazelbuild/bazel-gazelle/internal/module/BUILD.bazel new file mode 100644 index 000000000..2446f86c7 --- /dev/null +++ b/vendor/github.com/bazelbuild/bazel-gazelle/internal/module/BUILD.bazel @@ -0,0 +1,43 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "module", + srcs = ["module.go"], + importpath = "github.com/bazelbuild/bazel-gazelle/internal/module", + visibility = ["//:__subpackages__"], + deps = [ + "//label", + "@com_github_bazelbuild_buildtools//build", + ], +) + +filegroup( + name = "all_files", + testonly = True, + srcs = [ + "BUILD.bazel", + "module.go", + "module_test.go", + ], + visibility = ["//visibility:public"], +) + +alias( + name = "go_default_library", + actual = ":module", + visibility = ["//:__subpackages__"], +) + +go_test( + name = "module_test", + srcs = ["module_test.go"], + data = glob( + ["testdata/**"], + allow_empty = True, + ), + embed = [":module"], + deps = [ + "@com_github_google_go_cmp//cmp", + "@io_bazel_rules_go//go/runfiles", + ], +) diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/internal/version/BUILD.bazel b/vendor/github.com/bazelbuild/bazel-gazelle/internal/version/BUILD.bazel new file mode 100644 index 000000000..2d358d8d7 --- /dev/null +++ b/vendor/github.com/bazelbuild/bazel-gazelle/internal/version/BUILD.bazel @@ -0,0 +1,31 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "version", + srcs = ["version.go"], + importpath = "github.com/bazelbuild/bazel-gazelle/internal/version", + visibility = ["//:__subpackages__"], +) + +go_test( + name = "version_test", + srcs = ["version_test.go"], + embed = [":version"], +) + +filegroup( + name = "all_files", + testonly = True, + srcs = [ + "BUILD.bazel", + "version.go", + "version_test.go", + ], + visibility = ["//visibility:public"], +) + +alias( + name = "go_default_library", + actual = ":version", + visibility = ["//:__subpackages__"], +) diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/internal/wspace/BUILD.bazel b/vendor/github.com/bazelbuild/bazel-gazelle/internal/wspace/BUILD.bazel new file mode 100644 index 000000000..7cd120185 --- /dev/null +++ b/vendor/github.com/bazelbuild/bazel-gazelle/internal/wspace/BUILD.bazel @@ -0,0 +1,32 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "wspace", + srcs = ["finder.go"], + importpath = "github.com/bazelbuild/bazel-gazelle/internal/wspace", + visibility = ["//visibility:public"], +) + +go_test( + name = "wspace_test", + size = "small", + srcs = ["finder_test.go"], + embed = [":wspace"], +) + +filegroup( + name = "all_files", + testonly = True, + srcs = [ + "BUILD.bazel", + "finder.go", + "finder_test.go", + ], + visibility = ["//visibility:public"], +) + +alias( + name = "go_default_library", + actual = ":wspace", + visibility = ["//:__subpackages__"], +) diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/label/BUILD.bazel b/vendor/github.com/bazelbuild/bazel-gazelle/label/BUILD.bazel new file mode 100644 index 000000000..94aaa100d --- /dev/null +++ b/vendor/github.com/bazelbuild/bazel-gazelle/label/BUILD.bazel @@ -0,0 +1,35 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "label", + srcs = ["label.go"], + importpath = "github.com/bazelbuild/bazel-gazelle/label", + visibility = ["//visibility:public"], + deps = [ + "//pathtools", + "@com_github_bazelbuild_buildtools//build", + ], +) + +go_test( + name = "label_test", + srcs = ["label_test.go"], + embed = [":label"], +) + +filegroup( + name = "all_files", + testonly = True, + srcs = [ + "BUILD.bazel", + "label.go", + "label_test.go", + ], + visibility = ["//visibility:public"], +) + +alias( + name = "go_default_library", + actual = ":label", + visibility = ["//visibility:public"], +) diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/language/BUILD.bazel b/vendor/github.com/bazelbuild/bazel-gazelle/language/BUILD.bazel new file mode 100644 index 000000000..77e295069 --- /dev/null +++ b/vendor/github.com/bazelbuild/bazel-gazelle/language/BUILD.bazel @@ -0,0 +1,42 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "language", + srcs = [ + "base.go", + "lang.go", + "lifecycle.go", + "update.go", + ], + importpath = "github.com/bazelbuild/bazel-gazelle/language", + visibility = ["//visibility:public"], + deps = [ + "//config", + "//label", + "//repo", + "//resolve", + "//rule", + ], +) + +filegroup( + name = "all_files", + testonly = True, + srcs = [ + "BUILD.bazel", + "base.go", + "lang.go", + "lifecycle.go", + "update.go", + "//language/bazel:all_files", + "//language/go:all_files", + "//language/proto:all_files", + ], + visibility = ["//visibility:public"], +) + +alias( + name = "go_default_library", + actual = ":language", + visibility = ["//visibility:public"], +) diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/language/go/BUILD.bazel b/vendor/github.com/bazelbuild/bazel-gazelle/language/go/BUILD.bazel new file mode 100644 index 000000000..bdcc9ad64 --- /dev/null +++ b/vendor/github.com/bazelbuild/bazel-gazelle/language/go/BUILD.bazel @@ -0,0 +1,162 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("@bazel_skylib//rules:diff_test.bzl", "diff_test") +load("@bazel_skylib//rules:run_binary.bzl", "run_binary") +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") +load(":def.bzl", "std_package_list") + +# gazelle:exclude testdata + +# TODO(jayconrod): test that the checked-in static file matches the generated +# file. The generated code is checked in so that Gazelle can still be built +# with "go get". +std_package_list( + name = "std_package_list", + out = "std_package_list.go", +) + +# gazelle:exclude gen_platform_info.go +run_binary( + name = "gen_platform_info", + outs = ["gen_platform_info.go"], + args = ["$(location :gen_platform_info.go)"], + tool = "//language/go/platform_info_generator", +) + +diff_test( + name = "platform_info_diff_test", + file1 = ":gen_platform_info.go", + file2 = "platform_info.go", +) + +go_library( + name = "go", + srcs = [ + "build_constraints.go", + "config.go", + "constants.go", + "embed.go", + "fileinfo.go", + "fix.go", + "generate.go", + "kinds.go", + "lang.go", + "modules.go", + "package.go", + "platform_info.go", + "resolve.go", + "std_package_list.go", + "stdlib_links.go", + "update.go", + "utils.go", + "work.go", + ], + importpath = "github.com/bazelbuild/bazel-gazelle/language/go", + visibility = ["//visibility:public"], + deps = [ + "//config", + "//flag", + "//internal/module", + "//internal/version", + "//label", + "//language", + "//language/proto", + "//pathtools", + "//repo", + "//resolve", + "//rule", + "@com_github_bazelbuild_buildtools//build", + "@org_golang_x_mod//modfile", + "@org_golang_x_mod//module", + "@org_golang_x_sync//errgroup", + ], +) + +go_test( + name = "go_test", + srcs = [ + "build_constraints_test.go", + "config_test.go", + "fileinfo_go_test.go", + "fileinfo_test.go", + "fix_test.go", + "generate_test.go", + "resolve_test.go", + "stubs_test.go", + "update_import_test.go", + ], + data = glob( + ["testdata/**"], + # Empty when distributed. + allow_empty = True, + ), + embed = [":go"], + deps = [ + "//config", + "//label", + "//language", + "//language/proto", + "//merger", + "//pathtools", + "//repo", + "//resolve", + "//rule", + "//testtools", + "//walk", + "@com_github_bazelbuild_buildtools//build", + "@com_github_google_go_cmp//cmp", + "@io_bazel_rules_go//go/tools/bazel", + "@org_golang_x_tools_go_vcs//:vcs", + ], +) + +filegroup( + name = "all_files", + testonly = True, + srcs = [ + "BUILD.bazel", + "build_constraints.go", + "build_constraints_test.go", + "config.go", + "config_test.go", + "constants.go", + "def.bzl", + "embed.go", + "fileinfo.go", + "fileinfo_go_test.go", + "fileinfo_test.go", + "fix.go", + "fix_test.go", + "generate.go", + "generate_test.go", + "kinds.go", + "lang.go", + "modules.go", + "package.go", + "platform_info.go", + "resolve.go", + "resolve_test.go", + "std_package_list.go", + "stdlib_links.go", + "stubs_test.go", + "update.go", + "update_import_test.go", + "utils.go", + "work.go", + "//language/go/gen_std_package_list:all_files", + "//language/go/platform_info_generator:all_files", + ], + visibility = ["//visibility:public"], +) + +alias( + name = "go_default_library", + actual = ":go", + visibility = ["//visibility:public"], +) + +bzl_library( + name = "def", + srcs = ["def.bzl"], + visibility = ["//visibility:public"], + deps = ["@io_bazel_rules_go//go:def"], +) diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/language/proto/BUILD.bazel b/vendor/github.com/bazelbuild/bazel-gazelle/language/proto/BUILD.bazel new file mode 100644 index 000000000..d274b97e1 --- /dev/null +++ b/vendor/github.com/bazelbuild/bazel-gazelle/language/proto/BUILD.bazel @@ -0,0 +1,127 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") +load("//language/proto/gen:def.bzl", "known_imports") + +# gazelle:exclude testdata + +known_imports( + name = "known_imports", + src = "proto.csv", + out = "known_imports.go", + key = 0, + package = "proto", + value = 1, + var = "knownImports", +) + +known_imports( + name = "known_proto_imports", + src = "proto.csv", + out = "known_proto_imports.go", + key = 0, + package = "proto", + value = 3, + var = "knownProtoImports", +) + +known_imports( + name = "known_go_imports", + src = "proto.csv", + out = "known_go_imports.go", + key = 2, + package = "proto", + value = 3, + var = "knownGoProtoImports", +) + +go_library( + name = "proto", + srcs = [ + "config.go", + "constants.go", + "fileinfo.go", + "fix.go", + "generate.go", + "kinds.go", + "known_go_imports.go", + "known_imports.go", + "known_proto_imports.go", + "lang.go", + "package.go", + "resolve.go", + ], + importpath = "github.com/bazelbuild/bazel-gazelle/language/proto", + visibility = ["//visibility:public"], + deps = [ + "//config", + "//label", + "//language", + "//merger", + "//pathtools", + "//repo", + "//resolve", + "//rule", + ], +) + +go_test( + name = "proto_test", + srcs = [ + "config_test.go", + "fileinfo_test.go", + "generate_test.go", + "resolve_test.go", + ], + data = glob( + ["testdata/**"], + # Empty when distributed. + allow_empty = True, + ), + embed = [":proto"], + deps = [ + "//config", + "//label", + "//language", + "//merger", + "//repo", + "//resolve", + "//rule", + "//testtools", + "//walk", + "@com_github_bazelbuild_buildtools//build", + ], +) + +exports_files(["proto.csv"]) + +filegroup( + name = "all_files", + testonly = True, + srcs = [ + "BUILD.bazel", + "config.go", + "config_test.go", + "constants.go", + "fileinfo.go", + "fileinfo_test.go", + "fix.go", + "generate.go", + "generate_test.go", + "kinds.go", + "known_go_imports.go", + "known_imports.go", + "known_proto_imports.go", + "lang.go", + "package.go", + "proto.csv", + "resolve.go", + "resolve_test.go", + "//language/proto/gen:all_files", + ], + visibility = ["//visibility:public"], +) + +alias( + name = "go_default_library", + actual = ":proto", + visibility = ["//visibility:public"], +) diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/merger/BUILD.bazel b/vendor/github.com/bazelbuild/bazel-gazelle/merger/BUILD.bazel new file mode 100644 index 000000000..505838990 --- /dev/null +++ b/vendor/github.com/bazelbuild/bazel-gazelle/merger/BUILD.bazel @@ -0,0 +1,52 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "merger", + srcs = [ + "fix.go", + "merger.go", + ], + importpath = "github.com/bazelbuild/bazel-gazelle/merger", + visibility = ["//visibility:public"], + deps = [ + "//rule", + "@com_github_bazelbuild_buildtools//build", + ], +) + +go_test( + name = "merger_test", + size = "small", + srcs = [ + "fix_test.go", + "merger_test.go", + ], + deps = [ + ":merger", + "//language", + "//language/go", + "//language/proto", + "//rule", + "@com_github_bazelbuild_buildtools//build", + "@com_github_google_go_cmp//cmp", + ], +) + +filegroup( + name = "all_files", + testonly = True, + srcs = [ + "BUILD.bazel", + "fix.go", + "fix_test.go", + "merger.go", + "merger_test.go", + ], + visibility = ["//visibility:public"], +) + +alias( + name = "go_default_library", + actual = ":merger", + visibility = ["//visibility:public"], +) diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/pathtools/BUILD.bazel b/vendor/github.com/bazelbuild/bazel-gazelle/pathtools/BUILD.bazel new file mode 100644 index 000000000..a429f877b --- /dev/null +++ b/vendor/github.com/bazelbuild/bazel-gazelle/pathtools/BUILD.bazel @@ -0,0 +1,32 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "pathtools", + srcs = ["path.go"], + importpath = "github.com/bazelbuild/bazel-gazelle/pathtools", + visibility = ["//visibility:public"], +) + +go_test( + name = "pathtools_test", + srcs = ["path_test.go"], + embed = [":pathtools"], + deps = ["@com_github_google_go_cmp//cmp"], +) + +filegroup( + name = "all_files", + testonly = True, + srcs = [ + "BUILD.bazel", + "path.go", + "path_test.go", + ], + visibility = ["//visibility:public"], +) + +alias( + name = "go_default_library", + actual = ":pathtools", + visibility = ["//visibility:public"], +) diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/repo/BUILD.bazel b/vendor/github.com/bazelbuild/bazel-gazelle/repo/BUILD.bazel new file mode 100644 index 000000000..71396b236 --- /dev/null +++ b/vendor/github.com/bazelbuild/bazel-gazelle/repo/BUILD.bazel @@ -0,0 +1,54 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "repo", + srcs = [ + "remote.go", + "repo.go", + ], + importpath = "github.com/bazelbuild/bazel-gazelle/repo", + visibility = ["//visibility:public"], + deps = [ + "//label", + "//pathtools", + "//rule", + "@org_golang_x_mod//modfile", + "@org_golang_x_tools_go_vcs//:vcs", + ], +) + +go_test( + name = "repo_test", + srcs = [ + "remote_test.go", + "repo_test.go", + "stubs_test.go", + ], + embed = [":repo"], + deps = [ + "//pathtools", + "//rule", + "//testtools", + "@org_golang_x_tools_go_vcs//:vcs", + ], +) + +filegroup( + name = "all_files", + testonly = True, + srcs = [ + "BUILD.bazel", + "remote.go", + "remote_test.go", + "repo.go", + "repo_test.go", + "stubs_test.go", + ], + visibility = ["//visibility:public"], +) + +alias( + name = "go_default_library", + actual = ":repo", + visibility = ["//visibility:public"], +) diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/resolve/BUILD.bazel b/vendor/github.com/bazelbuild/bazel-gazelle/resolve/BUILD.bazel new file mode 100644 index 000000000..0acd29e3c --- /dev/null +++ b/vendor/github.com/bazelbuild/bazel-gazelle/resolve/BUILD.bazel @@ -0,0 +1,47 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "resolve", + srcs = [ + "config.go", + "index.go", + ], + importpath = "github.com/bazelbuild/bazel-gazelle/resolve", + visibility = ["//visibility:public"], + deps = [ + "//config", + "//label", + "//repo", + "//rule", + ], +) + +filegroup( + name = "all_files", + testonly = True, + srcs = [ + "BUILD.bazel", + "config.go", + "index.go", + "resolve_test.go", + ], + visibility = ["//visibility:public"], +) + +alias( + name = "go_default_library", + actual = ":resolve", + visibility = ["//visibility:public"], +) + +go_test( + name = "resolve_test", + srcs = ["resolve_test.go"], + embed = [":resolve"], + deps = [ + "//config", + "//label", + "//rule", + "@com_github_google_go_cmp//cmp", + ], +) diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/rule/BUILD.bazel b/vendor/github.com/bazelbuild/bazel-gazelle/rule/BUILD.bazel new file mode 100644 index 000000000..7884d50ef --- /dev/null +++ b/vendor/github.com/bazelbuild/bazel-gazelle/rule/BUILD.bazel @@ -0,0 +1,67 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "rule", + srcs = [ + "directives.go", + "expr.go", + "merge.go", + "platform.go", + "platform_strings.go", + "rule.go", + "sort_labels.go", + "types.go", + "value.go", + ], + importpath = "github.com/bazelbuild/bazel-gazelle/rule", + visibility = ["//visibility:public"], + deps = [ + "//label", + "@com_github_bazelbuild_buildtools//build", + "@com_github_bazelbuild_buildtools//tables", + ], +) + +go_test( + name = "rule_test", + srcs = [ + "directives_test.go", + "merge_test.go", + "rule_test.go", + "value_test.go", + ], + embed = [":rule"], + deps = [ + "//label", + "@com_github_bazelbuild_buildtools//build", + "@com_github_google_go_cmp//cmp", + ], +) + +filegroup( + name = "all_files", + testonly = True, + srcs = [ + "BUILD.bazel", + "directives.go", + "directives_test.go", + "expr.go", + "merge.go", + "merge_test.go", + "platform.go", + "platform_strings.go", + "rule.go", + "rule_test.go", + "sort_labels.go", + "types.go", + "value.go", + "value_test.go", + ], + visibility = ["//visibility:public"], +) + +alias( + name = "go_default_library", + actual = ":rule", + visibility = ["//visibility:public"], +) diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/testtools/BUILD.bazel b/vendor/github.com/bazelbuild/bazel-gazelle/testtools/BUILD.bazel new file mode 100644 index 000000000..bc3495030 --- /dev/null +++ b/vendor/github.com/bazelbuild/bazel-gazelle/testtools/BUILD.bazel @@ -0,0 +1,34 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "testtools", + testonly = True, + srcs = [ + "config.go", + "files.go", + ], + importpath = "github.com/bazelbuild/bazel-gazelle/testtools", + visibility = ["//visibility:public"], + deps = [ + "//config", + "//language", + "@com_github_google_go_cmp//cmp", + ], +) + +filegroup( + name = "all_files", + testonly = True, + srcs = [ + "BUILD.bazel", + "config.go", + "files.go", + ], + visibility = ["//visibility:public"], +) + +alias( + name = "go_default_library", + actual = ":testtools", + visibility = ["//visibility:public"], +) diff --git a/vendor/github.com/bazelbuild/bazel-gazelle/walk/BUILD.bazel b/vendor/github.com/bazelbuild/bazel-gazelle/walk/BUILD.bazel new file mode 100644 index 000000000..c1f181932 --- /dev/null +++ b/vendor/github.com/bazelbuild/bazel-gazelle/walk/BUILD.bazel @@ -0,0 +1,58 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "walk", + srcs = [ + "cache.go", + "config.go", + "dirinfo.go", + "walk.go", + ], + importpath = "github.com/bazelbuild/bazel-gazelle/walk", + visibility = ["//visibility:public"], + deps = [ + "//config", + "//flag", + "//pathtools", + "//rule", + "@com_github_bazelbuild_buildtools//build", + "@com_github_bmatcuk_doublestar_v4//:doublestar", + ], +) + +go_test( + name = "walk_test", + srcs = [ + "config_test.go", + "walk_test.go", + ], + embed = [":walk"], + deps = [ + "//config", + "//rule", + "//testtools", + "@com_github_bmatcuk_doublestar_v4//:doublestar", + "@com_github_google_go_cmp//cmp", + ], +) + +filegroup( + name = "all_files", + testonly = True, + srcs = [ + "BUILD.bazel", + "cache.go", + "config.go", + "config_test.go", + "dirinfo.go", + "walk.go", + "walk_test.go", + ], + visibility = ["//visibility:public"], +) + +alias( + name = "go_default_library", + actual = ":walk", + visibility = ["//visibility:public"], +) diff --git a/vendor/github.com/bazelbuild/buildtools/build/BUILD.bazel b/vendor/github.com/bazelbuild/buildtools/build/BUILD.bazel new file mode 100644 index 000000000..2b7fcf0dc --- /dev/null +++ b/vendor/github.com/bazelbuild/buildtools/build/BUILD.bazel @@ -0,0 +1,64 @@ +# gazelle:exclude parse.y.go +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") +load(":build_defs.bzl", "go_yacc") + +go_yacc( + src = "parse.y", + out = "parse.y.baz.go", +) + +go_library( + name = "build", + srcs = [ + "lex.go", + "parse.y.baz.go", # keep + "print.go", + "quote.go", + "rewrite.go", + "rule.go", + "syntax.go", + "utils.go", + "walk.go", + ], + importpath = "github.com/bazelbuild/buildtools/build", + visibility = ["//visibility:public"], + deps = [ + "//labels", + "//tables", + ], +) + +go_test( + name = "build_test", + size = "small", + srcs = [ + "checkfile_test.go", + "lex_test.go", + "parse_test.go", + "print_test.go", + "quote_test.go", + "rewrite_test.go", + "rule_test.go", + "walk_test.go", + ], + data = glob(["testdata/*"]) + [ + # parse.y.go is checked in to satisfy the Go community + # https://github.com/bazelbuild/buildtools/issues/14 + # this test ensures it doesn't get stale. + "parse.y.baz.go", + "parse.y.go", + "rewrite_test_files/original_formatted.star", + "rewrite_test_files/original.star", + ], + embed = [":build"], + deps = [ + "//tables", + "//testutils", + ], +) + +alias( + name = "go_default_library", + actual = ":build", + visibility = ["//visibility:public"], +) diff --git a/vendor/github.com/bazelbuild/buildtools/labels/BUILD.bazel b/vendor/github.com/bazelbuild/buildtools/labels/BUILD.bazel new file mode 100644 index 000000000..d6ed5b2c0 --- /dev/null +++ b/vendor/github.com/bazelbuild/buildtools/labels/BUILD.bazel @@ -0,0 +1,20 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "labels", + srcs = ["labels.go"], + importpath = "github.com/bazelbuild/buildtools/labels", + visibility = ["//visibility:public"], +) + +go_test( + name = "labels_test", + srcs = ["labels_test.go"], + embed = [":labels"], +) + +alias( + name = "go_default_library", + actual = ":labels", + visibility = ["//visibility:public"], +) diff --git a/vendor/github.com/bazelbuild/buildtools/tables/BUILD.bazel b/vendor/github.com/bazelbuild/buildtools/tables/BUILD.bazel new file mode 100644 index 000000000..5fa695cf4 --- /dev/null +++ b/vendor/github.com/bazelbuild/buildtools/tables/BUILD.bazel @@ -0,0 +1,25 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "tables", + srcs = [ + "jsonparser.go", + "tables.go", + ], + importpath = "github.com/bazelbuild/buildtools/tables", + visibility = ["//visibility:public"], +) + +go_test( + name = "tables_test", + size = "small", + srcs = ["jsonparser_test.go"], + data = glob(["testdata/*"]), + embed = [":tables"], +) + +alias( + name = "go_default_library", + actual = ":tables", + visibility = ["//visibility:public"], +) diff --git a/vendor/github.com/bazelbuild/rules_go/go/runfiles/BUILD.bazel b/vendor/github.com/bazelbuild/rules_go/go/runfiles/BUILD.bazel new file mode 100644 index 000000000..58e3b9bf6 --- /dev/null +++ b/vendor/github.com/bazelbuild/rules_go/go/runfiles/BUILD.bazel @@ -0,0 +1,61 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "runfiles", + srcs = [ + "directory.go", + "fs.go", + "global.go", + "manifest.go", + "runfiles.go", + ], + importpath = "github.com/bazelbuild/rules_go/go/runfiles", + visibility = ["//visibility:public"], +) + +alias( + name = "go_default_library", + actual = ":runfiles", + visibility = ["//visibility:public"], +) + +go_test( + name = "example_test", + srcs = [ + "caller_repository_example_test.go", + "example_test.go", + "rlocationpath_xdefs_example_test.go", + ], + deps = [":runfiles"], +) + +filegroup( + name = "all_files", + testonly = True, + srcs = glob(["**"]), + visibility = ["//visibility:public"], +) + +go_test( + name = "runfiles_test", + srcs = [ + "caller_repository_example_test.go", + "example_test.go", + "rlocationpath_xdefs_example_test.go", + ], + deps = [":runfiles"], +) diff --git a/vendor/github.com/bazelbuild/rules_go/go/tools/bazel/BUILD.bazel b/vendor/github.com/bazelbuild/rules_go/go/tools/bazel/BUILD.bazel new file mode 100644 index 000000000..674da86f8 --- /dev/null +++ b/vendor/github.com/bazelbuild/rules_go/go/tools/bazel/BUILD.bazel @@ -0,0 +1,34 @@ +load("//go:def.bzl", "go_library", "go_test") + +go_library( + name = "bazel", + srcs = [ + "bazel.go", + "runfiles.go", + ], + importpath = "github.com/bazelbuild/rules_go/go/tools/bazel", + visibility = ["//visibility:public"], +) + +go_test( + name = "bazel_test", + size = "small", + srcs = ["bazel_test.go"], + data = ["empty.txt"], + embed = [":bazel"], +) + +# Runfiles functionality in this package is tested by //tests/core/runfiles. + +filegroup( + name = "all_files", + testonly = True, + srcs = glob(["**"]), + visibility = ["//visibility:public"], +) + +alias( + name = "go_default_library", + actual = ":bazel", + visibility = ["//visibility:public"], +) From aa5f356ed4b76b6203ea1b2684590a3b9ef33c17 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Sun, 23 Nov 2025 11:29:53 -0700 Subject: [PATCH 03/12] Initial starlarkbundle lang --- cmd/gazelle/BUILD.bazel | 2 +- cmd/gazelle/langs.go | 4 +- .../{symbol => starlarkbundle}/BUILD.bazel | 9 +- language/starlarkbundle/language.go | 174 ++++++++++++++++++ .../starlark_library.go} | 151 ++++----------- rules/private/proto_repository_tools_srcs.bzl | 26 ++- rules/starlark_library.bzl | 9 + 7 files changed, 253 insertions(+), 122 deletions(-) rename language/{symbol => starlarkbundle}/BUILD.bazel (76%) create mode 100644 language/starlarkbundle/language.go rename language/{symbol/symbol.go => starlarkbundle/starlark_library.go} (58%) create mode 100644 rules/starlark_library.bzl diff --git a/cmd/gazelle/BUILD.bazel b/cmd/gazelle/BUILD.bazel index b0a060f2d..9451a3856 100644 --- a/cmd/gazelle/BUILD.bazel +++ b/cmd/gazelle/BUILD.bazel @@ -29,7 +29,7 @@ go_library( "//cmd/gazelle/internal/wspace", "//language/proto_go_modules", "//language/protobuf", - "//language/symbol", + "//language/starlarkbundle", "@bazel_gazelle//config:go_default_library", "@bazel_gazelle//flag:go_default_library", "@bazel_gazelle//label:go_default_library", diff --git a/cmd/gazelle/langs.go b/cmd/gazelle/langs.go index 7202e0a08..534498c63 100644 --- a/cmd/gazelle/langs.go +++ b/cmd/gazelle/langs.go @@ -21,7 +21,7 @@ import ( "github.com/bazelbuild/bazel-gazelle/language/proto" "github.com/stackb/rules_proto/language/proto_go_modules" "github.com/stackb/rules_proto/language/protobuf" - "github.com/stackb/rules_proto/language/symbol" + "github.com/stackb/rules_proto/language/starlarkbundle" ) var languages = []language.Language{ @@ -29,5 +29,5 @@ var languages = []language.Language{ protobuf.NewLanguage(), golang.NewLanguage(), proto_go_modules.NewLanguage(), - symbol.NewLanguage(), + starlarkbundle.NewLanguage(), } diff --git a/language/symbol/BUILD.bazel b/language/starlarkbundle/BUILD.bazel similarity index 76% rename from language/symbol/BUILD.bazel rename to language/starlarkbundle/BUILD.bazel index bd1ec100a..6fe09a86a 100644 --- a/language/symbol/BUILD.bazel +++ b/language/starlarkbundle/BUILD.bazel @@ -1,9 +1,12 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( - name = "symbol", - srcs = ["symbol.go"], - importpath = "github.com/stackb/rules_proto/language/symbol", + name = "starlarkbundle", + srcs = [ + "language.go", + "starlark_library.go", + ], + importpath = "github.com/stackb/rules_proto/language/starlarkbundle", visibility = ["//visibility:public"], deps = [ "@bazel_gazelle//config:go_default_library", diff --git a/language/starlarkbundle/language.go b/language/starlarkbundle/language.go new file mode 100644 index 000000000..1c56c56d7 --- /dev/null +++ b/language/starlarkbundle/language.go @@ -0,0 +1,174 @@ +/* Copyright 2020 The Bazel Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package symbol generates a `starlark_library` target for every `.bzl` file in +// each package. At the root of the module, a single starlark_bundle is +// populated with deps that include all other starlark_libraries. +// +// The original code for this gazelle extension started from +// https://github.com/bazelbuild/bazel-skylib/blob/main/gazelle/bzl/gazelle.go. +package starlarkbundle + +import ( + "flag" + "log" + "maps" + + "github.com/bazelbuild/bazel-gazelle/config" + "github.com/bazelbuild/bazel-gazelle/label" + "github.com/bazelbuild/bazel-gazelle/language" + "github.com/bazelbuild/bazel-gazelle/repo" + "github.com/bazelbuild/bazel-gazelle/resolve" + "github.com/bazelbuild/bazel-gazelle/rule" +) + +const ( + languageName = "starlark_bundle" + starlarkBundleRootDirectiveName = "starlark_bundle_root" +) + +type starlarkBundleLang struct { + starlarkBundleRoot *string + starlarkLibraries map[label.Label]*rule.Rule +} + +// NewLanguage is called by Gazelle to install this language extension in a +// binary. +func NewLanguage() language.Language { + return &starlarkBundleLang{ + starlarkLibraries: make(map[label.Label]*rule.Rule), + } +} + +// Name returns the name of the language. This should be a prefix of the kinds +// of rules generated by the language, e.g., "go" for the Go extension since it +// generates "go_library" rules. +func (*starlarkBundleLang) Name() string { + return languageName +} + +// The following methods are implemented to satisfy the +// https://pkg.go.dev/github.com/bazelbuild/bazel-gazelle/resolve?tab=doc#Resolver +// interface, but are otherwise unused. +func (l *starlarkBundleLang) RegisterFlags(fs *flag.FlagSet, cmd string, c *config.Config) { +} + +func (*starlarkBundleLang) CheckFlags(fs *flag.FlagSet, c *config.Config) error { + return nil +} + +func (*starlarkBundleLang) KnownDirectives() []string { + return []string{starlarkBundleRootDirectiveName} +} + +func (l *starlarkBundleLang) Configure(c *config.Config, rel string, f *rule.File) { + if f != nil { + for _, d := range f.Directives { + if d.Key == starlarkBundleRootDirectiveName { + if l.starlarkBundleRoot != nil { + log.Fatalf("gazelle:%s should only be set once (refusing to override %q with %q)", starlarkBundleRootDirectiveName, *l.starlarkBundleRoot, d.Value) + } + l.starlarkBundleRoot = &rel + } + } + } +} + +// Kinds returns a map of maps rule names (kinds) and information on how to +// match and merge attributes that may be found in rules of those kinds. All +// kinds of rules generated for this language may be found here. +func (*starlarkBundleLang) Kinds() map[string]rule.KindInfo { + kinds := map[string]rule.KindInfo{} + maps.Copy(kinds, starlarkLibraryKindInfo) + return kinds +} + +// Loads returns .bzl files and symbols they define. Every rule generated by +// GenerateRules, now or in the past, should be loadable from one of these +// files. +func (*starlarkBundleLang) Loads() []rule.LoadInfo { + return []rule.LoadInfo{ + starlarkLibraryLoadInfo, + } +} + +// Fix repairs deprecated usage of language-specific rules in f. This is called +// before the file is indexed. Unless c.ShouldFix is true, fixes that delete or +// rename rules should not be performed. +func (*starlarkBundleLang) Fix(c *config.Config, f *rule.File) { +} + +// Imports returns a list of ImportSpecs that can be used to import the rule r. +// This is used to populate RuleIndex. +// +// If nil is returned, the rule will not be indexed. If any non-nil slice is +// returned, including an empty slice, the rule will be indexed. +func (b *starlarkBundleLang) Imports(c *config.Config, r *rule.Rule, f *rule.File) []resolve.ImportSpec { + switch r.Kind() { + case starlarkLibraryKind: + return starlarkLibraryImports(c, r, f) + } + return nil +} + +// Embeds returns a list of labels of rules that the given rule embeds. If a +// rule is embedded by another importable rule of the same language, only the +// embedding rule will be indexed. The embedding rule will inherit the imports +// of the embedded rule. Since SkyLark doesn't support embedding this should +// always return nil. +func (*starlarkBundleLang) Embeds(r *rule.Rule, from label.Label) []label.Label { + return nil +} + +// Resolve translates imported libraries for a given rule into Bazel +// dependencies. Information about imported libraries is returned for each rule +// generated by language.GenerateRules in language.GenerateResult.Imports. +// Resolve generates a "deps" attribute (or the appropriate language-specific +// equivalent) for each import according to language-specific rules and +// heuristics. +func (*starlarkBundleLang) Resolve(c *config.Config, ix *resolve.RuleIndex, rc *repo.RemoteCache, r *rule.Rule, importsRaw interface{}, from label.Label) { + switch r.Kind() { + case starlarkLibraryKind: + starlarkLibraryResolve(c, ix, rc, r, importsRaw, from) + } +} + +// GenerateRules extracts build metadata from source files in a directory. +// GenerateRules is called in each directory where an update is requested in +// depth-first post-order. +// +// args contains the arguments for GenerateRules. This is passed as a struct to +// avoid breaking implementations in the future when new fields are added. +// +// A GenerateResult struct is returned. Optional fields may be added to this +// type in the future. +// +// Any non-fatal errors this function encounters should be logged using +// log.Print. +func (l *starlarkBundleLang) GenerateRules(args language.GenerateArgs) (result language.GenerateResult) { + if l.starlarkBundleRoot == nil { + return + } + + for _, r := range []language.GenerateResult{ + starlarkLibararyGenerate(args), + } { + result.Gen = append(result.Gen, r.Gen...) + result.Imports = append(result.Imports, r.Imports...) + result.Empty = append(result.Empty, r.Empty...) + } + + return +} diff --git a/language/symbol/symbol.go b/language/starlarkbundle/starlark_library.go similarity index 58% rename from language/symbol/symbol.go rename to language/starlarkbundle/starlark_library.go index addf2b215..aabca991e 100644 --- a/language/symbol/symbol.go +++ b/language/starlarkbundle/starlark_library.go @@ -13,16 +13,15 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package symbol generates a `symbol_library` target for every `.bzl` file in -// each package. At the root of the module, a single symbol_library is +// Package symbol generates a `starlark_library` target for every `.bzl` file in +// each package. At the root of the module, a single starlark_bundle is // populated with deps that include all other symbol_libraries. // // The original code for this gazelle extension started from // https://github.com/bazelbuild/bazel-skylib/blob/main/gazelle/bzl/gazelle.go. -package symbol +package starlarkbundle import ( - "flag" "fmt" "log" "os" @@ -41,9 +40,8 @@ import ( ) const ( - languageName = "symbol" - symbolLibraryKind = "symbol_library" - fileType = ".bzl" + starlarkLibraryKind = "starlark_library" + fileType = ".bzl" ) var ignoreSuffix = suffixes{ @@ -51,13 +49,18 @@ var ignoreSuffix = suffixes{ "_test.bzl", } -var kinds = map[string]rule.KindInfo{ - symbolLibraryKind: { +var starlarkLibraryKindInfo = map[string]rule.KindInfo{ + starlarkLibraryKind: { NonEmptyAttrs: map[string]bool{"srcs": true, "deps": true}, MergeableAttrs: map[string]bool{"srcs": true}, }, } +var starlarkLibraryLoadInfo = rule.LoadInfo{ + Name: "@build_stack_rules_proto//rules:starlark_library.bzl", + Symbols: []string{starlarkLibraryKind}, +} + type suffixes []string func (s suffixes) Matches(test string) bool { @@ -69,59 +72,31 @@ func (s suffixes) Matches(test string) bool { return false } -type symbolLang struct { - enabled bool -} +func starlarkLibraryRule(args language.GenerateArgs, f string) (*rule.Rule, []string) { + name := strings.TrimSuffix(f, fileType) + r := rule.NewRule(starlarkLibraryKind, name) -// NewLanguage is called by Gazelle to install this language extension in a -// binary. -func NewLanguage() language.Language { - return &symbolLang{} -} + r.SetAttr("srcs", []string{f}) -// Name returns the name of the language. This should be a prefix of the kinds -// of rules generated by the language, e.g., "go" for the Go extension since it -// generates "go_library" rules. -func (*symbolLang) Name() string { return languageName } + shouldSetVisibility := args.File == nil || !args.File.HasDefaultVisibility() + if shouldSetVisibility { + vis := checkInternalVisibility(args.Rel, "//visibility:public") + r.SetAttr("visibility", []string{vis}) + } -// The following methods are implemented to satisfy the -// https://pkg.go.dev/github.com/bazelbuild/bazel-gazelle/resolve?tab=doc#Resolver -// interface, but are otherwise unused. -func (l *symbolLang) RegisterFlags(fs *flag.FlagSet, cmd string, c *config.Config) { - fs.BoolVar(&l.enabled, "symbol_language_enabled", false, "whather this extension is turned on") -} -func (*symbolLang) CheckFlags(fs *flag.FlagSet, c *config.Config) error { return nil } -func (*symbolLang) KnownDirectives() []string { return nil } -func (*symbolLang) Configure(c *config.Config, rel string, f *rule.File) {} - -// Kinds returns a map of maps rule names (kinds) and information on how to -// match and merge attributes that may be found in rules of those kinds. All -// kinds of rules generated for this language may be found here. -func (*symbolLang) Kinds() map[string]rule.KindInfo { - return kinds -} + fullPath := filepath.Join(args.Dir, f) + loads, err := getBzlFileLoads(fullPath) -// Loads returns .bzl files and symbols they define. Every rule generated by -// GenerateRules, now or in the past, should be loadable from one of these -// files. -func (*symbolLang) Loads() []rule.LoadInfo { - return []rule.LoadInfo{{ - Name: "@build_stack_rules_proto//rules:symbol_library.bzl", - Symbols: []string{symbolLibraryKind}, - }} -} + if err != nil { + log.Printf("%s: contains syntax errors: %v", fullPath, err) + // don't return early since it is reasonable to create a target even + // without deps. + } -// Fix repairs deprecated usage of language-specific rules in f. This is called -// before the file is indexed. Unless c.ShouldFix is true, fixes that delete or -// rename rules should not be performed. -func (*symbolLang) Fix(c *config.Config, f *rule.File) {} + return r, loads +} -// Imports returns a list of ImportSpecs that can be used to import the rule r. -// This is used to populate RuleIndex. -// -// If nil is returned, the rule will not be indexed. If any non-nil slice is -// returned, including an empty slice, the rule will be indexed. -func (b *symbolLang) Imports(c *config.Config, r *rule.Rule, f *rule.File) []resolve.ImportSpec { +func starlarkLibraryImports(c *config.Config, r *rule.Rule, f *rule.File) []resolve.ImportSpec { srcs := r.AttrStrings("srcs") imports := make([]resolve.ImportSpec, 0, len(srcs)) @@ -140,20 +115,7 @@ func (b *symbolLang) Imports(c *config.Config, r *rule.Rule, f *rule.File) []res return imports } -// Embeds returns a list of labels of rules that the given rule embeds. If a -// rule is embedded by another importable rule of the same language, only the -// embedding rule will be indexed. The embedding rule will inherit the imports -// of the embedded rule. Since SkyLark doesn't support embedding this should -// always return nil. -func (*symbolLang) Embeds(r *rule.Rule, from label.Label) []label.Label { return nil } - -// Resolve translates imported libraries for a given rule into Bazel -// dependencies. Information about imported libraries is returned for each rule -// generated by language.GenerateRules in language.GenerateResult.Imports. -// Resolve generates a "deps" attribute (or the appropriate language-specific -// equivalent) for each import according to language-specific rules and -// heuristics. -func (*symbolLang) Resolve(c *config.Config, ix *resolve.RuleIndex, rc *repo.RemoteCache, r *rule.Rule, importsRaw interface{}, from label.Label) { +func starlarkLibraryResolve(c *config.Config, ix *resolve.RuleIndex, rc *repo.RemoteCache, r *rule.Rule, importsRaw interface{}, from label.Label) { imports := importsRaw.([]string) r.DelAttr("deps") @@ -213,30 +175,15 @@ func (*symbolLang) Resolve(c *config.Config, ix *resolve.RuleIndex, rc *repo.Rem } } -// GenerateRules extracts build metadata from source files in a directory. -// GenerateRules is called in each directory where an update is requested in -// depth-first post-order. -// -// args contains the arguments for GenerateRules. This is passed as a struct to -// avoid breaking implementations in the future when new fields are added. -// -// A GenerateResult struct is returned. Optional fields may be added to this -// type in the future. -// -// Any non-fatal errors this function encounters should be logged using -// log.Print. -func (l *symbolLang) GenerateRules(args language.GenerateArgs) language.GenerateResult { - if !l.enabled { - return language.GenerateResult{} - } - +func starlarkLibararyGenerate(args language.GenerateArgs) language.GenerateResult { var rules []*rule.Rule var imports []any + for _, f := range append(args.RegularFiles, args.GenFiles...) { if !isBzlSourceFile(f) { continue } - r, loads := makeSymbolLibraryRule(args, f) + r, loads := starlarkLibraryRule(args, f) rules = append(rules, r) imports = append(imports, loads) } @@ -248,30 +195,6 @@ func (l *symbolLang) GenerateRules(args language.GenerateArgs) language.Generate } } -func makeSymbolLibraryRule(args language.GenerateArgs, f string) (*rule.Rule, []string) { - name := strings.TrimSuffix(f, fileType) - r := rule.NewRule(symbolLibraryKind, name) - - r.SetAttr("srcs", []string{f}) - - shouldSetVisibility := args.File == nil || !args.File.HasDefaultVisibility() - if shouldSetVisibility { - vis := checkInternalVisibility(args.Rel, "//visibility:public") - r.SetAttr("visibility", []string{vis}) - } - - fullPath := filepath.Join(args.Dir, f) - loads, err := getBzlFileLoads(fullPath) - - if err != nil { - log.Printf("%s: contains syntax errors: %v", fullPath, err) - // Don't `continue` since it is reasonable to create a target even - // without deps. - } - - return r, loads -} - func getBzlFileLoads(path string) ([]string, error) { f, err := os.ReadFile(path) if err != nil { @@ -309,7 +232,7 @@ func generateEmpty(args language.GenerateArgs) []*rule.Rule { return ret } for _, r := range args.File.Rules { - if r.Kind() != symbolLibraryKind { + if r.Kind() != starlarkLibraryKind { continue } name := r.AttrString("name") @@ -330,7 +253,7 @@ func generateEmpty(args language.GenerateArgs) []*rule.Rule { } } if !srcsExist { - ret = append(ret, rule.NewRule(symbolLibraryKind, name)) + ret = append(ret, rule.NewRule(starlarkLibraryKind, name)) } } } diff --git a/rules/private/proto_repository_tools_srcs.bzl b/rules/private/proto_repository_tools_srcs.bzl index 062f2bcb1..af9ee3070 100644 --- a/rules/private/proto_repository_tools_srcs.bzl +++ b/rules/private/proto_repository_tools_srcs.bzl @@ -43,8 +43,9 @@ PROTO_REPOSITORY_TOOLS_SRCS = [ "@build_stack_rules_proto//language/proto_go_modules:rule.go", "@build_stack_rules_proto//language/protobuf:BUILD.bazel", "@build_stack_rules_proto//language/protobuf:protobuf.go", - "@build_stack_rules_proto//language/symbol:BUILD.bazel", - "@build_stack_rules_proto//language/symbol:symbol.go", + "@build_stack_rules_proto//language/starlarkbundle:BUILD.bazel", + "@build_stack_rules_proto//language/starlarkbundle:language.go", + "@build_stack_rules_proto//language/starlarkbundle:starlark_library.go", "@build_stack_rules_proto//pkg:BUILD.bazel", "@build_stack_rules_proto//pkg/goldentest:BUILD.bazel", "@build_stack_rules_proto//pkg/goldentest:cases.go", @@ -192,14 +193,22 @@ PROTO_REPOSITORY_TOOLS_SRCS = [ "@build_stack_rules_proto//toolchain:BUILD.bazel", "@build_stack_rules_proto//toolchain/scala:BUILD.bazel", "@build_stack_rules_proto//tools:BUILD.bazel", + "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/config:BUILD.bazel", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/config:config.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/config:constants.go", + "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/flag:BUILD.bazel", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/flag:flag.go", + "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/internal/module:BUILD.bazel", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/internal/module:module.go", + "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/internal/version:BUILD.bazel", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/internal/version:version.go", + "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/internal/wspace:BUILD.bazel", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/internal/wspace:finder.go", + "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/label:BUILD.bazel", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/label:label.go", + "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/language:BUILD.bazel", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/language:base.go", + "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/language/go:BUILD.bazel", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/language/go:build_constraints.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/language/go:config.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/language/go:constants.go", @@ -220,6 +229,7 @@ PROTO_REPOSITORY_TOOLS_SRCS = [ "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/language/go:work.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/language:lang.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/language:lifecycle.go", + "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/language/proto:BUILD.bazel", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/language/proto:config.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/language/proto:constants.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/language/proto:fileinfo.go", @@ -233,13 +243,18 @@ PROTO_REPOSITORY_TOOLS_SRCS = [ "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/language/proto:package.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/language/proto:resolve.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/language:update.go", + "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/merger:BUILD.bazel", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/merger:fix.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/merger:merger.go", + "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/pathtools:BUILD.bazel", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/pathtools:path.go", + "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/repo:BUILD.bazel", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/repo:remote.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/repo:repo.go", + "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/resolve:BUILD.bazel", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/resolve:config.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/resolve:index.go", + "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/rule:BUILD.bazel", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/rule:directives.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/rule:expr.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/rule:merge.go", @@ -249,12 +264,15 @@ PROTO_REPOSITORY_TOOLS_SRCS = [ "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/rule:sort_labels.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/rule:types.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/rule:value.go", + "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/testtools:BUILD.bazel", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/testtools:config.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/testtools:files.go", + "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/walk:BUILD.bazel", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/walk:cache.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/walk:config.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/walk:dirinfo.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/bazel-gazelle/walk:walk.go", + "@build_stack_rules_proto//vendor/github.com/bazelbuild/buildtools/build:BUILD.bazel", "@build_stack_rules_proto//vendor/github.com/bazelbuild/buildtools/build:lex.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/buildtools/build:parse.y.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/buildtools/build:print.go", @@ -264,14 +282,18 @@ PROTO_REPOSITORY_TOOLS_SRCS = [ "@build_stack_rules_proto//vendor/github.com/bazelbuild/buildtools/build:syntax.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/buildtools/build:utils.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/buildtools/build:walk.go", + "@build_stack_rules_proto//vendor/github.com/bazelbuild/buildtools/labels:BUILD.bazel", "@build_stack_rules_proto//vendor/github.com/bazelbuild/buildtools/labels:labels.go", + "@build_stack_rules_proto//vendor/github.com/bazelbuild/buildtools/tables:BUILD.bazel", "@build_stack_rules_proto//vendor/github.com/bazelbuild/buildtools/tables:jsonparser.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/buildtools/tables:tables.go", + "@build_stack_rules_proto//vendor/github.com/bazelbuild/rules_go/go/runfiles:BUILD.bazel", "@build_stack_rules_proto//vendor/github.com/bazelbuild/rules_go/go/runfiles:directory.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/rules_go/go/runfiles:fs.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/rules_go/go/runfiles:global.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/rules_go/go/runfiles:manifest.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/rules_go/go/runfiles:runfiles.go", + "@build_stack_rules_proto//vendor/github.com/bazelbuild/rules_go/go/tools/bazel:BUILD.bazel", "@build_stack_rules_proto//vendor/github.com/bazelbuild/rules_go/go/tools/bazel:bazel.go", "@build_stack_rules_proto//vendor/github.com/bazelbuild/rules_go/go/tools/bazel:runfiles.go", "@build_stack_rules_proto//vendor/github.com/bmatcuk/doublestar:doublestar.go", diff --git a/rules/starlark_library.bzl b/rules/starlark_library.bzl new file mode 100644 index 000000000..868ac818c --- /dev/null +++ b/rules/starlark_library.bzl @@ -0,0 +1,9 @@ +"""starlark_library.bzl is a thin wrapper over bzl_library.""" + +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + +def starlark_library(name, **kwargs): + bzl_library( + name = name, + **kwargs + ) From 0132f9f778a9f8eef518748bb5ceb3ec9429d4c1 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Sun, 23 Nov 2025 12:22:09 -0700 Subject: [PATCH 04/12] Add starlark_bundle rule --- language/starlarkbundle/BUILD.bazel | 1 + language/starlarkbundle/language.go | 55 ++++++++---- language/starlarkbundle/starlark_bundle.go | 85 +++++++++++++++++++ language/starlarkbundle/starlark_library.go | 38 ++++++--- rules/private/proto_repository_tools_srcs.bzl | 1 + rules/starlark_bundle.bzl | 32 +++++++ 6 files changed, 182 insertions(+), 30 deletions(-) create mode 100644 language/starlarkbundle/starlark_bundle.go create mode 100644 rules/starlark_bundle.bzl diff --git a/language/starlarkbundle/BUILD.bazel b/language/starlarkbundle/BUILD.bazel index 6fe09a86a..5971fd56e 100644 --- a/language/starlarkbundle/BUILD.bazel +++ b/language/starlarkbundle/BUILD.bazel @@ -4,6 +4,7 @@ go_library( name = "starlarkbundle", srcs = [ "language.go", + "starlark_bundle.go", "starlark_library.go", ], importpath = "github.com/stackb/rules_proto/language/starlarkbundle", diff --git a/language/starlarkbundle/language.go b/language/starlarkbundle/language.go index 1c56c56d7..5fc67d7db 100644 --- a/language/starlarkbundle/language.go +++ b/language/starlarkbundle/language.go @@ -25,6 +25,7 @@ import ( "flag" "log" "maps" + "strings" "github.com/bazelbuild/bazel-gazelle/config" "github.com/bazelbuild/bazel-gazelle/label" @@ -35,13 +36,15 @@ import ( ) const ( - languageName = "starlark_bundle" - starlarkBundleRootDirectiveName = "starlark_bundle_root" + languageName = "starlark_bundle" + starlarkBundleRootDirectiveName = "starlark_bundle_root" + starlarkBundleExcludeDirectiveName = "starlark_bundle_exclude" ) type starlarkBundleLang struct { - starlarkBundleRoot *string - starlarkLibraries map[label.Label]*rule.Rule + starlarkBundleRoot *string + starlarkBundleExcludeDirs []string + starlarkLibraries map[label.Label]*rule.Rule } // NewLanguage is called by Gazelle to install this language extension in a @@ -62,7 +65,7 @@ func (*starlarkBundleLang) Name() string { // The following methods are implemented to satisfy the // https://pkg.go.dev/github.com/bazelbuild/bazel-gazelle/resolve?tab=doc#Resolver // interface, but are otherwise unused. -func (l *starlarkBundleLang) RegisterFlags(fs *flag.FlagSet, cmd string, c *config.Config) { +func (ext *starlarkBundleLang) RegisterFlags(fs *flag.FlagSet, cmd string, c *config.Config) { } func (*starlarkBundleLang) CheckFlags(fs *flag.FlagSet, c *config.Config) error { @@ -70,17 +73,23 @@ func (*starlarkBundleLang) CheckFlags(fs *flag.FlagSet, c *config.Config) error } func (*starlarkBundleLang) KnownDirectives() []string { - return []string{starlarkBundleRootDirectiveName} + return []string{ + starlarkBundleRootDirectiveName, + starlarkBundleExcludeDirectiveName, + } } -func (l *starlarkBundleLang) Configure(c *config.Config, rel string, f *rule.File) { +func (ext *starlarkBundleLang) Configure(c *config.Config, rel string, f *rule.File) { if f != nil { for _, d := range f.Directives { - if d.Key == starlarkBundleRootDirectiveName { - if l.starlarkBundleRoot != nil { - log.Fatalf("gazelle:%s should only be set once (refusing to override %q with %q)", starlarkBundleRootDirectiveName, *l.starlarkBundleRoot, d.Value) + switch d.Key { + case starlarkBundleRootDirectiveName: + if ext.starlarkBundleRoot != nil { + log.Fatalf("gazelle:%s should only be set once (refusing to override %q with %q)", starlarkBundleRootDirectiveName, *ext.starlarkBundleRoot, d.Value) } - l.starlarkBundleRoot = &rel + ext.starlarkBundleRoot = &rel + case starlarkBundleExcludeDirectiveName: + ext.starlarkBundleExcludeDirs = append(ext.starlarkBundleExcludeDirs, d.Value) } } } @@ -92,6 +101,7 @@ func (l *starlarkBundleLang) Configure(c *config.Config, rel string, f *rule.Fil func (*starlarkBundleLang) Kinds() map[string]rule.KindInfo { kinds := map[string]rule.KindInfo{} maps.Copy(kinds, starlarkLibraryKindInfo) + maps.Copy(kinds, starlarkBundleKindInfo) return kinds } @@ -101,6 +111,7 @@ func (*starlarkBundleLang) Kinds() map[string]rule.KindInfo { func (*starlarkBundleLang) Loads() []rule.LoadInfo { return []rule.LoadInfo{ starlarkLibraryLoadInfo, + starlarkBundleLoadInfo, } } @@ -115,10 +126,12 @@ func (*starlarkBundleLang) Fix(c *config.Config, f *rule.File) { // // If nil is returned, the rule will not be indexed. If any non-nil slice is // returned, including an empty slice, the rule will be indexed. -func (b *starlarkBundleLang) Imports(c *config.Config, r *rule.Rule, f *rule.File) []resolve.ImportSpec { +func (ext *starlarkBundleLang) Imports(c *config.Config, r *rule.Rule, f *rule.File) []resolve.ImportSpec { switch r.Kind() { case starlarkLibraryKind: return starlarkLibraryImports(c, r, f) + case starlarkBundleKind: + return starlarkBundleImports(c, r, f) } return nil } @@ -138,10 +151,12 @@ func (*starlarkBundleLang) Embeds(r *rule.Rule, from label.Label) []label.Label // Resolve generates a "deps" attribute (or the appropriate language-specific // equivalent) for each import according to language-specific rules and // heuristics. -func (*starlarkBundleLang) Resolve(c *config.Config, ix *resolve.RuleIndex, rc *repo.RemoteCache, r *rule.Rule, importsRaw interface{}, from label.Label) { +func (ext *starlarkBundleLang) Resolve(c *config.Config, ix *resolve.RuleIndex, rc *repo.RemoteCache, r *rule.Rule, importsRaw interface{}, from label.Label) { switch r.Kind() { case starlarkLibraryKind: - starlarkLibraryResolve(c, ix, rc, r, importsRaw, from) + starlarkLibraryResolve(c, ix, r, importsRaw, from) + case starlarkBundleKind: + starlarkBundleResolve(r, ext.starlarkLibraries) } } @@ -157,13 +172,19 @@ func (*starlarkBundleLang) Resolve(c *config.Config, ix *resolve.RuleIndex, rc * // // Any non-fatal errors this function encounters should be logged using // log.Print. -func (l *starlarkBundleLang) GenerateRules(args language.GenerateArgs) (result language.GenerateResult) { - if l.starlarkBundleRoot == nil { +func (ext *starlarkBundleLang) GenerateRules(args language.GenerateArgs) (result language.GenerateResult) { + if ext.starlarkBundleRoot == nil { return } + for _, root := range ext.starlarkBundleExcludeDirs { + if strings.HasPrefix(args.Rel, root) { + return + } + } for _, r := range []language.GenerateResult{ - starlarkLibararyGenerate(args), + starlarkLibraryGenerate(args, ext.starlarkLibraries), + starlarkBundleGenerate(args, ext.starlarkBundleRoot), } { result.Gen = append(result.Gen, r.Gen...) result.Imports = append(result.Imports, r.Imports...) diff --git a/language/starlarkbundle/starlark_bundle.go b/language/starlarkbundle/starlark_bundle.go new file mode 100644 index 000000000..c19fb9400 --- /dev/null +++ b/language/starlarkbundle/starlark_bundle.go @@ -0,0 +1,85 @@ +/* Copyright 2020 The Bazel Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package symbol generates a `starlark_bundle` target for every `.bzl` file in +// each package. At the root of the module, a single starlark_bundle is +// populated with deps that include all other symbol_libraries. +// +// The original code for this gazelle extension started from +// https://github.com/bazelbuild/bazel-skylib/blob/main/gazelle/bzl/gazelle.go. +package starlarkbundle + +import ( + "sort" + + "github.com/bazelbuild/bazel-gazelle/config" + "github.com/bazelbuild/bazel-gazelle/label" + "github.com/bazelbuild/bazel-gazelle/language" + "github.com/bazelbuild/bazel-gazelle/resolve" + "github.com/bazelbuild/bazel-gazelle/rule" +) + +const ( + starlarkBundleKind = "starlark_bundle" +) + +var starlarkBundleKindInfo = map[string]rule.KindInfo{ + starlarkBundleKind: { + NonEmptyAttrs: map[string]bool{"deps": true}, + ResolveAttrs: map[string]bool{"deps": true}, + }, +} + +var starlarkBundleLoadInfo = rule.LoadInfo{ + Name: "@build_stack_rules_proto//rules:starlark_bundle.bzl", + Symbols: []string{starlarkBundleKind}, +} + +func starlarkBundleRule() (*rule.Rule, []string) { + r := rule.NewRule(starlarkBundleKind, starlarkBundleKind) + + r.SetAttr("visibility", []string{"//visibility:public"}) + + return r, []string{} +} + +func starlarkBundleImports(_ *config.Config, _ *rule.Rule, _ *rule.File) []resolve.ImportSpec { + return nil +} + +func starlarkBundleResolve(r *rule.Rule, starlarkLibraries map[label.Label]*rule.Rule) { + deps := make([]string, 0, len(starlarkLibraries)) + for dep := range starlarkLibraries { + deps = append(deps, dep.String()) + } + + sort.Strings(deps) + if len(deps) > 0 { + r.SetAttr("deps", deps) + } +} + +func starlarkBundleGenerate(args language.GenerateArgs, root *string) (result language.GenerateResult) { + if root == nil || *root != args.Rel { + return result + } + + r, loads := starlarkBundleRule() + + return language.GenerateResult{ + Gen: []*rule.Rule{r}, + Imports: []any{loads}, + } +} diff --git a/language/starlarkbundle/starlark_library.go b/language/starlarkbundle/starlark_library.go index aabca991e..ace522720 100644 --- a/language/starlarkbundle/starlark_library.go +++ b/language/starlarkbundle/starlark_library.go @@ -33,7 +33,6 @@ import ( "github.com/bazelbuild/bazel-gazelle/label" "github.com/bazelbuild/bazel-gazelle/language" "github.com/bazelbuild/bazel-gazelle/pathtools" - "github.com/bazelbuild/bazel-gazelle/repo" "github.com/bazelbuild/bazel-gazelle/resolve" "github.com/bazelbuild/bazel-gazelle/rule" "github.com/bazelbuild/buildtools/build" @@ -42,6 +41,7 @@ import ( const ( starlarkLibraryKind = "starlark_library" fileType = ".bzl" + visibilityPublic = "//visibility:public" ) var ignoreSuffix = suffixes{ @@ -73,14 +73,16 @@ func (s suffixes) Matches(test string) bool { } func starlarkLibraryRule(args language.GenerateArgs, f string) (*rule.Rule, []string) { - name := strings.TrimSuffix(f, fileType) + // prefix with 'lib' in case someone is also using bzl_library gazelle + // extension + name := "lib" + strings.TrimSuffix(f, fileType) r := rule.NewRule(starlarkLibraryKind, name) r.SetAttr("srcs", []string{f}) shouldSetVisibility := args.File == nil || !args.File.HasDefaultVisibility() if shouldSetVisibility { - vis := checkInternalVisibility(args.Rel, "//visibility:public") + vis := checkInternalVisibility(args.Rel, visibilityPublic) r.SetAttr("visibility", []string{vis}) } @@ -96,7 +98,7 @@ func starlarkLibraryRule(args language.GenerateArgs, f string) (*rule.Rule, []st return r, loads } -func starlarkLibraryImports(c *config.Config, r *rule.Rule, f *rule.File) []resolve.ImportSpec { +func starlarkLibraryImports(_ *config.Config, r *rule.Rule, f *rule.File) []resolve.ImportSpec { srcs := r.AttrStrings("srcs") imports := make([]resolve.ImportSpec, 0, len(srcs)) @@ -115,7 +117,7 @@ func starlarkLibraryImports(c *config.Config, r *rule.Rule, f *rule.File) []reso return imports } -func starlarkLibraryResolve(c *config.Config, ix *resolve.RuleIndex, rc *repo.RemoteCache, r *rule.Rule, importsRaw interface{}, from label.Label) { +func starlarkLibraryResolve(c *config.Config, ix *resolve.RuleIndex, r *rule.Rule, importsRaw interface{}, from label.Label) { imports := importsRaw.([]string) r.DelAttr("deps") @@ -175,7 +177,7 @@ func starlarkLibraryResolve(c *config.Config, ix *resolve.RuleIndex, rc *repo.Re } } -func starlarkLibararyGenerate(args language.GenerateArgs) language.GenerateResult { +func starlarkLibraryGenerate(args language.GenerateArgs, starlarkLibraries map[label.Label]*rule.Rule) language.GenerateResult { var rules []*rule.Rule var imports []any @@ -186,12 +188,18 @@ func starlarkLibararyGenerate(args language.GenerateArgs) language.GenerateResul r, loads := starlarkLibraryRule(args, f) rules = append(rules, r) imports = append(imports, loads) + + // populate the map so the bundle rule can use them later. + if isVisibilityPublic(r.AttrStrings("visibility")) { + from := label.New(args.Config.RepoName, args.Rel, r.Name()) + starlarkLibraries[from] = r + } } return language.GenerateResult{ Gen: rules, Imports: imports, - Empty: generateEmpty(args), + Empty: starlarkLibraryEmptyRules(args), } } @@ -221,12 +229,16 @@ func isBzlSourceFile(f string) bool { return strings.HasSuffix(f, fileType) && !ignoreSuffix.Matches(f) } -// generateEmpty generates the list of rules that don't need to exist in the -// BUILD file any more. For each symbol_library rule in args.File that only has -// srcs that aren't in args.RegularFiles or args.GenFiles, add a symbol_library -// with no srcs or deps. That will let Gazelle delete symbol_library rules after -// the corresponding .bzl files are deleted. -func generateEmpty(args language.GenerateArgs) []*rule.Rule { +func isVisibilityPublic(vis []string) bool { + return len(vis) == 1 && vis[0] == visibilityPublic +} + +// starlarkLibraryEmptyRules generates the list of rules that don't need to +// exist in the BUILD file any more. For each symbol_library rule in args.File +// that only has srcs that aren't in args.RegularFiles or args.GenFiles, add a +// symbol_library with no srcs or deps. That will let Gazelle delete +// symbol_library rules after the corresponding .bzl files are deleted. +func starlarkLibraryEmptyRules(args language.GenerateArgs) []*rule.Rule { var ret []*rule.Rule if args.File == nil { return ret diff --git a/rules/private/proto_repository_tools_srcs.bzl b/rules/private/proto_repository_tools_srcs.bzl index af9ee3070..bb53b1ef6 100644 --- a/rules/private/proto_repository_tools_srcs.bzl +++ b/rules/private/proto_repository_tools_srcs.bzl @@ -45,6 +45,7 @@ PROTO_REPOSITORY_TOOLS_SRCS = [ "@build_stack_rules_proto//language/protobuf:protobuf.go", "@build_stack_rules_proto//language/starlarkbundle:BUILD.bazel", "@build_stack_rules_proto//language/starlarkbundle:language.go", + "@build_stack_rules_proto//language/starlarkbundle:starlark_bundle.go", "@build_stack_rules_proto//language/starlarkbundle:starlark_library.go", "@build_stack_rules_proto//pkg:BUILD.bazel", "@build_stack_rules_proto//pkg/goldentest:BUILD.bazel", diff --git a/rules/starlark_bundle.bzl b/rules/starlark_bundle.bzl new file mode 100644 index 000000000..e25cddb35 --- /dev/null +++ b/rules/starlark_bundle.bzl @@ -0,0 +1,32 @@ +"""starlark_bundle.bzl aggreagates the transitive sources of dependencies + +Rule is needed because bzl_library strictly requires srcs. +""" + +load("@bazel_skylib//:bzl_library.bzl", "StarlarkLibraryInfo") + +def _starlark_bundle_impl(ctx): + deps = [d[StarlarkLibraryInfo] for d in ctx.attr.deps] + transitive_srcs = depset(transitive = [d.transitive_srcs for d in deps]) + + return [ + DefaultInfo( + files = transitive_srcs, + ), + StarlarkLibraryInfo( + srcs = [], + transitive_srcs = transitive_srcs, + ), + ] + +starlark_bundle = rule( + implementation = _starlark_bundle_impl, + attrs = { + "deps": attr.label_list( + doc = "list of starlark_library or bzl library rules", + providers = [StarlarkLibraryInfo], + ), + }, + doc = "", + provides = [DefaultInfo, StarlarkLibraryInfo], +) From 88af4181cb3c53556d31de6a2285e0e0d2eec2c9 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Mon, 24 Nov 2025 10:56:54 -0700 Subject: [PATCH 05/12] checkpoint: bazel_lib as starlark_repository actually builds! --- extensions/starlark_repository.bzl | 90 +++++++ language/starlarkbundle/BUILD.bazel | 16 +- language/starlarkbundle/language.go | 85 +++++- language/starlarkbundle/lifecycle.go | 43 ++++ language/starlarkbundle/lifecycle_test.go | 114 ++++++++ language/starlarkbundle/starlark_bundle.go | 2 +- language/starlarkbundle/starlark_library.go | 243 +++++++++++++----- .../starlarkbundle/starlark_library_test.go | 119 +++++++++ rules/private/proto_repository_tools.bzl | 4 +- rules/private/proto_repository_tools_srcs.bzl | 1 + rules/proto/starlark_repository.bzl | 31 +++ rules/starlark_bundle.bzl | 25 +- rules/starlark_library.bzl | 87 ++++++- 13 files changed, 772 insertions(+), 88 deletions(-) create mode 100644 extensions/starlark_repository.bzl create mode 100644 language/starlarkbundle/lifecycle.go create mode 100644 language/starlarkbundle/lifecycle_test.go create mode 100644 language/starlarkbundle/starlark_library_test.go create mode 100644 rules/proto/starlark_repository.bzl diff --git a/extensions/starlark_repository.bzl b/extensions/starlark_repository.bzl new file mode 100644 index 000000000..277ab42cb --- /dev/null +++ b/extensions/starlark_repository.bzl @@ -0,0 +1,90 @@ +"""proto_repostitory.bzl provides the starlark_repository rule.""" + +# Copyright 2014 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@bazel_features//:features.bzl", "bazel_features") +load("@build_stack_rules_proto//rules/proto:starlark_repository.bzl", "starlark_repository_attrs", starlark_repository_repo_rule = "starlark_repository") + +def _extension_metadata( + module_ctx, + *, + root_module_direct_deps = None, + root_module_direct_dev_deps = None, + reproducible = False): + """returns the module_ctx.extension_metadata in a bazel-version-aware way + + This function was copied from the bazel-gazelle repository. + """ + + if not hasattr(module_ctx, "extension_metadata"): + return None + metadata_kwargs = {} + if bazel_features.external_deps.extension_metadata_has_reproducible: + metadata_kwargs["reproducible"] = reproducible + return module_ctx.extension_metadata( + root_module_direct_deps = root_module_direct_deps, + root_module_direct_dev_deps = root_module_direct_dev_deps, + **metadata_kwargs + ) + +def _starlark_repository_impl(module_ctx): + # named_repos is a dict where V is the kwargs for the actual + # "starlark_repository" repo rule and K is the tag.name (the name given by the + # MODULE.bazel author) + named_archives = {} + + # iterate all the module tags and gather a list of named_archives. + # + # TODO(pcj): what is the best practice for version selection here? Do I need + # to check if module.is_root and handle that differently? + # + for module in module_ctx.modules: + for tag in module.tags.archive: + kwargs = { + attr: getattr(tag, attr) + for attr in _starlark_repository_archive_attrs.keys() + if hasattr(tag, attr) + } + named_archives[tag.name] = kwargs + + # declare a repository rule foreach one + for apparent_name, kwargs in named_archives.items(): + starlark_repository_repo_rule( + apparent_name = apparent_name, + **kwargs + ) + + return _extension_metadata( + module_ctx, + reproducible = True, + ) + +_starlark_repository_archive_attrs = starlark_repository_attrs | { + "name": attr.string( + doc = "The repo name.", + mandatory = True, + ), +} +_starlark_repository_archive_attrs.pop("apparent_name") + +starlark_repository = module_extension( + implementation = _starlark_repository_impl, + tag_classes = dict( + archive = tag_class( + doc = "declare an http_archive repository that is post-processed by a custom version of gazelle that includes the 'protobuf' language", + attrs = _starlark_repository_archive_attrs, + ), + ), +) diff --git a/language/starlarkbundle/BUILD.bazel b/language/starlarkbundle/BUILD.bazel index 5971fd56e..d1b3142c7 100644 --- a/language/starlarkbundle/BUILD.bazel +++ b/language/starlarkbundle/BUILD.bazel @@ -1,9 +1,10 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "starlarkbundle", srcs = [ "language.go", + "lifecycle.go", "starlark_bundle.go", "starlark_library.go", ], @@ -20,3 +21,16 @@ go_library( "@com_github_bazelbuild_buildtools//build:go_default_library", ], ) + +go_test( + name = "starlarkbundle_test", + srcs = [ + "lifecycle_test.go", + "starlark_library_test.go", + ], + embed = [":starlarkbundle"], + deps = [ + "@bazel_gazelle//config:go_default_library", + "@com_github_bazelbuild_buildtools//build:go_default_library", + ], +) diff --git a/language/starlarkbundle/language.go b/language/starlarkbundle/language.go index 5fc67d7db..81eaf4e62 100644 --- a/language/starlarkbundle/language.go +++ b/language/starlarkbundle/language.go @@ -23,8 +23,10 @@ package starlarkbundle import ( "flag" + "io" "log" "maps" + "os" "strings" "github.com/bazelbuild/bazel-gazelle/config" @@ -36,22 +38,29 @@ import ( ) const ( - languageName = "starlark_bundle" - starlarkBundleRootDirectiveName = "starlark_bundle_root" - starlarkBundleExcludeDirectiveName = "starlark_bundle_exclude" + languageName = "starlark_bundle" + starlarkBundleRootDirectiveName = "starlark_bundle_root" + starlarkBundleExcludeDirectiveName = "starlark_bundle_exclude" + starlarkBundleLogFileDirectiveName = "starlark_bundle_log_file" + starlarkModuleDependencyDirectiveName = "module_dependency" ) type starlarkBundleLang struct { starlarkBundleRoot *string starlarkBundleExcludeDirs []string + moduleDeps map[string]string starlarkLibraries map[label.Label]*rule.Rule + logFile string + logWriter *os.File + logger *log.Logger } // NewLanguage is called by Gazelle to install this language extension in a // binary. func NewLanguage() language.Language { return &starlarkBundleLang{ - starlarkLibraries: make(map[label.Label]*rule.Rule), + starlarkLibraries: map[label.Label]*rule.Rule{}, + moduleDeps: map[string]string{}, } } @@ -66,9 +75,17 @@ func (*starlarkBundleLang) Name() string { // https://pkg.go.dev/github.com/bazelbuild/bazel-gazelle/resolve?tab=doc#Resolver // interface, but are otherwise unused. func (ext *starlarkBundleLang) RegisterFlags(fs *flag.FlagSet, cmd string, c *config.Config) { + fs.StringVar(&ext.logFile, "starlark_bundle_log", "", "path to log file for starlark_bundle extension") } -func (*starlarkBundleLang) CheckFlags(fs *flag.FlagSet, c *config.Config) error { +func (ext *starlarkBundleLang) CheckFlags(fs *flag.FlagSet, c *config.Config) error { + if ext.logFile != "" { + ext.createLogger(c, ext.logFile) + ext.logf("CheckFlags: log file initialized from flag: %s", ext.logFile) + } + if ext.logger == nil { + ext.logger = log.New(io.Discard, "", 0) + } return nil } @@ -76,11 +93,14 @@ func (*starlarkBundleLang) KnownDirectives() []string { return []string{ starlarkBundleRootDirectiveName, starlarkBundleExcludeDirectiveName, + starlarkBundleLogFileDirectiveName, + starlarkModuleDependencyDirectiveName, } } func (ext *starlarkBundleLang) Configure(c *config.Config, rel string, f *rule.File) { if f != nil { + ext.logf("Configure: processing %d directives in %s", len(f.Directives), rel) for _, d := range f.Directives { switch d.Key { case starlarkBundleRootDirectiveName: @@ -90,11 +110,37 @@ func (ext *starlarkBundleLang) Configure(c *config.Config, rel string, f *rule.F ext.starlarkBundleRoot = &rel case starlarkBundleExcludeDirectiveName: ext.starlarkBundleExcludeDirs = append(ext.starlarkBundleExcludeDirs, d.Value) + case starlarkModuleDependencyDirectiveName: + nameVersion := strings.Fields(d.Value) + if len(nameVersion) != 2 { + log.Fatalf("malformed directive %s, should be NAME VERSION: %s", starlarkModuleDependencyDirectiveName, d.Value) + } + ext.moduleDeps[nameVersion[0]] = nameVersion[1] + ext.logf("Added module dependency: %q -> %q", nameVersion[0], nameVersion[1]) + case starlarkBundleLogFileDirectiveName: + ext.createLogger(c, d.Value) } } } } +func (ext *starlarkBundleLang) createLogger(c *config.Config, logFile string) { + if logFile != "" { + f, err := os.OpenFile(logFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666) + if err == nil { + ext.logWriter = f + ext.logger = log.New(f, "", log.LstdFlags) + } else { + log.Fatalf("attempting to open log file: %v", err) + } + } + if false && ext.logger == nil { + ext.logger = log.New(io.Discard, "", 0) + } + + ext.logf("Log initialized") +} + // Kinds returns a map of maps rule names (kinds) and information on how to // match and merge attributes that may be found in rules of those kinds. All // kinds of rules generated for this language may be found here. @@ -154,7 +200,7 @@ func (*starlarkBundleLang) Embeds(r *rule.Rule, from label.Label) []label.Label func (ext *starlarkBundleLang) Resolve(c *config.Config, ix *resolve.RuleIndex, rc *repo.RemoteCache, r *rule.Rule, importsRaw interface{}, from label.Label) { switch r.Kind() { case starlarkLibraryKind: - starlarkLibraryResolve(c, ix, r, importsRaw, from) + starlarkLibraryResolve(c, ix, r, importsRaw, from, ext) case starlarkBundleKind: starlarkBundleResolve(r, ext.starlarkLibraries) } @@ -182,8 +228,10 @@ func (ext *starlarkBundleLang) GenerateRules(args language.GenerateArgs) (result } } + ext.logf("GenerateRules: visiting %s", args.Rel) + for _, r := range []language.GenerateResult{ - starlarkLibraryGenerate(args, ext.starlarkLibraries), + starlarkLibraryGenerate(args, ext.starlarkLibraries, ext), starlarkBundleGenerate(args, ext.starlarkBundleRoot), } { result.Gen = append(result.Gen, r.Gen...) @@ -193,3 +241,26 @@ func (ext *starlarkBundleLang) GenerateRules(args language.GenerateArgs) (result return } + +func (ext *starlarkBundleLang) getModuleDependencyRepoName(repo string) (string, bool) { + ext.logf("getModuleDependencyRepoName called with repo=%q", repo) + if repo == "" { + ext.logf(" returning early for empty repo: @") + return "@", true + } + if mappedRepo, ok := ext.moduleDeps[repo]; ok { + ext.logf(" found mapping: %q -> %q", repo, mappedRepo) + return mappedRepo, true + } else { + ext.logf(" unknown module dependency: %q (known deps: %v)", repo, ext.moduleDeps) + log.Fatalf("unknown module dependency: %q", repo) + return repo, false + } +} + +// logf logs a message using the extension's logger if configured +func (ext *starlarkBundleLang) logf(format string, args ...any) { + if ext.logger != nil { + ext.logger.Printf(format, args...) + } +} diff --git a/language/starlarkbundle/lifecycle.go b/language/starlarkbundle/lifecycle.go new file mode 100644 index 000000000..fce02b638 --- /dev/null +++ b/language/starlarkbundle/lifecycle.go @@ -0,0 +1,43 @@ +/* Copyright 2020 The Bazel Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package starlarkbundle + +import ( + "context" +) + +// Before implements part of the language.LifecycleManager interface. +func (ext *starlarkBundleLang) Before(context.Context) { + ext.logf("Lifecycle: Before() called") +} + +// DoneGeneratingRules implements part of the language.FinishableLanguage interface. +func (ext *starlarkBundleLang) DoneGeneratingRules() { + ext.logf("Lifecycle: DoneGeneratingRules() called") +} + +// AfterResolvingDeps implements part of the language.LifecycleManager interface. +// This is where we flush and close the log file. +func (ext *starlarkBundleLang) AfterResolvingDeps(context.Context) { + ext.logf("Lifecycle: AfterResolvingDeps() called - flushing and closing log file") + if ext.logWriter != nil { + // Sync flushes any buffered data to disk + _ = ext.logWriter.Sync() + // Close the file + _ = ext.logWriter.Close() + ext.logWriter = nil + } +} diff --git a/language/starlarkbundle/lifecycle_test.go b/language/starlarkbundle/lifecycle_test.go new file mode 100644 index 000000000..a810dcf2c --- /dev/null +++ b/language/starlarkbundle/lifecycle_test.go @@ -0,0 +1,114 @@ +/* Copyright 2020 The Bazel Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package starlarkbundle + +import ( + "context" + "flag" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/bazelbuild/bazel-gazelle/config" +) + +func TestLoggerLifecycle(t *testing.T) { + // Create a temporary directory for the log file + tmpDir := t.TempDir() + logFile := filepath.Join(tmpDir, "test.log") + + // Create extension and register flags + ext := NewLanguage().(*starlarkBundleLang) + fs := flag.NewFlagSet("test", flag.ContinueOnError) + c := &config.Config{} + ext.RegisterFlags(fs, "update", c) + + // Set the log file flag + if err := fs.Set("starlark_bundle_log", logFile); err != nil { + t.Fatalf("failed to set flag: %v", err) + } + + // Initialize logger via CheckFlags + if err := ext.CheckFlags(fs, c); err != nil { + t.Fatalf("CheckFlags failed: %v", err) + } + + // Verify logger is initialized + if ext.logger == nil { + t.Fatal("logger should be initialized") + } + if ext.logWriter == nil { + t.Fatal("logWriter should be initialized") + } + + // Write some log messages + ext.logf("test message 1") + ext.logf("test message 2: %s", "with formatting") + + // Call lifecycle methods + ctx := context.Background() + ext.Before(ctx) + ext.DoneGeneratingRules() + ext.AfterResolvingDeps(ctx) + + // Verify log writer is closed + if ext.logWriter != nil { + t.Error("logWriter should be nil after AfterResolvingDeps") + } + + // Read the log file and verify content + content, err := os.ReadFile(logFile) + if err != nil { + t.Fatalf("failed to read log file: %v", err) + } + + logContent := string(content) + if !strings.Contains(logContent, "test message 1") { + t.Error("log file should contain 'test message 1'") + } + if !strings.Contains(logContent, "test message 2: with formatting") { + t.Error("log file should contain 'test message 2: with formatting'") + } +} + +func TestLoggerWithoutFile(t *testing.T) { + // Create extension without log file + ext := NewLanguage().(*starlarkBundleLang) + fs := flag.NewFlagSet("test", flag.ContinueOnError) + c := &config.Config{} + ext.RegisterFlags(fs, "update", c) + + // Initialize logger via CheckFlags (no log file set) + if err := ext.CheckFlags(fs, c); err != nil { + t.Fatalf("CheckFlags failed: %v", err) + } + + // Verify logger is initialized but logWriter is nil + if ext.logger == nil { + t.Fatal("logger should be initialized") + } + if ext.logWriter != nil { + t.Error("logWriter should be nil when no log file is specified") + } + + // Writing logs should not panic + ext.logf("this should not panic") + + // Lifecycle methods should not panic + ctx := context.Background() + ext.AfterResolvingDeps(ctx) +} diff --git a/language/starlarkbundle/starlark_bundle.go b/language/starlarkbundle/starlark_bundle.go index c19fb9400..80e711028 100644 --- a/language/starlarkbundle/starlark_bundle.go +++ b/language/starlarkbundle/starlark_bundle.go @@ -65,8 +65,8 @@ func starlarkBundleResolve(r *rule.Rule, starlarkLibraries map[label.Label]*rule deps = append(deps, dep.String()) } - sort.Strings(deps) if len(deps) > 0 { + sort.Strings(deps) r.SetAttr("deps", deps) } } diff --git a/language/starlarkbundle/starlark_library.go b/language/starlarkbundle/starlark_library.go index ace522720..30c2d25f2 100644 --- a/language/starlarkbundle/starlark_library.go +++ b/language/starlarkbundle/starlark_library.go @@ -23,7 +23,6 @@ package starlarkbundle import ( "fmt" - "log" "os" "path/filepath" "sort" @@ -39,20 +38,21 @@ import ( ) const ( - starlarkLibraryKind = "starlark_library" - fileType = ".bzl" - visibilityPublic = "//visibility:public" + starlarkLibraryKind = "starlark_library" + starlarkLibraryNamePrefix = "lib" + fileType = ".bzl" + visibilityPublic = "//visibility:public" ) var ignoreSuffix = suffixes{ - "_tests.bzl", - "_test.bzl", + // "_tests.bzl", + // "_test.bzl", } var starlarkLibraryKindInfo = map[string]rule.KindInfo{ starlarkLibraryKind: { - NonEmptyAttrs: map[string]bool{"srcs": true, "deps": true}, - MergeableAttrs: map[string]bool{"srcs": true}, + NonEmptyAttrs: map[string]bool{"src": true}, + ResolveAttrs: map[string]bool{"deps": true}, }, } @@ -72,13 +72,25 @@ func (s suffixes) Matches(test string) bool { return false } -func starlarkLibraryRule(args language.GenerateArgs, f string) (*rule.Rule, []string) { - // prefix with 'lib' in case someone is also using bzl_library gazelle - // extension - name := "lib" + strings.TrimSuffix(f, fileType) +func starlarkLibraryRule(args language.GenerateArgs, f string, ext *starlarkBundleLang) (*rule.Rule, []*build.LoadStmt) { + fullPath := filepath.Join(args.Dir, f) + name := starlarkLibraryNamePrefix + strings.TrimSuffix(f, fileType) + + ast, loads, err := getBzlFileLoadsStmts(fullPath) + if err != nil { + ext.logf("%s: contains syntax errors: %v", fullPath, err) + // don't return early since it is reasonable to create a target even + // without deps. + } + + // always castrate fail() exprs + if renameFailToPrint(ast) { + ext.emitBzlFile(fullPath, ast) + } + r := rule.NewRule(starlarkLibraryKind, name) - r.SetAttr("srcs", []string{f}) + r.SetAttr("src", f) shouldSetVisibility := args.File == nil || !args.File.HasDefaultVisibility() if shouldSetVisibility { @@ -86,56 +98,55 @@ func starlarkLibraryRule(args language.GenerateArgs, f string) (*rule.Rule, []st r.SetAttr("visibility", []string{vis}) } - fullPath := filepath.Join(args.Dir, f) - loads, err := getBzlFileLoads(fullPath) - - if err != nil { - log.Printf("%s: contains syntax errors: %v", fullPath, err) - // don't return early since it is reasonable to create a target even - // without deps. - } + r.SetPrivateAttr("full_path", fullPath) + r.SetPrivateAttr("ast", ast) return r, loads } func starlarkLibraryImports(_ *config.Config, r *rule.Rule, f *rule.File) []resolve.ImportSpec { - srcs := r.AttrStrings("srcs") - imports := make([]resolve.ImportSpec, 0, len(srcs)) - - for _, src := range srcs { - spec := resolve.ImportSpec{ - // Lang is the language in which the import string appears (this should - // match Resolver.Name). - Lang: languageName, - // Imp is an import string for the library. - Imp: fmt.Sprintf("//%s:%s", f.Pkg, src), - } - - imports = append(imports, spec) - } - - return imports + src := r.AttrString("src") + return []resolve.ImportSpec{{ + // Lang is the language in which the import string appears (this should + // match Resolver.Name). + Lang: languageName, + // Imp is an import string for the library. + Imp: fmt.Sprintf("//%s:%s", f.Pkg, src), + }} } -func starlarkLibraryResolve(c *config.Config, ix *resolve.RuleIndex, r *rule.Rule, importsRaw interface{}, from label.Label) { - imports := importsRaw.([]string) +func starlarkLibraryResolve(c *config.Config, ix *resolve.RuleIndex, r *rule.Rule, importsRaw interface{}, from label.Label, ext *starlarkBundleLang) { + loads := importsRaw.([]*build.LoadStmt) + fullPath := r.PrivateAttr("full_path").(string) + ast := r.PrivateAttr("ast").(*build.File) + + ext.logf("%s: starlarkLibraryResolve: %s with %d loads", fullPath, from.String(), len(loads)) r.DelAttr("deps") - if len(imports) == 0 { + if len(loads) == 0 { + ext.logf(" no loads to process") return } - deps := make([]string, 0, len(imports)) - for _, imp := range imports { + rewriteBzlSourceFile := false + var unknownDeps []string + var bazelToolsDeps []string + + deps := make([]string, 0, len(loads)) + for i, load := range loads { + imp := load.Module.Value + ext.logf(" processing load %d/%d: %q", i+1, len(loads), imp) + impLabel, err := label.Parse(imp) if err != nil { - log.Printf("%s: import of %q is invalid: %v", from.String(), imp, err) + ext.logf(" ERROR: import of %q is invalid: %v", imp, err) continue } // the index only contains absolute labels, not relative impLabel = impLabel.Abs(from.Repo, from.Pkg) + ext.logf(" absolute label: %s (repo=%q, pkg=%q, name=%q)", impLabel.String(), impLabel.Repo, impLabel.Pkg, impLabel.Name) if impLabel.Repo == "bazel_tools" { // The @bazel_tools repo is tricky because it is a part of the @@ -143,41 +154,100 @@ func starlarkLibraryResolve(c *config.Config, ix *resolve.RuleIndex, r *rule.Rul // outside world. This means that it can not depend on skylib. // Fortunately there is a fairly simple workaround for this, which // is that you can add those bzl files as `deps` entries. - deps = append(deps, imp) + // + // the bazel source code gathers them up in filegroups but not + // exposed publically. For this to work, caller must be able to use + // a modified version of bazel that adds public visibility to the targets (see `tools/build_defs/repo/BUILD.repo`) + bazelToolsLabel := label.New(impLabel.Repo, impLabel.Pkg, "bzl_srcs") + ext.logf(" bazel_tools dependency: adding to bazel_tools_deps") + // deps = append(deps, imp) + bazelToolsDeps = append(bazelToolsDeps, bazelToolsLabel.String()) + // unknownDeps = append(deps, imp) continue } if impLabel.Repo != "" || !c.IndexLibraries { - // This is a dependency that is external to the current repo, or - // indexing is disabled so take a guess at what the target name - // should be. - deps = append(deps, strings.TrimSuffix(imp, fileType)) + // This is a dependency that is external to the current repo. + // Rewrite the repo label to one suffixed by "_docs". We expect to + // find the starlark_library dependency that provides the file in + // that repo. Rewrite the load label because starlark_doc_extract + // will also expect to load the symbol from that location. + ext.logf(" external repo dependency: repo=%q, IndexLibraries=%v", impLabel.Repo, c.IndexLibraries) + extRepo, known := ext.getModuleDependencyRepoName(impLabel.Repo) + extLabel := label.New( + extRepo, + impLabel.Pkg, + starlarkLibraryNamePrefix+strings.TrimSuffix(impLabel.Name, fileType), + ) + + loadLabel := label.New(extLabel.Repo, extLabel.Pkg, impLabel.Name) + ext.logf(" rewriting load: %q -> %q", load.Module.Value, loadLabel.String()) + load.Module.Value = loadLabel.String() + rewriteBzlSourceFile = true + + if known { + ext.logf(" adding to deps: %s", extLabel.String()) + deps = append(deps, extLabel.String()) + } else { + ext.logf(" adding to unknownDeps: %s", extLabel.String()) + unknownDeps = append(unknownDeps, extLabel.String()) + } + continue } + ext.logf(" internal dependency: looking up in index") res := resolve.ImportSpec{ Lang: languageName, Imp: impLabel.String(), } matches := ix.FindRulesByImportWithConfig(c, res, languageName) + ext.logf(" found %d matches in index", len(matches)) if len(matches) == 0 { - log.Printf("%s: %q (%s) was not found in dependency index. Skipping. This may result in an incomplete deps section and require manual BUILD file intervention.\n", from.String(), imp, impLabel.String()) + ext.logf(" WARNING: %q (%s) was not found in dependency index", imp, impLabel.String()) + // unknownDeps = append(unknownDeps, impLabel.String()) } for _, m := range matches { depLabel := m.Label - depLabel = depLabel.Rel(from.Repo, from.Pkg) + ext.logf(" adding match to deps: %s", depLabel.String()) + // depLabel = depLabel.Rel(from.Repo, from.Pkg) deps = append(deps, depLabel.String()) } } - sort.Strings(deps) + ext.logf(" resolution complete: %d deps, %d unknown, %d bazel_tools", len(deps), len(unknownDeps), len(bazelToolsDeps)) + if len(deps) > 0 { + deps = deduplicateAndSort(deps) r.SetAttr("deps", deps) + ext.logf(" set deps attribute: %v", deps) + } + if len(unknownDeps) > 0 { + unknownDeps = deduplicateAndSort(unknownDeps) + r.SetAttr("unknown_deps", unknownDeps) + ext.logf(" set unknown_deps attribute: %v", unknownDeps) + } + if len(bazelToolsDeps) > 0 { + bazelToolsDeps = deduplicateAndSort(bazelToolsDeps) + r.SetAttr("bazel_tools_deps", bazelToolsDeps) + ext.logf(" set bazel_tools_deps attribute: %v", bazelToolsDeps) + } + + if rewriteBzlSourceFile { + ext.emitBzlFile(fullPath, ast) + } +} + +func (ext *starlarkBundleLang) emitBzlFile(fullPath string, ast *build.File) { + ext.logf("emitting source file: %s", fullPath) + data := build.Format(ast) + if err := os.WriteFile(fullPath, data, os.ModePerm); err != nil { + ext.logf(" ERROR: failed to emit rewritten bzl file: %v", err) } } -func starlarkLibraryGenerate(args language.GenerateArgs, starlarkLibraries map[label.Label]*rule.Rule) language.GenerateResult { +func starlarkLibraryGenerate(args language.GenerateArgs, starlarkLibraries map[label.Label]*rule.Rule, ext *starlarkBundleLang) language.GenerateResult { var rules []*rule.Rule var imports []any @@ -185,7 +255,11 @@ func starlarkLibraryGenerate(args language.GenerateArgs, starlarkLibraries map[l if !isBzlSourceFile(f) { continue } - r, loads := starlarkLibraryRule(args, f) + r, loads := starlarkLibraryRule(args, f, ext) + if r == nil { + continue + } + rules = append(rules, r) imports = append(imports, loads) @@ -203,26 +277,40 @@ func starlarkLibraryGenerate(args language.GenerateArgs, starlarkLibraries map[l } } -func getBzlFileLoads(path string) ([]string, error) { +func getBzlFileLoadsStmts(path string) (*build.File, []*build.LoadStmt, error) { f, err := os.ReadFile(path) if err != nil { - return nil, fmt.Errorf("os.ReadFile(%q) error: %v", path, err) + return nil, nil, fmt.Errorf("os.ReadFile(%q) error: %v", path, err) } - ast, err := build.ParseBuild(path, f) + ast, err := build.ParseBzl(path, f) if err != nil { - return nil, fmt.Errorf("build.Parse(%q) error: %v", f, err) + return nil, nil, fmt.Errorf("build.Parse(%q) error: %v", f, err) } - var loads []string + var loads []*build.LoadStmt build.WalkOnce(ast, func(expr *build.Expr) { n := *expr if l, ok := n.(*build.LoadStmt); ok { - loads = append(loads, l.Module.Value) + loads = append(loads, l) } }) - sort.Strings(loads) - return loads, nil + return ast, loads, nil +} + +func renameFailToPrint(ast *build.File) bool { + modified := false + build.Walk(ast, func(expr build.Expr, stack []build.Expr) { + if call, ok := expr.(*build.CallExpr); ok { + if callName, ok := call.X.(*build.Ident); ok { + if callName.Name == "fail" { + callName.Name = "print" + modified = true + } + } + } + }) + return modified } func isBzlSourceFile(f string) bool { @@ -257,14 +345,8 @@ func starlarkLibraryEmptyRules(args language.GenerateArgs) []*rule.Rule { exists[f] = true } for _, r := range args.File.Rules { - srcsExist := false - for _, f := range r.AttrStrings("srcs") { - if exists[f] { - srcsExist = true - break - } - } - if !srcsExist { + srcExist := exists[r.AttrString("src")] + if !srcExist { ret = append(ret, rule.NewRule(starlarkLibraryKind, name)) } } @@ -284,3 +366,26 @@ func checkInternalVisibility(rel, visibility string) string { } return visibility } + +func sanitizeName(name string) string { + name = strings.ReplaceAll(name, "/", "_") + name = strings.ReplaceAll(name, "+", "_") + return name +} + +// deduplicateAndSort removes duplicate entries and sorts the list +func deduplicateAndSort(in []string) (out []string) { + if len(in) == 0 { + return in + } + seen := make(map[string]bool) + for _, v := range in { + if seen[v] { + continue + } + seen[v] = true + out = append(out, v) + } + sort.Strings(out) + return +} diff --git a/language/starlarkbundle/starlark_library_test.go b/language/starlarkbundle/starlark_library_test.go new file mode 100644 index 000000000..e0d9223ab --- /dev/null +++ b/language/starlarkbundle/starlark_library_test.go @@ -0,0 +1,119 @@ +/* Copyright 2020 The Bazel Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package starlarkbundle + +import ( + "strings" + "testing" + + "github.com/bazelbuild/buildtools/build" +) + +func TestRenameFailToPrint(t *testing.T) { + tests := []struct { + name string + input string + want string + modified bool + }{ + { + name: "single fail call", + input: `fail("error message") +`, + want: `print("error message") +`, + modified: true, + }, + { + name: "multiple fail calls", + input: `if condition: + fail("error 1") +else: + fail("error 2") +`, + want: `if condition: + print("error 1") +else: + print("error 2") +`, + modified: true, + }, + { + name: "fail with multiple arguments", + input: `fail("error:", variable, "more text") +`, + want: `print("error:", variable, "more text") +`, + modified: true, + }, + { + name: "no fail calls", + input: `print("hello") +x = 1 +`, + want: `print("hello") +x = 1 +`, + modified: false, + }, + { + name: "mixed fail and print calls", + input: `print("info") +fail("error") +print("more info") +`, + want: `print("info") +print("error") +print("more info") +`, + modified: true, + }, + { + name: "fail in function definition", + input: `def my_function(param): + if not param: + fail("param is required") + return param +`, + want: `def my_function(param): + if not param: + print("param is required") + return param +`, + modified: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ast, err := build.ParseBzl("test.bzl", []byte(tt.input)) + if err != nil { + t.Fatalf("failed to parse input: %v", err) + } + + modified := renameFailToPrint(ast) + + if modified != tt.modified { + t.Errorf("renameFailToPrint() modified = %v, want %v", modified, tt.modified) + } + + got := string(build.Format(ast)) + if strings.TrimSpace(got) != strings.TrimSpace(tt.want) { + t.Errorf("renameFailToPrint() output mismatch\ngot:\n%s\n\nwant:\n%s", got, tt.want) + } + }) + } +} diff --git a/rules/private/proto_repository_tools.bzl b/rules/private/proto_repository_tools.bzl index fef1520c1..2f7bbfc6d 100644 --- a/rules/private/proto_repository_tools.bzl +++ b/rules/private/proto_repository_tools.bzl @@ -68,11 +68,11 @@ def _proto_repository_tools_impl(ctx): ctx.path(ctx.attr._list_repository_tools_srcs), "-dir=src/github.com/stackb/rules_proto", # Run it under 'check' to assert file is up-to-date - # "-check=rules/private/proto_repository_tools_srcs.bzl", + "-check=rules/private/proto_repository_tools_srcs.bzl", # Run it under 'skip' to not check (only for internal testing) # "-skip=rules/private/proto_repository_tools_srcs.bzl", # Run it under 'generate' to recreate the list - "-generate=rules/private/proto_repository_tools_srcs.bzl", + # "-generate=rules/private/proto_repository_tools_srcs.bzl", ], environment = env, ) diff --git a/rules/private/proto_repository_tools_srcs.bzl b/rules/private/proto_repository_tools_srcs.bzl index bb53b1ef6..5baab94d6 100644 --- a/rules/private/proto_repository_tools_srcs.bzl +++ b/rules/private/proto_repository_tools_srcs.bzl @@ -45,6 +45,7 @@ PROTO_REPOSITORY_TOOLS_SRCS = [ "@build_stack_rules_proto//language/protobuf:protobuf.go", "@build_stack_rules_proto//language/starlarkbundle:BUILD.bazel", "@build_stack_rules_proto//language/starlarkbundle:language.go", + "@build_stack_rules_proto//language/starlarkbundle:lifecycle.go", "@build_stack_rules_proto//language/starlarkbundle:starlark_bundle.go", "@build_stack_rules_proto//language/starlarkbundle:starlark_library.go", "@build_stack_rules_proto//pkg:BUILD.bazel", diff --git a/rules/proto/starlark_repository.bzl b/rules/proto/starlark_repository.bzl new file mode 100644 index 000000000..604b80e6c --- /dev/null +++ b/rules/proto/starlark_repository.bzl @@ -0,0 +1,31 @@ +"""proto_repository.bzl provides the proto_repository rule.""" + +# Copyright 2014 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@build_stack_rules_proto//rules/proto:proto_repository.bzl", "protobuf_go_repository", _proto_repository_attrs = "proto_repository_attrs") + +# TODO: narrow the set of available attrs +starlark_repository_attrs = _proto_repository_attrs + +def starlark_repository(**kwargs): + """starlark_repository wraps proto_repository and sets the language to starlark_bundle + + Args: + **kwargs: the kwargs dict passed to protobuf_go_repository + """ + name = kwargs.get("name") + kwargs.setdefault("apparent_name", name) + + protobuf_go_repository(**kwargs) diff --git a/rules/starlark_bundle.bzl b/rules/starlark_bundle.bzl index e25cddb35..acde10b03 100644 --- a/rules/starlark_bundle.bzl +++ b/rules/starlark_bundle.bzl @@ -4,29 +4,44 @@ Rule is needed because bzl_library strictly requires srcs. """ load("@bazel_skylib//:bzl_library.bzl", "StarlarkLibraryInfo") +load(":starlark_library.bzl", "StarlarkLibraryFileInfo") def _starlark_bundle_impl(ctx): - deps = [d[StarlarkLibraryInfo] for d in ctx.attr.deps] + deps = [d[StarlarkLibraryFileInfo] for d in ctx.attr.deps] transitive_srcs = depset(transitive = [d.transitive_srcs for d in deps]) + transitive_docs = depset(transitive = [d.transitive_docs for d in deps]) return [ DefaultInfo( - files = transitive_srcs, + files = transitive_docs, ), StarlarkLibraryInfo( srcs = [], transitive_srcs = transitive_srcs, ), + StarlarkLibraryFileInfo( + src = None, + transitive_srcs = transitive_srcs, + transitive_docs = transitive_docs, + broken = False, + ), ] -starlark_bundle = rule( +_starlark_bundle = rule( implementation = _starlark_bundle_impl, attrs = { "deps": attr.label_list( doc = "list of starlark_library or bzl library rules", - providers = [StarlarkLibraryInfo], + providers = [StarlarkLibraryFileInfo], ), }, doc = "", - provides = [DefaultInfo, StarlarkLibraryInfo], + provides = [DefaultInfo, StarlarkLibraryFileInfo, StarlarkLibraryInfo], ) + +def starlark_bundle(name, deps, **kwargs): + _starlark_bundle( + name = name, + deps = deps, + **kwargs + ) diff --git a/rules/starlark_library.bzl b/rules/starlark_library.bzl index 868ac818c..ec8208277 100644 --- a/rules/starlark_library.bzl +++ b/rules/starlark_library.bzl @@ -1,9 +1,90 @@ """starlark_library.bzl is a thin wrapper over bzl_library.""" -load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("@bazel_skylib//:bzl_library.bzl", "StarlarkLibraryInfo") -def starlark_library(name, **kwargs): - bzl_library( +StarlarkLibraryFileInfo = provider( + "Information on contained Starlark rules.", + fields = { + "src": "The top-level src file", + "doc": "The starlark_extract_doc output file", + "transitive_srcs": "Transitive closure of rules files required for " + + "interpretation of the src", + "transitive_docs": "Transitive closure of docs that have viable dependencies", + "broken": "If at last one of the transitive srcs has an unknown dependency.", + }, +) + +def _starlark_library_impl(ctx): + src = ctx.file.src + doc = ctx.file.doc + broken = len(ctx.attr.unknown_deps) > 0 + + deps = [d[StarlarkLibraryFileInfo] for d in ctx.attr.deps] + transitive_srcs = depset([src], order = "postorder", transitive = [d.transitive_srcs for d in deps]) + transitive_docs = depset([doc] if not broken else [], order = "postorder", transitive = [d.transitive_docs for d in deps]) + + for f in transitive_srcs.to_list(): + print(str(ctx.label), "src:", f.short_path) + + return [ + DefaultInfo( + files = transitive_srcs, + ), + OutputGroupInfo( + doc = [doc], + ), + StarlarkLibraryInfo( + srcs = [src], + transitive_srcs = transitive_srcs, + ), + StarlarkLibraryFileInfo( + src = src, + transitive_srcs = transitive_srcs, + transitive_docs = transitive_docs, + broken = broken, + ), + ] + +_starlark_library = rule( + implementation = _starlark_library_impl, + attrs = { + "src": attr.label( + doc = "label for the .bzl file", + allow_single_file = True, + ), + "doc": attr.label( + doc = "the output of the starlar_doc_extract rule", + allow_single_file = True, + ), + "unknown_deps": attr.string_list( + doc = "list of starlark_library rule dependencies that will not be able to resolve", + ), + "deps": attr.label_list( + doc = "list of starlark_library rule dependencies. These can be ", + providers = [StarlarkLibraryFileInfo], + ), + }, + doc = "", + provides = [DefaultInfo, StarlarkLibraryInfo, StarlarkLibraryFileInfo], +) + +def starlark_library(name, src, deps = [], **kwargs): + visibility = kwargs.pop("visibility", []) + bazel_tools_deps = kwargs.pop("bazel_tools_deps", []) + + doc_name = name + "_doc" + native.starlark_doc_extract( + name = doc_name, + src = src, + deps = deps + bazel_tools_deps, + visibility = ["//visibility:public"], + ) + + _starlark_library( name = name, + src = src, + doc = doc_name, + deps = deps, + visibility = ["//visibility:public"], **kwargs ) From c88be120f8697bfa7459f08ac7fb28da1511f31a Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Tue, 25 Nov 2025 17:48:12 -0700 Subject: [PATCH 06/12] checkpoint --- language/starlarkbundle/language.go | 2 +- language/starlarkbundle/starlark_bundle.go | 3 ++- language/starlarkbundle/starlark_library.go | 6 ------ 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/language/starlarkbundle/language.go b/language/starlarkbundle/language.go index 81eaf4e62..775e71e4d 100644 --- a/language/starlarkbundle/language.go +++ b/language/starlarkbundle/language.go @@ -253,7 +253,7 @@ func (ext *starlarkBundleLang) getModuleDependencyRepoName(repo string) (string, return mappedRepo, true } else { ext.logf(" unknown module dependency: %q (known deps: %v)", repo, ext.moduleDeps) - log.Fatalf("unknown module dependency: %q", repo) + // log.Fatalf("unknown module dependency: %q", repo) return repo, false } } diff --git a/language/starlarkbundle/starlark_bundle.go b/language/starlarkbundle/starlark_bundle.go index 80e711028..56e95b833 100644 --- a/language/starlarkbundle/starlark_bundle.go +++ b/language/starlarkbundle/starlark_bundle.go @@ -33,6 +33,7 @@ import ( const ( starlarkBundleKind = "starlark_bundle" + starlarkBundleName = "docs" ) var starlarkBundleKindInfo = map[string]rule.KindInfo{ @@ -48,7 +49,7 @@ var starlarkBundleLoadInfo = rule.LoadInfo{ } func starlarkBundleRule() (*rule.Rule, []string) { - r := rule.NewRule(starlarkBundleKind, starlarkBundleKind) + r := rule.NewRule(starlarkBundleKind, starlarkBundleName) r.SetAttr("visibility", []string{"//visibility:public"}) diff --git a/language/starlarkbundle/starlark_library.go b/language/starlarkbundle/starlark_library.go index 30c2d25f2..200ac73dd 100644 --- a/language/starlarkbundle/starlark_library.go +++ b/language/starlarkbundle/starlark_library.go @@ -367,12 +367,6 @@ func checkInternalVisibility(rel, visibility string) string { return visibility } -func sanitizeName(name string) string { - name = strings.ReplaceAll(name, "/", "_") - name = strings.ReplaceAll(name, "+", "_") - return name -} - // deduplicateAndSort removes duplicate entries and sorts the list func deduplicateAndSort(in []string) (out []string) { if len(in) == 0 { From 2c900b7555157080b60dfaaeeb6218ec06db1cb9 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Thu, 27 Nov 2025 19:40:31 -0700 Subject: [PATCH 07/12] checkpoint: before remove starlark_library gen --- language/starlarkbundle/bazelignore_test.go | 149 ++++++++++++++++++++ language/starlarkbundle/language.go | 88 +++++++++++- language/starlarkbundle/starlark_bundle.go | 53 ++++++- language/starlarkbundle/starlark_library.go | 36 ++++- rules/starlark_bundle.bzl | 9 +- rules/starlark_library.bzl | 32 +++-- 6 files changed, 343 insertions(+), 24 deletions(-) create mode 100644 language/starlarkbundle/bazelignore_test.go diff --git a/language/starlarkbundle/bazelignore_test.go b/language/starlarkbundle/bazelignore_test.go new file mode 100644 index 000000000..45d50fccc --- /dev/null +++ b/language/starlarkbundle/bazelignore_test.go @@ -0,0 +1,149 @@ +/* Copyright 2020 The Bazel Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package starlarkbundle + +import ( + "os" + "path/filepath" + "testing" +) + +func TestRemoveAbsolutePathsFromBazelignore(t *testing.T) { + tests := []struct { + name string + input string + expected string + removed bool + }{ + { + name: "removes absolute paths", + input: `.build/ +/.github/ +.swiftpm/ +/.vscode/ +`, + expected: `.build/ +.swiftpm/ +`, + removed: true, + }, + { + name: "keeps relative paths", + input: `.build/ +.github/ +.swiftpm/ +.vscode/ +`, + expected: `.build/ +.github/ +.swiftpm/ +.vscode/ +`, + removed: false, + }, + { + name: "handles empty lines", + input: `.build/ + +/.github/ + +.swiftpm/ +`, + expected: `.build/ + + +.swiftpm/ +`, + removed: true, + }, + { + name: "handles comments and absolute paths", + input: `# Comment +.build/ +/absolute/path +relative/path +/another/absolute +`, + expected: `# Comment +.build/ +relative/path +`, + removed: true, + }, + { + name: "empty file", + input: "", + expected: "", + removed: false, + }, + { + name: "only absolute paths", + input: `/path1 +/path2 +/path3 +`, + expected: ``, + removed: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create temp directory + tmpDir := t.TempDir() + bazelignorePath := filepath.Join(tmpDir, ".bazelignore") + + // Write input content + if err := os.WriteFile(bazelignorePath, []byte(tt.input), 0644); err != nil { + t.Fatalf("failed to write test file: %v", err) + } + + // Create extension + ext := NewLanguage().(*starlarkBundleLang) + ext.logger = nil // Disable logging for tests + + // Run the function + err := ext.removeAbsolutePathsFromBazelignore(bazelignorePath) + if err != nil { + t.Fatalf("removeAbsolutePathsFromBazelignore failed: %v", err) + } + + // Read result + result, err := os.ReadFile(bazelignorePath) + if err != nil { + t.Fatalf("failed to read result file: %v", err) + } + + if string(result) != tt.expected { + t.Errorf("Content mismatch:\ngot:\n%q\n\nwant:\n%q", string(result), tt.expected) + } + }) + } +} + +func TestRemoveAbsolutePathsFromBazelignore_NonExistent(t *testing.T) { + tmpDir := t.TempDir() + bazelignorePath := filepath.Join(tmpDir, ".bazelignore") + + ext := NewLanguage().(*starlarkBundleLang) + ext.logger = nil + + // Should not error on non-existent file + err := ext.removeAbsolutePathsFromBazelignore(bazelignorePath) + if err != nil { + t.Errorf("expected no error for non-existent file, got: %v", err) + } +} diff --git a/language/starlarkbundle/language.go b/language/starlarkbundle/language.go index 775e71e4d..4e06956b8 100644 --- a/language/starlarkbundle/language.go +++ b/language/starlarkbundle/language.go @@ -23,10 +23,12 @@ package starlarkbundle import ( "flag" + "fmt" "io" "log" "maps" "os" + "path/filepath" "strings" "github.com/bazelbuild/bazel-gazelle/config" @@ -39,17 +41,21 @@ import ( const ( languageName = "starlark_bundle" + starlarkBundleRepoNameDirectiveName = "starlark_bundle_repo_name" starlarkBundleRootDirectiveName = "starlark_bundle_root" starlarkBundleExcludeDirectiveName = "starlark_bundle_exclude" starlarkBundleLogFileDirectiveName = "starlark_bundle_log_file" starlarkModuleDependencyDirectiveName = "module_dependency" + coarseDependencies = true ) type starlarkBundleLang struct { starlarkBundleRoot *string + starlarkBundleRepoName string starlarkBundleExcludeDirs []string moduleDeps map[string]string starlarkLibraries map[label.Label]*rule.Rule + bzlFiles map[string]bool logFile string logWriter *os.File logger *log.Logger @@ -60,6 +66,7 @@ type starlarkBundleLang struct { func NewLanguage() language.Language { return &starlarkBundleLang{ starlarkLibraries: map[label.Label]*rule.Rule{}, + bzlFiles: map[string]bool{}, moduleDeps: map[string]string{}, } } @@ -80,7 +87,7 @@ func (ext *starlarkBundleLang) RegisterFlags(fs *flag.FlagSet, cmd string, c *co func (ext *starlarkBundleLang) CheckFlags(fs *flag.FlagSet, c *config.Config) error { if ext.logFile != "" { - ext.createLogger(c, ext.logFile) + ext.createLogger(ext.logFile) ext.logf("CheckFlags: log file initialized from flag: %s", ext.logFile) } if ext.logger == nil { @@ -108,6 +115,8 @@ func (ext *starlarkBundleLang) Configure(c *config.Config, rel string, f *rule.F log.Fatalf("gazelle:%s should only be set once (refusing to override %q with %q)", starlarkBundleRootDirectiveName, *ext.starlarkBundleRoot, d.Value) } ext.starlarkBundleRoot = &rel + case starlarkBundleRepoNameDirectiveName: + ext.starlarkBundleRepoName = d.Value case starlarkBundleExcludeDirectiveName: ext.starlarkBundleExcludeDirs = append(ext.starlarkBundleExcludeDirs, d.Value) case starlarkModuleDependencyDirectiveName: @@ -118,13 +127,13 @@ func (ext *starlarkBundleLang) Configure(c *config.Config, rel string, f *rule.F ext.moduleDeps[nameVersion[0]] = nameVersion[1] ext.logf("Added module dependency: %q -> %q", nameVersion[0], nameVersion[1]) case starlarkBundleLogFileDirectiveName: - ext.createLogger(c, d.Value) + ext.createLogger(d.Value) } } } } -func (ext *starlarkBundleLang) createLogger(c *config.Config, logFile string) { +func (ext *starlarkBundleLang) createLogger(logFile string) { if logFile != "" { f, err := os.OpenFile(logFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666) if err == nil { @@ -202,7 +211,12 @@ func (ext *starlarkBundleLang) Resolve(c *config.Config, ix *resolve.RuleIndex, case starlarkLibraryKind: starlarkLibraryResolve(c, ix, r, importsRaw, from, ext) case starlarkBundleKind: - starlarkBundleResolve(r, ext.starlarkLibraries) + if coarseDependencies { + starlarkBundleResolveCoarse(r, ext.starlarkLibraries, ext.moduleDeps) + } else { + starlarkBundleResolve(r, ext.starlarkLibraries) + + } } } @@ -230,6 +244,26 @@ func (ext *starlarkBundleLang) GenerateRules(args language.GenerateArgs) (result ext.logf("GenerateRules: visiting %s", args.Rel) + if args.Rel == "" { + // REPO.bazel causes visibility issues if the root BUILD file has been + // deleted but still referencing non-existent rules. This should be + // moved to the part where gazelle cleans up build files. + repoFile := filepath.Join(args.Config.RepoRoot, "REPO.bazel") + if _, err := os.Stat(repoFile); err == nil { + ext.logf("Removing REPO.bazel from root package") + if err := os.Remove(repoFile); err != nil { + ext.logf("ERROR: failed to remove REPO.bazel: %v", err) + } + } + + bazelIgnore := filepath.Join(args.Config.RepoRoot, "REPO.bazel") + if _, err := os.Stat(bazelIgnore); err == nil { + if err := ext.removeAbsolutePathsFromBazelignore(bazelIgnore); err != nil { + ext.logf("ERROR: failed to cleanup .bazelignore file: %v", err) + } + } + } + for _, r := range []language.GenerateResult{ starlarkLibraryGenerate(args, ext.starlarkLibraries, ext), starlarkBundleGenerate(args, ext.starlarkBundleRoot), @@ -264,3 +298,49 @@ func (ext *starlarkBundleLang) logf(format string, args ...any) { ext.logger.Printf(format, args...) } } + +// removeAbsolutePathsFromBazelignore reads a .bazelignore file and removes +// any lines that are absolute paths (starting with /). +func (ext *starlarkBundleLang) removeAbsolutePathsFromBazelignore(bazelignorePath string) error { + ext.logf("Checking .bazelignore file: %s", bazelignorePath) + + // Read the file + content, err := os.ReadFile(bazelignorePath) + if err != nil { + if os.IsNotExist(err) { + return nil // File doesn't exist, nothing to do + } + return fmt.Errorf("failed to read .bazelignore: %w", err) + } + + // Split into lines + lines := strings.Split(string(content), "\n") + var filteredLines []string + removedCount := 0 + + for _, line := range lines { + trimmed := strings.TrimSpace(line) + // Keep the line if it's not an absolute path + if trimmed == "" || !strings.HasPrefix(trimmed, "/") { + filteredLines = append(filteredLines, line) + } else { + ext.logf(" Removing absolute path from .bazelignore: %q", trimmed) + removedCount++ + } + } + + // If nothing changed, don't rewrite the file + if removedCount == 0 { + ext.logf(" No absolute paths found in .bazelignore") + return nil + } + + // Write back the filtered content + newContent := strings.Join(filteredLines, "\n") + if err := os.WriteFile(bazelignorePath, []byte(newContent), 0644); err != nil { + return fmt.Errorf("failed to write .bazelignore: %w", err) + } + + ext.logf(" Removed %d absolute path(s) from .bazelignore", removedCount) + return nil +} diff --git a/language/starlarkbundle/starlark_bundle.go b/language/starlarkbundle/starlark_bundle.go index 56e95b833..afcf8ca17 100644 --- a/language/starlarkbundle/starlark_bundle.go +++ b/language/starlarkbundle/starlark_bundle.go @@ -22,6 +22,7 @@ limitations under the License. package starlarkbundle import ( + "log" "sort" "github.com/bazelbuild/bazel-gazelle/config" @@ -33,7 +34,7 @@ import ( const ( starlarkBundleKind = "starlark_bundle" - starlarkBundleName = "docs" + starlarkBundleName = "bundle" ) var starlarkBundleKindInfo = map[string]rule.KindInfo{ @@ -60,6 +61,8 @@ func starlarkBundleImports(_ *config.Config, _ *rule.Rule, _ *rule.File) []resol return nil } +// starlarkBundleResolveResolve iterates the set of stalark_library rules and +// adds them to it's deps list. func starlarkBundleResolve(r *rule.Rule, starlarkLibraries map[label.Label]*rule.Rule) { deps := make([]string, 0, len(starlarkLibraries)) for dep := range starlarkLibraries { @@ -72,6 +75,54 @@ func starlarkBundleResolve(r *rule.Rule, starlarkLibraries map[label.Label]*rule } } +// starlarkBundleResolveCoarse iterates the set of stalark_library rules and +// adds them to it's deps list. It also takes out all external dependencies +// from the library rules and collects them into itself, but only as the :bundle +// target. This is because we cannot ensure that the fine-grained dependency +// structure is correct. However, we can (hopefully) ensure that the +// coarse-grained dependency graph of bundles is correct. +func starlarkBundleResolveCoarse(r *rule.Rule, starlarkLibraries map[label.Label]*rule.Rule, moduleDeps map[string]string) { + deps := make([]string, 0, len(starlarkLibraries)) + for dep, r := range starlarkLibraries { + deps = append(deps, dep.String()) + + ruleDeps := r.AttrStrings("deps") + p1, _ := partitionExternalDeps(ruleDeps) + r.SetAttr("deps", p1) + } + + if false { + for _, module := range moduleDeps { + dep := label.New(module, "", "bundle") + deps = append(deps, dep.String()) + } + } + + if len(deps) > 0 { + sort.Strings(deps) + r.SetAttr("deps", deps) + } +} + +func partitionExternalDeps(deps []string) ([]string, []label.Label) { + var p1 []string // first and third-party deps + var p3 []label.Label // first and third-party deps + + for _, dep := range deps { + from, err := label.Parse(dep) + if err != nil { + log.Panicf("invalid dep label: %v", dep) + } + if from.Repo != "" { + p3 = append(p3, from) + } else { + p1 = append(p1, dep) + } + } + + return p1, p3 +} + func starlarkBundleGenerate(args language.GenerateArgs, root *string) (result language.GenerateResult) { if root == nil || *root != args.Rel { return result diff --git a/language/starlarkbundle/starlark_library.go b/language/starlarkbundle/starlark_library.go index 200ac73dd..9c94070ce 100644 --- a/language/starlarkbundle/starlark_library.go +++ b/language/starlarkbundle/starlark_library.go @@ -72,9 +72,17 @@ func (s suffixes) Matches(test string) bool { return false } +func makeStarlarkLibraryRuleName(f string) string { + name := starlarkLibraryNamePrefix + strings.TrimSuffix(f, fileType) + if strings.HasSuffix(name, "_test") { + name += "rule" + } + return name +} + func starlarkLibraryRule(args language.GenerateArgs, f string, ext *starlarkBundleLang) (*rule.Rule, []*build.LoadStmt) { fullPath := filepath.Join(args.Dir, f) - name := starlarkLibraryNamePrefix + strings.TrimSuffix(f, fileType) + name := makeStarlarkLibraryRuleName(f) ast, loads, err := getBzlFileLoadsStmts(fullPath) if err != nil { @@ -83,14 +91,17 @@ func starlarkLibraryRule(args language.GenerateArgs, f string, ext *starlarkBund // without deps. } - // always castrate fail() exprs - if renameFailToPrint(ast) { - ext.emitBzlFile(fullPath, ast) - } + // // always castrate fail() exprs + // if renameFailToPrint(ast) { + // ext.emitBzlFile(fullPath, ast) + // } r := rule.NewRule(starlarkLibraryKind, name) r.SetAttr("src", f) + if ext.starlarkBundleRepoName != "" { + r.SetAttr("repo_name", ext.starlarkBundleRepoName) + } shouldSetVisibility := args.File == nil || !args.File.HasDefaultVisibility() if shouldSetVisibility { @@ -148,6 +159,10 @@ func starlarkLibraryResolve(c *config.Config, ix *resolve.RuleIndex, r *rule.Rul impLabel = impLabel.Abs(from.Repo, from.Pkg) ext.logf(" absolute label: %s (repo=%q, pkg=%q, name=%q)", impLabel.String(), impLabel.Repo, impLabel.Pkg, impLabel.Name) + if coarseDependencies && impLabel.Repo != "" { + continue + } + if impLabel.Repo == "bazel_tools" { // The @bazel_tools repo is tricky because it is a part of the // "shipped with bazel" core library for interacting with the @@ -214,6 +229,17 @@ func starlarkLibraryResolve(c *config.Config, ix *resolve.RuleIndex, r *rule.Rul // depLabel = depLabel.Rel(from.Repo, from.Pkg) deps = append(deps, depLabel.String()) } + + // in the source file, never use unqualified + if ext.starlarkBundleRepoName != "" { + // absLabel := impLabel.Abs(ext.starlarkBundleRepoName, from.Pkg) + absLabel := label.New(ext.starlarkBundleRepoName, impLabel.Pkg, impLabel.Name) + load.Module.Value = absLabel.String() + rewriteBzlSourceFile = true + ext.logf(" rewrote %v => %v", impLabel, absLabel) + } else { + ext.logf(" not abs'ing label (nostarlarkBundleRepoName)") + } } ext.logf(" resolution complete: %d deps, %d unknown, %d bazel_tools", len(deps), len(unknownDeps), len(bazelToolsDeps)) diff --git a/rules/starlark_bundle.bzl b/rules/starlark_bundle.bzl index acde10b03..11120576e 100644 --- a/rules/starlark_bundle.bzl +++ b/rules/starlark_bundle.bzl @@ -13,14 +13,16 @@ def _starlark_bundle_impl(ctx): return [ DefaultInfo( - files = transitive_docs, + files = transitive_srcs, ), StarlarkLibraryInfo( srcs = [], transitive_srcs = transitive_srcs, ), StarlarkLibraryFileInfo( + label = ctx.label, src = None, + deps = depset(deps), transitive_srcs = transitive_srcs, transitive_docs = transitive_docs, broken = False, @@ -35,11 +37,12 @@ _starlark_bundle = rule( providers = [StarlarkLibraryFileInfo], ), }, - doc = "", provides = [DefaultInfo, StarlarkLibraryFileInfo, StarlarkLibraryInfo], ) -def starlark_bundle(name, deps, **kwargs): +def starlark_bundle(name, **kwargs): + deps = kwargs.pop("deps", []) + _starlark_bundle( name = name, deps = deps, diff --git a/rules/starlark_library.bzl b/rules/starlark_library.bzl index ec8208277..a6ee97a10 100644 --- a/rules/starlark_library.bzl +++ b/rules/starlark_library.bzl @@ -5,8 +5,12 @@ load("@bazel_skylib//:bzl_library.bzl", "StarlarkLibraryInfo") StarlarkLibraryFileInfo = provider( "Information on contained Starlark rules.", fields = { + "label": "The label of the target rule", + "repo_name": "The original (non-canonical) repo (workspace!) name", "src": "The top-level src file", "doc": "The starlark_extract_doc output file", + "deps": "DepSet[StarlarkLibraryFileInfo]: deps of this file", + # "transitive_deps": "List[DepSet[StarlarkLibraryFileInfo]]: transitive deps of this file", "transitive_srcs": "Transitive closure of rules files required for " + "interpretation of the src", "transitive_docs": "Transitive closure of docs that have viable dependencies", @@ -20,12 +24,11 @@ def _starlark_library_impl(ctx): broken = len(ctx.attr.unknown_deps) > 0 deps = [d[StarlarkLibraryFileInfo] for d in ctx.attr.deps] + transitive_deps = [d.deps for d in deps] + transitive_srcs = depset([src], order = "postorder", transitive = [d.transitive_srcs for d in deps]) transitive_docs = depset([doc] if not broken else [], order = "postorder", transitive = [d.transitive_docs for d in deps]) - for f in transitive_srcs.to_list(): - print(str(ctx.label), "src:", f.short_path) - return [ DefaultInfo( files = transitive_srcs, @@ -38,7 +41,10 @@ def _starlark_library_impl(ctx): transitive_srcs = transitive_srcs, ), StarlarkLibraryFileInfo( + label = ctx.label, + repo_name = ctx.attr.repo_name, src = src, + deps = depset(deps, transitive = transitive_deps), transitive_srcs = transitive_srcs, transitive_docs = transitive_docs, broken = broken, @@ -48,6 +54,9 @@ def _starlark_library_impl(ctx): _starlark_library = rule( implementation = _starlark_library_impl, attrs = { + "repo_name": attr.string( + mandatory = True, + ), "src": attr.label( doc = "label for the .bzl file", allow_single_file = True, @@ -72,18 +81,19 @@ def starlark_library(name, src, deps = [], **kwargs): visibility = kwargs.pop("visibility", []) bazel_tools_deps = kwargs.pop("bazel_tools_deps", []) - doc_name = name + "_doc" - native.starlark_doc_extract( - name = doc_name, - src = src, - deps = deps + bazel_tools_deps, - visibility = ["//visibility:public"], - ) + # doc_name = name + "_doc" + # native.starlark_doc_extract( + # name = doc_name, + # src = src, + # deps = deps + bazel_tools_deps, + # visibility = ["//visibility:public"], + # ) _starlark_library( name = name, src = src, - doc = doc_name, + # doc = doc_name, + doc = src, deps = deps, visibility = ["//visibility:public"], **kwargs From 350fa87e73e0d6ec499f90f85c89f0f5fcb4fc45 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Sat, 29 Nov 2025 17:56:51 -0700 Subject: [PATCH 08/12] checkpoint: refactored for reduced scope --- cmd/gazelle/BUILD.bazel | 2 +- cmd/gazelle/langs.go | 4 +- language/starlarkbundle/bazelignore_test.go | 149 ------ language/starlarkbundle/language.go | 346 ------------- language/starlarkbundle/lifecycle.go | 43 -- language/starlarkbundle/lifecycle_test.go | 114 ----- language/starlarkbundle/starlark_bundle.go | 137 ----- language/starlarkbundle/starlark_library.go | 411 --------------- .../starlarkbundle/starlark_library_test.go | 119 ----- .../BUILD.bazel | 17 +- language/starlarklibrary/language.go | 470 ++++++++++++++++++ language/starlarklibrary/language_test.go | 252 ++++++++++ rules/private/proto_repository_tools_srcs.bzl | 7 +- rules/starlark_bundle.bzl | 50 -- rules/starlark_library.bzl | 81 +-- 15 files changed, 761 insertions(+), 1441 deletions(-) delete mode 100644 language/starlarkbundle/bazelignore_test.go delete mode 100644 language/starlarkbundle/language.go delete mode 100644 language/starlarkbundle/lifecycle.go delete mode 100644 language/starlarkbundle/lifecycle_test.go delete mode 100644 language/starlarkbundle/starlark_bundle.go delete mode 100644 language/starlarkbundle/starlark_library.go delete mode 100644 language/starlarkbundle/starlark_library_test.go rename language/{starlarkbundle => starlarklibrary}/BUILD.bazel (79%) create mode 100644 language/starlarklibrary/language.go create mode 100644 language/starlarklibrary/language_test.go delete mode 100644 rules/starlark_bundle.bzl diff --git a/cmd/gazelle/BUILD.bazel b/cmd/gazelle/BUILD.bazel index 9451a3856..bc620c057 100644 --- a/cmd/gazelle/BUILD.bazel +++ b/cmd/gazelle/BUILD.bazel @@ -29,7 +29,7 @@ go_library( "//cmd/gazelle/internal/wspace", "//language/proto_go_modules", "//language/protobuf", - "//language/starlarkbundle", + "//language/starlarklibrary", "@bazel_gazelle//config:go_default_library", "@bazel_gazelle//flag:go_default_library", "@bazel_gazelle//label:go_default_library", diff --git a/cmd/gazelle/langs.go b/cmd/gazelle/langs.go index 534498c63..e86762e50 100644 --- a/cmd/gazelle/langs.go +++ b/cmd/gazelle/langs.go @@ -21,7 +21,7 @@ import ( "github.com/bazelbuild/bazel-gazelle/language/proto" "github.com/stackb/rules_proto/language/proto_go_modules" "github.com/stackb/rules_proto/language/protobuf" - "github.com/stackb/rules_proto/language/starlarkbundle" + "github.com/stackb/rules_proto/language/starlarklibrary" ) var languages = []language.Language{ @@ -29,5 +29,5 @@ var languages = []language.Language{ protobuf.NewLanguage(), golang.NewLanguage(), proto_go_modules.NewLanguage(), - starlarkbundle.NewLanguage(), + starlarklibrary.NewLanguage(), } diff --git a/language/starlarkbundle/bazelignore_test.go b/language/starlarkbundle/bazelignore_test.go deleted file mode 100644 index 45d50fccc..000000000 --- a/language/starlarkbundle/bazelignore_test.go +++ /dev/null @@ -1,149 +0,0 @@ -/* Copyright 2020 The Bazel Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package starlarkbundle - -import ( - "os" - "path/filepath" - "testing" -) - -func TestRemoveAbsolutePathsFromBazelignore(t *testing.T) { - tests := []struct { - name string - input string - expected string - removed bool - }{ - { - name: "removes absolute paths", - input: `.build/ -/.github/ -.swiftpm/ -/.vscode/ -`, - expected: `.build/ -.swiftpm/ -`, - removed: true, - }, - { - name: "keeps relative paths", - input: `.build/ -.github/ -.swiftpm/ -.vscode/ -`, - expected: `.build/ -.github/ -.swiftpm/ -.vscode/ -`, - removed: false, - }, - { - name: "handles empty lines", - input: `.build/ - -/.github/ - -.swiftpm/ -`, - expected: `.build/ - - -.swiftpm/ -`, - removed: true, - }, - { - name: "handles comments and absolute paths", - input: `# Comment -.build/ -/absolute/path -relative/path -/another/absolute -`, - expected: `# Comment -.build/ -relative/path -`, - removed: true, - }, - { - name: "empty file", - input: "", - expected: "", - removed: false, - }, - { - name: "only absolute paths", - input: `/path1 -/path2 -/path3 -`, - expected: ``, - removed: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Create temp directory - tmpDir := t.TempDir() - bazelignorePath := filepath.Join(tmpDir, ".bazelignore") - - // Write input content - if err := os.WriteFile(bazelignorePath, []byte(tt.input), 0644); err != nil { - t.Fatalf("failed to write test file: %v", err) - } - - // Create extension - ext := NewLanguage().(*starlarkBundleLang) - ext.logger = nil // Disable logging for tests - - // Run the function - err := ext.removeAbsolutePathsFromBazelignore(bazelignorePath) - if err != nil { - t.Fatalf("removeAbsolutePathsFromBazelignore failed: %v", err) - } - - // Read result - result, err := os.ReadFile(bazelignorePath) - if err != nil { - t.Fatalf("failed to read result file: %v", err) - } - - if string(result) != tt.expected { - t.Errorf("Content mismatch:\ngot:\n%q\n\nwant:\n%q", string(result), tt.expected) - } - }) - } -} - -func TestRemoveAbsolutePathsFromBazelignore_NonExistent(t *testing.T) { - tmpDir := t.TempDir() - bazelignorePath := filepath.Join(tmpDir, ".bazelignore") - - ext := NewLanguage().(*starlarkBundleLang) - ext.logger = nil - - // Should not error on non-existent file - err := ext.removeAbsolutePathsFromBazelignore(bazelignorePath) - if err != nil { - t.Errorf("expected no error for non-existent file, got: %v", err) - } -} diff --git a/language/starlarkbundle/language.go b/language/starlarkbundle/language.go deleted file mode 100644 index 4e06956b8..000000000 --- a/language/starlarkbundle/language.go +++ /dev/null @@ -1,346 +0,0 @@ -/* Copyright 2020 The Bazel Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package symbol generates a `starlark_library` target for every `.bzl` file in -// each package. At the root of the module, a single starlark_bundle is -// populated with deps that include all other starlark_libraries. -// -// The original code for this gazelle extension started from -// https://github.com/bazelbuild/bazel-skylib/blob/main/gazelle/bzl/gazelle.go. -package starlarkbundle - -import ( - "flag" - "fmt" - "io" - "log" - "maps" - "os" - "path/filepath" - "strings" - - "github.com/bazelbuild/bazel-gazelle/config" - "github.com/bazelbuild/bazel-gazelle/label" - "github.com/bazelbuild/bazel-gazelle/language" - "github.com/bazelbuild/bazel-gazelle/repo" - "github.com/bazelbuild/bazel-gazelle/resolve" - "github.com/bazelbuild/bazel-gazelle/rule" -) - -const ( - languageName = "starlark_bundle" - starlarkBundleRepoNameDirectiveName = "starlark_bundle_repo_name" - starlarkBundleRootDirectiveName = "starlark_bundle_root" - starlarkBundleExcludeDirectiveName = "starlark_bundle_exclude" - starlarkBundleLogFileDirectiveName = "starlark_bundle_log_file" - starlarkModuleDependencyDirectiveName = "module_dependency" - coarseDependencies = true -) - -type starlarkBundleLang struct { - starlarkBundleRoot *string - starlarkBundleRepoName string - starlarkBundleExcludeDirs []string - moduleDeps map[string]string - starlarkLibraries map[label.Label]*rule.Rule - bzlFiles map[string]bool - logFile string - logWriter *os.File - logger *log.Logger -} - -// NewLanguage is called by Gazelle to install this language extension in a -// binary. -func NewLanguage() language.Language { - return &starlarkBundleLang{ - starlarkLibraries: map[label.Label]*rule.Rule{}, - bzlFiles: map[string]bool{}, - moduleDeps: map[string]string{}, - } -} - -// Name returns the name of the language. This should be a prefix of the kinds -// of rules generated by the language, e.g., "go" for the Go extension since it -// generates "go_library" rules. -func (*starlarkBundleLang) Name() string { - return languageName -} - -// The following methods are implemented to satisfy the -// https://pkg.go.dev/github.com/bazelbuild/bazel-gazelle/resolve?tab=doc#Resolver -// interface, but are otherwise unused. -func (ext *starlarkBundleLang) RegisterFlags(fs *flag.FlagSet, cmd string, c *config.Config) { - fs.StringVar(&ext.logFile, "starlark_bundle_log", "", "path to log file for starlark_bundle extension") -} - -func (ext *starlarkBundleLang) CheckFlags(fs *flag.FlagSet, c *config.Config) error { - if ext.logFile != "" { - ext.createLogger(ext.logFile) - ext.logf("CheckFlags: log file initialized from flag: %s", ext.logFile) - } - if ext.logger == nil { - ext.logger = log.New(io.Discard, "", 0) - } - return nil -} - -func (*starlarkBundleLang) KnownDirectives() []string { - return []string{ - starlarkBundleRootDirectiveName, - starlarkBundleExcludeDirectiveName, - starlarkBundleLogFileDirectiveName, - starlarkModuleDependencyDirectiveName, - } -} - -func (ext *starlarkBundleLang) Configure(c *config.Config, rel string, f *rule.File) { - if f != nil { - ext.logf("Configure: processing %d directives in %s", len(f.Directives), rel) - for _, d := range f.Directives { - switch d.Key { - case starlarkBundleRootDirectiveName: - if ext.starlarkBundleRoot != nil { - log.Fatalf("gazelle:%s should only be set once (refusing to override %q with %q)", starlarkBundleRootDirectiveName, *ext.starlarkBundleRoot, d.Value) - } - ext.starlarkBundleRoot = &rel - case starlarkBundleRepoNameDirectiveName: - ext.starlarkBundleRepoName = d.Value - case starlarkBundleExcludeDirectiveName: - ext.starlarkBundleExcludeDirs = append(ext.starlarkBundleExcludeDirs, d.Value) - case starlarkModuleDependencyDirectiveName: - nameVersion := strings.Fields(d.Value) - if len(nameVersion) != 2 { - log.Fatalf("malformed directive %s, should be NAME VERSION: %s", starlarkModuleDependencyDirectiveName, d.Value) - } - ext.moduleDeps[nameVersion[0]] = nameVersion[1] - ext.logf("Added module dependency: %q -> %q", nameVersion[0], nameVersion[1]) - case starlarkBundleLogFileDirectiveName: - ext.createLogger(d.Value) - } - } - } -} - -func (ext *starlarkBundleLang) createLogger(logFile string) { - if logFile != "" { - f, err := os.OpenFile(logFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666) - if err == nil { - ext.logWriter = f - ext.logger = log.New(f, "", log.LstdFlags) - } else { - log.Fatalf("attempting to open log file: %v", err) - } - } - if false && ext.logger == nil { - ext.logger = log.New(io.Discard, "", 0) - } - - ext.logf("Log initialized") -} - -// Kinds returns a map of maps rule names (kinds) and information on how to -// match and merge attributes that may be found in rules of those kinds. All -// kinds of rules generated for this language may be found here. -func (*starlarkBundleLang) Kinds() map[string]rule.KindInfo { - kinds := map[string]rule.KindInfo{} - maps.Copy(kinds, starlarkLibraryKindInfo) - maps.Copy(kinds, starlarkBundleKindInfo) - return kinds -} - -// Loads returns .bzl files and symbols they define. Every rule generated by -// GenerateRules, now or in the past, should be loadable from one of these -// files. -func (*starlarkBundleLang) Loads() []rule.LoadInfo { - return []rule.LoadInfo{ - starlarkLibraryLoadInfo, - starlarkBundleLoadInfo, - } -} - -// Fix repairs deprecated usage of language-specific rules in f. This is called -// before the file is indexed. Unless c.ShouldFix is true, fixes that delete or -// rename rules should not be performed. -func (*starlarkBundleLang) Fix(c *config.Config, f *rule.File) { -} - -// Imports returns a list of ImportSpecs that can be used to import the rule r. -// This is used to populate RuleIndex. -// -// If nil is returned, the rule will not be indexed. If any non-nil slice is -// returned, including an empty slice, the rule will be indexed. -func (ext *starlarkBundleLang) Imports(c *config.Config, r *rule.Rule, f *rule.File) []resolve.ImportSpec { - switch r.Kind() { - case starlarkLibraryKind: - return starlarkLibraryImports(c, r, f) - case starlarkBundleKind: - return starlarkBundleImports(c, r, f) - } - return nil -} - -// Embeds returns a list of labels of rules that the given rule embeds. If a -// rule is embedded by another importable rule of the same language, only the -// embedding rule will be indexed. The embedding rule will inherit the imports -// of the embedded rule. Since SkyLark doesn't support embedding this should -// always return nil. -func (*starlarkBundleLang) Embeds(r *rule.Rule, from label.Label) []label.Label { - return nil -} - -// Resolve translates imported libraries for a given rule into Bazel -// dependencies. Information about imported libraries is returned for each rule -// generated by language.GenerateRules in language.GenerateResult.Imports. -// Resolve generates a "deps" attribute (or the appropriate language-specific -// equivalent) for each import according to language-specific rules and -// heuristics. -func (ext *starlarkBundleLang) Resolve(c *config.Config, ix *resolve.RuleIndex, rc *repo.RemoteCache, r *rule.Rule, importsRaw interface{}, from label.Label) { - switch r.Kind() { - case starlarkLibraryKind: - starlarkLibraryResolve(c, ix, r, importsRaw, from, ext) - case starlarkBundleKind: - if coarseDependencies { - starlarkBundleResolveCoarse(r, ext.starlarkLibraries, ext.moduleDeps) - } else { - starlarkBundleResolve(r, ext.starlarkLibraries) - - } - } -} - -// GenerateRules extracts build metadata from source files in a directory. -// GenerateRules is called in each directory where an update is requested in -// depth-first post-order. -// -// args contains the arguments for GenerateRules. This is passed as a struct to -// avoid breaking implementations in the future when new fields are added. -// -// A GenerateResult struct is returned. Optional fields may be added to this -// type in the future. -// -// Any non-fatal errors this function encounters should be logged using -// log.Print. -func (ext *starlarkBundleLang) GenerateRules(args language.GenerateArgs) (result language.GenerateResult) { - if ext.starlarkBundleRoot == nil { - return - } - for _, root := range ext.starlarkBundleExcludeDirs { - if strings.HasPrefix(args.Rel, root) { - return - } - } - - ext.logf("GenerateRules: visiting %s", args.Rel) - - if args.Rel == "" { - // REPO.bazel causes visibility issues if the root BUILD file has been - // deleted but still referencing non-existent rules. This should be - // moved to the part where gazelle cleans up build files. - repoFile := filepath.Join(args.Config.RepoRoot, "REPO.bazel") - if _, err := os.Stat(repoFile); err == nil { - ext.logf("Removing REPO.bazel from root package") - if err := os.Remove(repoFile); err != nil { - ext.logf("ERROR: failed to remove REPO.bazel: %v", err) - } - } - - bazelIgnore := filepath.Join(args.Config.RepoRoot, "REPO.bazel") - if _, err := os.Stat(bazelIgnore); err == nil { - if err := ext.removeAbsolutePathsFromBazelignore(bazelIgnore); err != nil { - ext.logf("ERROR: failed to cleanup .bazelignore file: %v", err) - } - } - } - - for _, r := range []language.GenerateResult{ - starlarkLibraryGenerate(args, ext.starlarkLibraries, ext), - starlarkBundleGenerate(args, ext.starlarkBundleRoot), - } { - result.Gen = append(result.Gen, r.Gen...) - result.Imports = append(result.Imports, r.Imports...) - result.Empty = append(result.Empty, r.Empty...) - } - - return -} - -func (ext *starlarkBundleLang) getModuleDependencyRepoName(repo string) (string, bool) { - ext.logf("getModuleDependencyRepoName called with repo=%q", repo) - if repo == "" { - ext.logf(" returning early for empty repo: @") - return "@", true - } - if mappedRepo, ok := ext.moduleDeps[repo]; ok { - ext.logf(" found mapping: %q -> %q", repo, mappedRepo) - return mappedRepo, true - } else { - ext.logf(" unknown module dependency: %q (known deps: %v)", repo, ext.moduleDeps) - // log.Fatalf("unknown module dependency: %q", repo) - return repo, false - } -} - -// logf logs a message using the extension's logger if configured -func (ext *starlarkBundleLang) logf(format string, args ...any) { - if ext.logger != nil { - ext.logger.Printf(format, args...) - } -} - -// removeAbsolutePathsFromBazelignore reads a .bazelignore file and removes -// any lines that are absolute paths (starting with /). -func (ext *starlarkBundleLang) removeAbsolutePathsFromBazelignore(bazelignorePath string) error { - ext.logf("Checking .bazelignore file: %s", bazelignorePath) - - // Read the file - content, err := os.ReadFile(bazelignorePath) - if err != nil { - if os.IsNotExist(err) { - return nil // File doesn't exist, nothing to do - } - return fmt.Errorf("failed to read .bazelignore: %w", err) - } - - // Split into lines - lines := strings.Split(string(content), "\n") - var filteredLines []string - removedCount := 0 - - for _, line := range lines { - trimmed := strings.TrimSpace(line) - // Keep the line if it's not an absolute path - if trimmed == "" || !strings.HasPrefix(trimmed, "/") { - filteredLines = append(filteredLines, line) - } else { - ext.logf(" Removing absolute path from .bazelignore: %q", trimmed) - removedCount++ - } - } - - // If nothing changed, don't rewrite the file - if removedCount == 0 { - ext.logf(" No absolute paths found in .bazelignore") - return nil - } - - // Write back the filtered content - newContent := strings.Join(filteredLines, "\n") - if err := os.WriteFile(bazelignorePath, []byte(newContent), 0644); err != nil { - return fmt.Errorf("failed to write .bazelignore: %w", err) - } - - ext.logf(" Removed %d absolute path(s) from .bazelignore", removedCount) - return nil -} diff --git a/language/starlarkbundle/lifecycle.go b/language/starlarkbundle/lifecycle.go deleted file mode 100644 index fce02b638..000000000 --- a/language/starlarkbundle/lifecycle.go +++ /dev/null @@ -1,43 +0,0 @@ -/* Copyright 2020 The Bazel Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package starlarkbundle - -import ( - "context" -) - -// Before implements part of the language.LifecycleManager interface. -func (ext *starlarkBundleLang) Before(context.Context) { - ext.logf("Lifecycle: Before() called") -} - -// DoneGeneratingRules implements part of the language.FinishableLanguage interface. -func (ext *starlarkBundleLang) DoneGeneratingRules() { - ext.logf("Lifecycle: DoneGeneratingRules() called") -} - -// AfterResolvingDeps implements part of the language.LifecycleManager interface. -// This is where we flush and close the log file. -func (ext *starlarkBundleLang) AfterResolvingDeps(context.Context) { - ext.logf("Lifecycle: AfterResolvingDeps() called - flushing and closing log file") - if ext.logWriter != nil { - // Sync flushes any buffered data to disk - _ = ext.logWriter.Sync() - // Close the file - _ = ext.logWriter.Close() - ext.logWriter = nil - } -} diff --git a/language/starlarkbundle/lifecycle_test.go b/language/starlarkbundle/lifecycle_test.go deleted file mode 100644 index a810dcf2c..000000000 --- a/language/starlarkbundle/lifecycle_test.go +++ /dev/null @@ -1,114 +0,0 @@ -/* Copyright 2020 The Bazel Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package starlarkbundle - -import ( - "context" - "flag" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/bazelbuild/bazel-gazelle/config" -) - -func TestLoggerLifecycle(t *testing.T) { - // Create a temporary directory for the log file - tmpDir := t.TempDir() - logFile := filepath.Join(tmpDir, "test.log") - - // Create extension and register flags - ext := NewLanguage().(*starlarkBundleLang) - fs := flag.NewFlagSet("test", flag.ContinueOnError) - c := &config.Config{} - ext.RegisterFlags(fs, "update", c) - - // Set the log file flag - if err := fs.Set("starlark_bundle_log", logFile); err != nil { - t.Fatalf("failed to set flag: %v", err) - } - - // Initialize logger via CheckFlags - if err := ext.CheckFlags(fs, c); err != nil { - t.Fatalf("CheckFlags failed: %v", err) - } - - // Verify logger is initialized - if ext.logger == nil { - t.Fatal("logger should be initialized") - } - if ext.logWriter == nil { - t.Fatal("logWriter should be initialized") - } - - // Write some log messages - ext.logf("test message 1") - ext.logf("test message 2: %s", "with formatting") - - // Call lifecycle methods - ctx := context.Background() - ext.Before(ctx) - ext.DoneGeneratingRules() - ext.AfterResolvingDeps(ctx) - - // Verify log writer is closed - if ext.logWriter != nil { - t.Error("logWriter should be nil after AfterResolvingDeps") - } - - // Read the log file and verify content - content, err := os.ReadFile(logFile) - if err != nil { - t.Fatalf("failed to read log file: %v", err) - } - - logContent := string(content) - if !strings.Contains(logContent, "test message 1") { - t.Error("log file should contain 'test message 1'") - } - if !strings.Contains(logContent, "test message 2: with formatting") { - t.Error("log file should contain 'test message 2: with formatting'") - } -} - -func TestLoggerWithoutFile(t *testing.T) { - // Create extension without log file - ext := NewLanguage().(*starlarkBundleLang) - fs := flag.NewFlagSet("test", flag.ContinueOnError) - c := &config.Config{} - ext.RegisterFlags(fs, "update", c) - - // Initialize logger via CheckFlags (no log file set) - if err := ext.CheckFlags(fs, c); err != nil { - t.Fatalf("CheckFlags failed: %v", err) - } - - // Verify logger is initialized but logWriter is nil - if ext.logger == nil { - t.Fatal("logger should be initialized") - } - if ext.logWriter != nil { - t.Error("logWriter should be nil when no log file is specified") - } - - // Writing logs should not panic - ext.logf("this should not panic") - - // Lifecycle methods should not panic - ctx := context.Background() - ext.AfterResolvingDeps(ctx) -} diff --git a/language/starlarkbundle/starlark_bundle.go b/language/starlarkbundle/starlark_bundle.go deleted file mode 100644 index afcf8ca17..000000000 --- a/language/starlarkbundle/starlark_bundle.go +++ /dev/null @@ -1,137 +0,0 @@ -/* Copyright 2020 The Bazel Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package symbol generates a `starlark_bundle` target for every `.bzl` file in -// each package. At the root of the module, a single starlark_bundle is -// populated with deps that include all other symbol_libraries. -// -// The original code for this gazelle extension started from -// https://github.com/bazelbuild/bazel-skylib/blob/main/gazelle/bzl/gazelle.go. -package starlarkbundle - -import ( - "log" - "sort" - - "github.com/bazelbuild/bazel-gazelle/config" - "github.com/bazelbuild/bazel-gazelle/label" - "github.com/bazelbuild/bazel-gazelle/language" - "github.com/bazelbuild/bazel-gazelle/resolve" - "github.com/bazelbuild/bazel-gazelle/rule" -) - -const ( - starlarkBundleKind = "starlark_bundle" - starlarkBundleName = "bundle" -) - -var starlarkBundleKindInfo = map[string]rule.KindInfo{ - starlarkBundleKind: { - NonEmptyAttrs: map[string]bool{"deps": true}, - ResolveAttrs: map[string]bool{"deps": true}, - }, -} - -var starlarkBundleLoadInfo = rule.LoadInfo{ - Name: "@build_stack_rules_proto//rules:starlark_bundle.bzl", - Symbols: []string{starlarkBundleKind}, -} - -func starlarkBundleRule() (*rule.Rule, []string) { - r := rule.NewRule(starlarkBundleKind, starlarkBundleName) - - r.SetAttr("visibility", []string{"//visibility:public"}) - - return r, []string{} -} - -func starlarkBundleImports(_ *config.Config, _ *rule.Rule, _ *rule.File) []resolve.ImportSpec { - return nil -} - -// starlarkBundleResolveResolve iterates the set of stalark_library rules and -// adds them to it's deps list. -func starlarkBundleResolve(r *rule.Rule, starlarkLibraries map[label.Label]*rule.Rule) { - deps := make([]string, 0, len(starlarkLibraries)) - for dep := range starlarkLibraries { - deps = append(deps, dep.String()) - } - - if len(deps) > 0 { - sort.Strings(deps) - r.SetAttr("deps", deps) - } -} - -// starlarkBundleResolveCoarse iterates the set of stalark_library rules and -// adds them to it's deps list. It also takes out all external dependencies -// from the library rules and collects them into itself, but only as the :bundle -// target. This is because we cannot ensure that the fine-grained dependency -// structure is correct. However, we can (hopefully) ensure that the -// coarse-grained dependency graph of bundles is correct. -func starlarkBundleResolveCoarse(r *rule.Rule, starlarkLibraries map[label.Label]*rule.Rule, moduleDeps map[string]string) { - deps := make([]string, 0, len(starlarkLibraries)) - for dep, r := range starlarkLibraries { - deps = append(deps, dep.String()) - - ruleDeps := r.AttrStrings("deps") - p1, _ := partitionExternalDeps(ruleDeps) - r.SetAttr("deps", p1) - } - - if false { - for _, module := range moduleDeps { - dep := label.New(module, "", "bundle") - deps = append(deps, dep.String()) - } - } - - if len(deps) > 0 { - sort.Strings(deps) - r.SetAttr("deps", deps) - } -} - -func partitionExternalDeps(deps []string) ([]string, []label.Label) { - var p1 []string // first and third-party deps - var p3 []label.Label // first and third-party deps - - for _, dep := range deps { - from, err := label.Parse(dep) - if err != nil { - log.Panicf("invalid dep label: %v", dep) - } - if from.Repo != "" { - p3 = append(p3, from) - } else { - p1 = append(p1, dep) - } - } - - return p1, p3 -} - -func starlarkBundleGenerate(args language.GenerateArgs, root *string) (result language.GenerateResult) { - if root == nil || *root != args.Rel { - return result - } - - r, loads := starlarkBundleRule() - - return language.GenerateResult{ - Gen: []*rule.Rule{r}, - Imports: []any{loads}, - } -} diff --git a/language/starlarkbundle/starlark_library.go b/language/starlarkbundle/starlark_library.go deleted file mode 100644 index 9c94070ce..000000000 --- a/language/starlarkbundle/starlark_library.go +++ /dev/null @@ -1,411 +0,0 @@ -/* Copyright 2020 The Bazel Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package symbol generates a `starlark_library` target for every `.bzl` file in -// each package. At the root of the module, a single starlark_bundle is -// populated with deps that include all other symbol_libraries. -// -// The original code for this gazelle extension started from -// https://github.com/bazelbuild/bazel-skylib/blob/main/gazelle/bzl/gazelle.go. -package starlarkbundle - -import ( - "fmt" - "os" - "path/filepath" - "sort" - "strings" - - "github.com/bazelbuild/bazel-gazelle/config" - "github.com/bazelbuild/bazel-gazelle/label" - "github.com/bazelbuild/bazel-gazelle/language" - "github.com/bazelbuild/bazel-gazelle/pathtools" - "github.com/bazelbuild/bazel-gazelle/resolve" - "github.com/bazelbuild/bazel-gazelle/rule" - "github.com/bazelbuild/buildtools/build" -) - -const ( - starlarkLibraryKind = "starlark_library" - starlarkLibraryNamePrefix = "lib" - fileType = ".bzl" - visibilityPublic = "//visibility:public" -) - -var ignoreSuffix = suffixes{ - // "_tests.bzl", - // "_test.bzl", -} - -var starlarkLibraryKindInfo = map[string]rule.KindInfo{ - starlarkLibraryKind: { - NonEmptyAttrs: map[string]bool{"src": true}, - ResolveAttrs: map[string]bool{"deps": true}, - }, -} - -var starlarkLibraryLoadInfo = rule.LoadInfo{ - Name: "@build_stack_rules_proto//rules:starlark_library.bzl", - Symbols: []string{starlarkLibraryKind}, -} - -type suffixes []string - -func (s suffixes) Matches(test string) bool { - for _, v := range s { - if strings.HasSuffix(test, v) { - return true - } - } - return false -} - -func makeStarlarkLibraryRuleName(f string) string { - name := starlarkLibraryNamePrefix + strings.TrimSuffix(f, fileType) - if strings.HasSuffix(name, "_test") { - name += "rule" - } - return name -} - -func starlarkLibraryRule(args language.GenerateArgs, f string, ext *starlarkBundleLang) (*rule.Rule, []*build.LoadStmt) { - fullPath := filepath.Join(args.Dir, f) - name := makeStarlarkLibraryRuleName(f) - - ast, loads, err := getBzlFileLoadsStmts(fullPath) - if err != nil { - ext.logf("%s: contains syntax errors: %v", fullPath, err) - // don't return early since it is reasonable to create a target even - // without deps. - } - - // // always castrate fail() exprs - // if renameFailToPrint(ast) { - // ext.emitBzlFile(fullPath, ast) - // } - - r := rule.NewRule(starlarkLibraryKind, name) - - r.SetAttr("src", f) - if ext.starlarkBundleRepoName != "" { - r.SetAttr("repo_name", ext.starlarkBundleRepoName) - } - - shouldSetVisibility := args.File == nil || !args.File.HasDefaultVisibility() - if shouldSetVisibility { - vis := checkInternalVisibility(args.Rel, visibilityPublic) - r.SetAttr("visibility", []string{vis}) - } - - r.SetPrivateAttr("full_path", fullPath) - r.SetPrivateAttr("ast", ast) - - return r, loads -} - -func starlarkLibraryImports(_ *config.Config, r *rule.Rule, f *rule.File) []resolve.ImportSpec { - src := r.AttrString("src") - return []resolve.ImportSpec{{ - // Lang is the language in which the import string appears (this should - // match Resolver.Name). - Lang: languageName, - // Imp is an import string for the library. - Imp: fmt.Sprintf("//%s:%s", f.Pkg, src), - }} -} - -func starlarkLibraryResolve(c *config.Config, ix *resolve.RuleIndex, r *rule.Rule, importsRaw interface{}, from label.Label, ext *starlarkBundleLang) { - loads := importsRaw.([]*build.LoadStmt) - fullPath := r.PrivateAttr("full_path").(string) - ast := r.PrivateAttr("ast").(*build.File) - - ext.logf("%s: starlarkLibraryResolve: %s with %d loads", fullPath, from.String(), len(loads)) - - r.DelAttr("deps") - - if len(loads) == 0 { - ext.logf(" no loads to process") - return - } - - rewriteBzlSourceFile := false - var unknownDeps []string - var bazelToolsDeps []string - - deps := make([]string, 0, len(loads)) - for i, load := range loads { - imp := load.Module.Value - ext.logf(" processing load %d/%d: %q", i+1, len(loads), imp) - - impLabel, err := label.Parse(imp) - if err != nil { - ext.logf(" ERROR: import of %q is invalid: %v", imp, err) - continue - } - - // the index only contains absolute labels, not relative - impLabel = impLabel.Abs(from.Repo, from.Pkg) - ext.logf(" absolute label: %s (repo=%q, pkg=%q, name=%q)", impLabel.String(), impLabel.Repo, impLabel.Pkg, impLabel.Name) - - if coarseDependencies && impLabel.Repo != "" { - continue - } - - if impLabel.Repo == "bazel_tools" { - // The @bazel_tools repo is tricky because it is a part of the - // "shipped with bazel" core library for interacting with the - // outside world. This means that it can not depend on skylib. - // Fortunately there is a fairly simple workaround for this, which - // is that you can add those bzl files as `deps` entries. - // - // the bazel source code gathers them up in filegroups but not - // exposed publically. For this to work, caller must be able to use - // a modified version of bazel that adds public visibility to the targets (see `tools/build_defs/repo/BUILD.repo`) - bazelToolsLabel := label.New(impLabel.Repo, impLabel.Pkg, "bzl_srcs") - ext.logf(" bazel_tools dependency: adding to bazel_tools_deps") - // deps = append(deps, imp) - bazelToolsDeps = append(bazelToolsDeps, bazelToolsLabel.String()) - // unknownDeps = append(deps, imp) - continue - } - - if impLabel.Repo != "" || !c.IndexLibraries { - // This is a dependency that is external to the current repo. - // Rewrite the repo label to one suffixed by "_docs". We expect to - // find the starlark_library dependency that provides the file in - // that repo. Rewrite the load label because starlark_doc_extract - // will also expect to load the symbol from that location. - ext.logf(" external repo dependency: repo=%q, IndexLibraries=%v", impLabel.Repo, c.IndexLibraries) - extRepo, known := ext.getModuleDependencyRepoName(impLabel.Repo) - extLabel := label.New( - extRepo, - impLabel.Pkg, - starlarkLibraryNamePrefix+strings.TrimSuffix(impLabel.Name, fileType), - ) - - loadLabel := label.New(extLabel.Repo, extLabel.Pkg, impLabel.Name) - ext.logf(" rewriting load: %q -> %q", load.Module.Value, loadLabel.String()) - load.Module.Value = loadLabel.String() - rewriteBzlSourceFile = true - - if known { - ext.logf(" adding to deps: %s", extLabel.String()) - deps = append(deps, extLabel.String()) - } else { - ext.logf(" adding to unknownDeps: %s", extLabel.String()) - unknownDeps = append(unknownDeps, extLabel.String()) - } - - continue - } - - ext.logf(" internal dependency: looking up in index") - res := resolve.ImportSpec{ - Lang: languageName, - Imp: impLabel.String(), - } - matches := ix.FindRulesByImportWithConfig(c, res, languageName) - ext.logf(" found %d matches in index", len(matches)) - if len(matches) == 0 { - ext.logf(" WARNING: %q (%s) was not found in dependency index", imp, impLabel.String()) - // unknownDeps = append(unknownDeps, impLabel.String()) - } - - for _, m := range matches { - depLabel := m.Label - ext.logf(" adding match to deps: %s", depLabel.String()) - // depLabel = depLabel.Rel(from.Repo, from.Pkg) - deps = append(deps, depLabel.String()) - } - - // in the source file, never use unqualified - if ext.starlarkBundleRepoName != "" { - // absLabel := impLabel.Abs(ext.starlarkBundleRepoName, from.Pkg) - absLabel := label.New(ext.starlarkBundleRepoName, impLabel.Pkg, impLabel.Name) - load.Module.Value = absLabel.String() - rewriteBzlSourceFile = true - ext.logf(" rewrote %v => %v", impLabel, absLabel) - } else { - ext.logf(" not abs'ing label (nostarlarkBundleRepoName)") - } - } - - ext.logf(" resolution complete: %d deps, %d unknown, %d bazel_tools", len(deps), len(unknownDeps), len(bazelToolsDeps)) - - if len(deps) > 0 { - deps = deduplicateAndSort(deps) - r.SetAttr("deps", deps) - ext.logf(" set deps attribute: %v", deps) - } - if len(unknownDeps) > 0 { - unknownDeps = deduplicateAndSort(unknownDeps) - r.SetAttr("unknown_deps", unknownDeps) - ext.logf(" set unknown_deps attribute: %v", unknownDeps) - } - if len(bazelToolsDeps) > 0 { - bazelToolsDeps = deduplicateAndSort(bazelToolsDeps) - r.SetAttr("bazel_tools_deps", bazelToolsDeps) - ext.logf(" set bazel_tools_deps attribute: %v", bazelToolsDeps) - } - - if rewriteBzlSourceFile { - ext.emitBzlFile(fullPath, ast) - } -} - -func (ext *starlarkBundleLang) emitBzlFile(fullPath string, ast *build.File) { - ext.logf("emitting source file: %s", fullPath) - data := build.Format(ast) - if err := os.WriteFile(fullPath, data, os.ModePerm); err != nil { - ext.logf(" ERROR: failed to emit rewritten bzl file: %v", err) - } -} - -func starlarkLibraryGenerate(args language.GenerateArgs, starlarkLibraries map[label.Label]*rule.Rule, ext *starlarkBundleLang) language.GenerateResult { - var rules []*rule.Rule - var imports []any - - for _, f := range append(args.RegularFiles, args.GenFiles...) { - if !isBzlSourceFile(f) { - continue - } - r, loads := starlarkLibraryRule(args, f, ext) - if r == nil { - continue - } - - rules = append(rules, r) - imports = append(imports, loads) - - // populate the map so the bundle rule can use them later. - if isVisibilityPublic(r.AttrStrings("visibility")) { - from := label.New(args.Config.RepoName, args.Rel, r.Name()) - starlarkLibraries[from] = r - } - } - - return language.GenerateResult{ - Gen: rules, - Imports: imports, - Empty: starlarkLibraryEmptyRules(args), - } -} - -func getBzlFileLoadsStmts(path string) (*build.File, []*build.LoadStmt, error) { - f, err := os.ReadFile(path) - if err != nil { - return nil, nil, fmt.Errorf("os.ReadFile(%q) error: %v", path, err) - } - ast, err := build.ParseBzl(path, f) - if err != nil { - return nil, nil, fmt.Errorf("build.Parse(%q) error: %v", f, err) - } - - var loads []*build.LoadStmt - build.WalkOnce(ast, func(expr *build.Expr) { - n := *expr - if l, ok := n.(*build.LoadStmt); ok { - loads = append(loads, l) - } - }) - - return ast, loads, nil -} - -func renameFailToPrint(ast *build.File) bool { - modified := false - build.Walk(ast, func(expr build.Expr, stack []build.Expr) { - if call, ok := expr.(*build.CallExpr); ok { - if callName, ok := call.X.(*build.Ident); ok { - if callName.Name == "fail" { - callName.Name = "print" - modified = true - } - } - } - }) - return modified -} - -func isBzlSourceFile(f string) bool { - return strings.HasSuffix(f, fileType) && !ignoreSuffix.Matches(f) -} - -func isVisibilityPublic(vis []string) bool { - return len(vis) == 1 && vis[0] == visibilityPublic -} - -// starlarkLibraryEmptyRules generates the list of rules that don't need to -// exist in the BUILD file any more. For each symbol_library rule in args.File -// that only has srcs that aren't in args.RegularFiles or args.GenFiles, add a -// symbol_library with no srcs or deps. That will let Gazelle delete -// symbol_library rules after the corresponding .bzl files are deleted. -func starlarkLibraryEmptyRules(args language.GenerateArgs) []*rule.Rule { - var ret []*rule.Rule - if args.File == nil { - return ret - } - for _, r := range args.File.Rules { - if r.Kind() != starlarkLibraryKind { - continue - } - name := r.AttrString("name") - - exists := make(map[string]bool) - for _, f := range args.RegularFiles { - exists[f] = true - } - for _, f := range args.GenFiles { - exists[f] = true - } - for _, r := range args.File.Rules { - srcExist := exists[r.AttrString("src")] - if !srcExist { - ret = append(ret, rule.NewRule(starlarkLibraryKind, name)) - } - } - } - return ret -} - -// checkInternalVisibility overrides the given visibility if the package is -// internal. -func checkInternalVisibility(rel, visibility string) string { - if i := pathtools.Index(rel, "internal"); i > 0 { - visibility = fmt.Sprintf("//%s:__subpackages__", rel[:i-1]) - } else if i := pathtools.Index(rel, "private"); i > 0 { - visibility = fmt.Sprintf("//%s:__subpackages__", rel[:i-1]) - } else if pathtools.HasPrefix(rel, "internal") || pathtools.HasPrefix(rel, "private") { - visibility = "//:__subpackages__" - } - return visibility -} - -// deduplicateAndSort removes duplicate entries and sorts the list -func deduplicateAndSort(in []string) (out []string) { - if len(in) == 0 { - return in - } - seen := make(map[string]bool) - for _, v := range in { - if seen[v] { - continue - } - seen[v] = true - out = append(out, v) - } - sort.Strings(out) - return -} diff --git a/language/starlarkbundle/starlark_library_test.go b/language/starlarkbundle/starlark_library_test.go deleted file mode 100644 index e0d9223ab..000000000 --- a/language/starlarkbundle/starlark_library_test.go +++ /dev/null @@ -1,119 +0,0 @@ -/* Copyright 2020 The Bazel Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package starlarkbundle - -import ( - "strings" - "testing" - - "github.com/bazelbuild/buildtools/build" -) - -func TestRenameFailToPrint(t *testing.T) { - tests := []struct { - name string - input string - want string - modified bool - }{ - { - name: "single fail call", - input: `fail("error message") -`, - want: `print("error message") -`, - modified: true, - }, - { - name: "multiple fail calls", - input: `if condition: - fail("error 1") -else: - fail("error 2") -`, - want: `if condition: - print("error 1") -else: - print("error 2") -`, - modified: true, - }, - { - name: "fail with multiple arguments", - input: `fail("error:", variable, "more text") -`, - want: `print("error:", variable, "more text") -`, - modified: true, - }, - { - name: "no fail calls", - input: `print("hello") -x = 1 -`, - want: `print("hello") -x = 1 -`, - modified: false, - }, - { - name: "mixed fail and print calls", - input: `print("info") -fail("error") -print("more info") -`, - want: `print("info") -print("error") -print("more info") -`, - modified: true, - }, - { - name: "fail in function definition", - input: `def my_function(param): - if not param: - fail("param is required") - return param -`, - want: `def my_function(param): - if not param: - print("param is required") - return param -`, - modified: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ast, err := build.ParseBzl("test.bzl", []byte(tt.input)) - if err != nil { - t.Fatalf("failed to parse input: %v", err) - } - - modified := renameFailToPrint(ast) - - if modified != tt.modified { - t.Errorf("renameFailToPrint() modified = %v, want %v", modified, tt.modified) - } - - got := string(build.Format(ast)) - if strings.TrimSpace(got) != strings.TrimSpace(tt.want) { - t.Errorf("renameFailToPrint() output mismatch\ngot:\n%s\n\nwant:\n%s", got, tt.want) - } - }) - } -} diff --git a/language/starlarkbundle/BUILD.bazel b/language/starlarklibrary/BUILD.bazel similarity index 79% rename from language/starlarkbundle/BUILD.bazel rename to language/starlarklibrary/BUILD.bazel index d1b3142c7..21a8e92fb 100644 --- a/language/starlarkbundle/BUILD.bazel +++ b/language/starlarklibrary/BUILD.bazel @@ -1,14 +1,9 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( - name = "starlarkbundle", - srcs = [ - "language.go", - "lifecycle.go", - "starlark_bundle.go", - "starlark_library.go", - ], - importpath = "github.com/stackb/rules_proto/language/starlarkbundle", + name = "starlarklibrary", + srcs = ["language.go"], + importpath = "github.com/stackb/rules_proto/language/starlarklibrary", visibility = ["//visibility:public"], deps = [ "@bazel_gazelle//config:go_default_library", @@ -23,12 +18,14 @@ go_library( ) go_test( - name = "starlarkbundle_test", + name = "starlarklibrary_test", srcs = [ + "bazelignore_test.go", + "dict_test.go", "lifecycle_test.go", "starlark_library_test.go", ], - embed = [":starlarkbundle"], + embed = [":starlarklibrary"], deps = [ "@bazel_gazelle//config:go_default_library", "@com_github_bazelbuild_buildtools//build:go_default_library", diff --git a/language/starlarklibrary/language.go b/language/starlarklibrary/language.go new file mode 100644 index 000000000..0fa4c0e7a --- /dev/null +++ b/language/starlarklibrary/language.go @@ -0,0 +1,470 @@ +/* Copyright 2020 The Bazel Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package symbol generates a `starlark_library` target for every `.bzl` file in +// each package. At the root of the module, a single starlark_library is +// populated with deps that include all other starlark_libraries. +// +// The original code for this gazelle extension started from +// https://github.com/bazelbuild/bazel-skylib/blob/main/gazelle/bzl/gazelle.go. +package starlarklibrary + +import ( + "bufio" + "context" + "flag" + "fmt" + "io" + "log" + "maps" + "os" + "path" + "path/filepath" + "slices" + "strings" + + "github.com/bazelbuild/bazel-gazelle/config" + "github.com/bazelbuild/bazel-gazelle/label" + "github.com/bazelbuild/bazel-gazelle/language" + "github.com/bazelbuild/bazel-gazelle/repo" + "github.com/bazelbuild/bazel-gazelle/resolve" + "github.com/bazelbuild/bazel-gazelle/rule" + "github.com/bazelbuild/buildtools/build" +) + +const ( + languageName = "starlarklibrary" + repoNameDirectiveName = languageName + "_repo_name" + rootDirectiveName = languageName + "_root" + excludeDirectiveName = languageName + "_exclude" + logFileDirectiveName = languageName + "_log_file" + starlarkLibraryKind = "starlark_library" + starlarkLibraryName = "bzl_srcs" + fileType = ".bzl" + visibilityPublic = "//visibility:public" +) + +var ( + ignoreSuffix = suffixes{ + "_tests.bzl", + "_test.bzl", + } + starlarkLibraryKindInfo = map[string]rule.KindInfo{ + starlarkLibraryKind: { + NonEmptyAttrs: map[string]bool{"src": true}, + }, + } + starlarkLibraryLoadInfo = rule.LoadInfo{ + Name: "@build_stack_rules_proto//rules:starlark_library.bzl", + Symbols: []string{starlarkLibraryKind}, + } +) + +type starlarkLibraryLang struct { + roots []string + repoName string + excludeDirs []string + bzlFiles map[string]bool + bazelVersion string + bazelIgnore []string + logFile string + logWriter *os.File + logger *log.Logger +} + +// NewLanguage is called by Gazelle to install this language extension in a +// binary. +func NewLanguage() language.Language { + return &starlarkLibraryLang{ + bzlFiles: map[string]bool{}, + } +} + +// Name returns the name of the language. This should be a prefix of the kinds +// of rules generated by the language, e.g., "go" for the Go extension since it +// generates "go_library" rules. +func (*starlarkLibraryLang) Name() string { + return languageName +} + +// The following methods are implemented to satisfy the +// https://pkg.go.dev/github.com/bazelbuild/bazel-gazelle/resolve?tab=doc#Resolver +// interface, but are otherwise unused. +func (ext *starlarkLibraryLang) RegisterFlags(fs *flag.FlagSet, cmd string, c *config.Config) { + fs.StringVar(&ext.logFile, logFileDirectiveName, "", "path to log file for starlark_library extension") +} + +func (ext *starlarkLibraryLang) CheckFlags(fs *flag.FlagSet, c *config.Config) error { + if ext.logFile != "" { + ext.createLogger(ext.logFile) + ext.logf("CheckFlags: log file initialized from flag: %s", ext.logFile) + } + if ext.logger == nil { + ext.logger = log.New(io.Discard, "", 0) + } + + // REPO.bazel causes visibility issues if the root BUILD file has been + // deleted but still referencing non-existent rules. This should be + // moved to the part where gazelle cleans up build files. + repoBazelFilename := filepath.Join(c.RepoRoot, "REPO.bazel") + deleteFile(repoBazelFilename, ext.logf) + + bazelVersionFilename := filepath.Join(c.RepoRoot, ".bazelversion") + if lines, err := readFileLines(bazelVersionFilename, ext.logf); err == nil { + ext.bazelVersion = strings.Join(lines, ":") + deleteFile(bazelVersionFilename, ext.logf) + } + + bazelIgnoreFilename := filepath.Join(c.RepoRoot, ".bazelignore") + if lines, err := readFileLines(bazelIgnoreFilename, ext.logf); err == nil { + ext.bazelIgnore = lines + deleteFile(bazelIgnoreFilename, ext.logf) + } + + return nil +} + +func (*starlarkLibraryLang) KnownDirectives() []string { + return []string{ + rootDirectiveName, + excludeDirectiveName, + logFileDirectiveName, + } +} + +func (ext *starlarkLibraryLang) Configure(c *config.Config, rel string, f *rule.File) { + if f != nil { + ext.logf("Configure: processing %d directives in %s", len(f.Directives), rel) + for _, d := range f.Directives { + switch d.Key { + case rootDirectiveName: + ext.roots = append(ext.roots, d.Value) + case repoNameDirectiveName: + ext.repoName = d.Value + case excludeDirectiveName: + ext.excludeDirs = append(ext.excludeDirs, d.Value) + case logFileDirectiveName: + ext.createLogger(d.Value) + } + } + } +} + +func (ext *starlarkLibraryLang) createLogger(logFile string) { + if logFile != "" { + f, err := os.OpenFile(logFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666) + if err == nil { + ext.logWriter = f + ext.logger = log.New(f, "", log.LstdFlags) + } else { + log.Fatalf("attempting to open log file: %v", err) + } + } + if false && ext.logger == nil { + ext.logger = log.New(io.Discard, "", 0) + } + + ext.logf("Log initialized") +} + +// Kinds returns a map of maps rule names (kinds) and information on how to +// match and merge attributes that may be found in rules of those kinds. All +// kinds of rules generated for this language may be found here. +func (*starlarkLibraryLang) Kinds() map[string]rule.KindInfo { + kinds := map[string]rule.KindInfo{} + maps.Copy(kinds, starlarkLibraryKindInfo) + return kinds +} + +// Loads returns .bzl files and symbols they define. Every rule generated by +// GenerateRules, now or in the past, should be loadable from one of these +// files. +func (*starlarkLibraryLang) Loads() []rule.LoadInfo { + return []rule.LoadInfo{ + starlarkLibraryLoadInfo, + } +} + +// Fix repairs deprecated usage of language-specific rules in f. This is called +// before the file is indexed. Unless c.ShouldFix is true, fixes that delete or +// rename rules should not be performed. +func (*starlarkLibraryLang) Fix(c *config.Config, f *rule.File) { +} + +// Imports returns a list of ImportSpecs that can be used to import the rule r. +// This is used to populate RuleIndex. +// +// If nil is returned, the rule will not be indexed. If any non-nil slice is +// returned, including an empty slice, the rule will be indexed. +func (ext *starlarkLibraryLang) Imports(c *config.Config, r *rule.Rule, f *rule.File) []resolve.ImportSpec { + return nil +} + +// Embeds returns a list of labels of rules that the given rule embeds. If a +// rule is embedded by another importable rule of the same language, only the +// embedding rule will be indexed. The embedding rule will inherit the imports +// of the embedded rule. Since SkyLark doesn't support embedding this should +// always return nil. +func (*starlarkLibraryLang) Embeds(r *rule.Rule, from label.Label) []label.Label { + return nil +} + +// Resolve translates imported libraries for a given rule into Bazel +// dependencies. Information about imported libraries is returned for each rule +// generated by language.GenerateRules in language.GenerateResult.Imports. +// Resolve generates a "deps" attribute (or the appropriate language-specific +// equivalent) for each import according to language-specific rules and +// heuristics. +func (ext *starlarkLibraryLang) Resolve(c *config.Config, ix *resolve.RuleIndex, rc *repo.RemoteCache, r *rule.Rule, importsRaw interface{}, from label.Label) { +} + +// GenerateRules extracts build metadata from source files in a directory. +// GenerateRules is called in each directory where an update is requested in +// depth-first post-order. +// +// args contains the arguments for GenerateRules. This is passed as a struct to +// avoid breaking implementations in the future when new fields are added. +// +// A GenerateResult struct is returned. Optional fields may be added to this +// type in the future. +// +// Any non-fatal errors this function encounters should be logged using +// log.Print. +func (ext *starlarkLibraryLang) GenerateRules(args language.GenerateArgs) (result language.GenerateResult) { + // extension is effectively disabled without specifying at least one root + if len(ext.roots) == 0 { + return + } + // skip excluded dirs + for _, dir := range ext.excludeDirs { + if strings.HasPrefix(args.Rel, dir) { + return + } + } + + ext.logf("GenerateRules %v: visiting %s", ext.roots, args.Rel) + + // collect all .bzl files + for _, f := range append(args.RegularFiles, args.GenFiles...) { + if !isBzlSourceFile(f) { + continue + } + ext.bzlFiles[path.Join(args.Rel, f)] = true + } + + if slices.Contains(ext.roots, args.Rel) { + r, imports := ext.starlarkLibraryRule(args) + result.Gen = append(result.Gen, r) + result.Imports = append(result.Imports, imports) + } + + return +} + +func (ext *starlarkLibraryLang) starlarkLibraryRule(args language.GenerateArgs) (*rule.Rule, []any) { + + loadsByRelPath := make(map[string][]string) + for workspaceRelPath := range ext.bzlFiles { + if !strings.HasPrefix(workspaceRelPath, args.Rel) { + continue + } + fullPath := filepath.Join(args.Config.RepoRoot, workspaceRelPath) + relPath := strings.TrimPrefix(strings.TrimPrefix(workspaceRelPath, args.Rel), "/") + _, stmts, err := getBzlFileLoadsStmts(fullPath) + + if err != nil { + ext.logf("%s: contains syntax errors: %v", fullPath, err) + // don't return early since it is reasonable to create a target even + // without deps. + } + loads := make([]string, 0, len(stmts)) + for _, stmt := range stmts { + loads = append(loads, stmt.Module.Value) + } + loadsByRelPath[relPath] = loads + } + + r := rule.NewRule(starlarkLibraryKind, starlarkLibraryName) + r.SetAttr("srcs", slices.Sorted(maps.Keys(loadsByRelPath))) + r.SetAttr("loads", stringListDict(loadsByRelPath)) + if ext.repoName != "" { + r.SetAttr("repo_name", ext.repoName) + } + if ext.bazelVersion != "" { + r.SetAttr("bazelversion", ext.bazelVersion) + } + if len(ext.bazelIgnore) > 0 { + r.SetAttr("bazelignore", ext.bazelIgnore) + } + r.SetAttr("visibility", []string{visibilityPublic}) + + return r, []any{} +} + +// Before implements part of the language.LifecycleManager interface. +func (ext *starlarkLibraryLang) Before(context.Context) { + ext.logf("Lifecycle: Before() called") + ext.logf("roots: %v", ext.roots) + // if len(ext.roots) == 0 { + // log.Fatalf("at least one root must be specified") + // } +} + +// DoneGeneratingRules implements part of the language.FinishableLanguage interface. +func (ext *starlarkLibraryLang) DoneGeneratingRules() { + ext.logf("Lifecycle: DoneGeneratingRules() called") +} + +// AfterResolvingDeps implements part of the language.LifecycleManager interface. +// This is where we flush and close the log file. +func (ext *starlarkLibraryLang) AfterResolvingDeps(context.Context) { + ext.logf("Lifecycle: AfterResolvingDeps() called - flushing and closing log file") + if ext.logWriter != nil { + // Sync flushes any buffered data to disk + _ = ext.logWriter.Sync() + // Close the file + _ = ext.logWriter.Close() + ext.logWriter = nil + } +} + +// logf logs a message using the extension's logger if configured +func (ext *starlarkLibraryLang) logf(format string, args ...any) { + if ext.logger != nil { + ext.logger.Printf(format, args...) + } +} + +type suffixes []string + +func (s suffixes) Matches(test string) bool { + for _, v := range s { + if strings.HasSuffix(test, v) { + return true + } + } + return false +} + +// LogFunc is a function type for logging messages +type LogFunc func(format string, args ...any) + +// deleteFile removes a file and logs any errors +func deleteFile(filename string, logf LogFunc) { + if err := os.Remove(filename); err != nil { + logf("ERROR: failed to remove %s: %v", filename, err) + } +} + +// readFileLines reads a file and returns a slice of trimmed non-empty lines, +// excluding comments (lines starting with #). +func readFileLines(filePath string, logf LogFunc) ([]string, error) { + logf("Reading lines from file: %s", filePath) + + // Open the file + file, err := os.Open(filePath) + if err != nil { + if os.IsNotExist(err) { + logf(" File does not exist") + return nil, err + } + return nil, fmt.Errorf("failed to open file: %w", err) + } + defer file.Close() + + // Scan lines using bufio.Scanner + scanner := bufio.NewScanner(file) + var lines []string + + for scanner.Scan() { + line := scanner.Text() + trimmed := strings.TrimSpace(line) + + // Skip empty lines + if trimmed == "" { + continue + } + + // Skip comments + if strings.HasPrefix(trimmed, "#") { + continue + } + + lines = append(lines, trimmed) + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("failed to scan file: %w", err) + } + + logf(" Read %d non-empty lines", len(lines)) + return lines, nil +} + +func isBzlSourceFile(f string) bool { + return strings.HasSuffix(f, fileType) && !ignoreSuffix.Matches(f) +} + +func getBzlFileLoadsStmts(path string) (*build.File, []*build.LoadStmt, error) { + f, err := os.ReadFile(path) + if err != nil { + return nil, nil, fmt.Errorf("os.ReadFile(%q) error: %v", path, err) + } + ast, err := build.ParseBzl(path, f) + if err != nil { + return nil, nil, fmt.Errorf("build.Parse(%q) error: %v", f, err) + } + + var loads []*build.LoadStmt + build.WalkOnce(ast, func(expr *build.Expr) { + n := *expr + if l, ok := n.(*build.LoadStmt); ok { + loads = append(loads, l) + } + }) + + return ast, loads, nil +} + +// stringListDict creates a Dict expression with string keys and List values +// containing StringExpr elements. Used for representing load statements by file. +func stringListDict(data map[string][]string) *build.DictExpr { + if len(data) == 0 { + return &build.DictExpr{} + } + + // Sort keys for deterministic output + keys := slices.Sorted(maps.Keys(data)) + + var keyValues []*build.KeyValueExpr + for _, key := range keys { + values := data[key] + + // Create list of string expressions for the values + var listExprs []build.Expr + for _, value := range values { + listExprs = append(listExprs, &build.StringExpr{Value: value}) + } + + keyValues = append(keyValues, &build.KeyValueExpr{ + Key: &build.StringExpr{Value: key}, + Value: &build.ListExpr{List: listExprs}, + }) + } + + return &build.DictExpr{ + List: keyValues, + } +} diff --git a/language/starlarklibrary/language_test.go b/language/starlarklibrary/language_test.go new file mode 100644 index 000000000..fe48f650f --- /dev/null +++ b/language/starlarklibrary/language_test.go @@ -0,0 +1,252 @@ +/* Copyright 2020 The Bazel Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package starlarklibrary + +import ( + "os" + "path/filepath" + "testing" + + "github.com/bazelbuild/buildtools/build" +) + +// readFileLines tests + +func TestReadLines(t *testing.T) { + tests := []struct { + name string + input string + expected []string + }{ + { + name: "filters empty lines and comments", + input: `.build/ +# This is a comment +.swiftpm/ +.vscode/ + +`, + expected: []string{".build/", ".swiftpm/", ".vscode/"}, + }, + { + name: "handles only comments", + input: `# Comment 1 +# Comment 2 +# Comment 3 +`, + expected: []string{}, + }, + { + name: "handles mixed content", + input: ` .build/ +# Comment + +.swiftpm/ + # Another comment +.vscode/ +`, + expected: []string{".build/", ".swiftpm/", ".vscode/"}, + }, + { + name: "empty file", + input: "", + expected: []string{}, + }, + { + name: "only whitespace", + input: ` + + +`, + expected: []string{}, + }, + { + name: "trims whitespace", + input: ` .build/ + .swiftpm/ + .vscode/ +`, + expected: []string{".build/", ".swiftpm/", ".vscode/"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create temp directory + tmpDir := t.TempDir() + testFile := filepath.Join(tmpDir, "test.txt") + + // Write input content + if err := os.WriteFile(testFile, []byte(tt.input), 0644); err != nil { + t.Fatalf("failed to write test file: %v", err) + } + + // Create a no-op log function for tests + noop := func(format string, args ...any) {} + + // Run the function + lines, err := readFileLines(testFile, noop) + if err != nil { + t.Fatalf("readLines failed: %v", err) + } + + // Handle nil vs empty slice + if tt.expected == nil || len(tt.expected) == 0 { + if len(lines) != 0 { + t.Errorf("Expected empty result, got %d lines: %v", len(lines), lines) + } + return + } + + // Compare results + if len(lines) != len(tt.expected) { + t.Errorf("Length mismatch: got %d lines, want %d lines", len(lines), len(tt.expected)) + t.Errorf("got: %v", lines) + t.Errorf("want: %v", tt.expected) + return + } + + for i, line := range lines { + if line != tt.expected[i] { + t.Errorf("Line %d mismatch:\ngot: %q\nwant: %q", i, line, tt.expected[i]) + } + } + }) + } +} + +func TestReadLines_NonExistent(t *testing.T) { + tmpDir := t.TempDir() + testFile := filepath.Join(tmpDir, "nonexistent.txt") + + noop := func(format string, args ...any) {} + + // Should return error for non-existent file + lines, err := readFileLines(testFile, noop) + if err == nil { + t.Errorf("expected error for non-existent file, got nil") + } + if lines != nil { + t.Errorf("expected nil slice for non-existent file, got: %v", lines) + } +} + +// stringListDict tests + +func TestStringListDict(t *testing.T) { + tests := []struct { + name string + input map[string][]string + }{ + { + name: "empty map", + input: map[string][]string{}, + }, + { + name: "single key with single value", + input: map[string][]string{ + "foo.bzl": {"@repo//pkg:file.bzl"}, + }, + }, + { + name: "single key with multiple values", + input: map[string][]string{ + "foo.bzl": { + "@repo1//pkg:file1.bzl", + "@repo2//pkg:file2.bzl", + }, + }, + }, + { + name: "multiple keys sorted alphabetically", + input: map[string][]string{ + "zebra.bzl": {"@repo//z:z.bzl"}, + "alpha.bzl": {"@repo//a:a.bzl"}, + "beta.bzl": {"@repo//b:b.bzl"}, + }, + }, + { + name: "multiple keys with multiple values", + input: map[string][]string{ + "rules.bzl": { + "@rules_proto//proto:defs.bzl", + "@bazel_skylib//lib:paths.bzl", + }, + "providers.bzl": { + "@rules_proto//proto:providers.bzl", + }, + }, + }, + { + name: "empty values list", + input: map[string][]string{ + "empty.bzl": {}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := stringListDict(tt.input) + + // Verify the structure + if result == nil { + t.Fatal("result should not be nil") + } + + if len(tt.input) == 0 && len(result.List) != 0 { + t.Errorf("empty input should produce empty dict, got %d entries", len(result.List)) + } + + if len(tt.input) != 0 && len(result.List) != len(tt.input) { + t.Errorf("expected %d dict entries, got %d", len(tt.input), len(result.List)) + } + + // Verify each key-value pair + for _, kv := range result.List { + keyExpr, ok := kv.Key.(*build.StringExpr) + if !ok { + t.Errorf("key should be StringExpr, got %T", kv.Key) + continue + } + + listExpr, ok := kv.Value.(*build.ListExpr) + if !ok { + t.Errorf("value should be ListExpr, got %T", kv.Value) + continue + } + + // Verify all list elements are StringExpr + for i, elem := range listExpr.List { + if _, ok := elem.(*build.StringExpr); !ok { + t.Errorf("list element %d should be StringExpr, got %T", i, elem) + } + } + + // Verify the values match + expectedValues, exists := tt.input[keyExpr.Value] + if !exists { + t.Errorf("unexpected key %q in result", keyExpr.Value) + continue + } + + if len(listExpr.List) != len(expectedValues) { + t.Errorf("key %q: expected %d values, got %d", keyExpr.Value, len(expectedValues), len(listExpr.List)) + } + } + }) + } +} diff --git a/rules/private/proto_repository_tools_srcs.bzl b/rules/private/proto_repository_tools_srcs.bzl index 5baab94d6..b1398f296 100644 --- a/rules/private/proto_repository_tools_srcs.bzl +++ b/rules/private/proto_repository_tools_srcs.bzl @@ -43,11 +43,8 @@ PROTO_REPOSITORY_TOOLS_SRCS = [ "@build_stack_rules_proto//language/proto_go_modules:rule.go", "@build_stack_rules_proto//language/protobuf:BUILD.bazel", "@build_stack_rules_proto//language/protobuf:protobuf.go", - "@build_stack_rules_proto//language/starlarkbundle:BUILD.bazel", - "@build_stack_rules_proto//language/starlarkbundle:language.go", - "@build_stack_rules_proto//language/starlarkbundle:lifecycle.go", - "@build_stack_rules_proto//language/starlarkbundle:starlark_bundle.go", - "@build_stack_rules_proto//language/starlarkbundle:starlark_library.go", + "@build_stack_rules_proto//language/starlarklibrary:BUILD.bazel", + "@build_stack_rules_proto//language/starlarklibrary:language.go", "@build_stack_rules_proto//pkg:BUILD.bazel", "@build_stack_rules_proto//pkg/goldentest:BUILD.bazel", "@build_stack_rules_proto//pkg/goldentest:cases.go", diff --git a/rules/starlark_bundle.bzl b/rules/starlark_bundle.bzl deleted file mode 100644 index 11120576e..000000000 --- a/rules/starlark_bundle.bzl +++ /dev/null @@ -1,50 +0,0 @@ -"""starlark_bundle.bzl aggreagates the transitive sources of dependencies - -Rule is needed because bzl_library strictly requires srcs. -""" - -load("@bazel_skylib//:bzl_library.bzl", "StarlarkLibraryInfo") -load(":starlark_library.bzl", "StarlarkLibraryFileInfo") - -def _starlark_bundle_impl(ctx): - deps = [d[StarlarkLibraryFileInfo] for d in ctx.attr.deps] - transitive_srcs = depset(transitive = [d.transitive_srcs for d in deps]) - transitive_docs = depset(transitive = [d.transitive_docs for d in deps]) - - return [ - DefaultInfo( - files = transitive_srcs, - ), - StarlarkLibraryInfo( - srcs = [], - transitive_srcs = transitive_srcs, - ), - StarlarkLibraryFileInfo( - label = ctx.label, - src = None, - deps = depset(deps), - transitive_srcs = transitive_srcs, - transitive_docs = transitive_docs, - broken = False, - ), - ] - -_starlark_bundle = rule( - implementation = _starlark_bundle_impl, - attrs = { - "deps": attr.label_list( - doc = "list of starlark_library or bzl library rules", - providers = [StarlarkLibraryFileInfo], - ), - }, - provides = [DefaultInfo, StarlarkLibraryFileInfo, StarlarkLibraryInfo], -) - -def starlark_bundle(name, **kwargs): - deps = kwargs.pop("deps", []) - - _starlark_bundle( - name = name, - deps = deps, - **kwargs - ) diff --git a/rules/starlark_library.bzl b/rules/starlark_library.bzl index a6ee97a10..e7f8b4ed7 100644 --- a/rules/starlark_library.bzl +++ b/rules/starlark_library.bzl @@ -7,94 +7,67 @@ StarlarkLibraryFileInfo = provider( fields = { "label": "The label of the target rule", "repo_name": "The original (non-canonical) repo (workspace!) name", - "src": "The top-level src file", - "doc": "The starlark_extract_doc output file", + "srcs": "List of .bzl files", + "loads": "load statements per .bzl file", "deps": "DepSet[StarlarkLibraryFileInfo]: deps of this file", - # "transitive_deps": "List[DepSet[StarlarkLibraryFileInfo]]: transitive deps of this file", - "transitive_srcs": "Transitive closure of rules files required for " + - "interpretation of the src", - "transitive_docs": "Transitive closure of docs that have viable dependencies", - "broken": "If at last one of the transitive srcs has an unknown dependency.", + "transitive_deps": "DepSet[StarlarkLibraryFileInfo]: transitive deps of this file", + "bazelignore": "List[str] value of ctx.attr.bazelignore", + "bazelversion": "str: the value of ctx.attr.bazelversion", }, ) def _starlark_library_impl(ctx): - src = ctx.file.src - doc = ctx.file.doc - broken = len(ctx.attr.unknown_deps) > 0 - deps = [d[StarlarkLibraryFileInfo] for d in ctx.attr.deps] - transitive_deps = [d.deps for d in deps] + srcs = ctx.files.srcs + transitive_srcs = depset(srcs, order = "postorder", transitive = [d.transitive_srcs for d in deps]) - transitive_srcs = depset([src], order = "postorder", transitive = [d.transitive_srcs for d in deps]) - transitive_docs = depset([doc] if not broken else [], order = "postorder", transitive = [d.transitive_docs for d in deps]) + # loads = {} + # for k, v in ctx.attr.loads.items(): + # loads[Label(k)] = v return [ DefaultInfo( files = transitive_srcs, ), - OutputGroupInfo( - doc = [doc], - ), StarlarkLibraryInfo( - srcs = [src], + srcs = srcs, transitive_srcs = transitive_srcs, ), StarlarkLibraryFileInfo( label = ctx.label, repo_name = ctx.attr.repo_name, - src = src, - deps = depset(deps, transitive = transitive_deps), - transitive_srcs = transitive_srcs, - transitive_docs = transitive_docs, - broken = broken, + srcs = srcs, + loads = ctx.attr.loads, + bazelignore = ctx.attr.bazelignore, + bazelversion = ctx.attr.bazelversion, + transitive_deps = depset(deps, transitive = [d.transitive_deps for d in deps]), ), ] -_starlark_library = rule( +starlark_library = rule( implementation = _starlark_library_impl, attrs = { "repo_name": attr.string( mandatory = True, ), - "src": attr.label( + "srcs": attr.label_list( doc = "label for the .bzl file", - allow_single_file = True, + allow_files = True, ), - "doc": attr.label( - doc = "the output of the starlar_doc_extract rule", - allow_single_file = True, - ), - "unknown_deps": attr.string_list( - doc = "list of starlark_library rule dependencies that will not be able to resolve", + "loads": attr.string_list_dict( + doc = "load per file", ), "deps": attr.label_list( doc = "list of starlark_library rule dependencies. These can be ", providers = [StarlarkLibraryFileInfo], ), + "bazelignore": attr.string_list( + doc = "contents of the .bazelignore file, if present", + ), + "bazelversion": attr.string( + doc = "contents of the .bazelversion file, if present", + ), }, doc = "", provides = [DefaultInfo, StarlarkLibraryInfo, StarlarkLibraryFileInfo], ) - -def starlark_library(name, src, deps = [], **kwargs): - visibility = kwargs.pop("visibility", []) - bazel_tools_deps = kwargs.pop("bazel_tools_deps", []) - - # doc_name = name + "_doc" - # native.starlark_doc_extract( - # name = doc_name, - # src = src, - # deps = deps + bazel_tools_deps, - # visibility = ["//visibility:public"], - # ) - - _starlark_library( - name = name, - src = src, - # doc = doc_name, - doc = src, - deps = deps, - visibility = ["//visibility:public"], - **kwargs - ) From 171ef6d30b86a2c2c0fa41d27a15cb4e7f94aff3 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Sat, 29 Nov 2025 22:55:03 -0700 Subject: [PATCH 09/12] Cleanup starlark_library.bzl --- rules/starlark_library.bzl | 47 ++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/rules/starlark_library.bzl b/rules/starlark_library.bzl index e7f8b4ed7..dcd041d8f 100644 --- a/rules/starlark_library.bzl +++ b/rules/starlark_library.bzl @@ -1,29 +1,25 @@ -"""starlark_library.bzl is a thin wrapper over bzl_library.""" +"""starlark_library.bzl is similar to bzl_library but also provides load statement foreach file.""" load("@bazel_skylib//:bzl_library.bzl", "StarlarkLibraryInfo") StarlarkLibraryFileInfo = provider( "Information on contained Starlark rules.", fields = { + "bazelignore": "List[str] value of ctx.attr.bazelignore", + "bazelversion": "str: the value of ctx.attr.bazelversion", + "deps": "DepSet[StarlarkLibraryFileInfo]: deps of this file", "label": "The label of the target rule", - "repo_name": "The original (non-canonical) repo (workspace!) name", - "srcs": "List of .bzl files", "loads": "load statements per .bzl file", - "deps": "DepSet[StarlarkLibraryFileInfo]: deps of this file", + "srcs": "List of .bzl files", "transitive_deps": "DepSet[StarlarkLibraryFileInfo]: transitive deps of this file", - "bazelignore": "List[str] value of ctx.attr.bazelignore", - "bazelversion": "str: the value of ctx.attr.bazelversion", }, ) def _starlark_library_impl(ctx): - deps = [d[StarlarkLibraryFileInfo] for d in ctx.attr.deps] srcs = ctx.files.srcs + deps = [d[StarlarkLibraryFileInfo] for d in ctx.attr.deps] transitive_srcs = depset(srcs, order = "postorder", transitive = [d.transitive_srcs for d in deps]) - - # loads = {} - # for k, v in ctx.attr.loads.items(): - # loads[Label(k)] = v + transitive_deps = depset(deps, order = "postorder", transitive = [d.transitive_deps for d in deps]) return [ DefaultInfo( @@ -34,39 +30,36 @@ def _starlark_library_impl(ctx): transitive_srcs = transitive_srcs, ), StarlarkLibraryFileInfo( - label = ctx.label, - repo_name = ctx.attr.repo_name, - srcs = srcs, - loads = ctx.attr.loads, bazelignore = ctx.attr.bazelignore, bazelversion = ctx.attr.bazelversion, - transitive_deps = depset(deps, transitive = [d.transitive_deps for d in deps]), + deps = deps, + label = ctx.label, + loads = ctx.attr.loads, + srcs = srcs, + transitive_deps = transitive_deps, ), ] starlark_library = rule( implementation = _starlark_library_impl, attrs = { - "repo_name": attr.string( - mandatory = True, + "bazelignore": attr.string_list( + doc = "contents of the .bazelignore file, if present", ), - "srcs": attr.label_list( - doc = "label for the .bzl file", - allow_files = True, + "bazelversion": attr.string( + doc = "contents of the .bazelversion file, if present", ), "loads": attr.string_list_dict( doc = "load per file", ), + "srcs": attr.label_list( + doc = "label for the .bzl file", + allow_files = True, + ), "deps": attr.label_list( doc = "list of starlark_library rule dependencies. These can be ", providers = [StarlarkLibraryFileInfo], ), - "bazelignore": attr.string_list( - doc = "contents of the .bazelignore file, if present", - ), - "bazelversion": attr.string( - doc = "contents of the .bazelversion file, if present", - ), }, doc = "", provides = [DefaultInfo, StarlarkLibraryInfo, StarlarkLibraryFileInfo], From d96a7b31aefdb6723ba1579d79e17e1604cd6bd4 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Sat, 6 Dec 2025 11:14:08 -0700 Subject: [PATCH 10/12] refactor: generate package-level rules and aggegate deps at roots --- language/starlarklibrary/language.go | 118 ++++++++++++++++++++------- 1 file changed, 89 insertions(+), 29 deletions(-) diff --git a/language/starlarklibrary/language.go b/language/starlarklibrary/language.go index 0fa4c0e7a..60b793c10 100644 --- a/language/starlarklibrary/language.go +++ b/language/starlarklibrary/language.go @@ -30,9 +30,9 @@ import ( "log" "maps" "os" - "path" "path/filepath" "slices" + "sort" "strings" "github.com/bazelbuild/bazel-gazelle/config" @@ -63,7 +63,8 @@ var ( } starlarkLibraryKindInfo = map[string]rule.KindInfo{ starlarkLibraryKind: { - NonEmptyAttrs: map[string]bool{"src": true}, + NonEmptyAttrs: map[string]bool{"srcs": true}, + ResolveAttrs: map[string]bool{"deps": true}, }, } starlarkLibraryLoadInfo = rule.LoadInfo{ @@ -76,7 +77,6 @@ type starlarkLibraryLang struct { roots []string repoName string excludeDirs []string - bzlFiles map[string]bool bazelVersion string bazelIgnore []string logFile string @@ -87,9 +87,7 @@ type starlarkLibraryLang struct { // NewLanguage is called by Gazelle to install this language extension in a // binary. func NewLanguage() language.Language { - return &starlarkLibraryLang{ - bzlFiles: map[string]bool{}, - } + return &starlarkLibraryLang{} } // Name returns the name of the language. This should be a prefix of the kinds @@ -209,7 +207,25 @@ func (*starlarkLibraryLang) Fix(c *config.Config, f *rule.File) { // If nil is returned, the rule will not be indexed. If any non-nil slice is // returned, including an empty slice, the rule will be indexed. func (ext *starlarkLibraryLang) Imports(c *config.Config, r *rule.Rule, f *rule.File) []resolve.ImportSpec { - return nil + srcs := r.AttrStrings("srcs") + imports := make([]resolve.ImportSpec, 0, len(srcs)+1) + + for _, src := range srcs { + spec := resolve.ImportSpec{ + Lang: languageName, + Imp: fmt.Sprintf("//%s:%s", f.Pkg, src), + } + + imports = append(imports, spec) + } + + // add an import such the one can find *all* the stark_library rules. + imports = append(imports, resolve.ImportSpec{ + Lang: languageName, + Imp: starlarkLibraryKind, + }) + + return imports } // Embeds returns a list of labels of rules that the given rule embeds. If a @@ -228,6 +244,10 @@ func (*starlarkLibraryLang) Embeds(r *rule.Rule, from label.Label) []label.Label // equivalent) for each import according to language-specific rules and // heuristics. func (ext *starlarkLibraryLang) Resolve(c *config.Config, ix *resolve.RuleIndex, rc *repo.RemoteCache, r *rule.Rule, importsRaw interface{}, from label.Label) { + switch r.Kind() { + case starlarkLibraryKind: + ext.starlarkLibraryResolve(c, ix, rc, r, importsRaw, from) + } } // GenerateRules extracts build metadata from source files in a directory. @@ -256,33 +276,37 @@ func (ext *starlarkLibraryLang) GenerateRules(args language.GenerateArgs) (resul ext.logf("GenerateRules %v: visiting %s", ext.roots, args.Rel) - // collect all .bzl files + // collect all .bzl srcs + var srcs []string for _, f := range append(args.RegularFiles, args.GenFiles...) { - if !isBzlSourceFile(f) { - continue + if isBzlSourceFile(f) { + srcs = append(srcs, f) } - ext.bzlFiles[path.Join(args.Rel, f)] = true } - if slices.Contains(ext.roots, args.Rel) { - r, imports := ext.starlarkLibraryRule(args) - result.Gen = append(result.Gen, r) - result.Imports = append(result.Imports, imports) + if len(srcs) == 0 { + return } + r, imports := ext.starlarkLibraryRule(args, srcs) + result.Gen = append(result.Gen, r) + result.Imports = append(result.Imports, imports) + + // if slices.Contains(ext.roots, args.Rel) { + // r, imports := ext.starlarkLibraryRule(args, srcs) + // result.Gen = append(result.Gen, r) + // result.Imports = append(result.Imports, imports) + // } + return } -func (ext *starlarkLibraryLang) starlarkLibraryRule(args language.GenerateArgs) (*rule.Rule, []any) { - +func (ext *starlarkLibraryLang) starlarkLibraryRule(args language.GenerateArgs, srcs []string) (*rule.Rule, []any) { loadsByRelPath := make(map[string][]string) - for workspaceRelPath := range ext.bzlFiles { - if !strings.HasPrefix(workspaceRelPath, args.Rel) { - continue - } - fullPath := filepath.Join(args.Config.RepoRoot, workspaceRelPath) - relPath := strings.TrimPrefix(strings.TrimPrefix(workspaceRelPath, args.Rel), "/") - _, stmts, err := getBzlFileLoadsStmts(fullPath) + for _, src := range srcs { + relPath := filepath.Join(args.Rel, src) + fullPath := filepath.Join(args.Config.RepoRoot, relPath) + _, stmts, err := getBzlFileLoadsStmts(fullPath, args.Rel, ext.logf) if err != nil { ext.logf("%s: contains syntax errors: %v", fullPath, err) @@ -310,16 +334,47 @@ func (ext *starlarkLibraryLang) starlarkLibraryRule(args language.GenerateArgs) } r.SetAttr("visibility", []string{visibilityPublic}) - return r, []any{} + return r, []any{loadsByRelPath} +} + +func (ext *starlarkLibraryLang) starlarkLibraryResolve(c *config.Config, ix *resolve.RuleIndex, _ *repo.RemoteCache, r *rule.Rule, _ interface{}, from label.Label) { + // only perform resolve if this is one of the roots + var root *string + for _, dir := range ext.roots { + if dir == from.Pkg { + root = &dir + break + } + } + if root == nil { + ext.logf("skipping deps resolution for %v (not a root: %v)", from, ext.roots) + return + } + + var deps []string + + // fetch all rules, then filter by the ones below our root + matches := ix.FindRulesByImportWithConfig(c, resolve.ImportSpec{ + Lang: languageName, + Imp: starlarkLibraryKind, + }, languageName) + for _, m := range matches { + depLabel := m.Label.Rel(from.Repo, from.Pkg) + if strings.HasPrefix(depLabel.Pkg, *root) { + deps = append(deps, depLabel.String()) + } + } + + if len(deps) > 0 { + sort.Strings(deps) + r.SetAttr("deps", deps) + } } // Before implements part of the language.LifecycleManager interface. func (ext *starlarkLibraryLang) Before(context.Context) { ext.logf("Lifecycle: Before() called") ext.logf("roots: %v", ext.roots) - // if len(ext.roots) == 0 { - // log.Fatalf("at least one root must be specified") - // } } // DoneGeneratingRules implements part of the language.FinishableLanguage interface. @@ -417,7 +472,7 @@ func isBzlSourceFile(f string) bool { return strings.HasSuffix(f, fileType) && !ignoreSuffix.Matches(f) } -func getBzlFileLoadsStmts(path string) (*build.File, []*build.LoadStmt, error) { +func getBzlFileLoadsStmts(path, rel string, logf LogFunc) (*build.File, []*build.LoadStmt, error) { f, err := os.ReadFile(path) if err != nil { return nil, nil, fmt.Errorf("os.ReadFile(%q) error: %v", path, err) @@ -431,6 +486,11 @@ func getBzlFileLoadsStmts(path string) (*build.File, []*build.LoadStmt, error) { build.WalkOnce(ast, func(expr *build.Expr) { n := *expr if l, ok := n.(*build.LoadStmt); ok { + if lbl, err := label.Parse(l.Module.Value); err == nil { + l.Module.Value = lbl.Abs("", rel).String() + } else { + logf("rel=%q: load parse error: %s %v", rel, l.Module.Value, err) + } loads = append(loads, l) } }) From b770c9a7001354e25a363230bbe71c13bf49aa0c Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Sat, 6 Dec 2025 15:38:49 -0700 Subject: [PATCH 11/12] refactor extension with starlark_module and starlark_module_library --- cmd/gazelle/BUILD.bazel | 2 +- cmd/gazelle/langs.go | 4 +- .../BUILD.bazel | 21 +- .../language.go | 265 ++++++++++-------- .../language_test.go | 17 +- rules/private/proto_repository_tools.bzl | 4 +- rules/private/proto_repository_tools_srcs.bzl | 4 +- rules/starlark_library.bzl | 66 ----- rules/starlark_module.bzl | 39 +++ rules/starlark_module_library.bzl | 43 +++ 10 files changed, 244 insertions(+), 221 deletions(-) rename language/{starlarklibrary => starlarkrepository}/BUILD.bazel (56%) rename language/{starlarklibrary => starlarkrepository}/language.go (63%) rename language/{starlarklibrary => starlarkrepository}/language_test.go (89%) delete mode 100644 rules/starlark_library.bzl create mode 100644 rules/starlark_module.bzl create mode 100644 rules/starlark_module_library.bzl diff --git a/cmd/gazelle/BUILD.bazel b/cmd/gazelle/BUILD.bazel index bc620c057..4e650705d 100644 --- a/cmd/gazelle/BUILD.bazel +++ b/cmd/gazelle/BUILD.bazel @@ -29,7 +29,7 @@ go_library( "//cmd/gazelle/internal/wspace", "//language/proto_go_modules", "//language/protobuf", - "//language/starlarklibrary", + "//language/starlarkmodules", "@bazel_gazelle//config:go_default_library", "@bazel_gazelle//flag:go_default_library", "@bazel_gazelle//label:go_default_library", diff --git a/cmd/gazelle/langs.go b/cmd/gazelle/langs.go index e86762e50..26fab01c0 100644 --- a/cmd/gazelle/langs.go +++ b/cmd/gazelle/langs.go @@ -21,7 +21,7 @@ import ( "github.com/bazelbuild/bazel-gazelle/language/proto" "github.com/stackb/rules_proto/language/proto_go_modules" "github.com/stackb/rules_proto/language/protobuf" - "github.com/stackb/rules_proto/language/starlarklibrary" + "github.com/stackb/rules_proto/language/starlarkrepository" ) var languages = []language.Language{ @@ -29,5 +29,5 @@ var languages = []language.Language{ protobuf.NewLanguage(), golang.NewLanguage(), proto_go_modules.NewLanguage(), - starlarklibrary.NewLanguage(), + starlarkrepository.NewLanguage(), } diff --git a/language/starlarklibrary/BUILD.bazel b/language/starlarkrepository/BUILD.bazel similarity index 56% rename from language/starlarklibrary/BUILD.bazel rename to language/starlarkrepository/BUILD.bazel index 21a8e92fb..6ba3ede0c 100644 --- a/language/starlarklibrary/BUILD.bazel +++ b/language/starlarkrepository/BUILD.bazel @@ -1,9 +1,9 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") +load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( - name = "starlarklibrary", + name = "starlarkrepository", srcs = ["language.go"], - importpath = "github.com/stackb/rules_proto/language/starlarklibrary", + importpath = "github.com/stackb/rules_proto/language/starlarkrepository", visibility = ["//visibility:public"], deps = [ "@bazel_gazelle//config:go_default_library", @@ -16,18 +16,3 @@ go_library( "@com_github_bazelbuild_buildtools//build:go_default_library", ], ) - -go_test( - name = "starlarklibrary_test", - srcs = [ - "bazelignore_test.go", - "dict_test.go", - "lifecycle_test.go", - "starlark_library_test.go", - ], - embed = [":starlarklibrary"], - deps = [ - "@bazel_gazelle//config:go_default_library", - "@com_github_bazelbuild_buildtools//build:go_default_library", - ], -) diff --git a/language/starlarklibrary/language.go b/language/starlarkrepository/language.go similarity index 63% rename from language/starlarklibrary/language.go rename to language/starlarkrepository/language.go index 60b793c10..b59daf283 100644 --- a/language/starlarklibrary/language.go +++ b/language/starlarkrepository/language.go @@ -19,7 +19,7 @@ limitations under the License. // // The original code for this gazelle extension started from // https://github.com/bazelbuild/bazel-skylib/blob/main/gazelle/bzl/gazelle.go. -package starlarklibrary +package starlarkrepository import ( "bufio" @@ -45,15 +45,16 @@ import ( ) const ( - languageName = "starlarklibrary" - repoNameDirectiveName = languageName + "_repo_name" - rootDirectiveName = languageName + "_root" - excludeDirectiveName = languageName + "_exclude" - logFileDirectiveName = languageName + "_log_file" - starlarkLibraryKind = "starlark_library" - starlarkLibraryName = "bzl_srcs" - fileType = ".bzl" - visibilityPublic = "//visibility:public" + languageName = "starlarkrepository" + repoNameDirectiveName = languageName + "_repo_name" + rootDirectiveName = languageName + "_root" + excludeDirectiveName = languageName + "_exclude" + logFileDirectiveName = languageName + "_log_file" + starlarkModuleKind = "starlark_module" + starlarkModuleLibraryKind = "starlark_module_library" + starlarkModuleLibraryName = "modules" + fileType = ".bzl" + visibilityPublic = "//visibility:public" ) var ( @@ -61,19 +62,28 @@ var ( "_tests.bzl", "_test.bzl", } - starlarkLibraryKindInfo = map[string]rule.KindInfo{ - starlarkLibraryKind: { - NonEmptyAttrs: map[string]bool{"srcs": true}, - ResolveAttrs: map[string]bool{"deps": true}, + starlarkModuleLibraryKindInfo = map[string]rule.KindInfo{ + starlarkModuleLibraryKind: { + NonEmptyAttrs: map[string]bool{"modules": true}, + ResolveAttrs: map[string]bool{"modules": true}, }, } - starlarkLibraryLoadInfo = rule.LoadInfo{ - Name: "@build_stack_rules_proto//rules:starlark_library.bzl", - Symbols: []string{starlarkLibraryKind}, + starlarkModuleKindInfo = map[string]rule.KindInfo{ + starlarkModuleKind: { + NonEmptyAttrs: map[string]bool{"src": true}, + }, + } + starlarkModuleLibraryLoadInfo = rule.LoadInfo{ + Name: "@build_stack_rules_proto//rules:starlark_module_library.bzl", + Symbols: []string{starlarkModuleLibraryKind}, + } + starlarkModuleLoadInfo = rule.LoadInfo{ + Name: "@build_stack_rules_proto//rules:starlark_module.bzl", + Symbols: []string{starlarkModuleKind}, } ) -type starlarkLibraryLang struct { +type starlarkRepositoryLang struct { roots []string repoName string excludeDirs []string @@ -87,24 +97,26 @@ type starlarkLibraryLang struct { // NewLanguage is called by Gazelle to install this language extension in a // binary. func NewLanguage() language.Language { - return &starlarkLibraryLang{} + return &starlarkRepositoryLang{ + excludeDirs: []string{"javatests", "testdata"}, + } } // Name returns the name of the language. This should be a prefix of the kinds // of rules generated by the language, e.g., "go" for the Go extension since it // generates "go_library" rules. -func (*starlarkLibraryLang) Name() string { +func (*starlarkRepositoryLang) Name() string { return languageName } // The following methods are implemented to satisfy the // https://pkg.go.dev/github.com/bazelbuild/bazel-gazelle/resolve?tab=doc#Resolver // interface, but are otherwise unused. -func (ext *starlarkLibraryLang) RegisterFlags(fs *flag.FlagSet, cmd string, c *config.Config) { - fs.StringVar(&ext.logFile, logFileDirectiveName, "", "path to log file for starlark_library extension") +func (ext *starlarkRepositoryLang) RegisterFlags(fs *flag.FlagSet, cmd string, c *config.Config) { + fs.StringVar(&ext.logFile, logFileDirectiveName, "", "path to log file for this extension") } -func (ext *starlarkLibraryLang) CheckFlags(fs *flag.FlagSet, c *config.Config) error { +func (ext *starlarkRepositoryLang) CheckFlags(fs *flag.FlagSet, c *config.Config) error { if ext.logFile != "" { ext.createLogger(ext.logFile) ext.logf("CheckFlags: log file initialized from flag: %s", ext.logFile) @@ -114,8 +126,8 @@ func (ext *starlarkLibraryLang) CheckFlags(fs *flag.FlagSet, c *config.Config) e } // REPO.bazel causes visibility issues if the root BUILD file has been - // deleted but still referencing non-existent rules. This should be - // moved to the part where gazelle cleans up build files. + // deleted but still referencing non-existent rules. This should be moved + // to the part where gazelle cleans up build files. repoBazelFilename := filepath.Join(c.RepoRoot, "REPO.bazel") deleteFile(repoBazelFilename, ext.logf) @@ -134,7 +146,7 @@ func (ext *starlarkLibraryLang) CheckFlags(fs *flag.FlagSet, c *config.Config) e return nil } -func (*starlarkLibraryLang) KnownDirectives() []string { +func (*starlarkRepositoryLang) KnownDirectives() []string { return []string{ rootDirectiveName, excludeDirectiveName, @@ -142,7 +154,7 @@ func (*starlarkLibraryLang) KnownDirectives() []string { } } -func (ext *starlarkLibraryLang) Configure(c *config.Config, rel string, f *rule.File) { +func (ext *starlarkRepositoryLang) Configure(c *config.Config, rel string, f *rule.File) { if f != nil { ext.logf("Configure: processing %d directives in %s", len(f.Directives), rel) for _, d := range f.Directives { @@ -160,7 +172,7 @@ func (ext *starlarkLibraryLang) Configure(c *config.Config, rel string, f *rule. } } -func (ext *starlarkLibraryLang) createLogger(logFile string) { +func (ext *starlarkRepositoryLang) createLogger(logFile string) { if logFile != "" { f, err := os.OpenFile(logFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666) if err == nil { @@ -180,25 +192,27 @@ func (ext *starlarkLibraryLang) createLogger(logFile string) { // Kinds returns a map of maps rule names (kinds) and information on how to // match and merge attributes that may be found in rules of those kinds. All // kinds of rules generated for this language may be found here. -func (*starlarkLibraryLang) Kinds() map[string]rule.KindInfo { +func (*starlarkRepositoryLang) Kinds() map[string]rule.KindInfo { kinds := map[string]rule.KindInfo{} - maps.Copy(kinds, starlarkLibraryKindInfo) + maps.Copy(kinds, starlarkModuleLibraryKindInfo) + maps.Copy(kinds, starlarkModuleKindInfo) return kinds } // Loads returns .bzl files and symbols they define. Every rule generated by // GenerateRules, now or in the past, should be loadable from one of these // files. -func (*starlarkLibraryLang) Loads() []rule.LoadInfo { +func (*starlarkRepositoryLang) Loads() []rule.LoadInfo { return []rule.LoadInfo{ - starlarkLibraryLoadInfo, + starlarkModuleLibraryLoadInfo, + starlarkModuleLoadInfo, } } // Fix repairs deprecated usage of language-specific rules in f. This is called // before the file is indexed. Unless c.ShouldFix is true, fixes that delete or // rename rules should not be performed. -func (*starlarkLibraryLang) Fix(c *config.Config, f *rule.File) { +func (*starlarkRepositoryLang) Fix(c *config.Config, f *rule.File) { } // Imports returns a list of ImportSpecs that can be used to import the rule r. @@ -206,26 +220,21 @@ func (*starlarkLibraryLang) Fix(c *config.Config, f *rule.File) { // // If nil is returned, the rule will not be indexed. If any non-nil slice is // returned, including an empty slice, the rule will be indexed. -func (ext *starlarkLibraryLang) Imports(c *config.Config, r *rule.Rule, f *rule.File) []resolve.ImportSpec { - srcs := r.AttrStrings("srcs") - imports := make([]resolve.ImportSpec, 0, len(srcs)+1) - - for _, src := range srcs { - spec := resolve.ImportSpec{ - Lang: languageName, - Imp: fmt.Sprintf("//%s:%s", f.Pkg, src), - } - - imports = append(imports, spec) +func (ext *starlarkRepositoryLang) Imports(c *config.Config, r *rule.Rule, f *rule.File) []resolve.ImportSpec { + switch r.Kind() { + case starlarkModuleKind: + return ext.starlarkModuleImports(c, r, f) + default: + return nil } +} - // add an import such the one can find *all* the stark_library rules. - imports = append(imports, resolve.ImportSpec{ - Lang: languageName, - Imp: starlarkLibraryKind, - }) - - return imports +func (ext *starlarkRepositoryLang) starlarkModuleImports(_ *config.Config, r *rule.Rule, f *rule.File) []resolve.ImportSpec { + // return one for file level dep resolution, one for the rule kind itself + return []resolve.ImportSpec{ + {Lang: languageName, Imp: fmt.Sprintf("//%s:%s", f.Pkg, r.AttrString("src"))}, + {Lang: languageName, Imp: starlarkModuleKind}, + } } // Embeds returns a list of labels of rules that the given rule embeds. If a @@ -233,7 +242,7 @@ func (ext *starlarkLibraryLang) Imports(c *config.Config, r *rule.Rule, f *rule. // embedding rule will be indexed. The embedding rule will inherit the imports // of the embedded rule. Since SkyLark doesn't support embedding this should // always return nil. -func (*starlarkLibraryLang) Embeds(r *rule.Rule, from label.Label) []label.Label { +func (*starlarkRepositoryLang) Embeds(r *rule.Rule, from label.Label) []label.Label { return nil } @@ -243,10 +252,10 @@ func (*starlarkLibraryLang) Embeds(r *rule.Rule, from label.Label) []label.Label // Resolve generates a "deps" attribute (or the appropriate language-specific // equivalent) for each import according to language-specific rules and // heuristics. -func (ext *starlarkLibraryLang) Resolve(c *config.Config, ix *resolve.RuleIndex, rc *repo.RemoteCache, r *rule.Rule, importsRaw interface{}, from label.Label) { +func (ext *starlarkRepositoryLang) Resolve(c *config.Config, ix *resolve.RuleIndex, rc *repo.RemoteCache, r *rule.Rule, importsRaw interface{}, from label.Label) { switch r.Kind() { - case starlarkLibraryKind: - ext.starlarkLibraryResolve(c, ix, rc, r, importsRaw, from) + case starlarkModuleLibraryKind: + ext.starlarkModuleLibraryResolve(c, ix, rc, r, importsRaw, from) } } @@ -262,7 +271,7 @@ func (ext *starlarkLibraryLang) Resolve(c *config.Config, ix *resolve.RuleIndex, // // Any non-fatal errors this function encounters should be logged using // log.Print. -func (ext *starlarkLibraryLang) GenerateRules(args language.GenerateArgs) (result language.GenerateResult) { +func (ext *starlarkRepositoryLang) GenerateRules(args language.GenerateArgs) (result language.GenerateResult) { // extension is effectively disabled without specifying at least one root if len(ext.roots) == 0 { return @@ -274,55 +283,81 @@ func (ext *starlarkLibraryLang) GenerateRules(args language.GenerateArgs) (resul } } - ext.logf("GenerateRules %v: visiting %s", ext.roots, args.Rel) + ext.logf("GenerateRules %v: visiting %s: %v", ext.roots, args.Rel, args.RegularFiles) + mustListFiles(ext.logf, filepath.Join(args.Config.WorkDir, args.Rel)) + for _, f := range args.RegularFiles { + if !isBzlSourceFile(f) { + continue + } + relPath := filepath.Join(args.Rel, f) + fullPath := filepath.Join(args.Config.WorkDir, relPath) - // collect all .bzl srcs - var srcs []string - for _, f := range append(args.RegularFiles, args.GenFiles...) { - if isBzlSourceFile(f) { - srcs = append(srcs, f) + _, loadStmts, err := getBzlFileLoadsStmts(fullPath, args.Rel, ext.logf) + if err != nil { + ext.logf("%s: contains syntax errors: %v", fullPath, err) + continue } - } - if len(srcs) == 0 { - return + r, imports := ext.starlarkModuleRule(args, f, loadStmts) + result.Gen = append(result.Gen, r) + result.Imports = append(result.Imports, imports) + log.Printf("generated %s %s/%s", r.Kind(), args.Rel, r.Name()) } - r, imports := ext.starlarkLibraryRule(args, srcs) - result.Gen = append(result.Gen, r) - result.Imports = append(result.Imports, imports) - - // if slices.Contains(ext.roots, args.Rel) { - // r, imports := ext.starlarkLibraryRule(args, srcs) - // result.Gen = append(result.Gen, r) - // result.Imports = append(result.Imports, imports) - // } + if _, ok := getMatchingRoot(args.Rel, ext.roots); ok { + r, imports := ext.starlarkModuleLibraryRule(args) + result.Gen = append(result.Gen, r) + result.Imports = append(result.Imports, imports) + } return } -func (ext *starlarkLibraryLang) starlarkLibraryRule(args language.GenerateArgs, srcs []string) (*rule.Rule, []any) { - loadsByRelPath := make(map[string][]string) - for _, src := range srcs { - relPath := filepath.Join(args.Rel, src) - fullPath := filepath.Join(args.Config.RepoRoot, relPath) - _, stmts, err := getBzlFileLoadsStmts(fullPath, args.Rel, ext.logf) +// mustListFiles - convenience debugging function to log the files under a given +// dir, excluding proto files and the extra binaries here. +func mustListFiles(logf LogFunc, dir string) []string { + files := make([]string, 0) + if err := filepath.Walk(dir, func(relname string, info os.FileInfo, err error) error { if err != nil { - ext.logf("%s: contains syntax errors: %v", fullPath, err) - // don't return early since it is reasonable to create a target even - // without deps. + log.Fatal(err) + } + if info.IsDir() { + return nil } - loads := make([]string, 0, len(stmts)) - for _, stmt := range stmts { - loads = append(loads, stmt.Module.Value) + if filepath.Ext(relname) == ".proto" { + return nil } - loadsByRelPath[relPath] = loads + files = append(files, relname) + logf("file: %s", relname) + return nil + }); err != nil { + logf("walk error: %v", err) + } + + return files +} +func (ext *starlarkRepositoryLang) starlarkModuleRule(args language.GenerateArgs, src string, loadStmts []*build.LoadStmt) (*rule.Rule, []any) { + + name := strings.TrimSuffix(src, fileType) + ext.logf("generating %s rule for %s //%s:%s", starlarkModuleKind, src, args.Rel, name) + + loads := make([]string, 0, len(loadStmts)) + for _, stmt := range loadStmts { + loads = append(loads, stmt.Module.Value) } + sort.Strings(loads) + + r := rule.NewRule(starlarkModuleKind, name) + r.SetAttr("src", src) + r.SetAttr("loads", loads) + r.SetAttr("visibility", []string{visibilityPublic}) - r := rule.NewRule(starlarkLibraryKind, starlarkLibraryName) - r.SetAttr("srcs", slices.Sorted(maps.Keys(loadsByRelPath))) - r.SetAttr("loads", stringListDict(loadsByRelPath)) + return r, []any{} +} + +func (ext *starlarkRepositoryLang) starlarkModuleLibraryRule(_ language.GenerateArgs) (*rule.Rule, []any) { + r := rule.NewRule(starlarkModuleLibraryKind, starlarkModuleLibraryName) if ext.repoName != "" { r.SetAttr("repo_name", ext.repoName) } @@ -333,58 +368,51 @@ func (ext *starlarkLibraryLang) starlarkLibraryRule(args language.GenerateArgs, r.SetAttr("bazelignore", ext.bazelIgnore) } r.SetAttr("visibility", []string{visibilityPublic}) - - return r, []any{loadsByRelPath} + return r, []any{} } -func (ext *starlarkLibraryLang) starlarkLibraryResolve(c *config.Config, ix *resolve.RuleIndex, _ *repo.RemoteCache, r *rule.Rule, _ interface{}, from label.Label) { +func (ext *starlarkRepositoryLang) starlarkModuleLibraryResolve(c *config.Config, ix *resolve.RuleIndex, _ *repo.RemoteCache, r *rule.Rule, _ interface{}, from label.Label) { // only perform resolve if this is one of the roots - var root *string - for _, dir := range ext.roots { - if dir == from.Pkg { - root = &dir - break - } - } - if root == nil { - ext.logf("skipping deps resolution for %v (not a root: %v)", from, ext.roots) + root, isRoot := getMatchingRoot(from.Pkg, ext.roots) + if !isRoot { + ext.logf("skipping modules resolution for %v (not a root: %v)", from, ext.roots) return } - var deps []string + var modules []string - // fetch all rules, then filter by the ones below our root + // fetch all starlark_module rules, then filter by the ones below our root matches := ix.FindRulesByImportWithConfig(c, resolve.ImportSpec{ Lang: languageName, - Imp: starlarkLibraryKind, + Imp: starlarkModuleKind, }, languageName) for _, m := range matches { depLabel := m.Label.Rel(from.Repo, from.Pkg) - if strings.HasPrefix(depLabel.Pkg, *root) { - deps = append(deps, depLabel.String()) + if strings.HasPrefix(depLabel.Pkg, root) { + modules = append(modules, depLabel.String()) } } - if len(deps) > 0 { - sort.Strings(deps) - r.SetAttr("deps", deps) + if len(modules) > 0 { + sort.Strings(modules) + r.SetAttr("modules", modules) } } // Before implements part of the language.LifecycleManager interface. -func (ext *starlarkLibraryLang) Before(context.Context) { +func (ext *starlarkRepositoryLang) Before(context.Context) { ext.logf("Lifecycle: Before() called") ext.logf("roots: %v", ext.roots) } // DoneGeneratingRules implements part of the language.FinishableLanguage interface. -func (ext *starlarkLibraryLang) DoneGeneratingRules() { +func (ext *starlarkRepositoryLang) DoneGeneratingRules() { ext.logf("Lifecycle: DoneGeneratingRules() called") } // AfterResolvingDeps implements part of the language.LifecycleManager interface. // This is where we flush and close the log file. -func (ext *starlarkLibraryLang) AfterResolvingDeps(context.Context) { +func (ext *starlarkRepositoryLang) AfterResolvingDeps(context.Context) { ext.logf("Lifecycle: AfterResolvingDeps() called - flushing and closing log file") if ext.logWriter != nil { // Sync flushes any buffered data to disk @@ -396,12 +424,21 @@ func (ext *starlarkLibraryLang) AfterResolvingDeps(context.Context) { } // logf logs a message using the extension's logger if configured -func (ext *starlarkLibraryLang) logf(format string, args ...any) { +func (ext *starlarkRepositoryLang) logf(format string, args ...any) { if ext.logger != nil { ext.logger.Printf(format, args...) } } +func getMatchingRoot(rel string, roots []string) (string, bool) { + for _, root := range roots { + if root == rel { + return root, true + } + } + return "", false +} + type suffixes []string func (s suffixes) Matches(test string) bool { diff --git a/language/starlarklibrary/language_test.go b/language/starlarkrepository/language_test.go similarity index 89% rename from language/starlarklibrary/language_test.go rename to language/starlarkrepository/language_test.go index fe48f650f..3c0c61235 100644 --- a/language/starlarklibrary/language_test.go +++ b/language/starlarkrepository/language_test.go @@ -1,19 +1,4 @@ -/* Copyright 2020 The Bazel Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package starlarklibrary +package starlarkrepository import ( "os" diff --git a/rules/private/proto_repository_tools.bzl b/rules/private/proto_repository_tools.bzl index 2f7bbfc6d..fef1520c1 100644 --- a/rules/private/proto_repository_tools.bzl +++ b/rules/private/proto_repository_tools.bzl @@ -68,11 +68,11 @@ def _proto_repository_tools_impl(ctx): ctx.path(ctx.attr._list_repository_tools_srcs), "-dir=src/github.com/stackb/rules_proto", # Run it under 'check' to assert file is up-to-date - "-check=rules/private/proto_repository_tools_srcs.bzl", + # "-check=rules/private/proto_repository_tools_srcs.bzl", # Run it under 'skip' to not check (only for internal testing) # "-skip=rules/private/proto_repository_tools_srcs.bzl", # Run it under 'generate' to recreate the list - # "-generate=rules/private/proto_repository_tools_srcs.bzl", + "-generate=rules/private/proto_repository_tools_srcs.bzl", ], environment = env, ) diff --git a/rules/private/proto_repository_tools_srcs.bzl b/rules/private/proto_repository_tools_srcs.bzl index b1398f296..d3471cd4e 100644 --- a/rules/private/proto_repository_tools_srcs.bzl +++ b/rules/private/proto_repository_tools_srcs.bzl @@ -43,8 +43,8 @@ PROTO_REPOSITORY_TOOLS_SRCS = [ "@build_stack_rules_proto//language/proto_go_modules:rule.go", "@build_stack_rules_proto//language/protobuf:BUILD.bazel", "@build_stack_rules_proto//language/protobuf:protobuf.go", - "@build_stack_rules_proto//language/starlarklibrary:BUILD.bazel", - "@build_stack_rules_proto//language/starlarklibrary:language.go", + "@build_stack_rules_proto//language/starlarkrepository:BUILD.bazel", + "@build_stack_rules_proto//language/starlarkrepository:language.go", "@build_stack_rules_proto//pkg:BUILD.bazel", "@build_stack_rules_proto//pkg/goldentest:BUILD.bazel", "@build_stack_rules_proto//pkg/goldentest:cases.go", diff --git a/rules/starlark_library.bzl b/rules/starlark_library.bzl deleted file mode 100644 index dcd041d8f..000000000 --- a/rules/starlark_library.bzl +++ /dev/null @@ -1,66 +0,0 @@ -"""starlark_library.bzl is similar to bzl_library but also provides load statement foreach file.""" - -load("@bazel_skylib//:bzl_library.bzl", "StarlarkLibraryInfo") - -StarlarkLibraryFileInfo = provider( - "Information on contained Starlark rules.", - fields = { - "bazelignore": "List[str] value of ctx.attr.bazelignore", - "bazelversion": "str: the value of ctx.attr.bazelversion", - "deps": "DepSet[StarlarkLibraryFileInfo]: deps of this file", - "label": "The label of the target rule", - "loads": "load statements per .bzl file", - "srcs": "List of .bzl files", - "transitive_deps": "DepSet[StarlarkLibraryFileInfo]: transitive deps of this file", - }, -) - -def _starlark_library_impl(ctx): - srcs = ctx.files.srcs - deps = [d[StarlarkLibraryFileInfo] for d in ctx.attr.deps] - transitive_srcs = depset(srcs, order = "postorder", transitive = [d.transitive_srcs for d in deps]) - transitive_deps = depset(deps, order = "postorder", transitive = [d.transitive_deps for d in deps]) - - return [ - DefaultInfo( - files = transitive_srcs, - ), - StarlarkLibraryInfo( - srcs = srcs, - transitive_srcs = transitive_srcs, - ), - StarlarkLibraryFileInfo( - bazelignore = ctx.attr.bazelignore, - bazelversion = ctx.attr.bazelversion, - deps = deps, - label = ctx.label, - loads = ctx.attr.loads, - srcs = srcs, - transitive_deps = transitive_deps, - ), - ] - -starlark_library = rule( - implementation = _starlark_library_impl, - attrs = { - "bazelignore": attr.string_list( - doc = "contents of the .bazelignore file, if present", - ), - "bazelversion": attr.string( - doc = "contents of the .bazelversion file, if present", - ), - "loads": attr.string_list_dict( - doc = "load per file", - ), - "srcs": attr.label_list( - doc = "label for the .bzl file", - allow_files = True, - ), - "deps": attr.label_list( - doc = "list of starlark_library rule dependencies. These can be ", - providers = [StarlarkLibraryFileInfo], - ), - }, - doc = "", - provides = [DefaultInfo, StarlarkLibraryInfo, StarlarkLibraryFileInfo], -) diff --git a/rules/starlark_module.bzl b/rules/starlark_module.bzl new file mode 100644 index 000000000..60c857687 --- /dev/null +++ b/rules/starlark_module.bzl @@ -0,0 +1,39 @@ +"""starlark_module.bzl is similar to bzl_library but also provides load statement foreach file.""" + +load("@bazel_skylib//:bzl_library.bzl", "StarlarkLibraryInfo") + +StarlarkModuleInfo = provider( + "Information about a single .bzl file.", + fields = { + "label": "Label: The label of the target rule", + "loads": "List[str]: load statements for this file", + "src": "File: The .bzl file", + }, +) + +def _starlark_module_impl(ctx): + return [ + StarlarkLibraryInfo( + srcs = [ctx.file.src], + transitive_srcs = depset([ctx.file.src]), + ), + StarlarkModuleInfo( + label = ctx.label, + loads = ctx.attr.loads, + src = ctx.file.src, + ), + ] + +starlark_module = rule( + implementation = _starlark_module_impl, + attrs = { + "loads": attr.string_list( + doc = "the load statements in this file", + ), + "src": attr.label( + doc = "the .bzl source file", + allow_single_file = True, + ), + }, + provides = [StarlarkLibraryInfo, StarlarkModuleInfo], +) diff --git a/rules/starlark_module_library.bzl b/rules/starlark_module_library.bzl new file mode 100644 index 000000000..2dfa53e34 --- /dev/null +++ b/rules/starlark_module_library.bzl @@ -0,0 +1,43 @@ +"""starlark_module_library.bzl is similar to bzl_library but also provides load statement foreach file.""" + +load("//rules:starlark_module.bzl", "StarlarkModuleInfo") + +StarlarkModuleLibraryInfo = provider( + "Information on a set of starlark modules. This is a flat list, non-transitive.", + fields = { + "label": "The label of the target rule", + "modules": "List[StarlarkModuleInfo]: modules deps of this rule", + "srcs": "List[File]: source files for the modules, for convenience", + "bazelignore": "List[str] value of ctx.attr.bazelignore", + "bazelversion": "str: the value of ctx.attr.bazelversion", + }, +) + +def _starlark_module_library_impl(ctx): + modules = [m[StarlarkModuleInfo] for m in ctx.attr.modules] + return [ + StarlarkModuleLibraryInfo( + label = ctx.label, + bazelignore = ctx.attr.bazelignore, + bazelversion = ctx.attr.bazelversion, + modules = modules, + srcs = [m.src for m in modules], + ), + ] + +starlark_module_library = rule( + implementation = _starlark_module_library_impl, + attrs = { + "bazelignore": attr.string_list( + doc = "contents of the .bazelignore file, if present", + ), + "bazelversion": attr.string( + doc = "contents of the .bazelversion file, if present", + ), + "modules": attr.label_list( + doc = "list of starlark_module rule dependencies.", + providers = [StarlarkModuleInfo], + ), + }, + provides = [StarlarkModuleLibraryInfo], +) From 3bcf945833dc086651abb8ebfb83dc8bf9da90ce Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Sat, 13 Dec 2025 08:49:51 -0700 Subject: [PATCH 12/12] checkpoint functioning version --- language/starlarkrepository/language.go | 7 ++++++- rules/private/proto_repository_tools.bzl | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/language/starlarkrepository/language.go b/language/starlarkrepository/language.go index b59daf283..841f4931c 100644 --- a/language/starlarkrepository/language.go +++ b/language/starlarkrepository/language.go @@ -98,7 +98,12 @@ type starlarkRepositoryLang struct { // binary. func NewLanguage() language.Language { return &starlarkRepositoryLang{ - excludeDirs: []string{"javatests", "testdata"}, + excludeDirs: []string{ + "docs", + "javatests", + "testdata", + "tests", + }, } } diff --git a/rules/private/proto_repository_tools.bzl b/rules/private/proto_repository_tools.bzl index fef1520c1..2f7bbfc6d 100644 --- a/rules/private/proto_repository_tools.bzl +++ b/rules/private/proto_repository_tools.bzl @@ -68,11 +68,11 @@ def _proto_repository_tools_impl(ctx): ctx.path(ctx.attr._list_repository_tools_srcs), "-dir=src/github.com/stackb/rules_proto", # Run it under 'check' to assert file is up-to-date - # "-check=rules/private/proto_repository_tools_srcs.bzl", + "-check=rules/private/proto_repository_tools_srcs.bzl", # Run it under 'skip' to not check (only for internal testing) # "-skip=rules/private/proto_repository_tools_srcs.bzl", # Run it under 'generate' to recreate the list - "-generate=rules/private/proto_repository_tools_srcs.bzl", + # "-generate=rules/private/proto_repository_tools_srcs.bzl", ], environment = env, )