From 519aab65294af1b1c1a2bdd843c086e3dc6f08be Mon Sep 17 00:00:00 2001 From: Peter Ebden Date: Mon, 27 Jul 2026 10:31:12 +0100 Subject: [PATCH 01/16] Convert build parallelism to a more synchronous model --- .golangci.yml | 4 + src/build/build_step.go | 57 +- src/build/build_step_stress_test.go | 18 +- src/build/build_step_test.go | 18 +- src/build/incrementality.go | 15 +- src/build/incrementality_test.go | 1 + src/cache/async_cache_test.go | 4 +- src/cache/cmd_cache_test.go | 6 +- src/cache/dir_cache_test.go | 32 +- src/cache/http_cache_test.go | 2 +- src/clean/clean.go | 2 +- src/cmap/cerrmap.go | 39 +- src/cmap/cmap.go | 45 +- src/cmap/cmap_test.go | 42 +- src/core/build_env.go | 12 +- src/core/build_input.go | 6 +- src/core/build_label.go | 6 +- src/core/build_target.go | 580 ++++++++++----------- src/core/build_target_test.go | 138 ++--- src/core/command_replacements.go | 14 +- src/core/command_replacements_test.go | 15 +- src/core/cycle_detector.go | 5 +- src/core/cycle_detector_test.go | 23 - src/core/graph.go | 48 +- src/core/graph_benchmark_test.go | 16 +- src/core/package.go | 4 +- src/core/package_test.go | 14 +- src/core/stamp.go | 16 +- src/core/stamp_test.go | 12 +- src/core/state.go | 570 +------------------- src/core/state_test.go | 48 +- src/core/utils.go | 13 +- src/core/utils_benchmark_test.go | 2 - src/core/utils_test.go | 3 +- src/exec/exec.go | 2 +- src/export/export.go | 9 +- src/gc/gc.go | 8 +- src/generate/generate.go | 6 +- src/help/help.go | 2 +- src/output/interactive_display.go | 32 +- src/output/shell_output.go | 57 +- src/output/targets.go | 13 +- src/parse/BUILD | 13 +- src/parse/README.md | 14 +- src/parse/asp/builtins.go | 91 ++-- src/parse/asp/builtins_test.go | 6 - src/parse/asp/errors.go | 5 + src/parse/asp/interpreter.go | 49 +- src/parse/asp/interpreter_test.go | 6 +- src/parse/asp/logging_test.go | 2 +- src/parse/asp/main/main.go | 2 +- src/parse/asp/objects.go | 4 +- src/parse/asp/parser.go | 19 +- src/parse/asp/targets.go | 4 +- src/parse/init.go | 29 +- src/parse/parse_step.go | 141 +---- src/parse/parse_step_test.go | 166 ------ src/please.go | 11 +- src/plz/BUILD | 3 + src/plz/plz.go | 696 ++++++++++++++++++++----- src/query/changes_test.go | 3 - src/query/deps.go | 7 +- src/query/graph.go | 14 +- src/query/graph_test.go | 3 - src/query/outputs.go | 4 +- src/query/print.go | 7 +- src/query/reverse_deps.go | 2 +- src/query/reverse_deps_test.go | 2 - src/query/somepath.go | 2 +- src/query/whatoutputs.go | 6 +- src/query/whatoutputs_test.go | 6 +- src/remote/action.go | 12 +- src/remote/remote.go | 4 +- src/remote/remote_test.go | 10 +- src/remote/utils.go | 8 +- src/run/run_step.go | 6 +- src/test/coverage.go | 10 +- src/test/surefire.go | 5 +- src/watch/watch.go | 4 +- third_party/python/BUILD | 5 +- tools/build_langserver/lsp/lsp_test.go | 9 +- tools/performance/gen_parse_tree.py | 7 +- 82 files changed, 1445 insertions(+), 1911 deletions(-) delete mode 100644 src/parse/parse_step_test.go diff --git a/.golangci.yml b/.golangci.yml index a4cdaa60c6..251fd0f02e 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -107,6 +107,8 @@ linters: - third_party$ - builtin$ - examples$ + - tree + - plz-out formatters: enable: - gci @@ -124,3 +126,5 @@ formatters: - third_party$ - builtin$ - examples$ + - tree + - plz-out diff --git a/src/build/build_step.go b/src/build/build_step.go index 303c75097a..f77405950b 100644 --- a/src/build/build_step.go +++ b/src/build/build_step.go @@ -59,7 +59,7 @@ var successfulLocalTargetBuildDuration = metrics.NewHistogramVec( ) // Build implements the core logic for building a single target. -func Build(state *core.BuildState, target *core.BuildTarget, remote bool) { +func Build(state *core.BuildState, target *core.BuildTarget, remote bool) error { state = state.ForTarget(target) target.SetState(core.Building) start := time.Now() @@ -67,23 +67,21 @@ func Build(state *core.BuildState, target *core.BuildTarget, remote bool) { if errors.Is(err, errStop) { target.SetState(core.Stopped) state.LogBuildResult(target, core.TargetBuildStopped, "Build stopped") - return + return nil } state.LogBuildError(target.Label, core.TargetBuildFailed, err, "Build failed: %s", err) - if err := RemoveOutputs(target); err != nil { + if err := RemoveOutputs(state, target); err != nil { log.Errorf("Failed to remove outputs for %s: %s", target.Label, err) } target.SetState(core.Failed) - target.FinishBuild() - return + return err } if remote { successfulRemoteTargetBuildDuration.WithLabelValues(metrics.CILabel).Observe(float64(time.Since(start).Milliseconds())) } else { successfulLocalTargetBuildDuration.WithLabelValues(metrics.CILabel).Observe(float64(time.Since(start).Milliseconds())) } - // Mark the target as having finished building. - target.FinishBuild() + return nil } func validateBuildTargetBeforeBuild(state *core.BuildState, target *core.BuildTarget) error { @@ -92,7 +90,7 @@ func validateBuildTargetBeforeBuild(state *core.BuildState, target *core.BuildTa } // We can't do this check until build time, until then we don't know what all the outputs // will be (eg. for filegroups that collect outputs of other rules). - if err := target.CheckDuplicateOutputs(); err != nil { + if err := target.CheckDuplicateOutputs(state.Graph); err != nil { return err } @@ -136,7 +134,7 @@ func prepareOnly(state *core.BuildState, target *core.BuildTarget) error { return errStop } - if err := prepareDirectories(target); err != nil { + if err := prepareDirectories(state, target); err != nil { return err } if err := prepareSources(state, state.Graph, target); err != nil { @@ -278,7 +276,7 @@ func buildTarget(state *core.BuildState, target *core.BuildTarget, runRemotely b return nil } - if err := prepareDirectories(target); err != nil { + if err := prepareDirectories(state, target); err != nil { return fmt.Errorf("Error preparing directories for %s: %s", target.Label, err) } @@ -286,7 +284,7 @@ func buildTarget(state *core.BuildState, target *core.BuildTarget, runRemotely b // // N.B. Important we do not go through state.TargetHasher here since it memoises and // this calculation might be incorrect. - oldOutputHash := outputHashOrNil(target, target.FullOutputs(), state.PathHasher, state.PathHasher.NewHash) + oldOutputHash := outputHashOrNil(target, target.FullOutputs(state.Graph), state.PathHasher, state.PathHasher.NewHash) cacheKey = mustShortTargetHash(state, target) if state.Cache != nil && !runRemotely && !state.ShouldRebuild(target) { @@ -339,12 +337,12 @@ func buildTarget(state *core.BuildState, target *core.BuildTarget, runRemotely b } if target.PostBuildFunction != nil { - outs := target.Outputs() + outs := target.Outputs(state.Graph) if err := runPostBuildFunction(state, target, string(metadata.Stdout), postBuildOutput); err != nil { return err } - if runRemotely && len(outs) != len(target.Outputs()) { + if runRemotely && len(outs) != len(target.Outputs(state.Graph)) { // postBuildFunction has changed the target - must rebuild it log.Info("Rebuilding %s after post-build function", target) metadata, err = state.RemoteClient.Build(target) @@ -470,7 +468,7 @@ func retrieveArtifacts(state *core.BuildState, target *core.BuildTarget, oldOutp cacheKey := mustShortTargetHash(state, target) - if md := retrieveFromCache(state.Cache, target, cacheKey, target.Outputs()); md != nil { + if md := retrieveFromCache(state.Cache, target, cacheKey, target.Outputs(state.Graph)); md != nil { // Retrieve additional optional outputs from metadata if len(md.OptionalOutputs) > 0 { state.Cache.Retrieve(target, cacheKey, md.OptionalOutputs) @@ -481,7 +479,7 @@ func retrieveArtifacts(state *core.BuildState, target *core.BuildTarget, oldOutp newOutputHash, err := calculateAndCheckRuleHash(state, target) if err != nil { // Most likely hash verification failure log.Warning("Error retrieving cached artifacts for %s: %s", target.Label, err) - RemoveOutputs(target) + RemoveOutputs(state, target) return false } else if oldOutputHash == nil || !bytes.Equal(oldOutputHash, newOutputHash) { target.SetState(core.Cached) @@ -528,7 +526,7 @@ func runBuildCommand(state *core.BuildState, target *core.BuildTarget, command s // buildTextFile runs the build action for text_file() rules func buildTextFile(state *core.BuildState, target *core.BuildTarget) error { - outs := target.Outputs() + outs := target.Outputs(state.Graph) if len(outs) != 1 { return fmt.Errorf("text_file %s should have a single output, has %d", target.Label, len(outs)) } @@ -543,14 +541,14 @@ func buildTextFile(state *core.BuildState, target *core.BuildTarget) error { } // prepareOutputDirectories creates any directories the target has declared it will output into as a nicety -func prepareOutputDirectories(target *core.BuildTarget) error { +func prepareOutputDirectories(state *core.BuildState, target *core.BuildTarget) error { for _, dir := range target.OutputDirectories { if err := prepareParentDirs(target, dir.Dir()); err != nil { return err } } - for _, out := range target.Outputs() { + for _, out := range target.Outputs(state.Graph) { if err := prepareParentDirs(target, out); err != nil { return err } @@ -573,11 +571,11 @@ func prepareParentDirs(target *core.BuildTarget, out string) error { } // Prepares the temp and out directories for a target -func prepareDirectories(target *core.BuildTarget) error { +func prepareDirectories(state *core.BuildState, target *core.BuildTarget) error { if err := prepareDirectory(target.TmpDir(), true); err != nil { return err } - if err := prepareOutputDirectories(target); err != nil { + if err := prepareOutputDirectories(state, target); err != nil { return err } return prepareDirectory(target.OutDir(), false) @@ -604,7 +602,7 @@ func prepareSources(state *core.BuildState, graph *core.BuildGraph, target *core } } if target.Stamp { - if err := fs.WriteFile(bytes.NewReader(core.StampFile(state.Config, target)), filepath.Join(target.TmpDir(), target.StampFileName()), 0644); err != nil { + if err := fs.WriteFile(bytes.NewReader(core.StampFile(state, target)), filepath.Join(target.TmpDir(), target.StampFileName()), 0644); err != nil { return err } } @@ -699,7 +697,7 @@ func moveOutputs(state *core.BuildState, target *core.BuildTarget) ([]string, bo changed := false tmpDir := target.TmpDir() outDir := target.OutDir() - outs := target.Outputs() + outs := target.Outputs(state.Graph) allOuts := make([]string, len(outs), len(outs)+len(target.OutputDirectories)) for i, output := range outs { allOuts[i] = output @@ -779,8 +777,8 @@ func moveOutput(state *core.BuildState, target *core.BuildTarget, tmpOutput, rea } // RemoveOutputs removes all generated outputs for a rule. -func RemoveOutputs(target *core.BuildTarget) error { - for _, output := range target.Outputs() { +func RemoveOutputs(state *core.BuildState, target *core.BuildTarget) error { + for _, output := range target.Outputs(state.Graph) { out := filepath.Join(target.OutDir(), output) if err := fs.RemoveAll(out); err != nil { return err @@ -832,7 +830,7 @@ func calculateAndCheckRuleHash(state *core.BuildState, target *core.BuildTarget) } // Set appropriate permissions on outputs if target.IsBinary { - for _, output := range target.FullOutputs() { + for _, output := range target.FullOutputs(state.Graph) { // Walk through the output, // if the output is a directory, apply output mode to the file instead of the directory err := fs.Walk(output, func(path string, isDir bool) error { @@ -890,7 +888,7 @@ func (h *targetHasher) SetHash(target *core.BuildTarget, hash []byte) { // outputHash calculates the output hash for a target, choosing an appropriate strategy. func (h *targetHasher) outputHash(target *core.BuildTarget) ([]byte, error) { - outs := target.FullOutputs() + outs := target.FullOutputs(h.State.Graph) if len(outs) == 1 && fs.FileExists(outs[0]) { return outputHash(target, outs, h.State.PathHasher, nil) } @@ -930,7 +928,7 @@ func checkRuleHashes(state *core.BuildState, target *core.BuildTarget, hash []by if len(target.Hashes) == 0 { return nil // nothing to check } - outputs := target.FullOutputs() + outputs := target.FullOutputs(state.Graph) hashes := target.UnprefixedHashes() // Check if the hash we've already calculated matches any of these before we go off // trying any other combinations. @@ -1029,7 +1027,7 @@ func buildLinksOfType(state *core.BuildState, target *core.BuildTarget, prefix s for _, dest := range labels { destDir := filepath.Join(core.RepoRoot, os.Expand(dest, env.ReplaceEnvironment)) srcDir := filepath.Join(core.RepoRoot, target.OutDir()) - for _, out := range target.Outputs() { + for _, out := range target.Outputs(state.Graph) { if direct { fs.LinkDestination(filepath.Join(srcDir, out), destDir, f) } else { @@ -1082,7 +1080,8 @@ func fetchOneRemoteFile(state *core.BuildState, target *core.BuildTarget, url st env := core.BuildEnvironment(state, target, filepath.Join(core.RepoRoot, target.TmpDir())) url = os.Expand(url, env.ReplaceEnvironment) - tmpPath := filepath.Join(target.TmpDir(), target.Outputs()[0]) + outputs := target.Outputs(state.Graph) + tmpPath := filepath.Join(target.TmpDir(), outputs[0]) f, err := os.Create(tmpPath) if err != nil { return err diff --git a/src/build/build_step_stress_test.go b/src/build/build_step_stress_test.go index 6a4830e802..045db96d91 100644 --- a/src/build/build_step_stress_test.go +++ b/src/build/build_step_stress_test.go @@ -33,10 +33,10 @@ func TestBuildLotsOfTargets(t *testing.T) { pkg := core.NewPackage("pkg") state.Graph.AddPackage(pkg) + targets := []core.BuildLabel{} for i := 1; i <= size; i++ { - addTarget(state, i) + targets = append(targets, addTarget(state, i).Label) } - state.TaskDone() // Initial target adding counts as one. results := state.Results() // Consume and discard any results @@ -47,14 +47,13 @@ func TestBuildLotsOfTargets(t *testing.T) { } }() - plz.RunHost(nil, state) + plz.RunHost(targets, state) } func addTarget(state *core.BuildState, i int) *core.BuildTarget { // Create and add a new target, with a parent and a dependency. target := core.NewBuildTarget(label(i)) target.IsFilegroup = true // Will mean it doesn't have to shell out to anything. - target.SetState(core.Active) target.Test = new(core.TestFields) state.Graph.AddTarget(target) if i <= size { @@ -68,9 +67,6 @@ func addTarget(state *core.BuildState, i int) *core.BuildTarget { log.Info("Adding dependency %s -> %s", target.Label, dep) target.AddDependency(dep) } - } else { - // These are buildable now - state.QueueTarget(target.Label, core.OriginalTarget, false, core.ParseModeNormal) } } return target @@ -101,12 +97,8 @@ type fakeParser struct { PostBuildFunctions buildFunctionMap } -func (fake *fakeParser) RegisterPreload(core.BuildLabel) error { - return nil -} - // ParseFile stub -func (fake *fakeParser) ParseFile(pkg *core.Package, label, dependent *core.BuildLabel, mode core.ParseMode, fs iofs.FS, filename string) error { +func (fake *fakeParser) ParseFile(pkg *core.Package, label, dependent *core.BuildLabel, fs iofs.FS, filename string) error { return nil } @@ -125,7 +117,7 @@ func (fake *fakeParser) Init(state *core.BuildState) { } // ParseReader stub -func (fake *fakeParser) ParseReader(pkg *core.Package, r io.ReadSeeker, label, dependent *core.BuildLabel, mode core.ParseMode) error { +func (fake *fakeParser) ParseReader(pkg *core.Package, r io.ReadSeeker, label, dependent *core.BuildLabel) error { return nil } diff --git a/src/build/build_step_test.go b/src/build/build_step_test.go index 0e4642ce83..ba0a4a36c4 100644 --- a/src/build/build_step_test.go +++ b/src/build/build_step_test.go @@ -114,7 +114,7 @@ func TestPostBuildFunction(t *testing.T) { err := buildTarget(state, target, false) assert.NoError(t, err) assert.Equal(t, core.Built, target.State()) - assert.Equal(t, []string{"file7"}, target.Outputs()) + assert.Equal(t, []string{"file7"}, target.Outputs(state.Graph)) } func TestOutputDir(t *testing.T) { @@ -131,7 +131,7 @@ func TestOutputDir(t *testing.T) { err := buildTarget(state, target, false) require.NoError(t, err) - assert.Equal(t, []string{"file7"}, target.Outputs()) + assert.Equal(t, []string{"file7"}, target.Outputs(state.Graph)) md, err := loadTargetMetadata(target) require.NoError(t, err) @@ -143,7 +143,7 @@ func TestOutputDir(t *testing.T) { state, target = newTarget() err = buildTarget(state, target, false) require.NoError(t, err) - assert.Equal(t, []string{"file7"}, target.Outputs()) + assert.Equal(t, []string{"file7"}, target.Outputs(state.Graph)) assert.Equal(t, core.Reused, target.State()) } @@ -166,7 +166,7 @@ func TestOutputDirDoubleStar(t *testing.T) { err := buildTarget(state, target, false) require.NoError(t, err) - assert.Equal(t, []string{"foo"}, target.Outputs()) + assert.Equal(t, []string{"foo"}, target.Outputs(state.Graph)) md, err := loadTargetMetadata(target) require.NoError(t, err) @@ -182,7 +182,7 @@ func TestOutputDirDoubleStar(t *testing.T) { err = buildTarget(state, target, false) require.NoError(t, err) - assert.Equal(t, []string{"foo/file7"}, target.Outputs()) + assert.Equal(t, []string{"foo/file7"}, target.Outputs(state.Graph)) info, err = os.Lstat(filepath.Join(target.OutDir(), "foo/file7")) require.NoError(t, err) @@ -609,12 +609,8 @@ func (*mockCache) Shutdown() {} type fakeParser struct { } -func (fake *fakeParser) RegisterPreload(core.BuildLabel) error { - return nil -} - // ParseFile stub -func (fake *fakeParser) ParseFile(pkg *core.Package, label, dependent *core.BuildLabel, mode core.ParseMode, fs iofs.FS, filename string) error { +func (fake *fakeParser) ParseFile(pkg *core.Package, label, dependent *core.BuildLabel, fs iofs.FS, filename string) error { return nil } @@ -632,7 +628,7 @@ func (fake *fakeParser) NewParser(state *core.BuildState) { } // ParseReader stub -func (fake *fakeParser) ParseReader(pkg *core.Package, r io.ReadSeeker, label, dependent *core.BuildLabel, mode core.ParseMode) error { +func (fake *fakeParser) ParseReader(pkg *core.Package, r io.ReadSeeker, label, dependent *core.BuildLabel) error { return nil } diff --git a/src/build/incrementality.go b/src/build/incrementality.go index a98501a854..f7ba7ca4f7 100644 --- a/src/build/incrementality.go +++ b/src/build/incrementality.go @@ -81,7 +81,7 @@ func needsBuilding(state *core.BuildState, target *core.BuildTarget, postBuild b // Check the outputs of this rule exist. This would only happen if the user had // removed them but it's incredibly aggravating if you remove an output and the // rule won't rebuild itself. - for _, output := range target.Outputs() { + for _, output := range target.Outputs(state.Graph) { realOutput := filepath.Join(target.OutDir(), output) if !core.PathExists(realOutput) { log.Debug("Output %s doesn't exist for rule %s; will rebuild.", realOutput, target.Label) @@ -150,7 +150,14 @@ func RuleHash(state *core.BuildState, target *core.BuildTarget, runtime, postBui func ruleHash(state *core.BuildState, target *core.BuildTarget, runtime bool) []byte { h := sha1.New() h.Write([]byte(target.Label.String())) - for _, dep := range target.DeclaredDependencies() { + // Sort here so the hash is independent of the order deps were declared in the BUILD file; + // DeclaredDependencies yields them in declaration order. + var deps core.BuildLabels + for dep := range target.DeclaredDependencies() { + deps = append(deps, dep) + } + sort.Sort(deps) + for _, dep := range deps { h.Write([]byte(dep.String())) } for _, vis := range target.Visibility { @@ -293,7 +300,7 @@ type ruleHashes struct { // If postBuild is true then the rule hash will be the post-build one if present. func readRuleHashFromXattrs(state *core.BuildState, target *core.BuildTarget, postBuild bool) ruleHashes { var h []byte - for _, output := range target.FullOutputs() { + for _, output := range target.FullOutputs(state.Graph) { b := fs.ReadAttr(output, xattrName, state.XattrsSupported) if b == nil { return ruleHashes{} @@ -348,7 +355,7 @@ func writeRuleHash(state *core.BuildState, target *core.BuildTarget) error { return err } hash = append(hash, secretHash...) - outputs := target.FullOutputs() + outputs := target.FullOutputs(state.Graph) if len(outputs) == 0 { // Target has no outputs, have to use the fallback file. return fs.RecordAttrFile(filepath.Join(target.OutDir(), target.Label.Name), hash) diff --git a/src/build/incrementality_test.go b/src/build/incrementality_test.go index e74450413f..d1f222569f 100644 --- a/src/build/incrementality_test.go +++ b/src/build/incrementality_test.go @@ -111,6 +111,7 @@ var KnownFields = map[string]bool{ "mutex": true, "dependenciesRegistered": true, "finishedBuilding": true, + "ModifiedByCallback": true, // Used to save the rule hash rather than actually being hashed itself. "RuleHash": true, diff --git a/src/cache/async_cache_test.go b/src/cache/async_cache_test.go index f24d72966c..b1c0e3a7a8 100644 --- a/src/cache/async_cache_test.go +++ b/src/cache/async_cache_test.go @@ -14,7 +14,7 @@ import ( func TestStore(t *testing.T) { mCache, aCache := makeCaches() target := makeTarget1("//pkg1:test_store") - aCache.Store(target, nil, target.Outputs()) + aCache.Store(target, nil, target.Outputs(nil)) aCache.Shutdown() assert.False(t, mCache.inFlight[target]) assert.True(t, mCache.completed[target]) @@ -23,7 +23,7 @@ func TestStore(t *testing.T) { func TestRetrieve(t *testing.T) { mCache, aCache := makeCaches() target := makeTarget1("//pkg1:test_retrieve") - aCache.Retrieve(target, nil, target.Outputs()) + aCache.Retrieve(target, nil, target.Outputs(nil)) aCache.Shutdown() assert.False(t, mCache.inFlight[target]) assert.True(t, mCache.completed[target]) diff --git a/src/cache/cmd_cache_test.go b/src/cache/cmd_cache_test.go index 74b7913d52..57b39fd273 100644 --- a/src/cache/cmd_cache_test.go +++ b/src/cache/cmd_cache_test.go @@ -71,7 +71,7 @@ func TestCmdStoreInvalidCommand(t *testing.T) { // cache interface does not provide any result here... // but we should at least not panic or that alike - cache.Store(target, key, target.Outputs()) + cache.Store(target, key, target.Outputs(nil)) } func TestCmdStoreAndRetrieve(t *testing.T) { @@ -84,7 +84,7 @@ func TestCmdStoreAndRetrieve(t *testing.T) { key := []byte("TestCmdStoreAndRetrieve") os.Chdir("src/cache/test_data") - cache.Store(target, key, target.Outputs()) + cache.Store(target, key, target.Outputs(nil)) b, err := os.ReadFile("plz-out/gen/pkg/name/testfile2") assert.NoError(t, err) @@ -107,7 +107,7 @@ func TestCmdStoreAndRetrieveExitCode(t *testing.T) { key := []byte("TestCmdStoreAndRetrieveExitCode") os.Chdir("src/cache/test_data") - cache.Store(target, key, target.Outputs()) + cache.Store(target, key, target.Outputs(nil)) hit := cache.Retrieve(target, key, nil) // expected to fail because of "exit 1" diff --git a/src/cache/dir_cache_test.go b/src/cache/dir_cache_test.go index 092f55b849..c3c39df98f 100644 --- a/src/cache/dir_cache_test.go +++ b/src/cache/dir_cache_test.go @@ -29,7 +29,7 @@ func cachePath(target *core.BuildTarget, compress bool) string { if compress { return filepath.Join(".plz-cache-"+target.Label.PackageName, target.Label.PackageName, target.Label.Name, b64Hash+".tar.gz") } - return filepath.Join(".plz-cache-"+target.Label.PackageName, target.Label.PackageName, target.Label.Name, b64Hash, target.Outputs()[0]) + return filepath.Join(".plz-cache-"+target.Label.PackageName, target.Label.PackageName, target.Label.Name, b64Hash, target.Outputs(nil)[0]) } func inCache(target *core.BuildTarget) bool { @@ -47,23 +47,23 @@ func inCompressedCache(target *core.BuildTarget) bool { func TestStoreAndRetrieve(t *testing.T) { cache := makeCache(".plz-cache-test1", false) target := makeTarget2("//test1:target1", 20) - cache.Store(target, hash, target.Outputs()) + cache.Store(target, hash, target.Outputs(nil)) // Should now exist in cache at this path assert.True(t, inCache(target)) - assert.NotNil(t, cache.Retrieve(target, hash, target.Outputs())) + assert.NotNil(t, cache.Retrieve(target, hash, target.Outputs(nil))) // Should be able to store it again without problems - cache.Store(target, hash, target.Outputs()) + cache.Store(target, hash, target.Outputs(nil)) assert.True(t, inCache(target)) - assert.NotNil(t, cache.Retrieve(target, hash, target.Outputs())) + assert.NotNil(t, cache.Retrieve(target, hash, target.Outputs(nil))) } func TestCleanNoop(t *testing.T) { cache := makeCache(".plz-cache-test2", false) target1 := makeTarget2("//test2:target1", 2000) - cache.Store(target1, hash, target1.Outputs()) + cache.Store(target1, hash, target1.Outputs(nil)) assert.True(t, inCache(target1)) target2 := makeTarget2("//test2:target2", 2000) - cache.Store(target2, hash, target2.Outputs()) + cache.Store(target2, hash, target2.Outputs(nil)) assert.True(t, inCache(target2)) // Doesn't clean anything this time because the high water mark is sufficiently high totalSize := cache.clean(20000, 1000) @@ -75,10 +75,10 @@ func TestCleanNoop(t *testing.T) { func TestCleanNoop2(t *testing.T) { cache := makeCache(".plz-cache-test3", false) target1 := makeTarget2("//test3:target1", 2000) - cache.Store(target1, hash, target1.Outputs()) + cache.Store(target1, hash, target1.Outputs(nil)) assert.True(t, inCache(target1)) target2 := makeTarget2("//test3:target2", 2000) - cache.Store(target2, hash, target2.Outputs()) + cache.Store(target2, hash, target2.Outputs(nil)) assert.True(t, inCache(target2)) // Doesn't clean anything this time, the high water mark is lower but both targets have // just been built. @@ -91,7 +91,7 @@ func TestCleanNoop2(t *testing.T) { func TestCleanForReal(t *testing.T) { cache := makeCache(".plz-cache-test4", false) target1 := makeTarget2("//test4:target1", 2000) - cache.Store(target1, hash, target1.Outputs()) + cache.Store(target1, hash, target1.Outputs(nil)) assert.True(t, inCache(target1)) target2 := makeTarget2("//test4:target2", 2000) writeFile(cachePath(target2, false), 2000) @@ -109,7 +109,7 @@ func TestCleanForReal2(t *testing.T) { writeFile(cachePath(target1, false), 2000) assert.True(t, inCache(target1)) target2 := makeTarget2("//test5:target2", 2000) - cache.Store(target2, hash, target2.Outputs()) + cache.Store(target2, hash, target2.Outputs(nil)) assert.True(t, inCache(target2)) // This time it should clean target1, because target2 has just been stored totalSize := cache.clean(10000, 1000) @@ -121,14 +121,14 @@ func TestCleanForReal2(t *testing.T) { func TestStoreAndRetrieveCompressed(t *testing.T) { cache := makeCache(".plz-cache-test6", true) target := makeTarget2("//test6:target6", 20) - cache.Store(target, hash, target.Outputs()) + cache.Store(target, hash, target.Outputs(nil)) // Should now exist in cache at this path assert.True(t, inCompressedCache(target)) - assert.NotNil(t, cache.Retrieve(target, hash, target.Outputs())) + assert.NotNil(t, cache.Retrieve(target, hash, target.Outputs(nil))) // Should be able to store it again without problems - cache.Store(target, hash, target.Outputs()) + cache.Store(target, hash, target.Outputs(nil)) assert.True(t, inCompressedCache(target)) - assert.NotNil(t, cache.Retrieve(target, hash, target.Outputs())) + assert.NotNil(t, cache.Retrieve(target, hash, target.Outputs(nil))) } func TestCleanCompressed(t *testing.T) { @@ -137,7 +137,7 @@ func TestCleanCompressed(t *testing.T) { writeFile(cachePath(target1, true), 2000) assert.True(t, inCompressedCache(target1)) target2 := makeTarget2("//test7:target2", 2000) - cache.Store(target2, hash, target2.Outputs()) + cache.Store(target2, hash, target2.Outputs(nil)) assert.True(t, inCompressedCache(target2)) // Don't want to assert the size here since it depends on how well gzip compresses. // It's a bit hard to know exactly what the sizes here should be too but we'll guess diff --git a/src/cache/http_cache_test.go b/src/cache/http_cache_test.go index 9c25ac414a..8fbe02329d 100644 --- a/src/cache/http_cache_test.go +++ b/src/cache/http_cache_test.go @@ -35,7 +35,7 @@ func TestStoreAndRetrieveHTTP(t *testing.T) { cache := newHTTPCache(config) key := []byte("test_key") - cache.Store(target, key, target.Outputs()) + cache.Store(target, key, target.Outputs(nil)) b, err := os.ReadFile("plz-out/gen/pkg/name/testfile2") assert.NoError(t, err) diff --git a/src/clean/clean.go b/src/clean/clean.go index 79b09d7f6d..58b027371b 100644 --- a/src/clean/clean.go +++ b/src/clean/clean.go @@ -51,7 +51,7 @@ func Targets(state *core.BuildState, labels []core.BuildLabel) { } func cleanTarget(state *core.BuildState, target *core.BuildTarget) { - if err := build.RemoveOutputs(target); err != nil { + if err := build.RemoveOutputs(state, target); err != nil { log.Fatalf("Failed to remove output: %s", err) } if target.IsTest() { diff --git a/src/cmap/cerrmap.go b/src/cmap/cerrmap.go index 78c7dc8ed6..bccde4a9a7 100644 --- a/src/cmap/cerrmap.go +++ b/src/cmap/cerrmap.go @@ -1,5 +1,9 @@ package cmap +import ( + "context" +) + // A Limiter is the interface that we use to release/acquire workers while waiting. type Limiter interface { Acquire() @@ -32,13 +36,6 @@ func (m *ErrMap[K, V]) Add(key K, val V) bool { return m.m.Add(key, errV[V]{Val: val}) } -// AddOrGet either adds a new item (if the key doesn't exist) or gets the existing one. -// It returns true if the item was inserted, false if it already existed (in which case it won't be inserted) -func (m *ErrMap[K, V]) AddOrGet(key K, f func() V) (V, bool, error) { - v, present := m.m.AddOrGet(key, func() errV[V] { return errV[V]{Val: f()} }) - return v.Val, present, v.Err -} - // Set is the equivalent of `map[key] = val`. // It always overwrites any key that existed before. func (m *ErrMap[K, V]) Set(key K, val V) { @@ -60,7 +57,7 @@ func (m *ErrMap[K, V]) Get(key K) (V, error) { // GetOrSet returns the value if set, or an error if one has been set. // If nothing has been set for the key, it runs the given function to generate the value and then sets it. func (m *ErrMap[K, V]) GetOrSet(key K, f func() (V, error)) (V, error) { - v, wait, first := m.m.GetOrWait(key) + v, wait, first := m.m.getOrWait(key) if v.Err != nil { return v.Val, v.Err } else if first { @@ -79,6 +76,32 @@ func (m *ErrMap[K, V]) GetOrSet(key K, f func() (V, error)) (V, error) { return v.Val, v.Err } +// GetOrSetCtx is like GetOrSet but accepts a context that can be cancelled. +func (m *ErrMap[K, V]) GetOrSetCtx(ctx context.Context, key K, f func() (V, error)) (V, error) { + v, wait, first := m.m.getOrWait(key) + if v.Err != nil { + return v.Val, v.Err + } else if first { + val, err := f() + m.m.Set(key, errV[V]{Val: val, Err: err}) + return val, err + } else if wait != nil { + if m.l != nil { + // Release the limiter for the duration we're waiting + m.l.Release() + defer m.l.Acquire() + } + select { + case <-wait: + return m.Get(key) + case <-ctx.Done(): + var v V + return v, ctx.Err() + } + } + return v.Val, v.Err +} + // Range calls f for each key-value pair in the map. // No particular consistency guarantees are made during iteration. func (m *ErrMap[K, V]) Range(f func(key K, val V)) { diff --git a/src/cmap/cmap.go b/src/cmap/cmap.go index f8058ef732..31ad6504b8 100644 --- a/src/cmap/cmap.go +++ b/src/cmap/cmap.go @@ -53,12 +53,6 @@ func (m *Map[K, V]) Add(key K, val V) bool { return m.shards[m.hasher(key)&m.mask].Set(key, val, false) } -// AddOrGet either adds a new item (if the key doesn't exist, calling the given function to create it) or gets the existing one. -// It returns true if the item was inserted, false if it already existed (in which case it won't be inserted) -func (m *Map[K, V]) AddOrGet(key K, f func() V) (V, bool) { - return m.shards[m.hasher(key)&m.mask].LazySet(key, f) -} - // Set is the equivalent of `map[key] = val`. // It always overwrites any key that existed before. func (m *Map[K, V]) Set(key K, val V) { @@ -67,21 +61,15 @@ func (m *Map[K, V]) Set(key K, val V) { // Get returns the value corresponding to the given key, or its zero value if the key doesn't exist in the map. func (m *Map[K, V]) Get(key K) V { - v, _, _ := m.shards[m.hasher(key)&m.mask].Get(key) - return v + return m.shards[m.hasher(key)&m.mask].Get(key) } func (m *Map[K, V]) Contains(key K) bool { return m.shards[m.hasher(key)&m.mask].Contains(key) } -// GetOrWait returns the value or, if the key isn't present, a channel that it can be waited -// on for. The caller will need to call Get again after the channel closes. -// If the channel is non-nil, then val will exist in the map; otherwise it will have its zero value. -// The third return value is true if this is the first call that is awaiting this key. -// It's always false if the key exists. -func (m *Map[K, V]) GetOrWait(key K) (val V, wait <-chan struct{}, first bool) { - return m.shards[m.hasher(key)&m.mask].Get(key) +func (m *Map[K, V]) getOrWait(key K) (val V, wait <-chan struct{}, first bool) { + return m.shards[m.hasher(key)&m.mask].GetOrWait(key) } // Values returns a slice of all the current values in the map. @@ -138,32 +126,19 @@ func (s *shard[K, V]) Set(key K, val V, overwrite bool) bool { return true } -// LazySet is like Set but calls the given function to construct the object only if needed. -// It also returns the value that is now set in the map (whether overwritten or not). -func (s *shard[K, V]) LazySet(key K, f func() V) (V, bool) { - s.l.Lock() - defer s.l.Unlock() - if existing, present := s.m[key]; present { - if existing.Wait == nil { - return existing.Val, false // already added - } - // Hasn't been added, but something is waiting for it to be. - v := f() - s.m[key] = awaitableValue[V]{Val: v} - close(existing.Wait) - existing.Wait = nil - return v, true - } - v := f() - s.m[key] = awaitableValue[V]{Val: v} - return v, true +// get returns the value for a key, or its zero value if it isn't present. +// Unlike Get it never inserts anything, so it's safe for callers that only want to read. +func (s *shard[K, V]) Get(key K) V { + s.l.RLock() + defer s.l.RUnlock() + return s.m[key].Val } // Get returns the value for a key or, if not present, a channel that it can be waited // on for. // Exactly one of the target or channel will be returned. // The third value is true if it is the first call that is waiting on this value. -func (s *shard[K, V]) Get(key K) (val V, wait <-chan struct{}, first bool) { +func (s *shard[K, V]) GetOrWait(key K) (val V, wait <-chan struct{}, first bool) { s.l.RLock() if v, ok := s.m[key]; ok { s.l.RUnlock() diff --git a/src/cmap/cmap_test.go b/src/cmap/cmap_test.go index 2c4394f8a8..f06b26341b 100644 --- a/src/cmap/cmap_test.go +++ b/src/cmap/cmap_test.go @@ -26,47 +26,38 @@ func TestMap(t *testing.T) { assert.Equal(t, []int{5, 7}, vals) } +// TestWait covers the awaiting primitive directly; it's only reachable through ErrMap now, +// but it's the bit with the interesting concurrency so it's worth pinning down here. func TestWait(t *testing.T) { m := New[int, int](DefaultShardCount, hashInts) - v, ch, first := m.GetOrWait(5) + v, ch, first := m.getOrWait(5) assert.Equal(t, 0, v) // Should be the zero value assert.True(t, first) // We're the first to request it go func() { m.Set(5, 7) }() <-ch - v, ch, first = m.GetOrWait(5) + v, ch, first = m.getOrWait(5) assert.Nil(t, ch) assert.Equal(t, 7, v) assert.False(t, first) } +func TestGetDoesntInsert(t *testing.T) { + m := New[int, int](DefaultShardCount, hashInts) + assert.Equal(t, 0, m.Get(5)) + // A failed lookup must not leave an entry behind; anything that later tries to set this key + // would find something already waiting on it and never get to do the work. + assert.False(t, m.Contains(5)) +} + func TestReAdd(t *testing.T) { m := New[int, int](DefaultShardCount, hashInts) assert.True(t, m.Add(5, 7)) assert.False(t, m.Add(5, 7)) - v, ch, first := m.GetOrWait(5) - assert.Nil(t, ch) - assert.Equal(t, 7, v) - assert.False(t, first) + assert.Equal(t, 7, m.Get(5)) m.Set(5, 8) - v, ch, first = m.GetOrWait(5) - assert.Nil(t, ch) - assert.Equal(t, 8, v) - assert.False(t, first) -} - -func TestAddOrGet(t *testing.T) { - m := New[int, int](DefaultShardCount, hashInts) - x, inserted := m.AddOrGet(5, func() int { return 7 }) - assert.True(t, inserted) - assert.Equal(t, 7, x) - x, inserted = m.AddOrGet(5, func() int { return 8 }) - assert.False(t, inserted) - assert.Equal(t, 7, x) - x, inserted = m.AddOrGet(8, func() int { return 9 }) - assert.True(t, inserted) - assert.Equal(t, 9, x) + assert.Equal(t, 8, m.Get(5)) } func TestShardCount(t *testing.T) { @@ -91,10 +82,7 @@ func TestResize(t *testing.T) { m.Set(i, i) } for i := 0; i < n; i++ { - v, ch, first := m.GetOrWait(i) - assert.Equal(t, i, v, "Key %d appears to be not set or set incorrectly", i) - assert.Nil(t, ch) - assert.False(t, first) + assert.Equal(t, i, m.Get(i), "Key %d appears to be not set or set incorrectly", i) } }) } diff --git a/src/core/build_env.go b/src/core/build_env.go index b3c0bcfbb8..ff4a7fd288 100644 --- a/src/core/build_env.go +++ b/src/core/build_env.go @@ -73,7 +73,7 @@ func TargetEnvironment(state *BuildState, target *BuildTarget) BuildEnv { func BuildEnvironment(state *BuildState, target *BuildTarget, tmpDir string) BuildEnv { env := TargetEnvironment(state, target) sources := target.AllSourcePaths(state.Graph) - outEnv := target.GetTmpOutputAll(target.Outputs()) + outEnv := target.GetTmpOutputAll(target.Outputs(state.Graph)) abs := filepath.IsAbs(tmpDir) env["TMP_DIR"] = tmpDir @@ -174,8 +174,8 @@ func TestEnvironment(state *BuildState, target *BuildTarget, testDir string, run env["COVERAGE"] = "true" env["COVERAGE_FILE"] = filepath.Join(testDir, CoverageFile) } - if len(target.Outputs()) > 0 { - env["TEST"] = resolveOut(target.Outputs()[0], testDir, target.Test.Sandbox) + if outputs := target.Outputs(state.Graph); len(outputs) > 0 { + env["TEST"] = resolveOut(outputs[0], testDir, target.Test.Sandbox) } // Bit of a hack for gcov which needs access to its .gcno files. if target.HasLabel("cc") { @@ -197,7 +197,7 @@ func TestEnvironment(state *BuildState, target *BuildTarget, testDir string, run func RunEnvironment(state *BuildState, target *BuildTarget, inTmpDir bool) BuildEnv { env := RuntimeEnvironment(state, target, true, inTmpDir) - outEnv := target.Outputs() + outEnv := target.Outputs(state.Graph) env["OUTS"] = strings.Join(outEnv, " ") // The OUT variable is only available on rules that have a single output. if len(outEnv) == 1 { @@ -217,7 +217,7 @@ func ExecEnvironment(state *BuildState, target *BuildTarget, execDir string) Bui // of input and output in the terminal where the program is run. env["TERM"] = os.Getenv("TERM") - outEnv := target.Outputs() + outEnv := target.Outputs(state.Graph) // OUTS/OUT environment variables being always set is for backwards-compatibility. // Ideally, if the target is a test these variables shouldn't be set. env["OUTS"] = strings.Join(outEnv, " ") @@ -349,7 +349,7 @@ func toolPath(state *BuildState, tool BuildInput, abs bool) string { if o, ok := tool.(AnnotatedOutputLabel); ok { entryPoint = o.Annotation } - path := state.Graph.TargetOrDie(label).toolPath(abs, entryPoint) + path := state.Graph.TargetOrDie(label).toolPath(state.Graph, abs, entryPoint) if !strings.Contains(path, "/") { path = "./" + path } diff --git a/src/core/build_input.go b/src/core/build_input.go index 010e047b08..2500524c83 100644 --- a/src/core/build_input.go +++ b/src/core/build_input.go @@ -241,7 +241,7 @@ func (label AnnotatedOutputLabel) Paths(graph *BuildGraph) []string { return label.BuildLabel.Paths(graph) } - return addPathPrefix(target.NamedOutputs(label.Annotation), target.PackageDir()) + return addPathPrefix(target.NamedOutputs(graph, label.Annotation), target.PackageDir()) } // FullPaths is like Paths but includes the leading plz-out/gen directory. @@ -250,7 +250,7 @@ func (label AnnotatedOutputLabel) FullPaths(graph *BuildGraph) []string { if _, ok := target.EntryPoints[label.Annotation]; ok { return label.BuildLabel.FullPaths(graph) } - return addPathPrefix(target.NamedOutputs(label.Annotation), target.OutDir()) + return addPathPrefix(target.NamedOutputs(graph, label.Annotation), target.OutDir()) } // LocalPaths returns paths within the local package @@ -259,7 +259,7 @@ func (label AnnotatedOutputLabel) LocalPaths(graph *BuildGraph) []string { if _, ok := target.EntryPoints[label.Annotation]; ok { return label.BuildLabel.LocalPaths(graph) } - return target.NamedOutputs(label.Annotation) + return target.NamedOutputs(graph, label.Annotation) } // Label returns the build rule associated with this input. For a AnnotatedOutputLabel it's always non-nil. diff --git a/src/core/build_label.go b/src/core/build_label.go index 779e7302e0..d0c51d3d0a 100644 --- a/src/core/build_label.go +++ b/src/core/build_label.go @@ -342,13 +342,13 @@ func (label BuildLabel) Less(other BuildLabel) bool { // Paths is an implementation of BuildInput interface; we use build labels directly as inputs. func (label BuildLabel) Paths(graph *BuildGraph) []string { target := graph.TargetOrDie(label) - return addPathPrefix(target.Outputs(), target.PackageDir()) + return addPathPrefix(target.Outputs(graph), target.PackageDir()) } // FullPaths is an implementation of BuildInput interface. func (label BuildLabel) FullPaths(graph *BuildGraph) []string { target := graph.TargetOrDie(label) - return addPathPrefix(target.Outputs(), target.OutDir()) + return addPathPrefix(target.Outputs(graph), target.OutDir()) } // addPathPrefix adds a prefix to all the entries in a slice. @@ -362,7 +362,7 @@ func addPathPrefix(paths []string, prefix string) []string { // LocalPaths is an implementation of BuildInput interface. func (label BuildLabel) LocalPaths(graph *BuildGraph) []string { - return graph.TargetOrDie(label).Outputs() + return graph.TargetOrDie(label).Outputs(graph) } // Label is an implementation of BuildInput interface. It always returns this label. diff --git a/src/core/build_target.go b/src/core/build_target.go index 6fe7c01d0d..7d5e1b5f7b 100644 --- a/src/core/build_target.go +++ b/src/core/build_target.go @@ -13,8 +13,6 @@ import ( "sync/atomic" "time" - "golang.org/x/sync/errgroup" - "github.com/thought-machine/please/src/fs" ) @@ -116,7 +114,7 @@ type BuildTarget struct { // Dependencies of this target. // Maps the original declaration to whatever dependencies actually got attached, // which may be more than one in some cases. Also contains info about exporting etc. - dependencies []depInfo `name:"deps"` + dependencies []DeclaredDependency `name:"deps"` // The run-time dependencies of this target. runtimeDependencies []BuildLabel `name:"runtime_deps"` // Whether to consider the run-time dependencies of this target's sources to be additional @@ -207,17 +205,12 @@ type BuildTarget struct { EntryPoints map[string]string `name:"entry_points"` // Used to arbitrate concurrent access to dependencies, and to the test results. mutex sync.RWMutex `print:"false"` - // Used to notify once this target has built successfully. - finishedBuilding chan struct{} `print:"false"` // Env are any custom environment variables to set for this build target Env map[string]string `name:"env"` // The content of text_file() rules FileContent string `name:"content"` // Represents the state of this build target (see below) - state int32 `print:"false"` - // If true, the target is needed for a subinclude and therefore we will have to make sure its - // outputs are available locally when built. - neededForSubinclude atomic.Bool `print:"false"` + state atomic.Int32 `print:"false"` // The number of completed runs completedRuns uint16 `print:"false"` // True if this target is a binary (ie. runnable, will appear in plz-out/bin) @@ -254,6 +247,8 @@ type BuildTarget struct { IsTextFile bool `print:"false"` // Marks that the target was added in a post-build function. AddedPostBuild bool `print:"false"` + // Marks that this target was modified by a pre or post build function + ModifiedByCallback bool `print:"false"` // If true, skips generating environment variables for sources; instead files will be generated in // the build environment containing the lists of sources as follows: // - _plz/srcs (equivalent to $SRCS) always @@ -312,15 +307,14 @@ type PostBuildFunction interface { Call(target *BuildTarget, output string) error } -type depInfo struct { - declared *BuildLabel // the originally declared dependency - deps []*BuildTarget // list of actual deps - resolved bool // has the graph resolved it - exported bool // is it an exported dependency - internal bool // is it an internal dependency (that is not picked up implicitly by transitive searches) - runtime bool // is it a run-time (and therefore implicitly transitive) dependency - source bool // is it implicit because it's a source (not true if it's a dependency too) - data bool // is it a data item for a test +// A DeclaredDependency represents a dependency declared by a target. +type DeclaredDependency struct { + Label BuildLabel // the originally declared dependency + Exported bool // is it an exported dependency + Internal bool // is it an internal dependency (that is not picked up implicitly by transitive searches) + Runtime bool // is it a run-time (and therefore implicitly transitive) dependency + Source bool // is it implicit because it's a source (not true if it's a dependency too) + Data bool // is it a data item for a test } // OutputDirectory is an output directory for the build rule. It may have a suffix of /** which means that we should @@ -347,9 +341,6 @@ type BuildTargetState uint8 // The available states for a target. const ( Inactive BuildTargetState = iota // Target isn't used in current build - Semiactive // Target would be active if we needed a build - Active // Target is going to be used in current build - Pending // Target is ready to be built but not yet started. Building // Target is currently being built Stopped // We stopped building the target because we'd gone as far as needed. Built // Target has been successfully built @@ -367,12 +358,6 @@ func (s BuildTargetState) String() string { switch s { case Inactive: return "Inactive" - case Semiactive: - return "Semiactive" - case Active: - return "Active" - case Pending: - return "Pending" case Building: return "Building" case Stopped: @@ -406,9 +391,7 @@ func (s BuildTargetState) IsBuilt() bool { func NewBuildTarget(label BuildLabel) *BuildTarget { return &BuildTarget{ Label: label, - state: int32(Inactive), BuildingDescription: DefaultBuildingDescription, - finishedBuilding: make(chan struct{}), } } @@ -558,293 +541,268 @@ func (target *BuildTarget) AllURLs(state *BuildState) []string { return ret } -// resolveDependencies matches up all declared dependencies to the actual build targets. -// TODO(peterebden,tatskaari): Work out if we really want to have this and how the suite of *Dependencies functions -// -// below should behave (preferably nicely). -func (target *BuildTarget) resolveDependencies(graph *BuildGraph, callback func(*BuildTarget) error) error { - var g errgroup.Group - target.mutex.RLock() - for i := range target.dependencies { - dep := &target.dependencies[i] // avoid using a loop variable here as it mutates each iteration - if len(dep.deps) > 0 { - continue // already done - } - g.Go(func() error { - if err := target.resolveOneDependency(graph, dep); err != nil { - return err - } - for _, d := range dep.deps { - if err := callback(d); err != nil { - return err - } +// DeclaredDependencies returns all the targets this target declared any kind of dependency on (including sources and tools). +func (target *BuildTarget) DeclaredDependencies() iter.Seq[BuildLabel] { + return func(yield func(BuildLabel) bool) { + target.mutex.RLock() + defer target.mutex.RUnlock() + for _, dep := range target.dependencies { + if !yield(dep.Label) { + break } - return nil - }) + } } - target.mutex.RUnlock() - return g.Wait() } -func (target *BuildTarget) resolveOneDependency(graph *BuildGraph, dep *depInfo) error { - depTarget := graph.WaitForTarget(*dep.declared) - if depTarget == nil { - return fmt.Errorf("Couldn't find dependency %s", dep.declared) - } - dep.declared = &depTarget.Label // saves memory by not storing the label twice once resolved - - providesLabels, ok := depTarget.provideFor(target) - if !ok { - target.mutex.Lock() - defer target.mutex.Unlock() - - // Small optimisation to avoid re-looking-up the same target again. - dep.deps = []*BuildTarget{depTarget} - return nil - } - - deps := make([]*BuildTarget, 0, len(providesLabels)) - for _, l := range providesLabels { - providesTarget := graph.WaitForTarget(l) - if providesTarget == nil { - return fmt.Errorf("%s depends on %s (provided by %s), however that target doesn't exist", target, l, depTarget) +// DeclaredDependenciesStrict returns the original declaration of this target's dependencies. +func (target *BuildTarget) DeclaredDependenciesStrict() iter.Seq[BuildLabel] { + return func(yield func(BuildLabel) bool) { + target.mutex.RLock() + defer target.mutex.RUnlock() + for _, dep := range target.dependencies { + if !dep.Runtime && !dep.Exported && !dep.Source && !target.IsTool(dep.Label) { + if !yield(dep.Label) { + break + } + } } - deps = append(deps, providesTarget) } - - target.mutex.Lock() - defer target.mutex.Unlock() - - dep.deps = deps - - return nil } -// MustResolveDependencies is exposed only for testing purposes. -// TODO(peterebden, tatskaari): See if we can get rid of this. -func (target *BuildTarget) ResolveDependencies(graph *BuildGraph) error { - return target.resolveDependencies(graph, func(*BuildTarget) error { return nil }) -} - -// DeclaredDependencies returns all the targets this target declared any kind of dependency on (including sources and tools). -func (target *BuildTarget) DeclaredDependencies() []BuildLabel { +// Dependencies returns the resolved dependencies of this target, applying any require/provide +// relationships to map each declared dependency to the target(s) that actually satisfy it. +// It requires the graph to look targets up, since a BuildTarget no longer caches these itself. +// +// The second return is the labels of any dependencies that aren't in the graph. For most callers +// that indicates something has gone wrong and should be reported, but it's a legitimate state for +// anything that runs while the graph is still being built up (e.g. the cycle detector). +func (target *BuildTarget) Dependencies(graph *BuildGraph) ([]*BuildTarget, []BuildLabel) { target.mutex.RLock() - defer target.mutex.RUnlock() - ret := make(BuildLabels, len(target.dependencies)) + labels := make([]BuildLabel, len(target.dependencies)) for i, dep := range target.dependencies { - ret[i] = *dep.declared + labels[i] = dep.Label } - sort.Sort(ret) - return ret -} - -// DeclaredDependenciesStrict returns the original declaration of this target's dependencies. -func (target *BuildTarget) DeclaredDependenciesStrict() []BuildLabel { - target.mutex.RLock() - defer target.mutex.RUnlock() - ret := make(BuildLabels, 0, len(target.dependencies)) - for _, dep := range target.dependencies { - if !dep.runtime && !dep.exported && !dep.source && !target.IsTool(*dep.declared) { - ret = append(ret, *dep.declared) + target.mutex.RUnlock() + ret := make(BuildTargets, 0, len(labels)) + var unresolved []BuildLabel + for _, l := range labels { + depTarget := graph.Target(l) + if depTarget == nil { + unresolved = append(unresolved, l) + continue } - } - sort.Sort(ret) - return ret -} - -// Dependencies returns the resolved dependencies of this target. -func (target *BuildTarget) Dependencies() []*BuildTarget { - target.mutex.RLock() - defer target.mutex.RUnlock() - ret := make(BuildTargets, 0, len(target.dependencies)) - for _, deps := range target.dependencies { - for _, dep := range deps.deps { - ret = append(ret, dep) + for _, provided := range depTarget.ProvideFor(target) { + if t := graph.Target(provided); t != nil { + ret = append(ret, t) + } else { + unresolved = append(unresolved, provided) + } } } sort.Sort(ret) - return ret + return ret, unresolved } -// ExternalDependencies returns the non-internal dependencies of this target (i.e. not "_target#tag" ones). -func (target *BuildTarget) ExternalDependencies() []*BuildTarget { +// ExternalDependencies returns the resolved dependencies of this target, with any internal +// dependencies (i.e. "_target#tag" ones sharing this target's parent) flattened out to the +// external targets they in turn depend on. Require/provide relationships are applied as in Dependencies, +// as is the second return of any dependencies that aren't in the graph. +func (target *BuildTarget) ExternalDependencies(graph *BuildGraph) ([]*BuildTarget, []BuildLabel) { target.mutex.RLock() - defer target.mutex.RUnlock() - ret := make(BuildTargets, 0, len(target.dependencies)) - for _, deps := range target.dependencies { - for _, dep := range deps.deps { - if dep.Label.Parent() != target.Label { + labels := make([]BuildLabel, len(target.dependencies)) + for i, dep := range target.dependencies { + labels[i] = dep.Label + } + target.mutex.RUnlock() + ret := make(BuildTargets, 0, len(labels)) + var unresolved []BuildLabel + for _, l := range labels { + depTarget := graph.Target(l) + if depTarget == nil { + unresolved = append(unresolved, l) + continue + } + for _, provided := range depTarget.ProvideFor(target) { + dep := graph.Target(provided) + if dep == nil { + unresolved = append(unresolved, provided) + } else if dep.Label.Parent() != target.Label { ret = append(ret, dep) } else { - ret = append(ret, dep.ExternalDependencies()...) + deps, u := dep.ExternalDependencies(graph) + ret = append(ret, deps...) + unresolved = append(unresolved, u...) } } } sort.Sort(ret) - return ret + return ret, unresolved } -// BuildDependencies returns the build-time dependencies of this target (i.e. not run-time dependencies, data, internal nor source). -func (target *BuildTarget) BuildDependencies() []*BuildTarget { - target.mutex.RLock() - defer target.mutex.RUnlock() - ret := make(BuildTargets, 0, len(target.dependencies)) - for _, deps := range target.dependencies { - if !deps.runtime && !deps.data && !deps.internal && !deps.source { - for _, dep := range deps.deps { - ret = append(ret, dep) +// BuildDependencies returns the build-time dependency labels of this target (i.e. not run-time dependencies, data, internal nor source). +func (target *BuildTarget) BuildDependencyLabels() iter.Seq[BuildLabel] { + return func(yield func(BuildLabel) bool) { + target.mutex.RLock() + defer target.mutex.RUnlock() + for _, deps := range target.dependencies { + if !deps.Runtime && !deps.Data { + if !yield(deps.Label) { + break + } } } } - sort.Sort(ret) - return ret } // ExportedDependencies returns any exported dependencies of this target. -func (target *BuildTarget) ExportedDependencies() []BuildLabel { - target.mutex.RLock() - defer target.mutex.RUnlock() - ret := make(BuildLabels, 0, len(target.dependencies)) - for _, info := range target.dependencies { - if info.exported { - ret = append(ret, *info.declared) +func (target *BuildTarget) ExportedDependencies() iter.Seq[BuildLabel] { + return func(yield func(BuildLabel) bool) { + target.mutex.RLock() + defer target.mutex.RUnlock() + for _, deps := range target.dependencies { + if deps.Exported { + if !yield(deps.Label) { + break + } + } + } + } +} + +// BuildDependencies returns the build-time dependencies of this target (i.e. not run-time dependencies, data, internal nor source). +func (target *BuildTarget) BuildDependencies() iter.Seq[BuildLabel] { + return func(yield func(BuildLabel) bool) { + target.mutex.RLock() + defer target.mutex.RUnlock() + for _, deps := range target.dependencies { + if !deps.Runtime && !deps.Data && !deps.Internal && !deps.Source { + if !yield(deps.Label) { + break + } + } } } - return ret } // RuntimeDependencies returns any run-time dependencies of this target. // // Although run-time dependencies are transitive, RuntimeDependencies only returns this target's direct run-time // dependencies. Use IterAllRuntimeDependencies to iterate over the target's run-time dependencies transitively. -func (target *BuildTarget) RuntimeDependencies() []BuildLabel { - target.mutex.RLock() - defer target.mutex.RUnlock() - ret := make(BuildLabels, 0, len(target.dependencies)) - for _, deps := range target.dependencies { - if deps.runtime { - ret = append(ret, *deps.declared) +func (target *BuildTarget) RuntimeDependencies() iter.Seq[BuildLabel] { + return func(yield func(BuildLabel) bool) { + target.mutex.RLock() + defer target.mutex.RUnlock() + for _, deps := range target.dependencies { + if deps.Runtime { + if !yield(deps.Label) { + break + } + } } } - return ret } -// IterAllRuntimeDependencies returns an iterator over the transitive run-time dependencies of this target. -// Require/provide relationships between pairs of targets are resolved as they are with build-time dependencies. -func (target *BuildTarget) IterAllRuntimeDependencies(graph *BuildGraph) iter.Seq[BuildLabel] { - var ( - push func(*BuildTarget, func(BuildLabel) bool) bool - done = make(map[string]bool) - ) - push = func(t *BuildTarget, yield func(BuildLabel) bool) bool { - if done[t.String()] { - return true - } - done[t.String()] = true - for _, dep := range t.runtimeDependencies { - depLabel, _ := dep.Label() - for _, providedDep := range graph.TargetOrDie(depLabel).ProvideFor(t) { - if !yield(providedDep) { - return false - } - if !push(graph.TargetOrDie(providedDep), yield) { - return false +// RuntimeAndDataDependencies returns the direct run-time and data dependencies of this target, i.e. everything +// that has to be available when it's run or tested but not when it's built. +// N.B. This is not the same as RuntimeDependencies, which is what the target declared as runtime_deps. +func (target *BuildTarget) RuntimeAndDataDependencies() iter.Seq[BuildLabel] { + return func(yield func(BuildLabel) bool) { + target.mutex.RLock() + defer target.mutex.RUnlock() + for _, deps := range target.dependencies { + if deps.Runtime || deps.Data { + if !yield(deps.Label) { + break } } } - // Include the run-time dependencies of data targets, but not the data targets themselves. (We needn't worry - // about data files here - they can't have run-time dependencies of their own.) - for _, data := range t.AllData() { - dataLabel, ok := data.Label() - if !ok { - continue + } +} + +// IterAllRuntimeDependencies returns an iterator over the transitive run-time dependencies of this target. +// Require/provide relationships between pairs of targets are resolved as they are with build-time dependencies. +func (target *BuildTarget) IterAllRuntimeDependencies(graph *BuildGraph) iter.Seq[BuildLabel] { + return func(yield func(BuildLabel) bool) { + done := map[BuildLabel]bool{} + var push func(*BuildTarget) bool + push = func(t *BuildTarget) bool { + if done[t.Label] { + return true } - for _, providedDep := range graph.TargetOrDie(dataLabel).ProvideFor(t) { - if !push(graph.TargetOrDie(providedDep), yield) { - return false + done[t.Label] = true + for _, dep := range t.runtimeDependencies { + depLabel, _ := dep.Label() + for _, providedDep := range graph.TargetOrDie(depLabel).ProvideFor(t) { + if !yield(providedDep) { + return false + } + if !push(graph.TargetOrDie(providedDep)) { + return false + } } } - } - if t.Debug != nil { - for _, data := range t.AllDebugData() { + // Include the run-time dependencies of data targets, but not the data targets themselves. (We needn't worry + // about data files here - they can't have run-time dependencies of their own.) + for _, data := range t.AllData() { dataLabel, ok := data.Label() if !ok { continue } for _, providedDep := range graph.TargetOrDie(dataLabel).ProvideFor(t) { - if !push(graph.TargetOrDie(providedDep), yield) { + if !push(graph.TargetOrDie(providedDep)) { return false } } } - } - if t.RuntimeDependenciesFromSources || t.RuntimeDependenciesFromDependencies { - for _, dep := range t.dependencies { - // If required, include the run-time dependencies of sources, but not the sources themselves. - if t.RuntimeDependenciesFromSources && dep.source { - depLabel, _ := dep.declared.Label() - for _, providedDep := range graph.TargetOrDie(depLabel).ProvideFor(t) { - if !push(graph.TargetOrDie(providedDep), yield) { + if t.Debug != nil { + for _, data := range t.AllDebugData() { + dataLabel, ok := data.Label() + if !ok { + continue + } + for _, providedDep := range graph.TargetOrDie(dataLabel).ProvideFor(t) { + if !push(graph.TargetOrDie(providedDep)) { return false } } } - // If required, include the run-time dependencies of dependencies, but not the dependencies themselves. - if t.RuntimeDependenciesFromDependencies && !dep.exported && !dep.source && !dep.internal && !dep.runtime { - depLabel, _ := dep.declared.Label() - depTarget := graph.TargetOrDie(depLabel) - for _, providedDep := range depTarget.ProvideFor(t) { - if !push(graph.TargetOrDie(providedDep), yield) { - return false + } + if t.RuntimeDependenciesFromSources || t.RuntimeDependenciesFromDependencies { + for _, dep := range t.dependencies { + // If required, include the run-time dependencies of sources, but not the sources themselves. + if t.RuntimeDependenciesFromSources && dep.Source { + for _, providedDep := range graph.TargetOrDie(dep.Label).ProvideFor(t) { + if !push(graph.TargetOrDie(providedDep)) { + return false + } } } - // Also include the run-time dependencies of the target's exported dependencies, but not the - // exported dependencies themselves. - for _, exportedDep := range depTarget.ExportedDependencies() { - for _, providedDep := range graph.TargetOrDie(exportedDep).ProvideFor(t) { - if !push(graph.TargetOrDie(providedDep), yield) { + // If required, include the run-time dependencies of dependencies, but not the dependencies themselves. + if t.RuntimeDependenciesFromDependencies && !dep.Exported && !dep.Source && !dep.Internal && !dep.Runtime { + depTarget := graph.TargetOrDie(dep.Label) + for _, providedDep := range depTarget.ProvideFor(t) { + if !push(graph.TargetOrDie(providedDep)) { return false } } + // Also include the run-time dependencies of the target's exported dependencies, but not the + // exported dependencies themselves. + for exportedDep := range depTarget.ExportedDependencies() { + for _, providedDep := range graph.TargetOrDie(exportedDep).ProvideFor(t) { + if !push(graph.TargetOrDie(providedDep)) { + return false + } + } + } } } } + return true } - return true - } - return func(yield func(BuildLabel) bool) { - push(target, yield) + push(target) } } -// DependenciesFor returns the dependencies that relate to a given label. -func (target *BuildTarget) DependenciesFor(label BuildLabel) []*BuildTarget { - target.mutex.RLock() - defer target.mutex.RUnlock() - return target.dependenciesFor(label) -} - -func (target *BuildTarget) dependenciesFor(label BuildLabel) []*BuildTarget { - if info := target.dependencyInfo(label); info != nil { - return info.deps - } else if target.Label.Subrepo != "" && label.Subrepo == "" { - // Can implicitly use the target's subrepo. - label.Subrepo = target.Label.Subrepo - return target.dependenciesFor(label) - } - return nil -} - -// FinishBuild marks this target as having built. -func (target *BuildTarget) FinishBuild() { - close(target.finishedBuilding) -} - -// WaitForBuild blocks until this target has finished building. -func (target *BuildTarget) WaitForBuild(dependant BuildLabel) { - waitOnChan(target.finishedBuilding, "Still waiting on (target %v).WaitForBuild(dependant %v)", target.Label, dependant) -} - // DeclaredOutputs returns the outputs from this target's original declaration. // Hence it's similar to Outputs() but without the resolving of other rule names. func (target *BuildTarget) DeclaredOutputs() []string { @@ -890,20 +848,40 @@ func (target *BuildTarget) DeclaredSourceNames() []string { return ret } -func (target *BuildTarget) filegroupOutputs(srcs []BuildInput) []string { +// ResolveDependencySubrepo qualifies label with the target's subrepo if needed, returning the +// resolved label and whether the target actually declares a dependency on it. This handles labels +// (e.g. from command replacements like $(location :x)) that may be missing the subrepo the target +// itself was built in, as happens when cross-compiling. +func (target *BuildTarget) ResolveDependencySubrepo(label BuildLabel) (BuildLabel, bool) { + target.mutex.RLock() + defer target.mutex.RUnlock() + if target.dependencyInfo(label) != nil { + return label, true + } + if target.Label.Subrepo != "" && label.Subrepo == "" { + // Can implicitly use the target's subrepo. + label.Subrepo = target.Label.Subrepo + if target.dependencyInfo(label) != nil { + return label, true + } + } + return label, false +} + +func (target *BuildTarget) filegroupOutputs(graph *BuildGraph, srcs []BuildInput) []string { ret := make([]string, 0, len(srcs)) // Filegroups just re-output their inputs. for _, src := range srcs { if namedLabel, ok := src.(AnnotatedOutputLabel); ok { // Bit of a hack, but this needs different treatment from either of the others. - for _, dep := range target.DependenciesFor(namedLabel.BuildLabel) { - ret = append(ret, dep.NamedOutputs(namedLabel.Annotation)...) + for _, dep := range graph.TargetOrDie(namedLabel.BuildLabel).ProvideFor(target) { + ret = append(ret, graph.TargetOrDie(dep).NamedOutputs(graph, namedLabel.Annotation)...) } } else if label, ok := src.nonOutputLabel(); !ok { ret = append(ret, src.LocalPaths(nil)[0]) } else { - for _, dep := range target.DependenciesFor(label) { - ret = append(ret, dep.Outputs()...) + for _, dep := range graph.TargetOrDie(label).ProvideFor(target) { + ret = append(ret, graph.TargetOrDie(dep).Outputs(graph)...) } } } @@ -911,10 +889,10 @@ func (target *BuildTarget) filegroupOutputs(srcs []BuildInput) []string { } // Outputs returns a slice of all the outputs of this rule. -func (target *BuildTarget) Outputs() []string { +func (target *BuildTarget) Outputs(graph *BuildGraph) []string { var ret []string if target.IsFilegroup { - ret = target.filegroupOutputs(target.AllSources()) + ret = target.filegroupOutputs(graph, target.AllSources()) } else { // Must really copy the slice before sorting it ([:] is too shallow) ret = make([]string, len(target.outputs)) @@ -930,8 +908,8 @@ func (target *BuildTarget) Outputs() []string { } // FullOutputs returns a slice of all the outputs of this rule with the target's output directory prepended. -func (target *BuildTarget) FullOutputs() []string { - outs := target.Outputs() +func (target *BuildTarget) FullOutputs(graph *BuildGraph) []string { + outs := target.Outputs(graph) outDir := target.OutDir() for i, out := range outs { outs[i] = filepath.Join(outDir, out) @@ -941,8 +919,8 @@ func (target *BuildTarget) FullOutputs() []string { // AllOutputs returns a slice of all the outputs of this rule, including any output directories. // Outs are passed through GetTmpOutput as appropriate. -func (target *BuildTarget) AllOutputs() []string { - outs := target.Outputs() +func (target *BuildTarget) AllOutputs(graph *BuildGraph) []string { + outs := target.Outputs(graph) for i, out := range outs { outs[i] = target.GetTmpOutput(out) } @@ -954,13 +932,13 @@ func (target *BuildTarget) AllOutputs() []string { // NamedOutputs returns a slice of all the outputs of this rule with a given name. // If the name is not declared by this rule it panics. -func (target *BuildTarget) NamedOutputs(name string) []string { +func (target *BuildTarget) NamedOutputs(graph *BuildGraph, name string) []string { if target.IsFilegroup { if target.NamedSources == nil { return nil } if srcs, present := target.NamedSources[name]; present { - return target.filegroupOutputs(srcs) + return target.filegroupOutputs(graph, srcs) } return nil } @@ -1041,14 +1019,15 @@ func (target *BuildTarget) CanSee(state *BuildState, dep *BuildTarget) bool { // Returns an error if not, or nil if all's well. func (target *BuildTarget) CheckDependencyVisibility(state *BuildState) error { for _, d := range target.dependencies { - dep := state.Graph.TargetOrDie(*d.declared) - if !target.CanSee(state, dep) { - return fmt.Errorf("Target %s isn't visible to %s", dep.Label, target.Label) - } else if dep.TestOnly && !target.IsTest() && !target.TestOnly { - if target.Label.isExperimental(state) { - log.Info("Test-only restrictions suppressed for %s since %s is in the experimental tree", dep.Label, target.Label) - } else { - return fmt.Errorf("Target %s can't depend on %s, it's marked test_only", target.Label, dep.Label) + if dep := state.Graph.Target(d.Label); dep != nil { + if !target.CanSee(state, dep) { + return fmt.Errorf("Target %s isn't visible to %s", dep.Label, target.Label) + } else if dep.TestOnly && !target.IsTest() && !target.TestOnly { + if target.Label.isExperimental(state) { + log.Info("Test-only restrictions suppressed for %s since %s is in the experimental tree", dep.Label, target.Label) + } else { + return fmt.Errorf("Target %s can't depend on %s, it's marked test_only", target.Label, dep.Label) + } } } } @@ -1057,9 +1036,9 @@ func (target *BuildTarget) CheckDependencyVisibility(state *BuildState) error { // CheckDuplicateOutputs checks if any of the outputs of this target duplicate one another. // Returns an error if so, or nil if all's well. -func (target *BuildTarget) CheckDuplicateOutputs() error { +func (target *BuildTarget) CheckDuplicateOutputs(graph *BuildGraph) error { outputs := map[string]struct{}{} - for _, output := range target.Outputs() { + for _, output := range target.Outputs(graph) { if _, present := outputs[output]; present { return fmt.Errorf("Target %s declares output file %s multiple times", target.Label, output) } @@ -1077,7 +1056,7 @@ func (target *BuildTarget) CheckTargetOwnsBuildOutputs(state *BuildState) error return nil } - for _, output := range target.Outputs() { + for _, output := range target.Outputs(state.Graph) { targetPackage := target.Label.PackageName out := filepath.Join(targetPackage, output) @@ -1181,26 +1160,10 @@ func (target *BuildTarget) HasDependency(label BuildLabel) bool { return target.dependencyInfo(label) != nil } -// resolveDependency resolves a particular dependency on a target. -// TODO(jpoole): this is only used by tests: remove -func (target *BuildTarget) resolveDependency(label BuildLabel, dep *BuildTarget) { - target.mutex.Lock() - defer target.mutex.Unlock() - info := target.dependencyInfo(label) - if info == nil { - target.dependencies = append(target.dependencies, depInfo{declared: &label}) - info = &target.dependencies[len(target.dependencies)-1] - } - if dep != nil { - info.deps = append(info.deps, dep) - } - info.resolved = true -} - // dependencyInfo returns the information about a declared dependency, or nil if the target doesn't have it. -func (target *BuildTarget) dependencyInfo(label BuildLabel) *depInfo { +func (target *BuildTarget) dependencyInfo(label BuildLabel) *DeclaredDependency { for i, info := range target.dependencies { - if *info.declared == label { + if info.Label == label { return &target.dependencies[i] } } @@ -1210,26 +1173,17 @@ func (target *BuildTarget) dependencyInfo(label BuildLabel) *depInfo { // IsSourceOnlyDep returns true if the given dependency was only declared on the srcs of the target. func (target *BuildTarget) IsSourceOnlyDep(label BuildLabel) bool { info := target.dependencyInfo(label) - return info != nil && info.source + return info != nil && info.Source } // State returns the target's current state. func (target *BuildTarget) State() BuildTargetState { - return BuildTargetState(atomic.LoadInt32(&target.state)) + return BuildTargetState(target.state.Load()) } // SetState sets a target's current state. func (target *BuildTarget) SetState(state BuildTargetState) { - atomic.StoreInt32(&target.state, int32(state)) -} - -// SyncUpdateState oves the target's state from before to after via a lock. -// Returns true if successful, false if not (which implies something else changed the state first). -// The nature of our build graph ensures that most transitions are only attempted by -// one thread simultaneously, but this one can be attempted by several at once -// (eg. if a depends on b and c, which finish building simultaneously, they race to queue a). -func (target *BuildTarget) SyncUpdateState(before, after BuildTargetState) bool { - return atomic.CompareAndSwapInt32(&target.state, int32(before), int32(after)) + target.state.Store(int32(state)) } // AddLabel adds the given label to this target if it doesn't already have it. @@ -1497,7 +1451,7 @@ func (target *BuildTarget) AddDatum(datum BuildInput) { target.Data = append(target.Data, datum) if label, ok := datum.Label(); ok { target.AddDependency(label) - target.dependencyInfo(label).data = true + target.dependencyInfo(label).Data = true } } @@ -1509,7 +1463,7 @@ func (target *BuildTarget) AddNamedDatum(name string, datum BuildInput) { target.NamedData[name] = append(target.NamedData[name], datum) if label, ok := datum.Label(); ok { target.AddDependency(label) - target.dependencyInfo(label).data = true + target.dependencyInfo(label).Data = true } } @@ -1521,7 +1475,7 @@ func (target *BuildTarget) AddDebugDatum(datum BuildInput) { target.Debug.data = append(target.Debug.data, datum) if label, ok := datum.Label(); ok { target.AddDependency(label) - target.dependencyInfo(label).data = true + target.dependencyInfo(label).Data = true } } @@ -1536,7 +1490,7 @@ func (target *BuildTarget) AddDebugNamedDatum(name string, datum BuildInput) { target.Debug.namedData[name] = append(target.Debug.namedData[name], datum) if label, ok := datum.Label(); ok { target.AddDependency(label) - target.dependencyInfo(label).data = true + target.dependencyInfo(label).Data = true } } @@ -1790,19 +1744,19 @@ func (target *BuildTarget) AddMaybeExportedDependency(dep BuildLabel, exported, } info := target.dependencyInfo(dep) if info == nil { - target.dependencies = append(target.dependencies, depInfo{ - declared: &dep, - exported: exported, - source: source, - internal: internal, - runtime: runtime, + target.dependencies = append(target.dependencies, DeclaredDependency{ + Label: dep, + Exported: exported, + Source: source, + Internal: internal, + Runtime: runtime, }) } else { - info.exported = info.exported || exported - info.source = info.source && source - info.internal = info.internal && internal - info.runtime = info.runtime && runtime - info.data = false // It's not *only* data any more. + info.Exported = info.Exported || exported + info.Source = info.Source && source + info.Internal = info.Internal && internal + info.Runtime = info.Runtime && runtime + info.Data = false // It's not *only* data any more. } } @@ -1834,7 +1788,7 @@ func (target *BuildTarget) isTool(tool BuildLabel, tools []BuildInput, namedTool } // toolPath returns a path to this target when used as a tool. -func (target *BuildTarget) toolPath(abs bool, namedOutput string) string { +func (target *BuildTarget) toolPath(graph *BuildGraph, abs bool, namedOutput string) string { outToolPath := func(outputs ...string) string { ret := make([]string, len(outputs)) for i, o := range outputs { @@ -1856,7 +1810,7 @@ func (target *BuildTarget) toolPath(abs bool, namedOutput string) string { } panic(fmt.Sprintf("%v has no named output or entry point %v", target.Label, namedOutput)) } - return outToolPath(target.Outputs()...) + return outToolPath(target.Outputs(graph)...) } // AddOutput adds a new output to the target if it's not already there. @@ -1890,7 +1844,7 @@ func (target *BuildTarget) AddEntryPoint(name, output string) { if target.EntryPoints == nil { target.EntryPoints = make(map[string]string) } - if target.NamedOutputs(name) != nil { + if _, present := target.namedOutputs[name]; present { panic(fmt.Sprintf("%v already has a named output named %v; entry points may not have the same name as a named output", target.Label, name)) } if target.IsFilegroup && target.NamedSources[name] != nil { diff --git a/src/core/build_target_test.go b/src/core/build_target_test.go index 4360be1316..94a65eb874 100644 --- a/src/core/build_target_test.go +++ b/src/core/build_target_test.go @@ -131,12 +131,12 @@ func TestCheckDependencyVisibility(t *testing.T) { assert.NoError(t, target7.CheckDependencyVisibility(state)) // Now if we add a dep on this mock library, lib2 will fail because it's not a test. - target2.resolveDependency(target5.Label, target5) + target2.AddDependency(target5.Label) assert.Error(t, target2.CheckDependencyVisibility(state)) // Similarly to above test, if we add a dep on something that can't be seen, we should // get errors back from this function. - target3.resolveDependency(target1.Label, target1) + target3.AddDependency(target1.Label) assert.Error(t, target3.CheckDependencyVisibility(state)) } @@ -145,8 +145,8 @@ func TestAddOutput(t *testing.T) { target.AddOutput("thingy.py") target.AddOutput("thingy2.py") target.AddOutput("thingy.py") - if len(target.Outputs()) != 2 { - t.Errorf("Incorrect output length; should be 2, was %d", len(target.Outputs())) + if len(target.Outputs(nil)) != 2 { + t.Errorf("Incorrect output length; should be 2, was %d", len(target.Outputs(nil))) } } @@ -165,7 +165,7 @@ func TestAddOutputSorting(t *testing.T) { "3.py", "x.pyx", } - assert.Equal(t, expected, target.Outputs()) + assert.Equal(t, expected, target.Outputs(nil)) } func TestAddOutputPanics(t *testing.T) { @@ -182,7 +182,7 @@ func TestAddSource(t *testing.T) { target.AddSource(ParseBuildLabel("//src/test/python:lib3", "")) target.AddSource(ParseBuildLabel("//src/test/python:lib2", "")) assert.Equal(t, 2, len(target.Sources)) - assert.Equal(t, 2, len(target.DeclaredDependencies())) + assert.Equal(t, 2, len(slices.Collect(target.DeclaredDependencies()))) } func TestOutputs(t *testing.T) { @@ -195,17 +195,18 @@ func TestOutputs(t *testing.T) { target3 := makeFilegroup("//src/test:target3", "PUBLIC", target2) target3.AddSource(target2.Label) addFilegroupSource(target3, "file4.go") + graph := graphWith(target1, target2, target3) - assert.Equal(t, []string{"file1.go", "file2.go"}, target1.Outputs()) - assert.Equal(t, []string{"file1.go", "file2.go", "file3.go"}, target2.Outputs()) - assert.Equal(t, []string{"file1.go", "file2.go", "file3.go", "file4.go"}, target3.Outputs()) + assert.Equal(t, []string{"file1.go", "file2.go"}, target1.Outputs(graph)) + assert.Equal(t, []string{"file1.go", "file2.go", "file3.go"}, target2.Outputs(graph)) + assert.Equal(t, []string{"file1.go", "file2.go", "file3.go", "file4.go"}, target3.Outputs(graph)) } func TestFullOutputs(t *testing.T) { target := makeTarget1("//src/core:target1", "PUBLIC") target.AddOutput("file1.go") target.AddOutput("file2.go") - assert.Equal(t, []string{"plz-out/gen/src/core/file1.go", "plz-out/gen/src/core/file2.go"}, target.FullOutputs()) + assert.Equal(t, []string{"plz-out/gen/src/core/file1.go", "plz-out/gen/src/core/file2.go"}, target.FullOutputs(nil)) } func TestAllOutputs(t *testing.T) { @@ -213,7 +214,7 @@ func TestAllOutputs(t *testing.T) { target.AddOutput("please") target.AddOutput("plz") target.AddOutputDirectory("dir") - assert.Equal(t, []string{"please.out", "plz", "dir"}, target.AllOutputs()) + assert.Equal(t, []string{"please.out", "plz", "dir"}, target.AllOutputs(nil)) } func TestProvideFor(t *testing.T) { @@ -262,10 +263,10 @@ func TestAddDatum(t *testing.T) { target2 := makeTarget1("//src/core:target2", "PUBLIC") target1.AddDatum(target2.Label) assert.Equal(t, target1.Data, []BuildInput{target2.Label}) - assert.True(t, target1.dependencies[0].data) + assert.True(t, target1.dependencies[0].Data) // Now we add it as a dependency too, which unsets the data label target1.AddMaybeExportedDependency(target2.Label, false, false, false, false) - assert.False(t, target1.dependencies[0].data) + assert.False(t, target1.dependencies[0].Data) } func TestCheckDuplicateOutputs(t *testing.T) { @@ -274,14 +275,15 @@ func TestCheckDuplicateOutputs(t *testing.T) { target2 := makeFilegroup("//src/core:target2", "PUBLIC", target1, target3) addFilegroupSource(target1, "thingy.txt") addFilegroupSource(target3, "thingy.txt") - assert.NoError(t, target1.CheckDuplicateOutputs()) + graph := graphWith(target1, target2, target3) + assert.NoError(t, target1.CheckDuplicateOutputs(graph)) target2.AddSource(target1.Label) target2.AddSource(target1.Label) // Not an error yet because AddOutput deduplicates trivially identical outputs. - assert.NoError(t, target2.CheckDuplicateOutputs()) + assert.NoError(t, target2.CheckDuplicateOutputs(graph)) // Will fail now we add the same output to another target. target2.AddSource(target3.Label) - assert.Error(t, target2.CheckDuplicateOutputs()) + assert.Error(t, target2.CheckDuplicateOutputs(graph)) } func TestLabels(t *testing.T) { @@ -396,8 +398,8 @@ func TestToolPath(t *testing.T) { wd, _ := os.Getwd() RepoRoot = wd root := wd + "/plz-out/gen/src/core" - assert.Equal(t, fmt.Sprintf("%s/file1.go %s/file2.go", root, root), target.toolPath(true, "")) - assert.Equal(t, "src/core/file1.go src/core/file2.go", target.toolPath(false, "")) + assert.Equal(t, fmt.Sprintf("%s/file1.go %s/file2.go", root, root), target.toolPath(nil, true, "")) + assert.Equal(t, "src/core/file1.go src/core/file2.go", target.toolPath(nil, false, "")) } func TestToolPathWithEntryPoint(t *testing.T) { @@ -408,20 +410,21 @@ func TestToolPathWithEntryPoint(t *testing.T) { wd, _ := os.Getwd() RepoRoot = wd root := wd + "/plz-out/gen/src/core" - assert.Equal(t, root+"/file1.go", target.toolPath(true, "f1")) - assert.Equal(t, "src/core/file1.go", target.toolPath(false, "f1")) + assert.Equal(t, root+"/file1.go", target.toolPath(nil, true, "f1")) + assert.Equal(t, "src/core/file1.go", target.toolPath(nil, false, "f1")) } func TestDependencies(t *testing.T) { target1 := makeTarget1("//src/core:target1", "") target2 := makeTarget1("//src/core:target2", "", target1) target3 := makeTarget1("//src/core:target3", "", target1, target2) - assert.Equal(t, []BuildLabel{}, target1.DeclaredDependencies()) - assert.Equal(t, []*BuildTarget{}, target1.Dependencies()) - assert.Equal(t, []BuildLabel{target1.Label}, target2.DeclaredDependencies()) - assert.Equal(t, []*BuildTarget{target1}, target2.Dependencies()) - assert.Equal(t, []BuildLabel{target1.Label, target2.Label}, target3.DeclaredDependencies()) - assert.Equal(t, []*BuildTarget{target1, target2}, target3.Dependencies()) + graph := graphWith(target1, target2, target3) + assert.Empty(t, slices.Collect(target1.DeclaredDependencies())) + assert.Empty(t, resolved(target1.Dependencies(graph))) + assert.Equal(t, []BuildLabel{target1.Label}, slices.Collect(target2.DeclaredDependencies())) + assert.Equal(t, []*BuildTarget{target1}, resolved(target2.Dependencies(graph))) + assert.Equal(t, []BuildLabel{target1.Label, target2.Label}, slices.Collect(target3.DeclaredDependencies())) + assert.Equal(t, []*BuildTarget{target1, target2}, resolved(target3.Dependencies(graph))) } func TestBuildDependencies(t *testing.T) { @@ -434,10 +437,10 @@ func TestBuildDependencies(t *testing.T) { // BuildDependencies shouldn't return run-time dependencies: target5.IsBinary = true target5.AddMaybeExportedDependency(target4.Label, false, false, false, true) // runtime - assert.Equal(t, []*BuildTarget{}, target1.BuildDependencies()) - assert.Equal(t, []*BuildTarget{target1}, target2.BuildDependencies()) - assert.Equal(t, []*BuildTarget{target2}, target3.BuildDependencies()) - assert.Equal(t, []*BuildTarget{}, target5.BuildDependencies()) + assert.Empty(t, slices.Collect(target1.BuildDependencies())) + assert.Equal(t, []BuildLabel{target1.Label}, slices.Collect(target2.BuildDependencies())) + assert.Equal(t, []BuildLabel{target2.Label}, slices.Collect(target3.BuildDependencies())) + assert.Empty(t, slices.Collect(target5.BuildDependencies())) } func TestDeclaredDependenciesStrict(t *testing.T) { @@ -450,10 +453,10 @@ func TestDeclaredDependenciesStrict(t *testing.T) { // DeclaredDependenciesStrict shouldn't return run-time dependencies: target5.IsBinary = true target5.AddMaybeExportedDependency(target4.Label, false, false, false, true) // runtime - assert.Equal(t, []BuildLabel{}, target1.DeclaredDependenciesStrict()) - assert.Equal(t, []BuildLabel{target1.Label}, target2.DeclaredDependenciesStrict()) - assert.Equal(t, []BuildLabel{target2.Label}, target3.DeclaredDependenciesStrict()) - assert.Equal(t, []*BuildTarget{}, target5.BuildDependencies()) + assert.Empty(t, slices.Collect(target1.DeclaredDependenciesStrict())) + assert.Equal(t, []BuildLabel{target1.Label}, slices.Collect(target2.DeclaredDependenciesStrict())) + assert.Equal(t, []BuildLabel{target2.Label}, slices.Collect(target3.DeclaredDependenciesStrict())) + assert.Empty(t, slices.Collect(target5.DeclaredDependenciesStrict())) } func TestRuntimeDependencies(t *testing.T) { @@ -465,9 +468,9 @@ func TestRuntimeDependencies(t *testing.T) { target3.IsBinary = true target3.AddMaybeExportedDependency(target2.Label, false, false, false, true) // runtime // RuntimeDependencies shouldn't return transitive run-time dependencies. - assert.Equal(t, []BuildLabel{}, target1.RuntimeDependencies()) - assert.Equal(t, []BuildLabel{target1.Label}, target2.RuntimeDependencies()) - assert.Equal(t, []BuildLabel{target2.Label}, target3.RuntimeDependencies()) + assert.Empty(t, slices.Collect(target1.RuntimeDependencies())) + assert.Equal(t, []BuildLabel{target1.Label}, slices.Collect(target2.RuntimeDependencies())) + assert.Equal(t, []BuildLabel{target2.Label}, slices.Collect(target3.RuntimeDependencies())) } func TestIterAllRuntimeDependencies(t *testing.T) { @@ -491,17 +494,15 @@ func TestIterAllRuntimeDependencies(t *testing.T) { func TestAddDependency(t *testing.T) { target1 := makeTarget1("//src/core:target1", "") target2 := makeTarget1("//src/core:target2", "") - assert.Equal(t, []BuildLabel{}, target2.DeclaredDependencies()) - assert.Equal(t, []BuildLabel{}, target2.ExportedDependencies()) + assert.Empty(t, slices.Collect(target2.DeclaredDependencies())) + assert.Empty(t, slices.Collect(target2.ExportedDependencies())) target2.AddDependency(target1.Label) - assert.Equal(t, []BuildLabel{target1.Label}, target2.DeclaredDependencies()) - assert.Equal(t, []BuildLabel{}, target2.ExportedDependencies()) + assert.Equal(t, []BuildLabel{target1.Label}, slices.Collect(target2.DeclaredDependencies())) + assert.Empty(t, slices.Collect(target2.ExportedDependencies())) target2.AddMaybeExportedDependency(target1.Label, true, false, false, false) - assert.Equal(t, []BuildLabel{target1.Label}, target2.DeclaredDependencies()) - assert.Equal(t, []BuildLabel{target1.Label}, target2.ExportedDependencies()) - assert.Equal(t, []*BuildTarget{}, target2.Dependencies()) - target2.resolveDependency(target1.Label, target1) - assert.Equal(t, []*BuildTarget{target1}, target2.Dependencies()) + assert.Equal(t, []BuildLabel{target1.Label}, slices.Collect(target2.DeclaredDependencies())) + assert.Equal(t, []BuildLabel{target1.Label}, slices.Collect(target2.ExportedDependencies())) + assert.Equal(t, []*BuildTarget{target1}, resolved(target2.Dependencies(graphWith(target1, target2)))) } func TestAddRuntimeDependency(t *testing.T) { @@ -510,10 +511,10 @@ func TestAddRuntimeDependency(t *testing.T) { target1.IsBinary = true target1.AddMaybeExportedDependency(target2.Label, false, false, false, true) // runtime assert.Equal(t, target1.runtimeDependencies, []BuildLabel{target2.Label}) - assert.True(t, target1.dependencies[0].runtime) + assert.True(t, target1.dependencies[0].Runtime) // Now we add it as a build-time dependency too, which should unset the runtime flag. target1.AddMaybeExportedDependency(target2.Label, false, false, false, false) - assert.False(t, target1.dependencies[0].runtime) + assert.False(t, target1.dependencies[0].Runtime) } func TestAddDependencySource(t *testing.T) { @@ -526,11 +527,14 @@ func TestAddDependencySource(t *testing.T) { assert.False(t, target2.IsSourceOnlyDep(target1.Label)) } -func TestDependencyFor(t *testing.T) { +func TestResolveDependencySubrepo(t *testing.T) { target1 := makeTarget1("//src/core:target1", "") target2 := makeTarget1("//src/core:target2", "", target1) - assert.Equal(t, []*BuildTarget{target1}, target2.DependenciesFor(target1.Label)) - assert.Equal(t, []*BuildTarget(nil), target2.DependenciesFor(target2.Label)) + l, ok := target2.ResolveDependencySubrepo(target1.Label) + assert.True(t, ok) + assert.Equal(t, target1.Label, l) + _, ok = target2.ResolveDependencySubrepo(target2.Label) + assert.False(t, ok) assert.Equal(t, 1, len(target2.dependencies)) } @@ -582,7 +586,7 @@ func TestOutputOrdering(t *testing.T) { target2.AddOutput("file2.txt") target2.AddOutput("file1.txt") assert.Equal(t, target1.DeclaredOutputs(), target2.DeclaredOutputs()) - assert.Equal(t, target1.Outputs(), target2.Outputs()) + assert.Equal(t, target1.Outputs(nil), target2.Outputs(nil)) } func TestNamedOutputs(t *testing.T) { @@ -594,12 +598,12 @@ func TestNamedOutputs(t *testing.T) { target.AddNamedOutput("hdrs", "hdr1.h") target.AddNamedOutput("hdrs", "hdr2.h") target.AddNamedOutput("hdrs", "hdr2.h") // deliberate duplicate - assert.Equal(t, []string{"a.txt", "hdr1.h", "hdr2.h", "src1.c", "src2.c", "z.txt"}, target.Outputs()) + assert.Equal(t, []string{"a.txt", "hdr1.h", "hdr2.h", "src1.c", "src2.c", "z.txt"}, target.Outputs(nil)) assert.Equal(t, []string{"a.txt", "z.txt"}, target.DeclaredOutputs()) assert.Equal(t, map[string][]string{"srcs": {"src1.c", "src2.c"}, "hdrs": {"hdr1.h", "hdr2.h"}}, target.DeclaredNamedOutputs()) - assert.Equal(t, []string{"hdr1.h", "hdr2.h"}, target.NamedOutputs("hdrs")) - assert.Equal(t, []string{"src1.c", "src2.c"}, target.NamedOutputs("srcs")) - assert.Equal(t, 0, len(target.NamedOutputs("go_srcs"))) + assert.Equal(t, []string{"hdr1.h", "hdr2.h"}, target.NamedOutputs(nil, "hdrs")) + assert.Equal(t, []string{"src1.c", "src2.c"}, target.NamedOutputs(nil, "srcs")) + assert.Equal(t, 0, len(target.NamedOutputs(nil, "go_srcs"))) assert.Equal(t, []string{"hdrs", "srcs"}, target.DeclaredOutputNames()) } @@ -743,7 +747,8 @@ func TestExternalDependencies(t *testing.T) { t1 := makeTarget1("//src/core:target1", "PUBLIC", t1a) t2a := makeTarget1("//src/core:_target2#a", "PUBLIC", t1) t2 := makeTarget1("//src/core:target2", "PUBLIC", t2a) - assert.Equal(t, []*BuildTarget{t1}, t2.ExternalDependencies()) + graph := graphWith(t1a, t1, t2a, t2) + assert.Equal(t, []*BuildTarget{t1}, resolved(t2.ExternalDependencies(graph))) } func TestBuildTargetOwnBuildInputs(t *testing.T) { @@ -1035,11 +1040,28 @@ func makeTarget1(label, visibility string, deps ...*BuildTarget) *BuildTarget { } for _, dep := range deps { target.AddDependency(dep.Label) - target.resolveDependency(dep.Label, dep) } return target } +// resolved unwraps a call to Dependencies / ExternalDependencies, requiring that everything resolved. +func resolved(deps []*BuildTarget, unresolved []BuildLabel) []*BuildTarget { + if len(unresolved) > 0 { + panic(fmt.Sprintf("dependencies not in graph: %s", unresolved)) + } + return deps +} + +// graphWith returns a graph populated with the given targets, for tests that need dependency +// resolution (which now happens against the graph rather than being cached on the target). +func graphWith(targets ...*BuildTarget) *BuildGraph { + graph := NewGraph() + for _, target := range targets { + graph.AddTarget(target) + } + return graph +} + func makeTarget1WithLabels(name string, labels ...string) *BuildTarget { target := makeTarget1(name, "") for _, label := range labels { diff --git a/src/core/command_replacements.go b/src/core/command_replacements.go index 07f1b98e67..1ec0442802 100644 --- a/src/core/command_replacements.go +++ b/src/core/command_replacements.go @@ -252,22 +252,24 @@ func replaceSequenceLabel(state *BuildState, target *BuildTarget, label BuildLab } // TODO(jpoole): This doesn't handle tools when cross compiling. ///freebsd_amd64//tools:tool // will not match the tool //tools:tool - deps := target.DependenciesFor(label) - if len(deps) == 0 { + label, ok := target.ResolveDependencySubrepo(label) + if !ok { panic(fmt.Sprintf("Rule %s can't use %s; doesn't depend on target %s", target.Label, in, label)) } + deps := state.Graph.TargetOrDie(label).ProvideFor(target) // TODO(pebers): this does not correctly handle the case where there are multiple deps here // (but is better than the previous case where it never worked at all) - return checkAndReplaceSequence(state, target, deps[0], ep, in, runnable, multiple, dir, outPrefix, hash, test, allOutputs, target.IsTool(label)) + dep := state.Graph.TargetOrDie(deps[0]) + return checkAndReplaceSequence(state, target, dep, ep, in, runnable, multiple, dir, outPrefix, hash, test, allOutputs, target.IsTool(label)) } func checkAndReplaceSequence(state *BuildState, target, dep *BuildTarget, ep, in string, runnable, multiple, dir, outPrefix, hash, test, allOutputs, tool bool) string { - if allOutputs && !multiple && len(dep.Outputs()) > 1 && ep == "" { + if allOutputs && !multiple && len(dep.Outputs(state.Graph)) > 1 && ep == "" { // Label must have only one output. panic(fmt.Sprintf("Rule %s can't use %s; %s has multiple outputs.", target.Label, in, dep.Label)) } else if runnable && !dep.IsBinary { panic(fmt.Sprintf("Rule %s can't $(exe %s), it's not executable", target.Label, dep.Label)) - } else if runnable && len(dep.Outputs()) == 0 { + } else if runnable && len(dep.Outputs(state.Graph)) == 0 { panic(fmt.Sprintf("Rule %s is tagged as binary but produces no output.", dep.Label)) } else if test && tool { panic(fmt.Sprintf("Rule %s uses %s in its test command, but tools are not accessible at test time", target, dep)) @@ -281,7 +283,7 @@ func checkAndReplaceSequence(state *BuildState, target, dep *BuildTarget, ep, in } var outputBuilder strings.Builder if ep == "" { - for _, out := range dep.Outputs() { + for _, out := range dep.Outputs(state.Graph) { if allOutputs || out == in { if tool && !state.WillRunRemotely(target) { abs, err := filepath.Abs(handleDir(dep.OutDir(), out, dir)) diff --git a/src/core/command_replacements_test.go b/src/core/command_replacements_test.go index 4187278cdf..3111934f55 100644 --- a/src/core/command_replacements_test.go +++ b/src/core/command_replacements_test.go @@ -270,16 +270,13 @@ func makeTarget2(name string, command string, dep *BuildTarget) *BuildTarget { target := NewBuildTarget(ParseBuildLabel(name, "")) target.Command = command target.AddOutput(target.Label.Name + ".py") + // Dependency resolution now happens against the graph at replacement time rather than being + // cached on the target, so the targets must live in the state's graph. These tests share a + // global state and reuse labels across tests, so we replace rather than AddTarget (which would + // panic on a duplicate label). + state.Graph.targets.Set(target.Label, target) if dep != nil { target.AddDependency(dep.Label) - // This is a bit awkward but I don't want to add a public interface just for a test. - graph := NewGraph() - graph.AddTarget(target) - graph.AddTarget(dep) - target.AddDependency(dep.Label) - if err := target.ResolveDependencies(graph); err != nil { - log.Fatalf("Failed to resolve some dependencies for %s: %s", target, err) - } } return target } @@ -359,8 +356,10 @@ func TestTestCommand(t *testing.T) { }) t.Run("Combined sequence and placeholder replacement", func(t *testing.T) { + state := NewDefaultBuildState() target2 := makeTarget2("//path/to:target2", "", nil) target1 := makeTarget2("//path/to:target1", "$(location //path/to:target2) __TEST_ARGS__", target2) + state.Graph.AddTarget(target2) target1.Test = &TestFields{ Command: "$(location //path/to:target2) __TEST_ARGS__", ArgsPlaceholder: "__TEST_ARGS__", diff --git a/src/core/cycle_detector.go b/src/core/cycle_detector.go index 39dc99549e..12d76aff79 100644 --- a/src/core/cycle_detector.go +++ b/src/core/cycle_detector.go @@ -35,7 +35,10 @@ func (c *cycleDetector) Check() *errCycle { return []*BuildTarget{target}, false } partial[target] = struct{}{} - for _, dep := range target.Dependencies() { + // Ignore anything we can't resolve; we run while the build is still going on so it's + // entirely normal for parts of the graph not to exist yet. + deps, _ := target.Dependencies(c.graph) + for _, dep := range deps { if cycle, done := visit(dep); cycle != nil { if done || target == cycle[len(cycle)-1] { return cycle, true // This target is already in the cycle diff --git a/src/core/cycle_detector_test.go b/src/core/cycle_detector_test.go index 70506f59b7..58a9ca8cf8 100644 --- a/src/core/cycle_detector_test.go +++ b/src/core/cycle_detector_test.go @@ -2,7 +2,6 @@ package core import ( "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -15,29 +14,9 @@ func TestCycleDetector(t *testing.T) { target.AddDependency(ParseBuildLabel(dep, "")) } state.Graph.AddTarget(target) - state.QueueTarget(target.Label, OriginalTarget, true, ParseModeForSubinclude) return target } - waitForDeps := func(state *BuildState) { - // Wait for all targets to have resolved all their dependencies. - allDepsResolved := func() bool { - for _, target := range state.Graph.AllTargets() { - if len(target.DeclaredDependencies()) != len(target.Dependencies()) { - return false - } - } - return true - } - for i := 0; i < 1000; i++ { - if allDepsResolved() { - return - } - time.Sleep(2 * time.Millisecond) - } - panic("not all dependencies resolved") - } - t.Run("NoCycle", func(t *testing.T) { state := NewDefaultBuildState() newTarget(state, "//src:a", "//src:b", "//src:c") @@ -47,7 +26,6 @@ func TestCycleDetector(t *testing.T) { newTarget(state, "//src:e", "//src:f") newTarget(state, "//src:f", "//src:g") newTarget(state, "//src:g") - waitForDeps(state) detector := cycleDetector{graph: state.Graph} assert.Nil(t, detector.Check()) @@ -62,7 +40,6 @@ func TestCycleDetector(t *testing.T) { e := newTarget(state, "//src:e", "//src:f") f := newTarget(state, "//src:f", "//src:g") g := newTarget(state, "//src:g", "//src:e") - waitForDeps(state) detector := cycleDetector{graph: state.Graph} err := detector.Check() diff --git a/src/core/graph.go b/src/core/graph.go index f3f9705e9e..c30824c9f6 100644 --- a/src/core/graph.go +++ b/src/core/graph.go @@ -5,6 +5,8 @@ package core import ( + "context" + "fmt" "maps" "slices" "sort" @@ -30,7 +32,7 @@ type BuildGraph struct { // Map of all currently known targets by their label. targets *cmap.Map[BuildLabel, *BuildTarget] // Map of all currently known packages. - packages *cmap.Map[packageKey, *Package] + packages *cmap.ErrMap[packageKey, *Package] // Registered subrepos, as a map of their name to their root. subrepos *cmap.Map[string, *Subrepo] // Subincludes that are subincluded by other subincludes @@ -64,33 +66,11 @@ func (graph *BuildGraph) Target(label BuildLabel) *BuildTarget { func (graph *BuildGraph) TargetOrDie(label BuildLabel) *BuildTarget { target := graph.Target(label) if target == nil { - log.Fatalf("Target %s not found in build graph\n", label) + panic(fmt.Sprintf("Target %s not found in build graph\n", label)) } return target } -// WaitForTarget returns the given target, waiting for it to be added if it isn't yet. -// It returns nil if the target finally turns out not to exist. -func (graph *BuildGraph) WaitForTarget(label BuildLabel) *BuildTarget { - t, tch, _ := graph.targets.GetOrWait(label) - if t != nil { - return t - } - p, pch, _ := graph.packages.GetOrWait(packageKey{Name: label.PackageName, Subrepo: label.Subrepo}) - if p != nil { - // Check target again to avoid race conditions - return graph.Target(label) - } - // Now we need to wait for either (hopefully) the target or its package to exist. - // Either the target will, which is fine, or if the package appears but the target doesn't - // we will conclude it doesn't exist. - select { - case <-tch: - case <-pch: - } - return graph.Target(label) -} - // PackageByLabel retrieves a package from the graph using the appropriate parts of the given label. // The Name entry is ignored. func (graph *BuildGraph) PackageByLabel(label BuildLabel) *Package { @@ -99,7 +79,14 @@ func (graph *BuildGraph) PackageByLabel(label BuildLabel) *Package { // Package retrieves a package from the graph by name & subrepo, or nil if it can't be found. func (graph *BuildGraph) Package(name, subrepo string) *Package { - return graph.packages.Get(packageKey{Name: name, Subrepo: subrepo}) + pkg, _ := graph.packages.Get(packageKey{Name: name, Subrepo: subrepo}) + return pkg +} + +// GetOrSetPackage retrieves a package from the graph. +// If it doesn't exist, it calls the supplied function to create it. +func (graph *BuildGraph) GetOrSetPackage(ctx context.Context, label BuildLabel, f func() (*Package, error)) (*Package, error) { + return graph.packages.GetOrSetCtx(ctx, packageKey{Name: label.PackageName, Subrepo: label.Subrepo}, f) } // PackageOrDie retrieves a package by label, and dies if it can't be found. @@ -153,12 +140,13 @@ func (graph *BuildGraph) AllTargets() BuildTargets { return targets } -// PackageMap returns a copy of the graph's internal map of name to package. +// PackageMap returns a map of name to package. +// TODO(peterebden): Change this to an iterator. func (graph *BuildGraph) PackageMap() map[string]*Package { packages := map[string]*Package{} - for _, pkg := range graph.packages.Values() { - packages[packageKey{Subrepo: pkg.SubrepoName, Name: pkg.Name}.String()] = pkg - } + graph.packages.Range(func(k packageKey, v *Package) { + packages[k.String()] = v + }) return packages } @@ -166,7 +154,7 @@ func (graph *BuildGraph) PackageMap() map[string]*Package { func NewGraph() *BuildGraph { g := &BuildGraph{ targets: cmap.New[BuildLabel, *BuildTarget](cmap.DefaultShardCount, hashBuildLabel), - packages: cmap.New[packageKey, *Package](cmap.DefaultShardCount, hashPackageKey), + packages: cmap.NewErrMap[packageKey, *Package](cmap.DefaultShardCount, hashPackageKey, nil), subrepos: cmap.New[string, *Subrepo](cmap.SmallShardCount, cmap.XXHash), subincludeSubincludes: map[BuildLabel]labelSet{}, } diff --git a/src/core/graph_benchmark_test.go b/src/core/graph_benchmark_test.go index 0fcfc04a11..f00f9581c3 100644 --- a/src/core/graph_benchmark_test.go +++ b/src/core/graph_benchmark_test.go @@ -35,20 +35,10 @@ func BenchmarkTargetLookup(b *testing.B) { graph.TargetOrDie(targets[i&targetIndexMask].Label) } }) - - // This benchmarks the best case of calling WaitForTarget, where the targets already exist, - // so it should perform identically to Simple above. - b.Run("WaitForTargetFast", func(b *testing.B) { - b.ReportAllocs() - for i := 0; i < b.N; i++ { - graph.WaitForTarget(targets[i&targetIndexMask].Label) - } - }) } -// BenchmarkWaitForTargetSlow is a more complex benchmark that tests targets being added as they are -// being waited on. -func BenchmarkWaitForTargetSlow(b *testing.B) { +// BenchmarkConcurrentTargetLookup tests targets being looked up at the same time as they're added. +func BenchmarkConcurrentTargetLookup(b *testing.B) { const parallelism = 8 var wg sync.WaitGroup wg.Add(parallelism * 2) @@ -67,7 +57,7 @@ func BenchmarkWaitForTargetSlow(b *testing.B) { lookupTargets := func() { for _, target := range targets { - graph.WaitForTarget(target.Label) + graph.Target(target.Label) } wg.Done() } diff --git a/src/core/package.go b/src/core/package.go index 377022a482..4af9122ff6 100644 --- a/src/core/package.go +++ b/src/core/package.go @@ -254,9 +254,9 @@ func FindOwningPackage(state *BuildState, file string) BuildLabel { return BuildLabel{PackageName: "", Name: "all"} } -// suggestTargets suggests the targets in the given package that might be misspellings of +// SuggestTargets suggests the targets in the given package that might be misspellings of // the requested one. -func suggestTargets(pkg *Package, label, dependent BuildLabel) string { +func (pkg *Package) SuggestTargets(label, dependent BuildLabel) string { if pkg == nil { return "" } diff --git a/src/core/package_test.go b/src/core/package_test.go index 4b100fd127..bdc617e8d7 100644 --- a/src/core/package_test.go +++ b/src/core/package_test.go @@ -118,43 +118,43 @@ func TestVerifyOutputs(t *testing.T) { func TestSuggestNoTargetFromSamePackage(t *testing.T) { pkg := makePackage("src/core", "wobble", "wibble") - s := suggestTargets(pkg, bl("//src/core:target2"), bl("//src/core:wibble")) + s := pkg.SuggestTargets(bl("//src/core:target2"), bl("//src/core:wibble")) assert.Equal(t, s, "", "No suggestion because they're not similar at all.") } func TestSuggestSingleTargetFromSamePackage(t *testing.T) { pkg := makePackage("src/core", "target1", "wibble") - s := suggestTargets(pkg, bl("//src/core:target2"), bl("//src/core:wibble")) + s := pkg.SuggestTargets(bl("//src/core:target2"), bl("//src/core:wibble")) assert.Equal(t, s, "\nMaybe you meant :target1 ?") } func TestSuggestTwoTargetsFromSamePackage(t *testing.T) { pkg := makePackage("src/core", "target1", "target21", "wibble") - s := suggestTargets(pkg, bl("//src/core:target"), bl("//src/core:blibble")) + s := pkg.SuggestTargets(bl("//src/core:target"), bl("//src/core:blibble")) assert.Equal(t, s, "\nMaybe you meant :target1 or :target21 ?") } func TestSuggestSeveralTargetsFromSamePackage(t *testing.T) { pkg := makePackage("src/core", "target1", "target21", "target_21", "wibble") - s := suggestTargets(pkg, bl("//src/core:target"), bl("//src/core:blibble")) + s := pkg.SuggestTargets(bl("//src/core:target"), bl("//src/core:blibble")) assert.Equal(t, s, "\nMaybe you meant :target1 , :target21 or :target_21 ?") } func TestSuggestSingleTargetFromAnotherPackage(t *testing.T) { pkg := makePackage("src/core", "target1", "wibble") - s := suggestTargets(pkg, bl("//src/core:target2"), bl("//src/parse:wibble")) + s := pkg.SuggestTargets(bl("//src/core:target2"), bl("//src/parse:wibble")) assert.Equal(t, s, "\nMaybe you meant //src/core:target1 ?") } func TestSuggestTwoTargetsFromAnotherPackage(t *testing.T) { pkg := makePackage("src/core", "target1", "target21", "wibble") - s := suggestTargets(pkg, bl("//src/core:target"), bl("//src/parse:blibble")) + s := pkg.SuggestTargets(bl("//src/core:target"), bl("//src/parse:blibble")) assert.Equal(t, s, "\nMaybe you meant //src/core:target1 or //src/core:target21 ?") } func TestSuggestSeveralTargetsFromAnotherPackage(t *testing.T) { pkg := makePackage("src/core", "target1", "target21", "target_21", "wibble") - s := suggestTargets(pkg, bl("//src/core:target"), bl("//src/parse:blibble")) + s := pkg.SuggestTargets(bl("//src/core:target"), bl("//src/parse:blibble")) assert.Equal(t, s, "\nMaybe you meant //src/core:target1 , //src/core:target21 or //src/core:target_21 ?") } diff --git a/src/core/stamp.go b/src/core/stamp.go index e1e71e4b59..44a8e28c7a 100644 --- a/src/core/stamp.go +++ b/src/core/stamp.go @@ -8,11 +8,11 @@ import ( // a target that is marked stamp=True. // This file contains information about its transitive dependencies that can be used to // embed information into the output (for example information from labels or licences). -func StampFile(config *Configuration, target *BuildTarget) []byte { +func StampFile(state *BuildState, target *BuildTarget) []byte { info := &stampInfo{ Targets: map[BuildLabel]targetInfo{}, } - populateStampInfo(config, target, info) + populateStampInfo(state, target, info) b, err := json.MarshalIndent(info, "", " ") if err != nil { log.Fatalf("Failed to encode stamp file: %s", err) @@ -20,16 +20,20 @@ func StampFile(config *Configuration, target *BuildTarget) []byte { return b } -func populateStampInfo(config *Configuration, target *BuildTarget, info *stampInfo) { - accepted, _ := target.CheckLicences(config) +func populateStampInfo(state *BuildState, target *BuildTarget, info *stampInfo) { + accepted, _ := target.CheckLicences(state.Config) info.Targets[target.Label] = targetInfo{ Licences: target.Licences, AcceptedLicence: accepted, Labels: target.Labels, } - for _, dep := range target.Dependencies() { + deps, unresolved := target.Dependencies(state.Graph) + if len(unresolved) > 0 { + log.Fatalf("Can't stamp %s; dependencies not in build graph: %s", target.Label, unresolved) + } + for _, dep := range deps { if _, present := info.Targets[dep.Label]; !present { - populateStampInfo(config, dep, info) + populateStampInfo(state, dep, info) } } } diff --git a/src/core/stamp_test.go b/src/core/stamp_test.go index 1747b92309..59af293aa5 100644 --- a/src/core/stamp_test.go +++ b/src/core/stamp_test.go @@ -7,8 +7,8 @@ import ( ) func TestStampFile(t *testing.T) { - config := DefaultConfiguration() - config.Licences.Accept = []string{"bsd-2-clause"} + state := NewDefaultBuildState() + state.Config.Licences.Accept = []string{"bsd-2-clause"} t1 := NewBuildTarget(ParseBuildLabel("//src/core:core", "")) t2 := NewBuildTarget(ParseBuildLabel("//src/fs:fs", "")) t3 := NewBuildTarget(ParseBuildLabel("//third_party/go:errors", "")) @@ -16,11 +16,11 @@ func TestStampFile(t *testing.T) { t3.AddLabel("go_get:github.com/pkg/errors") t3.AddLicence("bsd-2-clause") t1.AddDependency(t2.Label) - t1.resolveDependency(t2.Label, t2) t1.AddDependency(t3.Label) - t1.resolveDependency(t3.Label, t3) t2.AddDependency(t3.Label) - t2.resolveDependency(t3.Label, t3) + state.Graph.AddTarget(t1) + state.Graph.AddTarget(t2) + state.Graph.AddTarget(t3) expected := []byte(`{ "targets": { "//src/core:core": { @@ -40,5 +40,5 @@ func TestStampFile(t *testing.T) { } } }`) - assert.Equal(t, expected, StampFile(config, t1)) + assert.Equal(t, expected, StampFile(state, t1)) } diff --git a/src/core/state.go b/src/core/state.go index 2a9ca6fb0a..22fa8733e3 100644 --- a/src/core/state.go +++ b/src/core/state.go @@ -2,8 +2,10 @@ package core import ( "bytes" + "context" "crypto/sha1" "crypto/sha256" + "errors" "fmt" "hash" "hash/crc32" @@ -21,31 +23,12 @@ import ( "github.com/cespare/xxhash/v2" "github.com/zeebo/blake3" - "golang.org/x/sync/errgroup" "github.com/thought-machine/please/src/cli" - "github.com/thought-machine/please/src/cmap" "github.com/thought-machine/please/src/fs" "github.com/thought-machine/please/src/process" ) -type ParseMode uint8 - -const ( - ParseModeNormal ParseMode = 1 << iota - ParseModeForSubinclude - ParseModeForPreload - ParseModeForceBuild -) - -func (m ParseMode) IsPreload() bool { - return m&ParseModeForPreload != 0 -} - -func (m ParseMode) IsForSubinclude() bool { - return m&ParseModeForSubinclude != 0 -} - // startTime is as close as we can conveniently get to process start time. var startTime = time.Now() @@ -55,7 +38,6 @@ const cycleCheckDuration = 5 * time.Second // ParseTask is the type for the parse task queue type ParseTask struct { Label, Dependent BuildLabel - Mode ParseMode } // A TaskType identifies whether a task is a build or test action. @@ -88,14 +70,13 @@ const ( // A Parser is the interface to reading and interacting with BUILD files. type Parser interface { // ParseFile parses a single BUILD file into the given package. - ParseFile(pkg *Package, forLabel, dependent *BuildLabel, mode ParseMode, fs iofs.FS, filename string) error + ParseFile(pkg *Package, forLabel, dependent *BuildLabel, fs iofs.FS, filename string) error // ParseReader parses a single BUILD file into the given package. - ParseReader(pkg *Package, reader io.ReadSeeker, forLabel, dependent *BuildLabel, mode ParseMode) error + ParseReader(pkg *Package, reader io.ReadSeeker, forLabel, dependent *BuildLabel) error // RunPreBuildFunction runs a pre-build function for a target. RunPreBuildFunction(state *BuildState, target *BuildTarget) error // RunPostBuildFunction runs a post-build function for a target. RunPostBuildFunction(state *BuildState, target *BuildTarget, output string) error - RegisterPreload(label BuildLabel) error } // A RemoteClient is the interface to a remote execution service. @@ -136,9 +117,6 @@ type TargetHasher interface { // Tasks are internally tracked by priority, which is determined by their type. type BuildState struct { Graph *BuildGraph - // Streams of pending tasks - pendingParses chan ParseTask - pendingActions chan Task // Timestamp that the build is considered to start at. StartTime time.Time // Various system statistics. Mostly used during remote communication. @@ -237,10 +215,19 @@ type BuildState struct { // EnableBreakpoints enablese the breakpoint() build-in, and drops Please into an interactive debugger when // they're encountered. EnableBreakpoints bool - // NeedDebugDeps is true if we're doing a `plz debug` and we need to build the debug tools and - // data + // NeedDebugDeps is true if we're doing a `plz debug` and we need to build the debug tools and data NeedDebugDeps bool + // Build is a callback to build a single target. It's set from outside here. + // TODO(peter): can we find a way of moving these off this struct? it feels weird here + // The second label is the dependent, i.e. whatever is asking for this to be built. + Build func(label, dependent BuildLabel) (*BuildTarget, error) + // Parse is a callback to parse a single package. It's also set from outside. + // The second label is the dependent, i.e. whatever is asking for this to be parsed. + Parse func(label, dependent BuildLabel) (*Package, error) + // Cancel is a cancel function called when the state detects a cycle. + Cancel func() + // initOnce is used to control loading the subrepo .plzconfig initOnce *sync.Once @@ -280,20 +267,8 @@ func (state *BuildState) Initialise(subrepo *Subrepo) (err error) { // A stateProgress records various points of progress for a State. // This is split out from above so we can share it between multiple instances. type stateProgress struct { - // Used to count the number of currently active/pending targets - numActive int64 - numPending int64 - numDone int64 - numParses atomic.Int64 mutex sync.Mutex - closeOnce sync.Once resultOnce sync.Once - // Used to track subinclude() calls that block until targets are built. Keyed by their label. - pendingTargets *cmap.Map[BuildLabel, chan struct{}] - // Used to track general package parsing requests. Keyed by a packageKey struct. - pendingPackages *cmap.Map[packageKey, chan struct{}] - // similar to pendingPackages but consumers haven't committed to parsing the package - packageWaits *cmap.Map[packageKey, chan struct{}] // The set of known states allStates []*BuildState // Targets that we were originally requested to build @@ -338,89 +313,6 @@ type lockedStats struct { Stats SystemStats } -// addActiveTargets increments the counter for a number of newly active build targets. -func (state *BuildState) addActiveTargets(n int) { - atomic.AddInt64(&state.progress.numActive, int64(n)) -} - -// addPendingParse adds a task for a pending parse of a build label. -func (state *BuildState) addPendingParse(label, dependent BuildLabel, mode ParseMode) { - atomic.AddInt64(&state.progress.numActive, 1) - atomic.AddInt64(&state.progress.numPending, 1) - - go func() { - defer func() { - recover() // Prevent death on 'send on closed channel' - }() - state.pendingParses <- ParseTask{Label: label, Dependent: dependent, Mode: mode} - }() -} - -// addPendingBuild adds a task for a pending build of a target. -func (state *BuildState) addPendingBuild(target *BuildTarget) { - atomic.AddInt64(&state.progress.numPending, 1) - go func() { - defer func() { - recover() // Prevent death on 'send on closed channel' - }() - state.pendingActions <- Task{Target: target, Type: BuildTask} - }() -} - -// AddPendingTest adds a task for a pending test of a target. -func (state *BuildState) AddPendingTest(target *BuildTarget) { - if state.TestSequentially { - state.addPendingTest(target, 1) - } else { - state.addPendingTest(target, int(state.NumTestRuns)) - } -} - -// Parses returns the number of current parse tasks -func (state *BuildState) Parses() *atomic.Int64 { - return &state.progress.numParses -} - -func (state *BuildState) addPendingTest(target *BuildTarget, numRuns int) { - atomic.AddInt64(&state.progress.numPending, int64(numRuns)) - go func() { - defer func() { - recover() // Prevent death on 'send on closed channel' - }() - for run := 1; run <= numRuns; run++ { - state.pendingActions <- Task{Target: target, Run: uint32(run), Type: TestTask} - } - }() -} - -// TaskQueues returns a set of channels to listen on for tasks of various types. -func (state *BuildState) TaskQueues() (parses <-chan ParseTask, actions <-chan Task) { - return state.pendingParses, state.pendingActions -} - -// TaskDone indicates that a single task is finished. Should be called after one is finished with -// a task returned from NextTask(). -func (state *BuildState) TaskDone() { - state.taskDone(false) -} - -func (state *BuildState) taskDone(wasSynthetic bool) { - if !wasSynthetic { - atomic.AddInt64(&state.progress.numDone, 1) - } - if atomic.AddInt64(&state.progress.numPending, -1) <= 0 { - state.Stop() - } -} - -// Stop stops the worker queues after any current tasks are done. -func (state *BuildState) Stop() { - state.progress.closeOnce.Do(func() { - close(state.pendingParses) - close(state.pendingActions) - }) -} - // CloseResults closes the result channels. func (state *BuildState) CloseResults() { state.progress.cycleDetector.Stop() @@ -433,6 +325,11 @@ func (state *BuildState) CloseResults() { } } +// AddOriginalTarget adds an original target to this state +func (state *BuildState) AddOriginalTarget(label BuildLabel) { + state.progress.originalTargets.Add(label) +} + // IsOriginalTarget returns true if a target is an original target, ie. one specified on the command line. func (state *BuildState) IsOriginalTarget(target *BuildTarget) bool { return state.isOriginalTarget(target, false) @@ -484,25 +381,6 @@ func (state *BuildState) ShouldInclude(target *BuildTarget) bool { return target.ShouldInclude(state.Include, state.Exclude) } -// AddOriginalTarget adds one of the original targets and enqueues it for parsing / building. -func (state *BuildState) AddOriginalTarget(label BuildLabel, addToList bool) { - _, arch := SplitSubrepoArch(label.Subrepo) - if arch != "" { - state.Graph.AddSubrepo(SubrepoForArch(state, cli.NewArchFromString(arch))) - } - - // Check it's not excluded first. - for _, e := range state.ExcludeTargets { - if e.Includes(label) { - return - } - } - if addToList { - state.progress.originalTargets.Add(label) - } - state.addPendingParse(label, OriginalTarget, ParseModeNormal) -} - // Hasher returns a PathHasher for the given function (e.g. "SHA1"). func (state *BuildState) Hasher(name string) *fs.PathHasher { hasher, present := state.hashers[name] @@ -525,14 +403,6 @@ func (state *BuildState) OutputHashCheckers() []*fs.PathHasher { // LogParseResult logs the result of a target parsing. func (state *BuildState) LogParseResult(label BuildLabel, status BuildResultStatus, description string) { if status == PackageParsed { - // We may have parse tasks waiting for this package to exist, check for them. - key := packageKey{Name: label.PackageName, Subrepo: label.Subrepo} - if ch := state.progress.pendingPackages.Get(key); ch != nil { - close(ch) // This signals to anyone waiting that it's done. - } - if ch := state.progress.packageWaits.Get(key); ch != nil { - close(ch) // This signals to anyone waiting that it's done. - } return // We don't notify anything else on these. } state.logResult(&BuildResult{ @@ -552,20 +422,6 @@ func (state *BuildState) LogBuildResult(target *BuildTarget, status BuildResultS Err: nil, Description: description, }) - if status == TargetBuilt || status == TargetCached { - // We may have parse tasks waiting for this guy to build, check for them. - if ch := state.progress.pendingTargets.Get(target.Label); ch != nil { - close(ch) // This signals to anyone waiting that it's done. - } - } -} - -// ArchSubrepoInitialised closes the pending target channel for the non-existent arch subrepo psudo-target -func (state *BuildState) ArchSubrepoInitialised(subrepoLabel BuildLabel) { - // We may have parse tasks waiting for this guy to build, check for them. - if ch := state.progress.pendingTargets.Get(subrepoLabel); ch != nil { - close(ch) // This signals to anyone waiting that it's done. - } } // LogTestRunning logs a target while its tests are running. @@ -611,6 +467,9 @@ func (state *BuildState) LogBuildError(label BuildLabel, status BuildResultStatu // logResult logs a build result directly to the state's queue. func (state *BuildState) logResult(result *BuildResult) { + if result.Err != nil && errors.Is(result.Err, context.Canceled) { + return + } result.Time = time.Now() state.progress.internalResults <- result if result.Status.IsFailure() { @@ -673,37 +532,11 @@ func (state *BuildState) forwardResults() { } } -// RegisterPreloads waits for all preloaded subinclude targets to be built, downloads them, and then registers them with -// the interpreter. We have to actually register them otherwise this will return before we build any -// transitive subincludes. -func (state *BuildState) RegisterPreloads() error { - var err error - state.preloadDownloadOnce.Do(func() { - var eg errgroup.Group - for _, inc := range state.GetPreloadedSubincludes() { - if inc.IsPseudoTarget() { - log.Fatalf("Can't preload pseudotarget %v", inc) - } - - // Queue them up asynchronously to feed the queues as quickly as possible - inc := inc - eg.Go(func() error { - state.WaitForTargetAndEnsureDownload(inc, OriginalTarget, true) - return state.Parser.RegisterPreload(inc) - }) - } - // We must wait for all the subinclude targets to be built otherwise updating the locals might race with parsing - // a package - err = eg.Wait() - }) - return err -} - // checkForCycles is run to detect a cycle in the graph. It converts any returned error into an async error. func (state *BuildState) checkForCycles() { if err := state.progress.cycleDetector.Check(); err != nil { state.LogBuildError(err.Cycle[0].Label, TargetBuildFailed, err, "") - state.Stop() + state.Cancel() } } @@ -722,17 +555,6 @@ func (state *BuildState) Results() <-chan *BuildResult { return state.progress.results } -// NumActive returns the number of currently active tasks (i.e. those that are -// scheduled to be built at some point, or have been built already). -func (state *BuildState) NumActive() int { - return int(atomic.LoadInt64(&state.progress.numActive)) -} - -// NumDone returns the number of tasks that have been completed so far. -func (state *BuildState) NumDone() int { - return int(atomic.LoadInt64(&state.progress.numDone)) -} - // ExpandOriginalLabels expands any pseudo-labels (ie. :all, ... has already been resolved to a bunch :all targets) // from the set of original labels. This will exclude non-test targets when we're building for test. func (state *BuildState) ExpandOriginalLabels() BuildLabels { @@ -841,87 +663,6 @@ func (state *BuildState) ExpandVisibleOriginalTargets() BuildLabels { return ret } -// SyncParsePackage either returns the given package which is already parsed and available, -// or returns nil indicating it is ready to be parsed. Everything subsequently calling this -// will block until the original caller parse it. -func (state *BuildState) SyncParsePackage(label BuildLabel) *Package { - if p := state.Graph.PackageByLabel(label); p != nil { - return p - } - if ch, inserted := state.progress.pendingPackages.AddOrGet(label.packageKey(), func() chan struct{} { - return make(chan struct{}) - }); !inserted { - waitOnChan(ch, "Still waiting for SyncParsePackage(%v)", label) - } - return state.Graph.PackageByLabel(label) // Important to check again; it's possible to race against this whole lot. -} - -func waitOnChan[T any](ch chan T, message string, args ...any) { - start := time.Now() - t := time.NewTimer(10 * time.Second) - defer t.Stop() - select { - case <-ch: - return - case <-t.C: - log.Debugf("%v (after %v)", fmt.Sprintf(message, args...), time.Since(start)) - } - <-ch -} - -// WaitForPackage is similar to WaitForBuiltTarget however it waits for the package to be parsed, queuing it for parse -// if necessary -func (state *BuildState) WaitForPackage(l, dependent BuildLabel, mode ParseMode) *Package { - if p := state.Graph.PackageByLabel(l); p != nil { - return p - } - key := packageKey{Name: l.PackageName, Subrepo: l.Subrepo} - - // If something has promised to parse it, wait for them to do so - if ch := state.progress.pendingPackages.Get(key); ch != nil { - waitOnChan(ch, "Still waiting for pending package in WaitForPackage(%v, %v, %v)", l, dependent, mode) - return state.Graph.PackageByLabel(l) - } - - // If something has already queued the package to be parsed, wait for them - // (atomically: a racing Get-then-Set here can orphan the first caller's channel) - if ch, inserted := state.progress.packageWaits.AddOrGet(key, func() chan struct{} { - return make(chan struct{}) - }); !inserted { - waitOnChan(ch, "Still waiting for package wait in WaitForPackage(%v, %v, %v)", l, dependent, mode) - return state.Graph.PackageByLabel(l) - } - - // Otherwise queue the target for parse and recurse - state.addPendingParse(l, dependent, mode) - - return state.WaitForPackage(l, dependent, mode) -} - -func (state *BuildState) WaitForBuiltTarget(l, dependent BuildLabel, mode ParseMode) *BuildTarget { - if t := state.Graph.Target(l); t != nil && t.State().IsBuilt() { - return t - } - - dependent.Name = "all" // Every target in this package depends on this one. - // okay, we need to register and wait for this guy. - if ch, inserted := state.progress.pendingTargets.AddOrGet(l, func() chan struct{} { - return make(chan struct{}) - }); !inserted { - // Something's already registered for this, get on the train - waitOnChan(ch, "Still waiting on WaitForBuiltTarget(label %v, dependant %v, ParseMode(%v))", l, dependent, mode) - return state.Graph.Target(l) - } - if err := state.queueTarget(l, dependent, mode.IsForSubinclude(), mode); err != nil { - log.Fatalf("%v", err) - } - - // Do this all over; the re-checking that happens here is actually fairly important to resolve - // a potential race condition if the target was built between us checking earlier and registering - // the channel just now. - return state.WaitForBuiltTarget(l, dependent, mode) -} - // AddTarget adds a new target to the build graph. func (state *BuildState) AddTarget(pkg *Package, target *BuildTarget) { pkg.AddTarget(target) @@ -979,256 +720,6 @@ func (state *BuildState) EnsureDownloaded(target *BuildTarget) error { return nil } -// WaitForTargetAndEnsureDownload waits for the target to be built and then downloads it if executing remotely -func (state *BuildState) WaitForTargetAndEnsureDownload(l, dependent BuildLabel, isForPreload bool) *BuildTarget { - mode := ParseModeForSubinclude - if isForPreload { - mode |= ParseModeForPreload - } - return state.waitForTargetAndEnsureDownload(l, dependent, mode) -} - -// WaitForInitialTargetAndEnsureDownload is like WaitForTargetAndEnsureDownload but is used for -// targets in the initial set. -func (state *BuildState) WaitForInitialTargetAndEnsureDownload(l, dependent BuildLabel) *BuildTarget { - // This may have been an architecture label from the CLI - if state.WaitForBuiltTarget(l, dependent, ParseModeNormal) == nil { - return nil - } - return state.waitForTargetAndEnsureDownload(l, dependent, ParseModeNormal) -} - -func (state *BuildState) waitForTargetAndEnsureDownload(l, dependent BuildLabel, mode ParseMode) *BuildTarget { - target := state.WaitForBuiltTarget(l, dependent, mode) - if !target.State().IsBuilt() { - return nil - } - if err := state.EnsureDownloaded(target); err != nil { - panic(fmt.Errorf("failed to download target outputs: %w", err)) - } - return target -} - -// ActivateTarget marks a target as active (ie. to be built) and adds its dependencies as pending parses. -func (state *BuildState) ActivateTarget(pkg *Package, label, dependent BuildLabel, mode ParseMode) error { - if !label.IsAllTargets() && state.Graph.Target(label) == nil { - if label.Subrepo == "" && label.PackageName == "" && label.Name == dependent.Subrepo { - if subrepo := state.CheckArchSubrepo(label.Name); subrepo != nil { - state.ArchSubrepoInitialised(label) - return nil - } - } - if state.Config.Bazel.Compatibility && mode.IsForSubinclude() { - // Bazel allows some things that look like build targets but aren't - notably the syntax - // to load(). It suits us to treat that as though it is one, but we now have to - // implicitly make it available. - if pkg != nil { - exportFile(state, pkg, label) - } - } else { - msg := fmt.Sprintf("Parsed build file %s but it doesn't contain target %s", pkg.Filename, label.Name) - if dependent != OriginalTarget { - msg += fmt.Sprintf(" (depended on by %s)", dependent) - } - return fmt.Errorf("%s", msg+suggestTargets(pkg, label, dependent)) - } - } - if state.ParsePackageOnly && !mode.IsForSubinclude() { - return nil // Some kinds of query don't need a full recursive parse. - } else if label.IsAllTargets() { - if pkg == nil { - return fmt.Errorf("Cannot use :all in this context") - } - if dependent == OriginalTarget { - for _, target := range pkg.AllTargets() { - // Don't activate targets that were added in a post-build function; that causes a race condition - // between the post-build functions running and other things trying to activate them too early. - if state.ShouldInclude(target) && !target.AddedPostBuild { - // Must always do this for coverage because we need to calculate sources of - // non-test targets later on. - if !state.NeedTests || target.IsTest() || state.NeedCoverage { - if err := state.QueueTarget(target.Label, dependent, dependent.IsAllTargets(), mode); err != nil { - return err - } - } - } - } - } - } else { - for _, l := range state.Graph.DependentTargets(dependent, label) { - // We use :all to indicate a dependency needed for parse. - if err := state.QueueTarget(l, dependent, dependent.IsAllTargets(), mode); err != nil { - return err - } - } - } - return nil -} - -// exportFile adds a single-file export target. This is primarily used for Bazel compat. -func exportFile(state *BuildState, pkg *Package, label BuildLabel) { - t := NewBuildTarget(label) - t.Subrepo = pkg.Subrepo - t.IsFilegroup = true - t.AddSource(NewFileLabel(label.Name, pkg)) - state.AddTarget(pkg, t) -} - -// CheckArchSubrepo checks if a target refers to a cross-compiling subrepo. -// Those don't have to be explicitly defined - maybe we should insist on that, but it's nicer not to have to. -func (state *BuildState) CheckArchSubrepo(name string) *Subrepo { - var arch cli.Arch - if err := arch.UnmarshalFlag(name); err == nil { - return state.Graph.MaybeAddSubrepo(SubrepoForArch(state, arch)) - } - return nil -} - -// QueueTarget adds a single target to the build queue. -func (state *BuildState) QueueTarget(label, dependent BuildLabel, forceBuild bool, mode ParseMode) error { - return state.queueTarget(label, dependent, forceBuild || mode.IsForSubinclude() || (mode&ParseModeForceBuild) != 0, mode) -} - -func (state *BuildState) queueTarget(label, dependent BuildLabel, forceBuild bool, mode ParseMode) error { - target := state.Graph.Target(label) - if target == nil { - // If the package isn't loaded yet, we need to queue a parse for it. - if state.Graph.PackageByLabel(label) == nil { - if forceBuild { - mode |= ParseModeForceBuild - } - // Queue the target up for parse. The parse step activates the target for us if it needs to be built, so we - // don't need to do this here. - state.addPendingParse(label, dependent, mode) - return nil - } - // Package is loaded but target doesn't exist in it. Check again to avoid nasty races. - target = state.Graph.Target(label) - if target == nil { - return fmt.Errorf("Target %s (referenced by %s) doesn't exist", label, dependent) - } - } - if dependent.IsAllTargets() || dependent == OriginalTarget { - return state.queueResolvedTarget(target, forceBuild, mode) - } - for _, l := range target.ProvideFor(state.Graph.TargetOrDie(dependent)) { - if l == label { - if err := state.queueResolvedTarget(target, forceBuild, mode); err != nil { - return err - } - } else if err := state.queueTarget(l, dependent, forceBuild, mode); err != nil { - return err - } - } - return nil -} - -// QueueTestTarget adds a target to the queue to be tested. -func (state *BuildState) QueueTestTarget(target *BuildTarget) { - state.queueTargetData(target) - state.AddPendingTest(target) -} - -// queueTargetData queues up builds of the target's runtime data. -func (state *BuildState) queueTargetData(target *BuildTarget) { - for _, data := range target.AllData() { - if l, ok := data.Label(); ok { - state.WaitForBuiltTarget(l, target.Label, ParseModeForSubinclude) - } - } -} - -// queueResolvedTarget is like queueTarget but once we have a resolved target. -func (state *BuildState) queueResolvedTarget(target *BuildTarget, forceBuild bool, mode ParseMode) error { - if mode.IsForSubinclude() { - target.neededForSubinclude.Store(true) - } - if target.State() >= Active && !forceBuild { - return nil // Target is already tagged to be built and likely on the queue. - } - - queueAsync := func(shouldBuild bool) { - if target.IsTest() && state.NeedTests { - if state.TestSequentially { - state.addActiveTargets(2) // One for build & one for test - } else { - // Tests count however many times we're going to run them if parallel. - state.addActiveTargets(int(1 + state.NumTestRuns)) - } - } else { - state.addActiveTargets(1) - } - // Actual queuing stuff now happens asynchronously in here. - atomic.AddInt64(&state.progress.numPending, 1) - go state.queueTargetAsync(target, forceBuild, shouldBuild, mode) - } - - // Here we want to ensure we don't queue the target every time; ideally we only do it once. - // However we might need to do it twice if the initial request doesn't require it to be built - // but a later one does. - if state.NeedBuild || forceBuild { - if target.SyncUpdateState(Inactive, Active) || target.SyncUpdateState(Semiactive, Active) { - queueAsync(true) - } - } else if target.SyncUpdateState(Inactive, Semiactive) { - queueAsync(false) - } - return nil -} - -// queueTarget enqueues a target's dependencies and the target itself once they are done. -func (state *BuildState) queueTargetAsync(target *BuildTarget, forceBuild, building bool, mode ParseMode) { - defer state.taskDone(true) - for _, dep := range target.DeclaredDependencies() { - if err := state.queueTarget(dep, target.Label, forceBuild, mode); err != nil { - state.asyncError(dep, err) - return - } - } - for { - var called atomic.Bool - if err := target.resolveDependencies(state.Graph, func(t *BuildTarget) error { - called.Store(true) - return state.queueResolvedTarget(t, forceBuild, ParseModeNormal) - }); err != nil { - state.asyncError(target.Label, err) - return - } - // Wait for these targets to actually build. - if building { - for _, t := range target.Dependencies() { - t.WaitForBuild(target.Label) - if t.State() >= DependencyFailed { // Either the target failed or its dependencies failed - // Give up and set the original target as dependency failed - target.SetState(DependencyFailed) - state.LogBuildResult(target, TargetBuilt, "Dependency failed") - target.FinishBuild() - return - } - } - } - if !called.Load() { - // We are now ready to go, we have nothing to wait for. - if building && target.SyncUpdateState(Active, Pending) { - // If we're going to run the target, we need its runtime data to be done. This has to - // happen before we build it otherwise remote downloads will fail. - if state.NeedRun && state.IsOriginalTarget(target) { - state.queueTargetData(target) - } - state.addPendingBuild(target) - } - return - } - } -} - -// asyncError reports an error that's happened in an asynchronous function. -func (state *BuildState) asyncError(label BuildLabel, err error) { - log.Error("Error queuing %s: %s", label, err) - state.LogBuildError(label, TargetBuildFailed, err, "") - state.Stop() -} - // ForTarget returns the state associated with a given target. // This differs if the target is in a subrepo for a different architecture. func (state *BuildState) ForTarget(target *BuildTarget) *BuildState { @@ -1411,10 +902,6 @@ func (state *BuildState) Root() *BuildState { return state.ParentState.Root() } -func (state *BuildState) IsPendingTarget(label BuildLabel) bool { - return state.progress.pendingTargets.Contains(label) -} - func newCRC32() hash.Hash { return hash.Hash(crc32.NewIEEE()) } @@ -1456,9 +943,7 @@ func executorFromConfig(config *Configuration) *process.Executor { func NewBuildState(config *Configuration) *BuildState { graph := NewGraph() state := &BuildState{ - Graph: graph, - pendingParses: make(chan ParseTask, 10000), - pendingActions: make(chan Task, 1000), + Graph: graph, hashers: map[string]*fs.PathHasher{ // For compatibility reasons the sha1 hasher has no suffix. "sha1": fs.NewPathHasher(RepoRoot, config.Build.Xattrs, sha1.New, "sha1"), @@ -1480,11 +965,6 @@ func NewBuildState(config *Configuration) *BuildState { Arch: cli.HostArch(), stats: &lockedStats{}, progress: &stateProgress{ - numActive: 1, // One for the initial target adding on the main thread. - numPending: 1, - pendingTargets: cmap.New[BuildLabel, chan struct{}](cmap.DefaultShardCount, hashBuildLabel), - pendingPackages: cmap.New[packageKey, chan struct{}](cmap.DefaultShardCount, hashPackageKey), - packageWaits: cmap.New[packageKey, chan struct{}](cmap.DefaultShardCount, hashPackageKey), internalResults: make(chan *BuildResult, 1000), cycleDetector: cycleDetector{graph: graph}, originalTargets: NewTargetSet(), diff --git a/src/core/state_test.go b/src/core/state_test.go index 1500de9971..50885deae4 100644 --- a/src/core/state_test.go +++ b/src/core/state_test.go @@ -11,8 +11,8 @@ import ( func TestExpandOriginalLabels(t *testing.T) { state := NewDefaultBuildState() - state.AddOriginalTarget(BuildLabel{PackageName: "src/core", Name: "all"}, true) - state.AddOriginalTarget(BuildLabel{PackageName: "src/parse", Name: "parse"}, true) + state.AddOriginalTarget(BuildLabel{PackageName: "src/core", Name: "all"}) + state.AddOriginalTarget(BuildLabel{PackageName: "src/parse", Name: "parse"}) state.Include = []string{"go"} state.Exclude = []string{"py"} @@ -36,7 +36,7 @@ func TestExpandOriginalLabels(t *testing.T) { func TestExpandOriginalTestLabels(t *testing.T) { state := NewDefaultBuildState() - state.AddOriginalTarget(BuildLabel{PackageName: "src/core", Name: "all"}, true) + state.AddOriginalTarget(BuildLabel{PackageName: "src/core", Name: "all"}) state.NeedTests = true state.Include = []string{"go"} state.Exclude = []string{"py"} @@ -52,7 +52,7 @@ func TestExpandOriginalTestLabels(t *testing.T) { func TestExpandVisibleOriginalTargets(t *testing.T) { state := NewDefaultBuildState() - state.AddOriginalTarget(BuildLabel{PackageName: "src/core", Name: "all"}, true) + state.AddOriginalTarget(BuildLabel{PackageName: "src/core", Name: "all"}) addTarget(state, "//src/core:target1", "py") addTarget(state, "//src/core:_target1#zip", "py") @@ -61,8 +61,8 @@ func TestExpandVisibleOriginalTargets(t *testing.T) { func TestExpandOriginalSubLabels(t *testing.T) { state := NewDefaultBuildState() - state.AddOriginalTarget(BuildLabel{PackageName: "src/core", Name: "all"}, true) - state.AddOriginalTarget(BuildLabel{PackageName: "src/core/tests", Name: "all"}, true) + state.AddOriginalTarget(BuildLabel{PackageName: "src/core", Name: "all"}) + state.AddOriginalTarget(BuildLabel{PackageName: "src/core/tests", Name: "all"}) state.Include = []string{"go"} state.Exclude = []string{"py"} addTarget(state, "//src/core:target1", "go") @@ -78,10 +78,10 @@ func TestExpandOriginalSubLabels(t *testing.T) { func TestExpandOriginalLabelsOrdering(t *testing.T) { state := NewDefaultBuildState() - state.AddOriginalTarget(BuildLabel{PackageName: "src/parse", Name: "parse"}, true) - state.AddOriginalTarget(BuildLabel{PackageName: "src/core", Name: "all"}, true) - state.AddOriginalTarget(BuildLabel{PackageName: "src/core/tests", Name: "all"}, true) - state.AddOriginalTarget(BuildLabel{PackageName: "src/build", Name: "build"}, true) + state.AddOriginalTarget(BuildLabel{PackageName: "src/parse", Name: "parse"}) + state.AddOriginalTarget(BuildLabel{PackageName: "src/core", Name: "all"}) + state.AddOriginalTarget(BuildLabel{PackageName: "src/core/tests", Name: "all"}) + state.AddOriginalTarget(BuildLabel{PackageName: "src/build", Name: "build"}) addTarget(state, "//src/core:target1", "go") addTarget(state, "//src/core:target2", "py") addTarget(state, "//src/core/tests:target3", "go") @@ -111,24 +111,6 @@ func TestAddTargetFilegroupPackageOutputs(t *testing.T) { assert.True(t, exists) } -func TestAddDepsToTarget(t *testing.T) { - state := NewDefaultBuildState() - _, builds := state.TaskQueues() - pkg := NewPackage("src/core") - target1 := addTargetDeps(state, pkg, "//src/core:target1", "//src/core:target2") - target2 := addTargetDeps(state, pkg, "//src/core:target2") - state.Graph.AddPackage(pkg) - state.QueueTarget(target1.Label, OriginalTarget, false, ParseModeNormal) - task := <-builds - assert.Equal(t, Task{Target: target2}, task) - // Now simulate target2 being built and adding a new dep to target1 in its post-build function. - target3 := addTargetDeps(state, pkg, "//src/core:target3") - target1.AddDependency(target3.Label) - target2.FinishBuild() - task = <-builds - assert.Equal(t, Task{Target: target3}, task) -} - func addTarget(state *BuildState, name string, labels ...string) { target := NewBuildTarget(ParseBuildLabel(name, "")) target.Labels = labels @@ -144,16 +126,6 @@ func addTarget(state *BuildState, name string, labels ...string) { state.Graph.AddTarget(target) } -func addTargetDeps(state *BuildState, pkg *Package, name string, deps ...string) *BuildTarget { - target := NewBuildTarget(ParseBuildLabel(name, "")) - for _, d := range deps { - target.AddDependency(ParseBuildLabel(d, "")) - } - pkg.AddTarget(target) - state.Graph.AddTarget(target) - return target -} - func TestCopyPlugin(t *testing.T) { plugin := &Plugin{ ExtraValues: map[string][]string{ diff --git a/src/core/utils.go b/src/core/utils.go index 3c63f53516..e59f4f9cdf 100644 --- a/src/core/utils.go +++ b/src/core/utils.go @@ -171,8 +171,8 @@ func IterInputs(state *BuildState, graph *BuildGraph, target *BuildTarget, inclu done[dependency.Label] = true if target == dependency || (target.NeedsTransitiveDependencies && !dependency.OutputIsComplete) { - for _, dep := range dependency.BuildDependencies() { - for dep2 := range recursivelyProvideFor(graph, target, dependency, dep.Label) { + for dep := range dependency.BuildDependencies() { + for dep2 := range recursivelyProvideFor(graph, target, dependency, dep) { if !done[dep2] && !dependency.IsTool(dep2) { if !inner(graph.TargetOrDie(dep2)) { return false @@ -181,7 +181,7 @@ func IterInputs(state *BuildState, graph *BuildGraph, target *BuildTarget, inclu } } } else { - for _, dep := range dependency.ExportedDependencies() { + for dep := range dependency.ExportedDependencies() { for dep2 := range recursivelyProvideFor(graph, target, dependency, dep) { if !done[dep2] { if !inner(graph.TargetOrDie(dep2)) { @@ -261,7 +261,7 @@ func IterRuntimeFiles(graph *BuildGraph, target *BuildTarget, absoluteOuts bool, } outDir := target.OutDir() - for _, out := range target.Outputs() { + for _, out := range target.Outputs(graph) { if !pushOut(filepath.Join(outDir, out), out) { return } @@ -431,8 +431,9 @@ func IterInputPaths(graph *BuildGraph, target *BuildTarget) iter.Seq[string] { } // Finally recurse for all the deps of this rule. - for _, dep := range target.Dependencies() { - for d := range recursivelyProvideFor(graph, target, dep, dep.Label) { + for dep := range target.DeclaredDependencies() { + t := graph.TargetOrDie(dep) + for d := range recursivelyProvideFor(graph, target, t, t.Label) { if !inner(graph.TargetOrDie(d)) { return false } diff --git a/src/core/utils_benchmark_test.go b/src/core/utils_benchmark_test.go index cec960cf11..64d22a0917 100644 --- a/src/core/utils_benchmark_test.go +++ b/src/core/utils_benchmark_test.go @@ -31,7 +31,6 @@ func BenchmarkIterInputsSimple(b *testing.B) { } state.Graph.AddTarget(dep) target.AddDependency(dep.Label) - target.resolveDependency(target.Label, dep) } for i := 0; i < 25; i++ { @@ -63,7 +62,6 @@ func BenchmarkIterInputsNamedSources(b *testing.B) { } state.Graph.AddTarget(dep) target.AddDependency(dep.Label) - target.resolveDependency(target.Label, dep) } for i := 0; i < 5; i++ { diff --git a/src/core/utils_test.go b/src/core/utils_test.go index adfa07dd63..831c12fb9a 100644 --- a/src/core/utils_test.go +++ b/src/core/utils_test.go @@ -72,8 +72,8 @@ func TestIterSources(t *testing.T) { assert.Equal(t, []SourcePair{ {"src/output/output2.go", "plz-out/tmp/src/output/output2._build/src/output/output2.go"}, - {"plz-out/gen/src/core/target2.a", "plz-out/tmp/src/output/output2._build/src/core/target2.a"}, {"plz-out/gen/src/output/output1.a", "plz-out/tmp/src/output/output2._build/src/output/output1.a"}, + {"plz-out/gen/src/core/target2.a", "plz-out/tmp/src/output/output2._build/src/core/target2.a"}, }, iterSources("//src/output:output2")) assert.Equal(t, []SourcePair{ @@ -172,7 +172,6 @@ func makeTarget4(graph *BuildGraph, label string, deps ...string) *BuildTarget { for _, dep := range deps { t := graph.TargetOrDie(ParseBuildLabel(dep, "")) target.AddDependency(t.Label) - target.resolveDependency(target.Label, t) } target.Sources = append(target.Sources, FileLabel{ File: target.Label.Name + ".go", diff --git a/src/exec/exec.go b/src/exec/exec.go index 93ca0deab3..7d1c08b51d 100644 --- a/src/exec/exec.go +++ b/src/exec/exec.go @@ -134,7 +134,7 @@ func resolveCmd(state *core.BuildState, target *core.BuildTarget, overrideCmdArg return core.ReplaceSequences(state, target, strings.Join(overrideCmdArgs, " ")) } - outs := target.Outputs() + outs := target.Outputs(state.Graph) if len(outs) != 1 { return "", fmt.Errorf("Target %s cannot be executed as it has %d outputs", target.Label, len(outs)) } diff --git a/src/export/export.go b/src/export/export.go index 2863dfef0c..46bad491e8 100644 --- a/src/export/export.go +++ b/src/export/export.go @@ -186,7 +186,12 @@ func (e *export) export(target *core.BuildTarget) { } e.exportedTargets[target.Label] = true - for _, dep := range target.Dependencies() { + deps, unresolved := target.Dependencies(e.state.Graph) + if len(unresolved) > 0 { + // Carrying on would silently produce an exported repo that doesn't build. + log.Fatalf("Can't export %s; dependencies not in build graph: %s", target.Label, unresolved) + } + for _, dep := range deps { e.export(dep) } for _, subinclude := range e.state.Graph.PackageOrDie(target.Label).AllSubincludes(e.state.Graph) { @@ -201,7 +206,7 @@ func (e *export) export(target *core.BuildTarget) { func Outputs(state *core.BuildState, dir string, targets []core.BuildLabel) { for _, label := range targets { target := state.Graph.TargetOrDie(label) - for _, out := range target.Outputs() { + for _, out := range target.Outputs(state.Graph) { fullPath := filepath.Join(dir, out) outDir := filepath.Dir(fullPath) if err := os.MkdirAll(outDir, core.DirPermissions); err != nil { diff --git a/src/gc/gc.go b/src/gc/gc.go index 1bdc7463df..66e7859a6c 100644 --- a/src/gc/gc.go +++ b/src/gc/gc.go @@ -167,10 +167,12 @@ func addTarget(graph *core.BuildGraph, m targetMap, target *core.BuildTarget) { } log.Debug(" %s", target.Label) m[target] = true - for _, dep := range target.DeclaredDependencies() { + for dep := range target.DeclaredDependencies() { addTarget(graph, m, graph.Target(dep)) } - for _, dep := range target.Dependencies() { + // As above, anything we can't resolve is simply skipped. + deps, _ := target.Dependencies(graph) + for _, dep := range deps { addTarget(graph, m, dep) } if target.Subrepo != nil && target.Subrepo.Target != nil { @@ -199,7 +201,7 @@ func anyInclude(labels []core.BuildLabel, label core.BuildLabel) bool { // it will return //src/test:test for //src/test:container_test. func publicDependencies(graph *core.BuildGraph, target *core.BuildTarget) []*core.BuildTarget { ret := []*core.BuildTarget{} - for _, dep := range target.DeclaredDependencies() { + for dep := range target.DeclaredDependencies() { if depTarget := graph.Target(dep); depTarget != nil { if depTarget.Label.Parent() == target.Label.Parent() { ret = append(ret, publicDependencies(graph, depTarget)...) diff --git a/src/generate/generate.go b/src/generate/generate.go index 33a6582953..44516416a5 100644 --- a/src/generate/generate.go +++ b/src/generate/generate.go @@ -24,7 +24,7 @@ func UpdateGitignore(graph *core.BuildGraph, labels []core.BuildLabel, gitignore if !t.HasLabel("codegen") { continue } - for _, out := range t.Outputs() { + for _, out := range t.Outputs(graph) { relativePkg := t.Label.PackageName if pkg != "." { if !strings.HasPrefix(t.Label.PackageName, pkg) { @@ -49,7 +49,7 @@ func allLabelGenOuts(graph *core.BuildGraph, labels []core.BuildLabel) []string if !t.HasLabel("codegen") { continue } - outs = append(outs, t.Outputs()...) + outs = append(outs, t.Outputs(graph)...) } return outs } @@ -72,7 +72,7 @@ func LinkGeneratedSources(state *core.BuildState, labels []core.BuildLabel) { if !target.HasLabel("codegen") { continue } - for _, out := range target.Outputs() { + for _, out := range target.Outputs(state.Graph) { destDir := filepath.Join(core.RepoRoot, target.Label.PackageDir()) srcDir := filepath.Join(core.RepoRoot, target.OutDir()) fs.LinkDestination(filepath.Join(srcDir, out), filepath.Join(destDir, out), linker) diff --git a/src/help/help.go b/src/help/help.go index eb1bc303fc..b105ccd5b9 100644 --- a/src/help/help.go +++ b/src/help/help.go @@ -139,7 +139,7 @@ func getSubrepoOrDie(name string, target core.BuildLabel) *core.Subrepo { // This is sufficient to get everything we need. The parsing of the build def files happens in getPluginBuildDefs. state.ParsePackageOnly = true - plz.Run([]core.BuildLabel{target}, nil, state, state.Config, state.TargetArch) + plz.Run([]core.BuildLabel{target}, nil, state, &plz.Progress{}, state.TargetArch) return state.Graph.SubrepoOrDie(name) } diff --git a/src/output/interactive_display.go b/src/output/interactive_display.go index 1940c9e590..e3e6197701 100644 --- a/src/output/interactive_display.go +++ b/src/output/interactive_display.go @@ -26,13 +26,17 @@ type displayer interface { Frequency() time.Duration } -func setupDisplayer(state *core.BuildState, plain bool) displayer { +func setupDisplayer(state *core.BuildState, progress Progress, plain bool) displayer { if plain { - return &plainDisplay{state: state} + return &plainDisplay{ + state: state, + progress: progress, + } } cli.CurrentBackend.SetPassthrough(false, state.Config.Display.MaxWorkers, state.Watch) return &interactiveDisplay{ state: state, + progress: progress, numWorkers: state.Config.Please.NumThreads, maxWorkers: state.Config.Display.MaxWorkers, numRemote: state.Config.NumRemoteExecutors(), @@ -41,7 +45,8 @@ func setupDisplayer(state *core.BuildState, plain bool) displayer { } type plainDisplay struct { - state *core.BuildState + state *core.BuildState + progress Progress } func (d *plainDisplay) Update(targets []buildingTarget) { @@ -49,11 +54,11 @@ func (d *plainDisplay) Update(targets []buildingTarget) { log.Notice( "Build running for %s, %d / %d tasks done (%d left), %s busy, parsing %s", time.Since(d.state.StartTime).Round(time.Second), - d.state.NumDone(), - d.state.NumActive(), - d.state.NumActive()-d.state.NumDone(), + d.progress.NumDone(), + d.progress.NumTotal(), + d.progress.NumTotal()-d.progress.NumDone(), pluralise(localbusy+remotebusy, "worker", "workers"), - pluralise(int(d.state.Parses().Load()), "BUILD file", "BUILD files"), + pluralise(d.progress.NumParsing(), "BUILD file", "BUILD files"), ) } @@ -78,6 +83,7 @@ func (d *plainDisplay) Close() {} type interactiveDisplay struct { state *core.BuildState + progress Progress numWorkers, maxWorkers, numRemote, maxRows, maxCols int stats bool lines, lastLines int // mutable - records how many rows we've printed this time @@ -85,7 +91,7 @@ type interactiveDisplay struct { } func (d *interactiveDisplay) Close() { - setWindowTitle(d.state, false) + setWindowTitle(d.state, d.progress, false) d.moveToFirstLine() d.printf("${CLEAR_END}") d.flush() @@ -111,7 +117,7 @@ func (d *interactiveDisplay) Update(targets []buildingTarget) { } d.printf("\x1b[%dA", d.lastLines-d.lines) // Move back up again } - setWindowTitle(d.state, true) + setWindowTitle(d.state, d.progress, true) d.flush() } @@ -129,9 +135,9 @@ func (d *interactiveDisplay) printLines(targets []buildingTarget) { localActive, remoteActive := countActive(targets) totalActive := localActive + remoteActive if d.numRemote > 0 { - d.printf("Building [%d/%d, %2d/%d local, %3d/%d remote, %3.1fs]:\n", d.state.NumDone(), d.state.NumActive(), localActive, d.numWorkers, remoteActive, d.numRemote, time.Since(d.state.StartTime).Seconds()) + d.printf("Building [%d/%d, %2d/%d local, %3d/%d remote, %3.1fs]:\n", d.progress.NumDone(), d.progress.NumTotal(), localActive, d.numWorkers, remoteActive, d.numRemote, time.Since(d.state.StartTime).Seconds()) } else { - d.printf("Building [%d/%d, %3.1fs]:\n", d.state.NumDone(), d.state.NumActive(), time.Since(d.state.StartTime).Seconds()) + d.printf("Building [%d/%d, %3.1fs]:\n", d.progress.NumDone(), d.progress.NumTotal(), time.Since(d.state.StartTime).Seconds()) } d.lines++ if d.stats { @@ -276,14 +282,14 @@ func (d *interactiveDisplay) lprintfPrepare(cols int, s string) string { } // setWindowTitle sets the title of the current shell window based on the current build state. -func setWindowTitle(state *core.BuildState, running bool) { +func setWindowTitle(state *core.BuildState, progress Progress, running bool) { if !state.Config.Display.UpdateTitle { return } if running { SetWindowTitle("plz: finishing up") } else { - SetWindowTitle(fmt.Sprintf("plz: %d / %d tasks, %3.1fs", state.NumDone(), state.NumActive(), time.Since(state.StartTime).Seconds())) + SetWindowTitle(fmt.Sprintf("plz: %d / %d tasks, %3.1fs", progress.NumDone(), progress.NumTotal(), time.Since(state.StartTime).Seconds())) } } diff --git a/src/output/shell_output.go b/src/output/shell_output.go index 88aceddff5..46850dd0e3 100644 --- a/src/output/shell_output.go +++ b/src/output/shell_output.go @@ -25,9 +25,15 @@ import ( const durationGranularity = 10 * time.Millisecond const testDurationGranularity = time.Millisecond +type Progress interface { + NumDone() int + NumTotal() int + NumParsing() int +} + // MonitorState monitors the build while it's running and prints output until the results // channel of state has completed. -func MonitorState(state *core.BuildState, plainOutput, detailedTests, streamTestResults, shell, shellRun bool, traceFile string) { +func MonitorState(state *core.BuildState, progress Progress, plainOutput, detailedTests, streamTestResults, shell, shellRun bool, traceFile string) { initPrintf(state.Config) if len(state.Config.Please.Motd) != 0 { @@ -41,11 +47,11 @@ func MonitorState(state *core.BuildState, plainOutput, detailedTests, streamTest defer tw.Close() } - displayer := setupDisplayer(state, plainOutput) + displayer := setupDisplayer(state, progress, plainOutput) t := time.NewTicker(displayer.Frequency()) defer t.Stop() results := state.Results() - bt := newBuildingTargets(state, plainOutput) + bt := newBuildingTargets(state, progress, plainOutput) displayer.Update(bt.Targets()) loop: for { @@ -80,7 +86,7 @@ loop: } else if (state.NeedHashesOnly || state.PrepareOnly || shell) && target.State() == core.Stopped { // Do nothing, we will output about this shortly. } else if target.State() < core.Built && len(bt.FailedTargets) == 0 && !target.AddedPostBuild { - log.Fatalf("Target %s hasn't built but we have no pending tasks left.\n%s", label, unbuiltTargetsMessage(state.Graph)) + log.Fatalf("Target %s hasn't built but we have no pending tasks left.\n%s", label, unbuiltDepsMessage(state.Graph, target)) } } } @@ -367,7 +373,7 @@ func testResultMessage(results *core.TestSuite, showDuration bool) string { func printUnformattedBuildResults(state *core.BuildState) { for _, label := range state.ExpandVisibleOriginalTargets() { - for _, result := range buildResult(state.Graph.TargetOrDie(label)) { + for _, result := range buildResult(state, state.Graph.TargetOrDie(label)) { fmt.Printf("%s\n", result) } } @@ -398,7 +404,7 @@ func printBuildResults(state *core.BuildState, duration time.Duration) { for _, label := range state.ExpandVisibleOriginalTargets() { target := state.Graph.TargetOrDie(label) fmt.Printf("%s:\n", label) - for _, result := range buildResult(target) { + for _, result := range buildResult(state, target) { fmt.Printf(" %s\n", result) } } @@ -470,10 +476,10 @@ func printTempDirs(state *core.BuildState, duration time.Duration, shell, shellR } } -func buildResult(target *core.BuildTarget) []string { +func buildResult(state *core.BuildState, target *core.BuildTarget) []string { results := []string{} if target != nil { - for _, out := range target.Outputs() { + for _, out := range target.Outputs(state.Graph) { if core.StartedAtRepoRoot() { results = append(results, filepath.Join(target.OutDir(), out)) } else { @@ -637,20 +643,33 @@ func colouriseError(err error) error { // errorMessageRe is a regex to find lines that look like they're specifying a file. var errorMessageRe = deferredregex.DeferredRegex{Re: `^([^ ]+\.[^: /]+):([0-9]+):(?:([0-9]+):)? *(?:([a-z-_ ]+):)? (.*)$`} -// unbuiltTargetsMessage returns a message for any targets that are supposed to build but haven't yet. -func unbuiltTargetsMessage(graph *core.BuildGraph) string { - var msgBuilder strings.Builder - for _, target := range graph.AllTargets() { - if target.State() == core.Active { - _, _ = fmt.Fprintf(&msgBuilder, " %s", target.Label) - } else if target.State() == core.Pending { - _, _ = fmt.Fprintf(&msgBuilder, " %s (pending build)\n", target.Label) +// unbuiltDepsMessage returns a message describing why the given target hasn't built, by listing +// any of its transitive dependencies that aren't built either. +func unbuiltDepsMessage(graph *core.BuildGraph, target *core.BuildTarget) string { + var b strings.Builder + seen := map[*core.BuildTarget]bool{} + var walk func(*core.BuildTarget) + walk = func(t *core.BuildTarget) { + if seen[t] { + return + } + seen[t] = true + deps, unresolved := t.Dependencies(graph) + for _, l := range unresolved { + fmt.Fprintf(&b, " %s (not in the build graph)\n", l) + } + for _, dep := range deps { + if !dep.State().IsBuilt() { + fmt.Fprintf(&b, " %s (%s)\n", dep.Label, dep.State()) + walk(dep) + } } } - if msgBuilder.Len() == 0 { - return "\nThe following targets have not yet built:\n" + msgBuilder.String() + walk(target) + if b.Len() == 0 { + return "" } - return "" + return "\nThe following dependencies have not built:\n" + b.String() } // shortError returns the message for an error, shortening it if the error supports that. diff --git a/src/output/targets.go b/src/output/targets.go index 0a3a3a7c31..9ac480c62f 100644 --- a/src/output/targets.go +++ b/src/output/targets.go @@ -34,6 +34,7 @@ type buildingTargets struct { plain bool anyRemote bool state *core.BuildState + progress Progress targets []buildingTarget currentTargets map[buildingTargetKey]int localAvailable map[int]struct{} @@ -42,12 +43,13 @@ type buildingTargets struct { FailedNonTests []core.BuildLabel } -func newBuildingTargets(state *core.BuildState, plainOutput bool) *buildingTargets { +func newBuildingTargets(state *core.BuildState, progress Progress, plainOutput bool) *buildingTargets { n := state.Config.Please.NumThreads + state.Config.NumRemoteExecutors() return &buildingTargets{ plain: plainOutput, anyRemote: state.Config.NumRemoteExecutors() > 0, state: state, + progress: progress, targets: make([]buildingTarget, n), currentTargets: make(map[buildingTargetKey]int, n), localAvailable: makeAvailable(state.Config.Please.NumThreads, 0), @@ -101,11 +103,6 @@ func (bt *buildingTargets) handleOutput(result *core.BuildResult) { if result.Status != core.TargetTestFailed { // Reset colour so the entire compiler error output doesn't appear red. log.Errorf("%s failed:\x1b[0m\n%s", label, shortError(result.Err)) - // TODO(rgodden): make sure we close off any pending targets when their package fails to parse e.g. because - // a subrepo failed to build. - if !bt.state.KeepGoing || result.Status == core.ParseFailed { - bt.state.Stop() - } } else if msg := shortError(result.Err); msg != "" { log.Errorf("%s failed: %s", result.Label, msg) } else { @@ -166,8 +163,8 @@ func (bt *buildingTargets) updateTarget(idx int, result *core.BuildResult, t *co if bt.plain { if !active { - active := pluralise(bt.state.NumActive(), "task", "tasks") - log.Info("[%d/%s] %s: %s [%3.1fs]", bt.state.NumDone(), active, result.Label, result.Description, time.Since(target.Started).Seconds()) + active := pluralise(bt.progress.NumTotal(), "task", "tasks") + log.Info("[%d/%s] %s: %s [%3.1fs]", bt.progress.NumDone(), active, result.Label, result.Description, time.Since(target.Started).Seconds()) } else { log.Info("%s: %s", result.Label, result.Description) } diff --git a/src/parse/BUILD b/src/parse/BUILD index bc3cc09ee8..d7a74907be 100644 --- a/src/parse/BUILD +++ b/src/parse/BUILD @@ -6,7 +6,7 @@ go_library( "parse_step.go", ], pgo_file = "//:pgo", - resources = glob(["internal.tmpl"]), + resources = ["internal.tmpl"], visibility = ["PUBLIC"], deps = [ "//rules", @@ -19,14 +19,3 @@ go_library( "//src/version", ], ) - -go_test( - name = "parse_step_test", - srcs = ["parse_step_test.go"], - resources = ["internal.tmpl"], - deps = [ - ":parse", - "///third_party/go/github.com_stretchr_testify//assert", - "//src/core", - ], -) diff --git a/src/parse/README.md b/src/parse/README.md index 9fba09c81d..f551f34462 100644 --- a/src/parse/README.md +++ b/src/parse/README.md @@ -2,17 +2,17 @@ *This readme is a stub. Please raise an issue on this repo if you would like something expanded on.* -This package defines the parse step used to parse, and interpret a BUILD file to populate the build graph. You can find -the actual interpreter for asp, the python dialect used in build files, in the asp subfolder. +This package defines the parse step used to parse, and interpret a BUILD file to populate the build graph. You can find +the actual interpreter for asp, the python dialect used in build files, in the asp subfolder. Parsing does the following basic things: -1. Synchronise on package parsing by calling `state.SyncParsePackage(label)`. This will either return the existing - package, blocking if the parse is in flight, or return nil, if we're the first. When this return nil, we MUST parse - the package. +1. Synchronise on package parsing by calling `state.SyncParsePackage(label)`. This will either return the existing + package, blocking if the parse is in flight, or return nil, if we're the first. When this return nil, we MUST parse + the package. 2. Check to see if we have a subrepo label. When we do, the subrepo package must be parsed first. This involves waiting - for the subrepo target that defiens this package to be built. + for the subrepo target that defiens this package to be built. 3. Parse the package, and add the package to the build graph. We also mark the pacakge as parsed which unblocks any other - calls to `state.SyncParsePackage(label)`. + calls to `state.SyncParsePackage(label)`. 4. If we queued up a specific target to be built, activate the target and queue it again. This will trigger a build. see [src/core](../core/README.md) for more information on how this works. diff --git a/src/parse/asp/builtins.go b/src/parse/asp/builtins.go index 8be2cbc8e4..fb7359b38b 100644 --- a/src/parse/asp/builtins.go +++ b/src/parse/asp/builtins.go @@ -213,18 +213,6 @@ func buildRule(s *scope, args []pyObject) pyObject { if s.Callback { target.AddedPostBuild = true } - - if s.parsingFor != nil && s.parsingFor.label == target.Label { - if err := s.state.ActivateTarget(s.pkg, s.parsingFor.label, s.parsingFor.dependent, s.mode); err != nil { - s.Error("%v", err) - } - } - if s.state.IsPendingTarget(target.Label) { - if err := s.state.ActivateTarget(s.pkg, target.Label, target.Label, s.mode); err != nil { - s.Error("%v", err) - } - } - return pyString(":" + target.Label.Name) } @@ -314,11 +302,10 @@ func bazelLoad(s *scope, args []pyObject) pyObject { // WaitForSubincludedTarget drops the interpreter lock and waits for the subincluded target to be built. This is // important to keep us from deadlocking all available parser threads (easy to happen if they're all waiting on a // single target which now can't start) -func (s *scope) WaitForSubincludedTarget(l, dependent core.BuildLabel) *core.BuildTarget { +func (s *scope) WaitForSubincludedTarget(l, dependent core.BuildLabel) (*core.BuildTarget, error) { s.interpreter.limiter.Release() defer s.interpreter.limiter.Acquire() - - return s.state.WaitForTargetAndEnsureDownload(l, dependent, s.mode.IsPreload()) + return s.state.Build(l, dependent) } // builtinFail raises an immediate error that can't be intercepted. @@ -361,9 +348,9 @@ func subinclude(s *scope, args []pyObject) pyObject { var outs []string if len(annotation) > 0 { - outs = t.NamedOutputs(annotation) + outs = t.NamedOutputs(s.state.Graph, annotation) } else { - outs = t.Outputs() + outs = t.Outputs(s.state.Graph) } for _, out := range outs { s.SetAll(s.interpreter.Subinclude(s, filepath.Join(t.OutDir(), out), t.Label, false), false) @@ -394,7 +381,9 @@ func subincludeTarget(s *scope, l core.BuildLabel) *core.BuildTarget { Subrepo: subrepoLabel.Subrepo, Name: "all", } - s.state.WaitForPackage(subrepoPackageLabel, pkgLabel, s.mode|core.ParseModeForSubinclude) + if _, err := s.state.Parse(subrepoPackageLabel, pkgLabel); err != nil { + s.Error("Failed to parse subrepo target: %w", err) + } } // isLocal is true when this subinclude target in the current package being parsed @@ -404,17 +393,13 @@ func subincludeTarget(s *scope, l core.BuildLabel) *core.BuildTarget { // but isn't activated, we should activate it otherwise WaitForSubincludedTarget might block. This can happen when // another package also subincludes this target, and queues it first. t := s.state.Graph.Target(l) - if t != nil { - if t.State() < core.Active { - if err := s.state.ActivateTarget(s.pkg, l, pkgLabel, s.mode|core.ParseModeForSubinclude); err != nil { - s.Error("Failed to activate subinclude target: %v", err) - } - } - } else if isLocal { + if t == nil && isLocal { s.Error("Target :%s is not defined in this package; it has to be defined before the subinclude() call", l.Name) } - t = s.WaitForSubincludedTarget(l, pkgLabel) - if s.pkg != nil { + t, err := s.WaitForSubincludedTarget(l, pkgLabel) + if err != nil { + s.Error("Failed to build subincluded target: %w", err) + } else if s.pkg != nil { s.pkg.RegisterSubinclude(l) } else if s.subincludeLabel != nil { // If this is nil, that indicates a preloadedSubinclude s.state.Graph.RegisterTransitiveSubinclude(*s.subincludeLabel, l) @@ -1152,10 +1137,10 @@ func getLabels(s *scope, args []pyObject) pyObject { } if core.LooksLikeABuildLabel(name) { label := core.ParseBuildLabel(name, s.pkg.Name) - return getLabelsInternal(s.state.Graph.TargetOrDie(label), prefix, core.Built, all, maxDepth) + return getLabelsInternal(s.state.Graph, s.state.Graph.TargetOrDie(label), prefix, core.Built, all, maxDepth) } target := getTargetPost(s, name) - return getLabelsInternal(target, prefix, core.Building, all, maxDepth) + return getLabelsInternal(s.state.Graph, target, prefix, core.Building, all, maxDepth) } // addLabel adds a set of labels to the named rule @@ -1175,7 +1160,7 @@ func addLabel(s *scope, args []pyObject) pyObject { return None } -func getLabelsInternal(target *core.BuildTarget, prefix string, minState core.BuildTargetState, all bool, maxDepth int) pyObject { +func getLabelsInternal(graph *core.BuildGraph, target *core.BuildTarget, prefix string, minState core.BuildTargetState, all bool, maxDepth int) pyObject { if target.State() < minState { log.Fatalf("get_labels called on a target that is not yet built: %s", target.Label) } @@ -1196,7 +1181,12 @@ func getLabelsInternal(target *core.BuildTarget, prefix string, minState core.Bu return } if !t.OutputIsComplete || t == target || all { - for _, dep := range t.Dependencies() { + deps, unresolved := t.Dependencies(graph) + if len(unresolved) > 0 { + // Shouldn't happen; by the time this is callable the target's dependencies are built. + log.Fatalf("get_labels called on %s, but its dependencies aren't in the build graph: %s", t.Label, unresolved) + } + for _, dep := range deps { if !done[dep] { getLabels(dep, max(depth-1, -1)) } @@ -1234,32 +1224,10 @@ func addDep(s *scope, args []pyObject) pyObject { exported := args[2].IsTruthy() runtime := args[3].IsTruthy() target.AddMaybeExportedDependency(dep, exported, false, false, runtime) - // Queue this dependency if it'll be needed. - if target.State() > core.Inactive { - err := s.state.QueueTarget(dep, target.Label, false, core.ParseModeNormal) - s.Assert(err == nil, "%s", err) - } + target.ModifiedByCallback = true return None } -func addDatumToTargetAndMaybeQueue(s *scope, target *core.BuildTarget, datum core.BuildInput, systemAllowed, tool bool) { - target.AddDatum(datum) - // Queue this dependency if it'll be needed. - if l, ok := datum.Label(); ok && target.State() > core.Inactive { - err := s.state.QueueTarget(l, target.Label, false, core.ParseModeNormal) - s.Assert(err == nil, "%s", err) - } -} - -func addNamedDatumToTargetAndMaybeQueue(s *scope, name string, target *core.BuildTarget, datum core.BuildInput, systemAllowed, tool bool) { - target.AddNamedDatum(name, datum) - // Queue this dependency if it'll be needed. - if l, ok := datum.Label(); ok && target.State() > core.Inactive { - err := s.state.QueueTarget(l, target.Label, false, core.ParseModeNormal) - s.Assert(err == nil, "%s", err) - } -} - // Add runtime dependencies to target func addData(s *scope, args []pyObject) pyObject { s.Assert(s.Callback, "can only be called from a pre- or post-build callback") @@ -1274,25 +1242,26 @@ func addData(s *scope, args []pyObject) pyObject { // add_data() builtin can take a string, list, or dict if isType(datum, "str") { if bi := parseBuildInput(s, datum, string(label.(pyString)), systemAllowed, tool); bi != nil { - addDatumToTargetAndMaybeQueue(s, target, bi, systemAllowed, tool) + target.AddDatum(bi) } } else if isType(datum, "list") { for _, str := range datum.(pyList) { if bi := parseBuildInput(s, str, string(label.(pyString)), systemAllowed, tool); bi != nil { - addDatumToTargetAndMaybeQueue(s, target, bi, systemAllowed, tool) + target.AddDatum(bi) } } } else if isType(datum, "dict") { for name, v := range datum.(pyDict) { for _, str := range v.(pyList) { if bi := parseBuildInput(s, str, string(label.(pyString)), systemAllowed, tool); bi != nil { - addNamedDatumToTargetAndMaybeQueue(s, name, target, bi, systemAllowed, tool) + target.AddNamedDatum(name, bi) } } } } else { - log.Fatal("Unrecognised data type passed to add_data") + s.Error("Unrecognised data type passed to add_data") } + target.ModifiedByCallback = true return None } @@ -1324,7 +1293,7 @@ func getOuts(s *scope, args []pyObject) pyObject { target = getTargetPost(s, name) } - outs := target.Outputs() + outs := target.Outputs(s.state.Graph) ret := make(pyList, len(outs)) for i, out := range outs { ret[i] = pyString(out) @@ -1498,8 +1467,8 @@ func subrepo(s *scope, args []pyObject) pyObject { // N.B. The target must be already registered on this package. target = s.pkg.TargetOrDie(s.parseLabelInPackage(dep, s.pkg).Name) root = target.Label.Name - if len(target.Outputs()) == 1 { - root = target.Outputs()[0] + if outputs := target.Outputs(s.state.Graph); len(outputs) == 1 { + root = outputs[0] } if target.Local || s.state.RemoteClient == nil { root = filepath.Join(target.OutDir(), root) diff --git a/src/parse/asp/builtins_test.go b/src/parse/asp/builtins_test.go index 02e622b0c2..d687c86e7e 100644 --- a/src/parse/asp/builtins_test.go +++ b/src/parse/asp/builtins_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/thought-machine/please/src/core" ) @@ -40,11 +39,6 @@ func TestGetLabels(t *testing.T) { state.Graph.AddTarget(middle) state.Graph.AddTarget(top) - err := middle.ResolveDependencies(state.Graph) - require.NoError(t, err) - err = top.ResolveDependencies(state.Graph) - require.NoError(t, err) - s := &scope{state: state, pkg: core.NewPackage("pkg")} ls := getLabels(s, []pyObject{pyString(":top"), pyString("target:"), False, True, pyInt(-1)}).(pyList) // transitive=True assert.Equal(t, pyList{pyString("bottom"), pyString("middle"), pyString("top")}, ls) diff --git a/src/parse/asp/errors.go b/src/parse/asp/errors.go index 1d458f0d40..148988d458 100644 --- a/src/parse/asp/errors.go +++ b/src/parse/asp/errors.go @@ -94,6 +94,11 @@ func (stack *errorStack) ShortError() string { return stack.err.Error() } +// Unwrap implements the errors interface so this can be unwrapped to get the contained error +func (stack *errorStack) Unwrap() error { + return stack.err +} + // stackTrace returns the lines of stacktrace from the error. func (stack *errorStack) stackTrace() string { ret := make([]string, len(stack.Stack)) diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index 93e4ca0a32..f4a526f882 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -7,7 +7,6 @@ import ( "path/filepath" "reflect" "regexp" - "runtime/debug" "runtime/pprof" "strings" "sync" @@ -34,6 +33,10 @@ type interpreter struct { stringMethods, dictMethods, configMethods map[string]*pyFunc regexCache *cmap.Map[string, *regexp.Regexp] + + // TODO(peter): rethink what we can do here, we don't really need build labels for this, + // we should be able to store a preloaded set of symbols or smthn that we can wang into scopes as needed. + preloads []core.BuildLabel } // newInterpreter creates and returns a new interpreter instance. @@ -87,7 +90,7 @@ func (i *interpreter) getConfig(state *core.BuildState) *pyConfig { // LoadBuiltins loads a set of builtins from a file, optionally with its contents. func (i *interpreter) LoadBuiltins(filename string, contents []byte, statements []*Statement) error { - s := i.scope.NewScope(filename, 0) + s := i.scope.NewScope(filename) // Gentle hack - attach the native code once we have loaded the correct file. // Needs to be after this file is loaded but before any of the others that will // use functions from it. @@ -129,8 +132,7 @@ func (i *interpreter) loadBuiltinStatements(s *scope, statements []*Statement, e } func (i *interpreter) preloadSubincludes(s *scope) error { - // We should have ensured these targets are downloaded by this point in `parse_step.go` - for _, label := range s.state.GetPreloadedSubincludes() { + for _, label := range i.preloads { if err := i.preloadSubinclude(s, label); err != nil { return err } @@ -154,7 +156,7 @@ func (i *interpreter) preloadSubinclude(s *scope, label core.BuildLabel) (err er } s.interpreter.loadPluginConfig(s, includeState) - for _, out := range t.FullOutputs() { + for _, out := range t.FullOutputs(s.state.Graph) { s.SetAll(s.interpreter.Subinclude(s, out, t.Label, true), false) } return nil @@ -162,8 +164,8 @@ func (i *interpreter) preloadSubinclude(s *scope, label core.BuildLabel) (err er // interpretAll runs a series of statements in the scope of the given package. // The first return value is for testing only. -func (i *interpreter) interpretAll(pkg *core.Package, forLabel, dependent *core.BuildLabel, mode core.ParseMode, statements []*Statement) (*scope, error) { - s := i.scope.NewPackagedScope(pkg, mode, 1) +func (i *interpreter) interpretAll(pkg *core.Package, forLabel, dependent *core.BuildLabel, statements []*Statement) (*scope, error) { + s := i.scope.NewPackagedScope(pkg, 1) s.config = i.getConfig(s.state).Copy() // Config needs a little separate tweaking. @@ -180,10 +182,8 @@ func (i *interpreter) interpretAll(pkg *core.Package, forLabel, dependent *core. defer pprof.SetGoroutineLabels(old) } - if !mode.IsPreload() { - if err := i.preloadSubincludes(s); err != nil { - return nil, err - } + if err := i.preloadSubincludes(s); err != nil { + return nil, err } s.Set("CONFIG", s.config) @@ -200,7 +200,6 @@ func handleErrors(r interface{}) (err error) { } else { err = fmt.Errorf("%s", r) } - log.Debug("%v:\n %s", err, debug.Stack()) return } @@ -225,11 +224,7 @@ func (i *interpreter) Subinclude(pkgScope *scope, path string, label core.BuildL return nil, err } - mode := pkgScope.mode - if preload { - mode |= core.ParseModeForPreload - } - s := i.scope.NewScope(path, mode) + s := i.scope.NewScope(path) s.state = pkgScope.state // Scope needs a local version of CONFIG @@ -237,7 +232,7 @@ func (i *interpreter) Subinclude(pkgScope *scope, path string, label core.BuildL s.Set("CONFIG", s.config) s.subincludeLabel = &label - if !mode.IsPreload() { + if !preload { if err := i.preloadSubincludes(s); err != nil { return nil, err } @@ -309,7 +304,6 @@ type scope struct { globber *fs.Globber // True if this scope is for a pre- or post-build callback. Callback bool - mode core.ParseMode } // parseAnnotatedLabelInPackage similarly to parseLabelInPackage, parses the label contextualising it to the provided @@ -403,17 +397,17 @@ func (s *scope) subincludePackage() *core.Package { } // NewScope creates a new child scope of this one. -func (s *scope) NewScope(filename string, mode core.ParseMode) *scope { - return s.newScope(s.pkg, mode, filename, 0) +func (s *scope) NewScope(filename string) *scope { + return s.newScope(s.pkg, filename, 0) } // NewPackagedScope creates a new child scope of this one pointing to the given package. // hint is a size hint for the new set of locals. -func (s *scope) NewPackagedScope(pkg *core.Package, mode core.ParseMode, hint int) *scope { - return s.newScope(pkg, mode, pkg.Filename, hint) +func (s *scope) NewPackagedScope(pkg *core.Package, hint int) *scope { + return s.newScope(pkg, pkg.Filename, hint) } -func (s *scope) newScope(pkg *core.Package, mode core.ParseMode, filename string, hint int) *scope { +func (s *scope) newScope(pkg *core.Package, filename string, hint int) *scope { s2 := &scope{ ctx: s.ctx, filename: filename, @@ -425,7 +419,6 @@ func (s *scope) newScope(pkg *core.Package, mode core.ParseMode, filename string locals: make(pyDict, hint), config: s.config, Callback: s.Callback, - mode: mode, } if pkg != nil && pkg.Subrepo != nil && pkg.Subrepo.State != nil { s2.state = pkg.Subrepo.State @@ -709,7 +702,7 @@ func (s *scope) interpretJoin(base string, list *List) pyObject { } // Has a comprehension. Note that there is only ever one level; by the anecdata, two-level ones // are rare in this context so not worth worrying about here. - cs := s.NewScope(s.filename, s.mode) + cs := s.NewScope(s.filename) it := s.iterable(list.Comprehension.Expr) first := true cs.evaluateComprehension(it, list.Comprehension, func(li pyObject) { @@ -932,7 +925,7 @@ func (s *scope) interpretList(expr *List) pyList { if expr.Comprehension == nil { return pyList(s.evaluateExpressions(expr.Values)) } - cs := s.NewScope(s.filename, s.mode) + cs := s.NewScope(s.filename) it, l := s.iterableLen(expr.Comprehension.Expr) ret := make(pyList, 0, l) cs.evaluateComprehension(it, expr.Comprehension, func(li pyObject) { @@ -953,7 +946,7 @@ func (s *scope) interpretDict(expr *Dict) pyObject { } return d } - cs := s.NewScope(s.filename, s.mode) + cs := s.NewScope(s.filename) it, l := s.iterableLen(expr.Comprehension.Expr) ret := make(pyDict, l) cs.evaluateComprehension(it, expr.Comprehension, func(li pyObject) { diff --git a/src/parse/asp/interpreter_test.go b/src/parse/asp/interpreter_test.go index 511f00b053..11c720b05a 100644 --- a/src/parse/asp/interpreter_test.go +++ b/src/parse/asp/interpreter_test.go @@ -34,7 +34,7 @@ func parseFileToStatementsInPkg(filename string, pkg *core.Package) (*scope, []* } statements = parser.optimise(statements) parser.interpreter.optimiseExpressions(statements) - s, err := parser.interpreter.interpretAll(pkg, nil, nil, 0, statements) + s, err := parser.interpreter.interpretAll(pkg, nil, nil, statements) return s, statements, err } @@ -607,7 +607,7 @@ func TestJSON(t *testing.T) { statements = parser.optimise(statements) parser.interpreter.optimiseExpressions(statements) - s := parser.interpreter.scope.NewScope("BUILD", core.ParseModeNormal) + s := parser.interpreter.scope.NewScope("BUILD") list := pyList{pyString("foo"), pyInt(5)} dict := pyDict{"foo": pyString("bar")} @@ -677,7 +677,7 @@ func TestLogConfigVariable(t *testing.T) { confBase := &pyConfigBase{dict: dict} config := &pyConfig{base: confBase, overlay: pyDict{"baz": pyInt(6)}} - s := parser.interpreter.scope.NewScope("BUILD", core.ParseModeNormal) + s := parser.interpreter.scope.NewScope("BUILD") s.config = config s.Set("CONFIG", config) diff --git a/src/parse/asp/logging_test.go b/src/parse/asp/logging_test.go index fb8ac0bf15..3ac63c070e 100644 --- a/src/parse/asp/logging_test.go +++ b/src/parse/asp/logging_test.go @@ -34,7 +34,7 @@ func parseFile2(filename string) (*scope, error) { if err != nil { panic(err) } - return parser.interpreter.interpretAll(pkg, nil, nil, 0, statements) + return parser.interpreter.interpretAll(pkg, nil, nil, statements) } // assertRecords asserts equality of a series of logging records. diff --git a/src/parse/asp/main/main.go b/src/parse/asp/main/main.go index 03ff09ba6a..336dc6faab 100644 --- a/src/parse/asp/main/main.go +++ b/src/parse/asp/main/main.go @@ -63,7 +63,7 @@ func parseFile(pkg *core.Package, p *asp.Parser, filename string) error { } return err } - return p.ParseFile(pkg, nil, nil, 0, nil, filename) + return p.ParseFile(pkg, nil, nil, nil, filename) } type assignment struct { diff --git a/src/parse/asp/objects.go b/src/parse/asp/objects.go index a783b55b5c..b7d69612ce 100644 --- a/src/parse/asp/objects.go +++ b/src/parse/asp/objects.go @@ -694,11 +694,11 @@ func (f *pyFunc) String() string { func (f *pyFunc) Call(s *scope, c *Call) pyObject { if f.nativeCode != nil { if f.kwargs { - return f.callNative(s.NewScope("", 0), c) + return f.callNative(s.NewScope(""), c) } return f.callNative(s, c) } - s2 := f.scope.newScope(s.pkg, s.mode, f.scope.filename, len(f.args)+1) + s2 := f.scope.newScope(s.pkg, f.scope.filename, len(f.args)+1) s2.config = s.config s2.Set("CONFIG", s.config) // This needs to be copied across too :( s2.Callback = s.Callback diff --git a/src/parse/asp/parser.go b/src/parse/asp/parser.go index f67a8605ae..4d52615ad8 100644 --- a/src/parse/asp/parser.go +++ b/src/parse/asp/parser.go @@ -73,7 +73,7 @@ func (p *Parser) MustLoadBuiltins(filename string, contents []byte) { // ParseFile parses the contents of a single file in the BUILD language. // It returns true if the call was deferred at some point awaiting target to build, // along with any error encountered. -func (p *Parser) ParseFile(pkg *core.Package, label, dependent *core.BuildLabel, mode core.ParseMode, fs iofs.FS, filename string) error { +func (p *Parser) ParseFile(pkg *core.Package, label, dependent *core.BuildLabel, fs iofs.FS, filename string) error { p.limiter.Acquire() defer p.limiter.Release() @@ -81,7 +81,7 @@ func (p *Parser) ParseFile(pkg *core.Package, label, dependent *core.BuildLabel, if err != nil { return err } - _, err = p.interpreter.interpretAll(pkg, label, dependent, mode, statements) + _, err = p.interpreter.interpretAll(pkg, label, dependent, statements) if err != nil { f, _ := p.open(fs, filename) p.annotate(err, f) @@ -89,22 +89,27 @@ func (p *Parser) ParseFile(pkg *core.Package, label, dependent *core.BuildLabel, return err } -// RegisterPreload pre-registers a preload, forcing us to build any transitive preloads before we move on -func (p *Parser) RegisterPreload(label core.BuildLabel) error { +// PreloadSubinclude pre-registers a preload, forcing us to build any transitive preloads before we move on +func (p *Parser) PreloadSubinclude(label core.BuildLabel) error { p.limiter.Acquire() defer p.limiter.Release() // This is a throw away scope. We're just doing this to avoid race conditions setting this on the main scope. - s := p.interpreter.scope.newScope(nil, p.interpreter.scope.mode, "", 0) + s := p.interpreter.scope.newScope(nil, "", 0) s.config = p.interpreter.scope.config.Copy() s.Set("CONFIG", s.config) return p.interpreter.preloadSubinclude(s, label) } +// RegisterPreloads registers the set of preloaded subincludes. +func (p *Parser) RegisterPreloads(labels []core.BuildLabel) { + p.interpreter.preloads = labels +} + // ParseReader parses the contents of the given ReadSeeker as a BUILD file. // The first return value is true if parsing succeeds - if the error is still non-nil // that indicates that interpretation failed. -func (p *Parser) ParseReader(pkg *core.Package, r io.ReadSeeker, forLabel, dependent *core.BuildLabel, mode core.ParseMode) (bool, error) { +func (p *Parser) ParseReader(pkg *core.Package, r io.ReadSeeker, forLabel, dependent *core.BuildLabel) (bool, error) { p.limiter.Acquire() defer p.limiter.Release() @@ -112,7 +117,7 @@ func (p *Parser) ParseReader(pkg *core.Package, r io.ReadSeeker, forLabel, depen if err != nil { return false, err } - _, err = p.interpreter.interpretAll(pkg, forLabel, dependent, mode, stmts) + _, err = p.interpreter.interpretAll(pkg, forLabel, dependent, stmts) return true, err } diff --git a/src/parse/asp/targets.go b/src/parse/asp/targets.go index b6bd02dc09..e92d1588e3 100644 --- a/src/parse/asp/targets.go +++ b/src/parse/asp/targets.go @@ -623,7 +623,7 @@ type preBuildFunction struct { } func (f *preBuildFunction) Call(target *core.BuildTarget) error { - s := f.f.scope.NewPackagedScope(f.f.scope.state.Graph.PackageOrDie(target.Label), f.f.scope.mode, 1) + s := f.f.scope.NewPackagedScope(f.f.scope.state.Graph.PackageOrDie(target.Label), 1) s.config = f.s.config s.Set("CONFIG", f.s.config) s.Callback = true @@ -643,7 +643,7 @@ type postBuildFunction struct { } func (f *postBuildFunction) Call(target *core.BuildTarget, output string) error { - s := f.f.scope.NewPackagedScope(f.f.scope.state.Graph.PackageOrDie(target.Label), f.f.scope.mode, 2) + s := f.f.scope.NewPackagedScope(f.f.scope.state.Graph.PackageOrDie(target.Label), 2) s.config = f.s.config s.Set("CONFIG", f.s.config) s.Callback = true diff --git a/src/parse/init.go b/src/parse/init.go index ee67dacda5..8bef84d787 100644 --- a/src/parse/init.go +++ b/src/parse/init.go @@ -16,13 +16,11 @@ import ( "github.com/thought-machine/please/src/parse/asp" ) -// InitParser initialises the parser engine. This is guaranteed to be called exactly once before any calls to Parse(). -func InitParser(state *core.BuildState) *core.BuildState { - if state.Parser == nil { - p := &aspParser{parser: newAspParser(state)} - state.Parser = p - } - return state +// InitParser initialises the parser engine. +func InitParser(state *core.BuildState) *asp.Parser { + p := newAspParser(state) + state.Parser = &aspParser{parser: p} + return p } // GetAspParser returns the underlying asp.Parser from the state's parser. @@ -69,12 +67,12 @@ func newAspParser(state *core.BuildState) *asp.Parser { return p } -func (p *aspParser) ParseFile(pkg *core.Package, forLabel, dependent *core.BuildLabel, mode core.ParseMode, fs iofs.FS, filename string) error { - return p.parser.ParseFile(pkg, forLabel, dependent, mode, fs, filename) +func (p *aspParser) ParseFile(pkg *core.Package, forLabel, dependent *core.BuildLabel, fs iofs.FS, filename string) error { + return p.parser.ParseFile(pkg, forLabel, dependent, fs, filename) } -func (p *aspParser) ParseReader(pkg *core.Package, reader io.ReadSeeker, forLabel, dependent *core.BuildLabel, mode core.ParseMode) error { - _, err := p.parser.ParseReader(pkg, reader, forLabel, dependent, mode) +func (p *aspParser) ParseReader(pkg *core.Package, reader io.ReadSeeker, forLabel, dependent *core.BuildLabel) error { + _, err := p.parser.ParseReader(pkg, reader, forLabel, dependent) return err } @@ -91,15 +89,12 @@ func (p *aspParser) RunPostBuildFunction(state *core.BuildState, target *core.Bu }) } -// RegisterPreload pre-registers a preload, forcing us to build any transitive preloads before we move on -func (p *aspParser) RegisterPreload(label core.BuildLabel) error { - return p.parser.RegisterPreload(label) -} - // runBuildFunction runs either the pre- or post-build function. func (p *aspParser) runBuildFunction(state *core.BuildState, target *core.BuildTarget, callbackType string, f func() error) error { state.LogBuildResult(target, core.PackageParsing, fmt.Sprintf("Running %s-build function for %s", callbackType, target.Label)) - state.SyncParsePackage(target.Label) + if _, err := state.Parse(target.Label, target.Label); err != nil { + return err + } if err := f(); err != nil { state.LogBuildError(target.Label, core.ParseFailed, err, "Failed %s-build function for %s", callbackType, target.Label) return err diff --git a/src/parse/parse_step.go b/src/parse/parse_step.go index 2c9689feab..3f1082c94f 100644 --- a/src/parse/parse_step.go +++ b/src/parse/parse_step.go @@ -22,81 +22,39 @@ var log = logging.Log var ErrMissingBuildFile = errors.New("build file not found") // Parse parses the package corresponding to a single build label. The label can be :all to add all targets in a package. -// It is not an error if the package has already been parsed. -// -// By default, after the package is parsed, any targets that are now needed for the build and ready -// to be built are queued, and any new packages are queued for parsing. When a specific label is requested -// this is straightforward, but when parsing for pseudo-targets like :all and ..., various flags affect it: -// 'include' and 'exclude' refer to the labels of targets to be added. If 'include' is non-empty then only -// targets with at least one matching label are added. Any targets with a label in 'exclude' are not added. -// 'forSubinclude' is set when the parse is required for a subinclude target so should proceed -// even when we're not otherwise building targets. -func Parse(state *core.BuildState, label, dependent core.BuildLabel, mode core.ParseMode) { - if err := parse(state, label, dependent, mode); err != nil { - state.LogBuildError(label, core.ParseFailed, err, "Failed to parse package") - } -} - -func parse(state *core.BuildState, label, dependent core.BuildLabel, mode core.ParseMode) error { - if t := state.Graph.Target(label); t != nil && t.State() < core.Active { - return state.ActivateTarget(nil, label, dependent, mode) - } - - subrepo, err := checkSubrepo(state, label, dependent, mode) +func Parse(state *core.BuildState, label, dependent core.BuildLabel) (*core.Package, error) { + subrepo, err := checkSubrepo(state, label) if err != nil { - return err + return nil, err } if subrepo != nil { state = subrepo.State } - // Ensure that all the preloaded targets are built before we sync the package parse. If we don't do this, we might - // take the package lock for a package involved in a subinclude, and end up in a deadlock - if !mode.IsPreload() { - if err := state.RegisterPreloads(); err != nil { - return err - } - } - - // See if something else has parsed this package first. - pkg := state.SyncParsePackage(label) - if pkg != nil { - // Does exist, all we need to do is toggle on this target - return state.ActivateTarget(pkg, label, dependent, mode) - } - // If we get here then it falls to us to parse this package. state.LogParseResult(label, core.PackageParsing, "Parsing...") if subrepo != nil && subrepo.Target != nil { // We have got the definition of the subrepo, but it depends on something, make sure that has been built. - state.WaitForBuiltTarget(subrepo.Target.Label, label, mode|core.ParseModeForSubinclude) - if !subrepo.Target.State().IsBuilt() { - return fmt.Errorf("%v: failed to build subrepo", label) + if _, err := state.Build(subrepo.Target.Label, dependent); err != nil { + return nil, err } if err := subrepo.State.Initialise(subrepo); err != nil { - return err + return nil, err } } // Subrepo & nothing else means we just want to ensure that subrepo is present. if label.Subrepo != "" && label.PackageName == "" && label.Name == "" { - return nil + // TODO(peter): is this relevant still? + return nil, nil } - pkg, err = parsePackage(state, label, dependent, subrepo, mode) - + pkg, err := parsePackage(state, label, dependent, subrepo) if err != nil { - return err + return nil, err } state.LogParseResult(label, core.PackageParsed, "Parsed package") - - // The target likely got activated already, however we activate here to handle pseudo-targets (:all), and to let - // this error when the target doesn't exist. - return state.ActivateTarget(pkg, label, dependent, mode) -} - -func inSamePackage(label, dependent core.BuildLabel) bool { - return !dependent.IsOriginalTarget() && label.Subrepo == dependent.Subrepo && label.PackageName == dependent.PackageName + return pkg, nil } // checkSubrepo checks if the label we're parsing is within a subrepo, returning that subrepo, if present in the label. @@ -104,83 +62,21 @@ func inSamePackage(label, dependent core.BuildLabel) bool { // The subrepo target can be inferred from the subrepo name using convention i.e. ///foo/bar//:baz has a subrepo label // //foo:bar. checkSubrepo parses package foo, expecting a call to `subrepo()` that registers a subrepo named foo/bar, // so it can return it. -func checkSubrepo(state *core.BuildState, label, dependent core.BuildLabel, mode core.ParseMode) (*core.Subrepo, error) { +func checkSubrepo(state *core.BuildState, label core.BuildLabel) (*core.Subrepo, error) { if label.Subrepo == "" { return nil, nil } - // Check if we already have it + // Check if we already have it (we expect the higher-level driver code to have arranged for it to be parsed + // before we get here) if subrepo := state.Graph.Subrepo(label.Subrepo); subrepo != nil { return subrepo, nil } - - // SubrepoLabel returns the expected build label for the subrepo's target. Parsing its package should give us the - // subrepo we're looking for. - sl := label.SubrepoLabel(state) - - // This can happen when we subinclude() a target in a subrepo from the same package the subrepo is defined in. In, - // this case, the subrepo must be registered by now. We shouldn't continue to try and parse the subrepo package, as - // it's the current package we're parsing, which would result in a lockup. - if inSamePackage(sl, dependent) { - return nil, fmt.Errorf("subrepo %v is not defined in this package yet. It must appear before it is used by %v", label.Subrepo, dependent) - } - - // Try parsing the package in the host repo first. - s, err := maybeParseSubrepoPackage(state, sl.PackageName, sl.Subrepo, label, mode) - if err != nil || s != nil { - return s, err - } - - if sl.Subrepo != dependent.Subrepo { - // They may have meant a subrepo that was defined in the dependent label's subrepo rather than the host repo - s, err = maybeParseSubrepoPackage(state, sl.PackageName, dependent.Subrepo, label, mode) - if err != nil || s != nil { - return s, err - } - } - - return nil, fmt.Errorf("Subrepo %s is not defined (referenced by %s)", label.Subrepo, dependent) -} - -// maybeParseSubrepoPackage parses a package to make sure subrepos are available, returning the subrepo if it exists. -// Returns nothing if the package doesn't exist, or the package doesn't define the subrepo. -func maybeParseSubrepoPackage(state *core.BuildState, subrepoPkg, subrepoSubrepo string, dependent core.BuildLabel, mode core.ParseMode) (*core.Subrepo, error) { - // First, check whether this is an architecture subrepo. The built-in architecture subrepos - which are implicitly - // defined at the top level - should be registered regardless of whether a top-level BUILD file exists, so we need to - // perform this check before we check whether the subrepo package exists. - s := state.CheckArchSubrepo(dependent.Subrepo) - if s != nil { - return s, nil - } - - // Check if the subrepo package exists - if state.Graph.Package(subrepoPkg, subrepoSubrepo) == nil { - // Don't have it already, must parse. - label := core.BuildLabel{Subrepo: subrepoSubrepo, PackageName: subrepoPkg, Name: "all"} - if err := parse(state, label, dependent, mode|core.ParseModeForSubinclude); err != nil { - // When we try and parse a subrepo package, but the BUILD file or directory doesn't exist, return nil so - // this gets handled later on, in the same way as when the package does exist but doesn't define the subrepo - if errors.Is(err, ErrMissingBuildFile) { - return nil, nil - } - return nil, err - } - } - - // Now that we know its package is parsed, we expect the subrepo to be registered - // - // NB: even if we didn't parse the package above (i.e. package was non-nil), this might've been parsed by a - // different thread, so we need to check if the subrepo exists again. - // - // We last checked if the subrepo existed at the beginning of the checkSubrepo() function, however we haven't - // acquired the package lock. That means another thread may have parsed the package, adding it to the graph by the - // time we check above. This means, we need to check if the subrepo exists again here regardless of whether we - // actually parsed the package in this thread. - return state.Graph.Subrepo(dependent.Subrepo), nil + return nil, fmt.Errorf("Subrepo %s is not defined", label.Subrepo) } // parsePackage parses a BUILD file and adds the package to the build graph -func parsePackage(state *core.BuildState, label, dependent core.BuildLabel, subrepo *core.Subrepo, mode core.ParseMode) (*core.Package, error) { +func parsePackage(state *core.BuildState, label, dependent core.BuildLabel, subrepo *core.Subrepo) (*core.Package, error) { packageName := label.PackageName pkg := core.NewPackage(packageName) pkg.Subrepo = subrepo @@ -195,14 +91,15 @@ func parsePackage(state *core.BuildState, label, dependent core.BuildLabel, subr if err != nil { return nil, fmt.Errorf("failed to generate internal package: %w", err) } - if err := state.Parser.ParseReader(pkg, strings.NewReader(pkgStr), &label, &dependent, mode); err != nil { + if err := state.Parser.ParseReader(pkg, strings.NewReader(pkgStr), &label, &dependent); err != nil { return nil, fmt.Errorf("failed to parse internal package: %w", err) } } else { filename, dir := buildFileName(state, subrepo, fileSystem, label.PackageName) if filename != "" { pkg.Filename = filename - if err := state.Parser.ParseFile(pkg, &label, &dependent, mode, fileSystem, filename); err != nil { + log.Debug("Parsing build file %s %s", label, filename) + if err := state.Parser.ParseFile(pkg, &label, &dependent, fileSystem, filename); err != nil { return nil, err } } else { diff --git a/src/parse/parse_step_test.go b/src/parse/parse_step_test.go deleted file mode 100644 index d21034f54e..0000000000 --- a/src/parse/parse_step_test.go +++ /dev/null @@ -1,166 +0,0 @@ -// Tests for general parse functions. - -package parse - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/thought-machine/please/src/core" -) - -func TestAddDepSimple(t *testing.T) { - // Simple case with only one package parsed and one target added - state := makeState(true, false) - state.ActivateTarget(nil, buildLabel("//package1:target1"), core.OriginalTarget, core.ParseModeNormal) - - time.Sleep(time.Millisecond * 100) - - assertPendingParses(t, state, "//package2:target1", "//package2:target1") - assertPendingBuilds(t, state) // None until package2 parses - assert.Equal(t, 5, state.NumActive()) -} - -func TestAddDepMultiple(t *testing.T) { - // Similar to above but doing all targets in that package - state := makeState(true, false) - state.ActivateTarget(nil, buildLabel("//package1:target1"), core.OriginalTarget, core.ParseModeNormal) - state.ActivateTarget(nil, buildLabel("//package1:target2"), core.OriginalTarget, core.ParseModeNormal) - state.ActivateTarget(nil, buildLabel("//package1:target3"), core.OriginalTarget, core.ParseModeNormal) - - time.Sleep(time.Millisecond * 100) - - // We get an additional dep on target2, but not another on package2:target1 because target2 - // is already activated since package1:target1 depends on it - assertPendingParses(t, state, "//package2:target1", "//package2:target1", "//package2:target2") - assertPendingBuilds(t, state) // None until package2 parses - assert.Equal(t, 7, state.NumActive()) -} - -func TestAddDepMultiplePackages(t *testing.T) { - // This time we already have package2 parsed - state := makeState(true, true) - state.ActivateTarget(nil, buildLabel("//package1:target1"), core.OriginalTarget, core.ParseModeNormal) - - time.Sleep(time.Millisecond * 100) - - assertPendingBuilds(t, state, "//package2:target2") // This is the only candidate target - assertPendingParses(t, state) // None, we have both packages already - assert.Equal(t, 6, state.NumActive()) -} - -func TestAddDepNoBuild(t *testing.T) { - // Tag state as not needing build. We shouldn't get any pending builds at this point. - state := makeState(true, true) - state.NeedBuild = false - state.ActivateTarget(nil, buildLabel("//package1:target1"), core.OriginalTarget, core.ParseModeNormal) - - time.Sleep(time.Millisecond * 100) - - assertPendingParses(t, state) // None, we have both packages already - assertPendingBuilds(t, state) // Nothing because we don't need to build. -} - -func TestAddParseDep(t *testing.T) { - // Tag state as not needing build. Any target that needs to be built to complete parse - // should still get queued for build though. Recall that we indicate this with :all... - state := makeState(true, true) - state.NeedBuild = false - state.ActivateTarget(nil, buildLabel("//package2:target2"), buildLabel("//package3:all"), core.ParseModeNormal) - - time.Sleep(time.Millisecond * 100) - - assertPendingBuilds(t, state, "//package2:target2") // Queued because it's needed for parse - assertPendingParses(t, state) // None, we have both packages already - assert.Equal(t, 2, state.NumActive()) -} - -func TestBuildFileNames(t *testing.T) { - assert.Equal(t, "BUILD", buildFileNames([]string{"BUILD"})) - assert.Equal(t, "BUILD or BUILD.plz", buildFileNames([]string{"BUILD", "BUILD.plz"})) - assert.Equal(t, "BUILD, BUILD.plz or BUILD.test", buildFileNames([]string{"BUILD", "BUILD.plz", "BUILD.test"})) -} - -func makeTarget(label string, deps ...string) *core.BuildTarget { - target := core.NewBuildTarget(core.ParseBuildLabel(label, "")) - for _, dep := range deps { - target.AddDependency(core.ParseBuildLabel(dep, "")) - } - return target -} - -// makeState creates a new build state with optionally one or two packages in it. -// Used in various tests above. -func makeState(withPackage1, withPackage2 bool) *core.BuildState { - state := core.NewDefaultBuildState() - if withPackage1 { - pkg := core.NewPackage("package1") - state.Graph.AddPackage(pkg) - pkg.AddTarget(makeTarget("//package1:target1", "//package1:target2", "//package2:target1")) - pkg.AddTarget(makeTarget("//package1:target2", "//package2:target1")) - pkg.AddTarget(makeTarget("//package1:target3", "//package2:target2")) - state.Graph.AddTarget(pkg.Target("target1")) - state.Graph.AddTarget(pkg.Target("target2")) - state.Graph.AddTarget(pkg.Target("target3")) - addDeps(state.Graph, pkg) - } - if withPackage2 { - pkg := core.NewPackage("package2") - state.Graph.AddPackage(pkg) - pkg.AddTarget(makeTarget("//package2:target1", "//package2:target2", "//package1:target3")) - pkg.AddTarget(makeTarget("//package2:target2")) - state.Graph.AddTarget(pkg.Target("target1")) - state.Graph.AddTarget(pkg.Target("target2")) - addDeps(state.Graph, pkg) - } - return state -} - -func addDeps(graph *core.BuildGraph, pkg *core.Package) { - for _, target := range pkg.AllTargets() { - for _, dep := range target.DeclaredDependencies() { - target.AddDependency(dep) - } - } -} - -func assertPendingParses(t *testing.T, state *core.BuildState, targets ...string) { - t.Helper() - parses, _ := getAllPending(state) - assert.ElementsMatch(t, targets, parses) -} - -func assertPendingBuilds(t *testing.T, state *core.BuildState, targets ...string) { - t.Helper() - _, builds := getAllPending(state) - assert.ElementsMatch(t, targets, builds) -} - -func getAllPending(state *core.BuildState) ([]string, []string) { - parses, builds := state.TaskQueues() - state.Stop() - var pendingParses, pendingBuilds []string - for parses != nil || builds != nil { - select { - case p, ok := <-parses: - if !ok { - parses = nil - break - } - pendingParses = append(pendingParses, p.Label.String()) - case t, ok := <-builds: - if !ok { - builds = nil - break - } - pendingBuilds = append(pendingBuilds, t.Target.Label.String()) - } - } - return pendingParses, pendingBuilds -} - -func buildLabel(bl string) core.BuildLabel { - return core.ParseBuildLabel(bl, "") -} diff --git a/src/please.go b/src/please.go index 9757c50244..4c54e845ab 100644 --- a/src/please.go +++ b/src/please.go @@ -481,7 +481,7 @@ var buildFunctions = map[string]func() int{ } for _, label := range state.ExpandOriginalLabels() { target := state.Graph.TargetOrDie((label)) - for _, out := range target.Outputs() { + for _, out := range target.Outputs(state.Graph) { from := filepath.Join(target.OutDir(), out) fm, err := os.Lstat(from) if err != nil { @@ -1223,14 +1223,17 @@ func runPlease(state *core.BuildState, targets []core.BuildLabel) { state.Cache = cache.NewCache(state) // Run the display - state.Results() // important this is called now, don't ask... + var progress plz.Progress var wg sync.WaitGroup wg.Add(1) go func() { - output.MonitorState(state, !pretty, detailedTests, streamTests, shell, shellRun, string(opts.OutputFlags.TraceFile)) + output.MonitorState(state, &progress, !pretty, detailedTests, streamTests, shell, shellRun, string(opts.OutputFlags.TraceFile)) wg.Done() }() - plz.Run(targets, opts.BuildFlags.PreTargets, state, config, state.TargetArch) + if err := plz.Run(targets, opts.BuildFlags.PreTargets, state, &progress, state.TargetArch); err != nil { + // TODO(peter): we might want to do something else with this + log.Error("%s", err) + } wg.Wait() } diff --git a/src/plz/BUILD b/src/plz/BUILD index bfaeffee93..c70633f2f8 100644 --- a/src/plz/BUILD +++ b/src/plz/BUILD @@ -4,13 +4,16 @@ go_library( pgo_file = "//:pgo", visibility = ["PUBLIC"], deps = [ + "///third_party/go/golang.org_x_sync//errgroup", "///third_party/go/github.com_peterebden_go-cli-init_v5//flags", "//src/build", + "//src/cmap", "//src/cli", "//src/cli/logging", "//src/core", "//src/fs", "//src/metrics", + "//src/parse/asp", "//src/parse", "//src/remote", "//src/test", diff --git a/src/plz/plz.go b/src/plz/plz.go index 7c893a8165..09eac34f84 100644 --- a/src/plz/plz.go +++ b/src/plz/plz.go @@ -1,19 +1,28 @@ package plz import ( + "context" + "errors" + "fmt" + "iter" "path/filepath" + "slices" "strings" "sync" + "sync/atomic" "github.com/peterebden/go-cli-init/v5/flags" + "golang.org/x/sync/errgroup" "github.com/thought-machine/please/src/build" "github.com/thought-machine/please/src/cli" "github.com/thought-machine/please/src/cli/logging" + "github.com/thought-machine/please/src/cmap" "github.com/thought-machine/please/src/core" "github.com/thought-machine/please/src/fs" "github.com/thought-machine/please/src/metrics" "github.com/thought-machine/please/src/parse" + "github.com/thought-machine/please/src/parse/asp" "github.com/thought-machine/please/src/remote" "github.com/thought-machine/please/src/test" ) @@ -25,166 +34,505 @@ var log = logging.Log // afterwards to find success / failure. // To get detailed results as it runs, use state.Results. You should call that *before* // starting this (otherwise a sufficiently fast build may bypass you completely). -func Run(targets, preTargets []core.BuildLabel, state *core.BuildState, config *core.Configuration, arch cli.Arch) { +func Run(targets, preTargets []core.BuildLabel, state *core.BuildState, progress *Progress, arch cli.Arch) error { build.Init(state) if state.Config.Remote.URL != "" { state.RemoteClient = remote.New(state) } - if config.Display.SystemStats { + if state.Config.Display.SystemStats { go state.UpdateResources() } - parse.InitParser(state) + parser := parse.InitParser(state) - // Start looking for the initial targets to kick the build off - go findOriginalTasks(state, preTargets, targets, arch) + // This must happen however we exit; anything reading state.Results() (e.g. the display) + // waits for that channel to be closed, so it would hang forever if we returned an error first. + defer func() { + if state.Cache != nil { + state.Cache.Shutdown() + } + if state.RemoteClient != nil { + _, _, in, out := state.RemoteClient.DataRate() + log.Info("Total remote RPC data in: %d out: %d", in, out) + } + state.CloseResults() + metrics.Push(state.Config.Metrics, state.Config.IsRemoteExecution()) + }() + + ctx, cancel := context.WithCancel(context.Background()) + state.Cancel = cancel + + r := runner{ + state: state, + arch: arch, + progress: progress, + buildOnce: cmap.NewErrMap[core.BuildLabel, *core.BuildTarget](cmap.DefaultShardCount, func(l core.BuildLabel) uint64 { + return cmap.XXHashes(l.Subrepo, l.PackageName, l.Name) + }, nil), + parseOnce: cmap.New[core.BuildLabel, struct{}](cmap.DefaultShardCount, func(l core.BuildLabel) uint64 { + return cmap.XXHashes(l.Subrepo, l.PackageName, l.Name) + }), + localLimiter: make(limiter, state.Config.Please.NumThreads), + remoteLimiter: make(limiter, state.Config.NumRemoteExecutors()), + anyRemote: state.Config.NumRemoteExecutors() > 0, + } + g, ctx := r.group(ctx) + r.tasks = g - parses, actions := state.TaskQueues() + // We don't have context as an argument to this, because they're not fully plumbed through (but probably should be) + state.Build = func(label, dependent core.BuildLabel) (*core.BuildTarget, error) { + return r.Build(ctx, label, dependent) + } + state.Parse = func(label, dependent core.BuildLabel) (*core.Package, error) { + return r.Parse(ctx, label, dependent) + } - localLimiter := make(limiter, config.Please.NumThreads) - remoteLimiter := make(limiter, config.NumRemoteExecutors()) - anyRemote := config.NumRemoteExecutors() > 0 + // Register the preloaded targets with the parser + if err := r.RegisterPreloads(ctx, state, parser); err != nil { + return err + } - completeAction := func(remote bool, task core.Task) { - if remote { - remoteLimiter.Release() - } else { - localLimiter.Release() + if state.Config.Bazel.Compatibility && fs.FileExists("WORKSPACE") { + // We have to parse the WORKSPACE file before anything else to understand subrepos. + // This is a bit crap really since it inhibits parallelism for the first step. + if _, err := r.Parse(ctx, core.NewBuildLabel("workspace", "all"), core.OriginalTarget); err != nil { + return err + } + } + if arch.Arch != "" && arch != cli.HostArch() { + // Set up a new subrepo for this architecture. + state.Graph.AddSubrepo(core.SubrepoForArch(state, arch)) + } + if len(preTargets) > 0 { + r.FindOriginalTaskSet(ctx, preTargets, false, true) + if err := g.Wait(); err != nil { + return err } + // Reset the group & context for next time + ctx, cancel = context.WithCancel(context.Background()) + g, ctx = r.group(ctx) + state.Cancel = cancel + r.tasks = g + } + r.FindOriginalTaskSet(ctx, targets, r.state.NeedTests, r.state.NeedBuild) + if state.NeedDebugDeps { + if len(targets) != 1 { + return fmt.Errorf("expected exactly 1 target in debug mode; got %d", len(targets)) + } + g.Go(func() error { + return r.queueTargetsForDebug(ctx, targets[0]) + }) + } - if task.Type != core.BuildTask { - state.TaskDone() - return + return g.Wait() +} + +// RunHost is a convenience function that uses the host architecture, the given state's +// configuration and no pre targets. It is otherwise identical to Run. +func RunHost(targets []core.BuildLabel, state *core.BuildState) { + Run(targets, nil, state, &Progress{}, cli.HostArch()) +} + +type runner struct { + tasks *errgroup.Group + state *core.BuildState + arch cli.Arch + progress *Progress + buildOnce *cmap.ErrMap[core.BuildLabel, *core.BuildTarget] + parseOnce *cmap.Map[core.BuildLabel, struct{}] + localLimiter limiter + remoteLimiter limiter + anyRemote bool +} + +// Parse parses for a target. It can be called more than once for the same build label. +// The dependent is whatever is asking for this to be parsed; it's used to produce better error +// messages, and to detect a package that is asking to parse itself. +func (r *runner) Parse(ctx context.Context, label, dependent core.BuildLabel) (*core.Package, error) { + return r.parse(ctx, label, dependent, false) +} + +// tryParse is like Parse but doesn't report failures. It's used where a failure isn't necessarily an +// error, i.e. when we're speculatively looking for the package that might define a subrepo; the caller +// is responsible for reporting anything it can't handle itself. +func (r *runner) tryParse(ctx context.Context, label, dependent core.BuildLabel) (*core.Package, error) { + return r.parse(ctx, label, dependent, true) +} + +func (r *runner) parse(ctx context.Context, label, dependent core.BuildLabel, quiet bool) (*core.Package, error) { + return r.state.Graph.GetOrSetPackage(ctx, label, func() (*core.Package, error) { + r.progress.numParsing.Add(1) + defer r.progress.numParsing.Add(-1) + pkg, err := func() (*core.Package, error) { + // If the target is in a subrepo that we don't know about yet, we must make sure that is defined first. + // If we already have it there's nothing to do here; it's been registered by whatever parse defined it. + if label.Subrepo != "" && r.state.Graph.Subrepo(label.Subrepo) == nil { + if err := r.ensureSubrepo(ctx, label, dependent); err != nil { + return nil, err + } + } + return parse.Parse(r.state, label, dependent) + }() + if err != nil && !quiet { + r.state.LogBuildError(label, core.ParseFailed, err, "Failed to parse package") + } + return pkg, err + }) +} + +// ensureSubrepo makes sure that the subrepo the given label is in has been defined. +// +// A name like `linux_amd64` is ambiguous: it could be a subrepo defined by a target somewhere, or one +// of the architecture subrepos, which are implicitly defined and so have no defining target anywhere. +// We resolve that by always preferring a real definition, and only falling back to the architecture +// interpretation once we know there isn't one. +func (r *runner) ensureSubrepo(ctx context.Context, label, dependent core.BuildLabel) error { + sl := label.SubrepoLabel(r.state) + // The subrepo would be defined by a target in the dependent's package, which means that package is + // the one currently being parsed - and since we didn't find the subrepo, the call that defines it + // hasn't been reached yet. We can't wait for that parse because we are that parse. + if inSamePackage(sl, dependent) { + return fmt.Errorf("subrepo %v is not defined in this package yet. It must appear before it is used by %v", label.Subrepo, dependent) + } + // Parsing the package that should define it registers the subrepo as a side effect. A missing BUILD + // file isn't fatal yet; that's exactly what we'd expect for an architecture subrepo. + _, err := r.tryParse(ctx, sl, label) + if err != nil && !errors.Is(err, parse.ErrMissingBuildFile) { + return err + } + if r.state.Graph.Subrepo(label.Subrepo) != nil { + return nil // The parse above defined it, we're done. + } + // Nothing defines it, so the only remaining possibility is an architecture subrepo. + if arch, ok := couldBeArch(label.Subrepo); ok { + r.state.Graph.MaybeAddSubrepo(core.SubrepoForArch(r.state, arch)) + return nil + } else if err != nil { + return err + } + return fmt.Errorf("Subrepo %s is not defined (referenced by %s)", label.Subrepo, dependent) +} + +// group returns an errgroup to run a set of tasks in, and the context to run them with. +// +// Normally that context is cancelled as soon as any of them fails, which stops us starting more work. +// With --keep_going we don't cancel anything, so everything that can still be built gets built; the +// group waits for all of it either way and Wait still returns the first error. +func (r *runner) group(ctx context.Context) (*errgroup.Group, context.Context) { + if r.state.KeepGoing { + return &errgroup.Group{}, ctx + } + return errgroup.WithContext(ctx) +} + +// inSamePackage returns true if the two labels are in the same package (and hence, if one of them is +// currently being parsed, both are). +func inSamePackage(label, dependent core.BuildLabel) bool { + return !dependent.IsOriginalTarget() && label.Subrepo == dependent.Subrepo && label.PackageName == dependent.PackageName +} + +// RecursiveParse is like Parse but recurses down into all dependencies of the target as well. +func (r *runner) RecursiveParse(ctx context.Context, label, dependent core.BuildLabel) error { + if !label.IsAllTargets() { + return r.recursiveParse(ctx, label, dependent) + } + pkg, err := r.Parse(ctx, label, dependent) + if err != nil { + return err + } + g, gctx := r.group(ctx) + for _, target := range pkg.AllTargets() { + for dep := range target.DeclaredDependencies() { + g.Go(func() error { + // N.B. No need to deduplicate these; recursiveParse does that for the whole walk. + return r.recursiveParse(gctx, dep, target.Label) + }) } + } + return g.Wait() +} - if !task.Target.State().IsBuilt() { - state.TaskDone() +// recursiveParse parses a target and, transitively, everything it depends on. +func (r *runner) recursiveParse(ctx context.Context, label, dependent core.BuildLabel) error { + if !r.parseOnce.Add(label, struct{}{}) { + return nil // Someone else has this one; they're in the same errgroup so we needn't wait for them. + } + target, err := r.parseTarget(ctx, label, dependent) + if err != nil { + return err + } + g, gctx := r.group(ctx) + for dep := range target.DeclaredDependencies() { + g.Go(func() error { + return r.recursiveParse(gctx, dep, target.Label) + }) + } + return g.Wait() +} + +func (r *runner) parseTarget(ctx context.Context, label, dependent core.BuildLabel) (*core.BuildTarget, error) { + if target := r.state.Graph.Target(label); target != nil { + return target, nil + } + pkg, err := r.Parse(ctx, label, dependent) + if err != nil { + return nil, err + } + if target := pkg.Target(label.Name); target != nil { + return target, nil + } + err = fmt.Errorf("Parsed build file %s but it doesn't contain target %s%s", pkg.Filename, label.Name, pkg.SuggestTargets(label, dependent)) + r.state.LogBuildError(label, core.ParseFailed, err, "%s", err) + return nil, err +} + +// resolveTarget resolves a target, dealing with require/provide as needed. +func (r *runner) resolveTarget(ctx context.Context, label core.BuildLabel, dependent *core.BuildTarget) iter.Seq2[*core.BuildTarget, error] { + return func(yield func(*core.BuildTarget, error) bool) { + target, err := r.parseTarget(ctx, label, dependent.Label) + if err != nil { + yield(nil, err) return } + // TODO(peter): We might want the minor optimisation here to avoid creating a slice in the common case + provided := target.ProvideFor(dependent) + if len(provided) == 1 && provided[0] == target.Label { + yield(target, nil) + return + } + // TODO(peter): Would parallelism here be useful? + for _, p := range provided { + if !yield(r.parseTarget(ctx, p, dependent.Label)) { + break + } + } + } +} - if state.NeedTests && task.Target.IsTest() && state.IsOriginalTarget(task.Target) { - state.QueueTestTarget(task.Target) +// buildDep builds a single dependency of a target (which might of course turn into multiple when resolved) +func (r *runner) buildDep(ctx context.Context, dep core.BuildLabel, target *core.BuildTarget) error { + for t, err := range r.resolveTarget(ctx, dep, target) { + if err != nil { + return err + } + if _, err := r.Build(ctx, t.Label, target.Label); err != nil { + return err } - state.TaskDone() } + return nil +} - startAction := func(remote bool) { - if remote { - remoteLimiter.Acquire() - } else { - localLimiter.Acquire() +// buildOne builds a single target (which cannot be a pseudo-label like :all) +func (r *runner) buildOne(ctx context.Context, target *core.BuildTarget) error { + g, gctx := r.group(ctx) + for dep := range target.BuildDependencyLabels() { + g.Go(func() error { + return r.buildDep(gctx, dep, target) + }) + } + for _, src := range target.AllSources() { + if l, ok := src.Label(); ok { + g.Go(func() error { + return r.buildDep(gctx, l, target) + }) } } + if err := g.Wait(); err != nil { + return err + } - // Start up all the build workers - var wg sync.WaitGroup - wg.Add(2) - go func() { - for task := range parses { - go func(task core.ParseTask) { - state.Parses().Add(1) - parse.Parse(state, task.Label, task.Dependent, task.Mode) - state.Parses().Add(-1) - state.TaskDone() - }(task) - } - wg.Done() - }() - go func() { - for task := range actions { - wg.Add(1) - go func(task core.Task) { - defer wg.Done() - - isRemote := anyRemote && !task.Target.Local - startAction(isRemote) - defer completeAction(isRemote, task) - - switch task.Type { - case core.TestTask: - test.Test(state, task.Target, isRemote, int(task.Run)) - case core.BuildTask: - build.Build(state, task.Target, isRemote) - } - }(task) + if target.ModifiedByCallback { + // A pre- or post-build function modified this target post parse, so we need to check its dependencies again. + g, gctx := r.group(ctx) + for dep := range target.BuildDependencyLabels() { + g.Go(func() error { + return r.buildDep(gctx, dep, target) + }) + } + if err := g.Wait(); err != nil { + return err + } + } + + // Okay, now the runtime dependencies can happen in parallel with the target itself. + // N.B. Even when there are none we can't just build the target and return; its own callbacks + // can add some, which we won't know about until it's built. + if deps := slices.Collect(target.RuntimeAndDataDependencies()); len(deps) == 0 { + if err := r.buildJustOne(target); err != nil { + return err + } + } else { + g, gctx = r.group(ctx) + g.Go(func() error { + return r.buildJustOne(target) + }) + for _, dep := range deps { + g.Go(func() error { + return r.buildDep(gctx, dep, target) + }) + } + if err := g.Wait(); err != nil { + return err } - wg.Done() - }() - // Wait until they've all exited, which they'll do once they have no tasks left. - wg.Wait() - if state.Cache != nil { - state.Cache.Shutdown() } - if state.RemoteClient != nil { - _, _, in, out := state.RemoteClient.DataRate() - log.Info("Total remote RPC data in: %d out: %d", in, out) + + if !target.ModifiedByCallback { + return nil + } + // It could have modified itself with its own post-build function, so we have to check runtime dpendencies again. + // This is a little unfortunate that we can't immediately distinguish from the case we checked above. + g, gctx = r.group(ctx) + for dep := range target.RuntimeAndDataDependencies() { + g.Go(func() error { + return r.buildDep(gctx, dep, target) + }) } - state.CloseResults() - metrics.Push(config.Metrics, config.IsRemoteExecution()) + return g.Wait() } -// RunHost is a convenience function that uses the host architecture, the given state's -// configuration and no pre targets. It is otherwise identical to Run. -func RunHost(targets []core.BuildLabel, state *core.BuildState) { - Run(targets, nil, state, state.Config, cli.HostArch()) +// buildJustOne calls the build for a single target. +func (r *runner) buildJustOne(target *core.BuildTarget) error { + remote := r.anyRemote && !target.Local + limiter := r.limiter(remote) + limiter.Acquire() + defer limiter.Release() + return build.Build(r.state, target, remote) } -// findOriginalTasks finds the original parse tasks for the original set of targets. -func findOriginalTasks(state *core.BuildState, preTargets, targets []core.BuildLabel, arch cli.Arch) { - if state.Config.Bazel.Compatibility && fs.FileExists("WORKSPACE") { - // We have to parse the WORKSPACE file before anything else to understand subrepos. - // This is a bit crap really since it inhibits parallelism for the first step. - parse.Parse(state, core.NewBuildLabel("workspace", "all"), core.OriginalTarget, core.ParseModeNormal) +// buildAll builds all the targets specified by the given label (which can be :all, but can't be ...). +func (r *runner) buildAll(ctx context.Context, label, dependent core.BuildLabel) error { + pkg, err := r.Parse(ctx, label, dependent) + if err != nil { + return err } - if arch.Arch != "" && arch != cli.HostArch() { - // Set up a new subrepo for this architecture. - state.Graph.AddSubrepo(core.SubrepoForArch(state, arch)) + g, gctx := r.group(ctx) + for _, target := range pkg.AllTargets() { + if r.state.ShouldInclude(target) { + g.Go(func() error { + // N.B. This must go through Build, not buildOne, so we don't build a target twice + // if it's reached both via :all and as a dependency of something else. + _, err := r.Build(gctx, target.Label, dependent) + return err + }) + } } - if len(preTargets) > 0 { - findOriginalTaskSet(state, preTargets, false, arch) - for _, target := range preTargets { - if target.IsAllTargets() { - log.Debug("Waiting for pre-target %s...", target) - state.SyncParsePackage(target) - log.Debug("Pre-target %s parsed, continuing...", target) - } + return g.Wait() +} + +// Build is the main entrypoint to build a label +func (r *runner) Build(ctx context.Context, label, dependent core.BuildLabel) (*core.BuildTarget, error) { + if label.IsAllTargets() { + return r.buildOnce.GetOrSetCtx(ctx, label, func() (*core.BuildTarget, error) { + return nil, r.buildAll(ctx, label, dependent) + }) + } + // N.B. We must parse the target _before_ claiming its entry in buildOnce; parsing its package can + // re-enter here for the same label (e.g. a BUILD file that subincludes a target it defines + // earlier in the same file) and we'd then deadlock waiting on ourselves. + target, err := r.parseTarget(ctx, label, dependent) + if err != nil { + return nil, err + } + return r.buildOnce.GetOrSetCtx(ctx, label, func() (*core.BuildTarget, error) { + r.progress.numTotal.Add(1) + defer r.progress.numDone.Add(1) + return target, r.buildOne(ctx, target) + }) +} + +// testOne tests one single target +func (r *runner) testOne(ctx context.Context, target *core.BuildTarget, dependent core.BuildLabel) error { + if target.IsTest() { + r.progress.numTotal.Add(int64(r.state.NumTestRuns)) + } + if _, err := r.Build(ctx, target.Label, dependent); err != nil { + return err + } + if !target.IsTest() { + return nil + } + // Now we're ready to test this target. + // TODO(peter): Is it okay for none of these to return errors? I _think_ so and we will capture it later? + remote := r.anyRemote && !target.Local + limiter := r.limiter(remote) + if r.state.TestSequentially || r.state.NumTestRuns == 1 { // minor optimisation to avoid creating unnecessary goroutines + limiter.Acquire() + defer limiter.Release() + for run := range int(r.state.NumTestRuns) { + test.Test(r.state, target, remote, run+1) + r.progress.numDone.Add(1) } - for _, target := range state.ExpandLabels(preTargets) { - log.Debug("Waiting for pre-target %s...", target) - state.WaitForInitialTargetAndEnsureDownload(target, targets[0]) - log.Debug("Pre-target %s built, continuing...", target) + return nil + } + var wg sync.WaitGroup + for run := range int(r.state.NumTestRuns) { + wg.Go(func() { + limiter.Acquire() + defer limiter.Release() + test.Test(r.state, target, remote, run+1) + r.progress.numDone.Add(1) + }) + } + wg.Wait() + return nil +} + +// Test is the main entrypoint to run tests for a label +func (r *runner) Test(ctx context.Context, label, dependent core.BuildLabel) error { + if !label.IsAllTargets() { + target, err := r.parseTarget(ctx, label, dependent) + if err != nil { + return err } + return r.testOne(ctx, target, dependent) } - findOriginalTaskSet(state, targets, true, arch) - log.Debug("Original target scan complete") - if state.NeedDebugDeps { - if len(targets) != 1 { - log.Fatalf("expected exactly 1 target in debug mode; got %d", len(targets)) + pkg, err := r.Parse(ctx, label, dependent) + if err != nil { + return err + } + g, ctx := r.group(ctx) + for _, target := range pkg.AllTargets() { + if r.state.ShouldInclude(target) { + g.Go(func() error { + return r.testOne(ctx, target, dependent) + }) } - queueTargetsForDebug(state, targets[0]) } - state.TaskDone() // initial target adding counts as one. + return g.Wait() } -func findOriginalTaskSet(state *core.BuildState, targets []core.BuildLabel, addToList bool, arch cli.Arch) { +// limiter returns either a local or remote limiter that ensures we don't build too many things at once. +func (r *runner) limiter(remote bool) limiter { + if remote { + return r.remoteLimiter + } + return r.localLimiter +} + +func (r *runner) FindOriginalTaskSet(ctx context.Context, targets []core.BuildLabel, needTest, needBuild bool) { for _, target := range ReadStdinLabels(targets) { - findOriginalTask(state, target, addToList, arch) + r.tasks.Go(func() error { + return r.findOriginalTask(ctx, target, needTest, needBuild) + }) } } -func queueTargetsForDebug(state *core.BuildState, target core.BuildLabel) { - parse.Parse(state, target, core.OriginalTarget, core.ParseModeNormal) - t := state.Graph.TargetOrDie(target) +func (r *runner) queueTargetsForDebug(ctx context.Context, target core.BuildLabel) error { + if _, err := r.Parse(ctx, target, core.OriginalTarget); err != nil { + return err + } + t := r.state.Graph.TargetOrDie(target) for _, tool := range t.AllDebugTools() { if l, ok := tool.Label(); ok { - state.AddOriginalTarget(l, false) + r.findOriginalTask(ctx, l, false, true) } } for _, data := range t.AllDebugData() { if l, ok := data.Label(); ok { - state.AddOriginalTarget(l, false) + r.findOriginalTask(ctx, l, false, true) } } + return nil } func stripHostRepoName(config *core.Configuration, label core.BuildLabel) core.BuildLabel { @@ -207,35 +555,83 @@ func stripHostRepoName(config *core.Configuration, label core.BuildLabel) core.B return label } -func findOriginalTask(state *core.BuildState, target core.BuildLabel, addToList bool, arch cli.Arch) { - if arch != cli.HostArch() { - target = core.LabelToArch(target, arch) - } - target = stripHostRepoName(state.Config, target) - if target.IsAllSubpackages() { - // Any command-line labels with subrepos and ... require us to know where they are in order to - // walk the directory tree, so we have to make sure the subrepo exists first. - dir := target.PackageName - prefix := "" - if target.Subrepo != "" { - subrepoLabel := target.SubrepoLabel(state) - if state.WaitForInitialTargetAndEnsureDownload(subrepoLabel, target) != nil { - // Targets now get activated during parsing, so can be built before we finish parsing their package. - state.WaitForPackage(subrepoLabel, target, core.ParseModeNormal) - subrepo := state.Graph.SubrepoOrDie(target.Subrepo) - dir = subrepo.Dir(dir) - prefix = subrepo.Dir(prefix) - } +func (r *runner) findOriginalTask(ctx context.Context, target core.BuildLabel, needTest, needBuild bool) error { + if r.arch != cli.HostArch() { + target = core.LabelToArch(target, r.arch) + } + target = stripHostRepoName(r.state.Config, target) + if !target.IsAllSubpackages() { + r.queueTask(ctx, target, needTest, needBuild) + return nil + } + // Any command-line labels with subrepos and ... require us to know where they are in order to + // walk the directory tree, so we have to make sure the subrepo exists first. + dir := target.PackageName + prefix := "" + if target.Subrepo != "" { + subrepoLabel := target.SubrepoLabel(r.state) + if target, err := r.Build(ctx, subrepoLabel, core.OriginalTarget); err != nil { + return err + } else if err := r.state.EnsureDownloaded(target); err != nil { + return err } - for filename := range FindAllBuildFiles(state.Config, dir, "") { - dirname, _ := filepath.Split(filename) - l := core.NewBuildLabel(strings.TrimLeft(strings.TrimPrefix(strings.TrimRight(dirname, "/"), prefix), "/"), "all") - l.Subrepo = target.Subrepo - state.AddOriginalTarget(l, addToList) + // Targets now get activated during parsing, so can be built before we finish parsing their package. + pkg, err := r.Parse(ctx, subrepoLabel, core.OriginalTarget) + if err != nil { + return err } - } else { - state.AddOriginalTarget(target, addToList) + dir = pkg.Subrepo.Dir(dir) + prefix = pkg.Subrepo.Dir(prefix) + } + for filename := range FindAllBuildFiles(r.state.Config, dir, "") { + dirname, _ := filepath.Split(filename) + l := core.NewBuildLabel(strings.TrimLeft(strings.TrimPrefix(strings.TrimRight(dirname, "/"), prefix), "/"), "all") + l.Subrepo = target.Subrepo + r.queueTask(ctx, l, needTest, needBuild) + } + return nil +} + +func (r *runner) queueTask(ctx context.Context, target core.BuildLabel, needTest, needBuild bool) { + r.state.AddOriginalTarget(target) + r.tasks.Go(func() error { + if needTest { + return r.Test(ctx, target, core.OriginalTarget) + } else if needBuild { + _, err := r.Build(ctx, target, core.OriginalTarget) + // TODO(peter): Ensure this gets downloaded if needed + return err + } + return r.RecursiveParse(ctx, target, core.OriginalTarget) + }) +} + +// RegisterPreloads waits for all preloaded subinclude targets to be built, downloads them, and then registers them with +// the interpreter. We have to actually register them otherwise this will return before we build any +// transitive subincludes. +func (r *runner) RegisterPreloads(ctx context.Context, state *core.BuildState, parser *asp.Parser) error { + g, ctx := r.group(ctx) + preloads := state.GetPreloadedSubincludes() + for _, inc := range preloads { + if inc.IsPseudoTarget() { + return fmt.Errorf("Can't preload pseudotarget %v", inc) + } + + // Queue them up asynchronously to feed the queues as quickly as possible + g.Go(func() error { + if _, err := r.Build(ctx, inc, core.OriginalTarget); err != nil { + return err + } + return parser.PreloadSubinclude(inc) + }) + } + // We must wait for all the subinclude targets to be built otherwise updating the locals might race with parsing + // a package + if err := g.Wait(); err != nil { + return err } + parser.RegisterPreloads(preloads) + return nil } // FindAllBuildFiles finds all BUILD files under a particular path. @@ -320,3 +716,35 @@ func (l limiter) Acquire() { func (l limiter) Release() { <-l } + +// Progress records some numerical progress in regard to tasks we have performed / yet to perform. +type Progress struct { + numTotal, numDone, numParsing atomic.Int64 +} + +// NumTotal returns the total number of tasks for this execution. +// These are discovered as we go so this can increase over time. +func (p *Progress) NumTotal() int { + return int(p.numTotal.Load()) +} + +// NumDone returns the number of tasks completed for this execution. +func (p *Progress) NumDone() int { + return int(p.numDone.Load()) +} + +// NumParsing returns the number of BUILD files currently being parsed. +func (p *Progress) NumParsing() int { + return int(p.numParsing.Load()) +} + +// couldBeArch returns the architecture for a potential subrepo name, if it could be one for +// cross-compiling. Note that this is only a syntactic check; a real subrepo can be named this way too, +// so a caller must satisfy itself that nothing else defines it before treating it as an architecture. +func couldBeArch(name string) (cli.Arch, bool) { + var arch cli.Arch + if err := arch.UnmarshalFlag(name); err != nil { + return arch, false + } + return arch, true +} diff --git a/src/query/changes_test.go b/src/query/changes_test.go index ed4124b684..7aa54850d1 100644 --- a/src/query/changes_test.go +++ b/src/query/changes_test.go @@ -155,9 +155,6 @@ func addTarget(state *core.BuildState, label string, dep *core.BuildTarget, sour t.AddDependency(dep.Label) } state.Graph.AddTarget(t) - if err := t.ResolveDependencies(state.Graph); err != nil { - log.Fatalf("Failed to resolve dependency %s -> %s: %s", t, dep, err) - } pkg := state.Graph.PackageByLabel(t.Label) if pkg == nil { pkg = core.NewPackageSubrepo(t.Label.PackageName, t.Label.Subrepo) diff --git a/src/query/deps.go b/src/query/deps.go index a67d884d61..559174a5a3 100644 --- a/src/query/deps.go +++ b/src/query/deps.go @@ -3,6 +3,8 @@ package query import ( "fmt" "io" + "slices" + "sort" "strings" "github.com/thought-machine/please/src/core" @@ -31,7 +33,10 @@ func deps(out io.Writer, state *core.BuildState, target *core.BuildTarget, done if currentLevel == targetLevel { return } - for _, l := range target.DeclaredDependencies() { + // Sort so output is stable and readable; DeclaredDependencies yields in declaration order. + declaredDeps := core.BuildLabels(slices.Collect(target.DeclaredDependencies())) + sort.Sort(declaredDeps) + for _, l := range declaredDeps { dep := state.Graph.TargetOrDie(l) for _, l := range dep.ProvideFor(target) { if !state.ShouldInclude(dep) || done[l] { diff --git a/src/query/graph.go b/src/query/graph.go index 4b24517827..0e32d07b38 100644 --- a/src/query/graph.go +++ b/src/query/graph.go @@ -133,7 +133,11 @@ func addJSONTarget(state *core.BuildState, graph *JSONGraph, label core.BuildLab }, } } - for _, dep := range target.Dependencies() { + deps, unresolved := target.Dependencies(state.Graph) + if len(unresolved) > 0 { + log.Fatalf("Can't generate graph for %s; dependencies not in build graph: %s", target.Label, unresolved) + } + for _, dep := range deps { addJSONTarget(state, graph, dep.Label, done) } } @@ -154,10 +158,14 @@ func makeJSONTarget(state *core.BuildState, target *core.BuildTarget) JSONTarget for in := range core.IterSources(state, state.Graph, target, false) { t.Inputs = append(t.Inputs, in) } - for _, out := range target.Outputs() { + for _, out := range target.Outputs(state.Graph) { t.Outputs = append(t.Outputs, filepath.Join(target.Label.PackageName, out)) } - for _, dep := range target.Dependencies() { + deps, unresolved := target.Dependencies(state.Graph) + if len(unresolved) > 0 { + log.Fatalf("Can't generate graph for %s; dependencies not in build graph: %s", target.Label, unresolved) + } + for _, dep := range deps { t.Deps = append(t.Deps, dep.Label.String()) } // just use run 1 as this is only used to print the test dir diff --git a/src/query/graph_test.go b/src/query/graph_test.go index 1b755ebcaa..749e0f35c5 100644 --- a/src/query/graph_test.go +++ b/src/query/graph_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/thought-machine/please/src/core" ) @@ -55,8 +54,6 @@ func makeGraph(t *testing.T) *core.BuildState { pkg2.AddTarget(t3) graph.AddTarget(pkg2.Target("target3")) graph.AddPackage(pkg2) - require.NoError(t, t2.ResolveDependencies(graph)) - require.NoError(t, t3.ResolveDependencies(graph)) return state } diff --git a/src/query/outputs.go b/src/query/outputs.go index 3ab38034cb..10fe6559b9 100644 --- a/src/query/outputs.go +++ b/src/query/outputs.go @@ -21,7 +21,7 @@ func TargetOutputs(graph *core.BuildGraph, labels []core.BuildLabel, useJSON boo func targetOutputsFlat(graph *core.BuildGraph, labels []core.BuildLabel) { for _, label := range labels { target := graph.TargetOrDie(label) - for _, out := range target.Outputs() { + for _, out := range target.Outputs(graph) { fmt.Printf("%s\n", filepath.Join(target.OutDir(), out)) } } @@ -31,7 +31,7 @@ func targetOutputsJSON(graph *core.BuildGraph, labels []core.BuildLabel) { data := map[string][]string{} for _, label := range labels { target := graph.TargetOrDie(label) - for _, out := range target.Outputs() { + for _, out := range target.Outputs(graph) { data[label.String()] = append(data[label.String()], filepath.Join(target.OutDir(), out)) } } diff --git a/src/query/print.go b/src/query/print.go index 6ecd7e63b0..f06b6b170d 100644 --- a/src/query/print.go +++ b/src/query/print.go @@ -6,6 +6,7 @@ import ( "io" "os" "reflect" + "slices" "sort" "strconv" "strings" @@ -132,13 +133,13 @@ func specialFields() specialFieldsMap { return "" }, "deps": func(target *core.BuildTarget) interface{} { - return target.DeclaredDependenciesStrict() + return slices.Collect(target.DeclaredDependenciesStrict()) }, "exported_deps": func(target *core.BuildTarget) interface{} { - return target.ExportedDependencies() + return slices.Collect(target.ExportedDependencies()) }, "runtime_deps": func(target *core.BuildTarget) interface{} { - return target.RuntimeDependencies() + return slices.Collect(target.RuntimeDependencies()) }, "visibility": func(target *core.BuildTarget) interface{} { if len(target.Visibility) == 1 && target.Visibility[0] == core.WholeGraph[0] { diff --git a/src/query/reverse_deps.go b/src/query/reverse_deps.go index cca1ea2f4c..e74c10c3c7 100644 --- a/src/query/reverse_deps.go +++ b/src/query/reverse_deps.go @@ -111,7 +111,7 @@ func buildRevdeps(graph *core.BuildGraph, includeSubrepos bool) map[core.BuildLa targets := graph.AllTargets() revdeps := make(map[core.BuildLabel][]*core.BuildTarget, len(targets)) for _, t := range targets { - for _, d := range t.DeclaredDependencies() { + for d := range t.DeclaredDependencies() { if t2 := graph.Target(d); t2 == nil { revdeps[d] = append(revdeps[d], t2) } else { diff --git a/src/query/reverse_deps_test.go b/src/query/reverse_deps_test.go index 1f1f4cbbbe..b0b558485a 100644 --- a/src/query/reverse_deps_test.go +++ b/src/query/reverse_deps_test.go @@ -20,8 +20,6 @@ func TestReverseDeps(t *testing.T) { graph.AddTarget(root) graph.AddTarget(branch) graph.AddTarget(leaf) - branch.ResolveDependencies(graph) - leaf.ResolveDependencies(graph) pkg := core.NewPackage("package") graph.AddPackage(pkg) diff --git a/src/query/somepath.go b/src/query/somepath.go index 75b91bdc10..dc0f12cce1 100644 --- a/src/query/somepath.go +++ b/src/query/somepath.go @@ -92,7 +92,7 @@ func somePath(graph *core.BuildGraph, target1, target2 *core.BuildTarget, seen, return nil } seen[target1.Label] = struct{}{} - for _, dep := range target1.DeclaredDependencies() { + for dep := range target1.DeclaredDependencies() { if t := graph.Target(dep); t != nil { if _, present := except[t.Label]; present { continue diff --git a/src/query/whatoutputs.go b/src/query/whatoutputs.go index c55517949b..ddea630ffc 100644 --- a/src/query/whatoutputs.go +++ b/src/query/whatoutputs.go @@ -12,7 +12,7 @@ import ( func WhatOutputs(graph *core.BuildGraph, files []string, printFiles bool) { targets := graph.AllTargets() for _, f := range files { - if t := whatOutputs(targets, f); len(t) > 0 { + if t := whatOutputs(graph, targets, f); len(t) > 0 { for _, l := range t { if printFiles { fmt.Printf("%s ", f) @@ -29,10 +29,10 @@ func WhatOutputs(graph *core.BuildGraph, files []string, printFiles bool) { } } -func whatOutputs(targets []*core.BuildTarget, file string) []core.BuildLabel { +func whatOutputs(graph *core.BuildGraph, targets []*core.BuildTarget, file string) []core.BuildLabel { ret := []core.BuildLabel{} for _, t := range targets { - for _, output := range t.FullOutputs() { + for _, output := range t.FullOutputs(graph) { if output == file { ret = append(ret, t.Label) } diff --git a/src/query/whatoutputs_test.go b/src/query/whatoutputs_test.go index 4dcc9dd838..09242f1200 100644 --- a/src/query/whatoutputs_test.go +++ b/src/query/whatoutputs_test.go @@ -37,7 +37,7 @@ func makeTarget2(g *core.BuildGraph, label string, filegroup bool, outputs ...st func TestDetectsOutputs(t *testing.T) { graph := core.NewGraph() makeTarget2(graph, "//package1:target1", false, "out1", "out2") - targets := whatOutputs(graph.AllTargets(), "plz-out/gen/package1/out1") + targets := whatOutputs(graph, graph.AllTargets(), "plz-out/gen/package1/out1") assert.Equal(t, []core.BuildLabel{{PackageName: "package1", Name: "target1"}}, targets) } @@ -47,11 +47,11 @@ func TestDetectOutputsFilegroup(t *testing.T) { graph := core.NewGraph() makeTarget2(graph, "//package1:target1", true, "out1", "out2") makeTarget2(graph, "//package1:target2", true, "out1") - targets := whatOutputs(graph.AllTargets(), "plz-out/gen/package1/out1") + targets := whatOutputs(graph, graph.AllTargets(), "plz-out/gen/package1/out1") assert.Equal(t, []core.BuildLabel{ {PackageName: "package1", Name: "target1"}, {PackageName: "package1", Name: "target2"}, }, targets) - targets = whatOutputs(graph.AllTargets(), "plz-out/gen/package1/out2") + targets = whatOutputs(graph, graph.AllTargets(), "plz-out/gen/package1/out2") assert.Equal(t, []core.BuildLabel{{PackageName: "package1", Name: "target1"}}, targets) } diff --git a/src/remote/action.go b/src/remote/action.go index 917a4641a7..7bf1b4ea51 100644 --- a/src/remote/action.go +++ b/src/remote/action.go @@ -106,8 +106,8 @@ func (c *Client) buildCommand(target *core.BuildTarget, inputRoot *pb.Directory, } } - outs := target.AllOutputs() - if len(target.Outputs()) == 1 { // $OUT is relative when running remotely; make it absolute + outs := target.AllOutputs(c.state.Graph) + if len(target.Outputs(c.state.Graph)) == 1 { // $OUT is relative when running remotely; make it absolute commandPrefixBuilder.WriteString(`export OUT="$TMP_DIR/$OUT" && `) } if target.IsRemoteFile { @@ -156,7 +156,7 @@ func (c *Client) buildTestCommand(state *core.BuildState, target *core.BuildTarg paths = append(paths, core.TestResultsFile) } commandPrefix := "export TMP_DIR=\"`pwd`\" TEST_DIR=\"`pwd`\" && " - if outs := target.Outputs(); len(outs) > 0 { + if outs := target.Outputs(state.Graph); len(outs) > 0 { commandPrefix += `export TEST="$TEST_DIR/` + outs[0] + `" && ` } cmd, err := core.TestCommand(state, target) @@ -177,7 +177,7 @@ func (c *Client) buildTestCommand(state *core.BuildState, target *core.BuildTarg // buildRunCommand builds the command to run a target remotely. func (c *Client) buildRunCommand(state *core.BuildState, target *core.BuildTarget) (*pb.Command, error) { - outs := target.Outputs() + outs := target.Outputs(state.Graph) if len(outs) == 0 { return nil, fmt.Errorf("Target %s has no outputs, it can't be run with `plz run`", target) } @@ -287,7 +287,7 @@ func (c *Client) uploadInputDir(ch chan<- *uploadinfo.Entry, target *core.BuildT } } if !isTest && target.Stamp { - stamp := core.StampFile(c.state.Config, target) + stamp := core.StampFile(c.state, target) entry := uploadinfo.EntryFromBlob(stamp) if ch != nil { ch <- entry @@ -539,7 +539,7 @@ func (c *Client) verifyActionResult(target *core.BuildTarget, command *pb.Comman // uploadLocalTarget uploads the outputs of a target that was built locally. func (c *Client) uploadLocalTarget(target *core.BuildTarget) error { - m, ar, err := c.client.ComputeOutputsToUpload(target.OutDir(), ".", target.Outputs(), filemetadata.NewNoopCache(), command.PreserveSymlink, map[string]*cpb.NodeProperties{}) + m, ar, err := c.client.ComputeOutputsToUpload(target.OutDir(), ".", target.Outputs(c.state.Graph), filemetadata.NewNoopCache(), command.PreserveSymlink, map[string]*cpb.NodeProperties{}) if err != nil { return err } diff --git a/src/remote/remote.go b/src/remote/remote.go index 6840e4adca..1f755cc68b 100644 --- a/src/remote/remote.go +++ b/src/remote/remote.go @@ -496,7 +496,7 @@ func (c *Client) download(target *core.BuildTarget, f func() error) error { func (c *Client) reallyDownload(target *core.BuildTarget, digest *pb.Digest, ar *pb.ActionResult) error { log.Debug("Downloading outputs for %s", target) - if err := removeOutputs(target); err != nil { + if err := c.removeOutputs(target); err != nil { return err } if err := c.downloadActionOutputs(context.Background(), ar, target); err != nil { @@ -904,7 +904,7 @@ func (c *Client) fetchRemoteFile(target *core.BuildTarget, actionDigest *pb.Dige } c.state.LogBuildResult(target, core.TargetBuilding, "Downloaded.") // If we get here, the blob exists in the CAS. Create an ActionResult corresponding to it. - outs := target.Outputs() + outs := target.Outputs(c.state.Graph) ar := &pb.ActionResult{ OutputFiles: []*pb.OutputFile{{ Path: outs[0], diff --git a/src/remote/remote_test.go b/src/remote/remote_test.go index 21b703a3d1..07a35f90be 100644 --- a/src/remote/remote_test.go +++ b/src/remote/remote_test.go @@ -83,7 +83,7 @@ func TestExecutePostBuildFunction(t *testing.T) { }) _, err := c.Build(target) assert.NoError(t, err) - assert.Equal(t, []string{"somefile"}, target.Outputs()) + assert.Equal(t, []string{"somefile"}, target.Outputs(c.state.Graph)) } func TestExecuteFetch(t *testing.T) { @@ -255,7 +255,7 @@ func TestOutDirsSetOutsOnTarget(t *testing.T) { Name: "out_dir_target", }) - c.state.AddOriginalTarget(outDirTarget.Label, true) + c.state.AddOriginalTarget(outDirTarget.Label) c.state.OutputDownload = core.OriginalOutputDownload require.True(t, c.state.ShouldDownload(outDirTarget)) @@ -266,9 +266,9 @@ func TestOutDirsSetOutsOnTarget(t *testing.T) { _, err := c.Build(outDirTarget) require.NoError(t, err) - assert.Len(t, outDirTarget.Outputs(), 2) - assert.ElementsMatch(t, []string{"foo.txt", "bar.txt"}, outDirTarget.Outputs()) - for _, out := range outDirTarget.Outputs() { + assert.Len(t, outDirTarget.Outputs(c.state.Graph), 2) + assert.ElementsMatch(t, []string{"foo.txt", "bar.txt"}, outDirTarget.Outputs(c.state.Graph)) + for _, out := range outDirTarget.Outputs(c.state.Graph) { assert.True(t, fs.FileExists(filepath.Join(outDirTarget.OutDir(), out)), "output %s doesn't exist in target out folder", out) } } diff --git a/src/remote/utils.go b/src/remote/utils.go index d81d70700a..d5fc6ff835 100644 --- a/src/remote/utils.go +++ b/src/remote/utils.go @@ -296,7 +296,7 @@ func (c *Client) retrieveLocalResults(target *core.BuildTarget, digest *pb.Diges // outputsExist returns true if the outputs for this target exist and are up to date. func (c *Client) outputsExist(target *core.BuildTarget, digest *pb.Digest) bool { hash, _ := hex.DecodeString(digest.Hash) - for _, out := range target.FullOutputs() { + for _, out := range target.FullOutputs(c.state.Graph) { if !bytes.Equal(hash, fs.ReadAttr(out, xattrName, c.state.XattrsSupported)) { return false } @@ -307,7 +307,7 @@ func (c *Client) outputsExist(target *core.BuildTarget, digest *pb.Digest) bool // recordAttrs sets the xattrs on output files which we will use in outputsExist in future runs. func (c *Client) recordAttrs(target *core.BuildTarget, digest *pb.Digest) { hash, _ := hex.DecodeString(digest.Hash) - for _, out := range target.FullOutputs() { + for _, out := range target.FullOutputs(c.state.Graph) { fs.RecordAttr(out, hash, xattrName, c.state.XattrsSupported) } } @@ -571,9 +571,9 @@ func (c *Client) targetPlatformProperties(target *core.BuildTarget) *pb.Platform } // removeOutputs removes all outputs for a target. -func removeOutputs(target *core.BuildTarget) error { +func (c *Client) removeOutputs(target *core.BuildTarget) error { outDir := target.OutDir() - for _, out := range target.Outputs() { + for _, out := range target.Outputs(c.state.Graph) { if err := fs.RemoveAll(filepath.Join(outDir, out)); err != nil { return fmt.Errorf("Failed to remove output for %s: %s", target, err) } diff --git a/src/run/run_step.go b/src/run/run_step.go index 8c471997d6..8ffe8c3267 100644 --- a/src/run/run_step.go +++ b/src/run/run_step.go @@ -105,8 +105,8 @@ func run(ctx context.Context, state *core.BuildState, label core.AnnotatedOutput if !target.IsBinary && overrideCmd == "" { log.Fatalf("Target %s cannot be run; it's not marked as binary", label) } - if label.Annotation == "" && len(target.Outputs()) != 1 { - log.Fatalf("Targets %s cannot be run as it has %d outputs.", label, len(target.Outputs())) + if label.Annotation == "" && len(target.Outputs(state.Graph)) != 1 { + log.Fatalf("Targets %s cannot be run as it has %d outputs.", label, len(target.Outputs(state.Graph))) } if remote { // Send this off to be done remotely. @@ -148,7 +148,7 @@ func run(ctx context.Context, state *core.BuildState, label core.AnnotatedOutput // out_exe handles java binary stuff by invoking the .jar with java as necessary var command string if tmpDir { - command = filepath.Join(dir, target.Outputs()[0]) + command = filepath.Join(dir, target.Outputs(state.Graph)[0]) } else { command, _ = core.ReplaceSequences(state, target, fmt.Sprintf("$(out_exe %s)", target.Label)) command = strings.Trim(command, "\"") diff --git a/src/test/coverage.go b/src/test/coverage.go index 74a4f39d41..1386b094fc 100644 --- a/src/test/coverage.go +++ b/src/test/coverage.go @@ -59,7 +59,9 @@ func collectCoverageFiles(state *core.BuildState, includeAllFiles bool) map[stri doneTargets := map[*core.BuildTarget]bool{} coverageFiles := map[string]bool{} for _, label := range state.ExpandAllOriginalLabels() { - collectAllFiles(state, state.Graph.TargetOrDie(label), coverageFiles, includeAllFiles, true, doneTargets) + if target := state.Graph.Target(label); target != nil { // It won't be if it failed to parse + collectAllFiles(state, target, coverageFiles, includeAllFiles, true, doneTargets) + } } return coverageFiles } @@ -74,7 +76,11 @@ func collectAllFiles(state *core.BuildState, target *core.BuildTarget, coverageF } } if deps { - for _, dep := range target.ExternalDependencies() { + extDeps, unresolved := target.ExternalDependencies(state.Graph) + if len(unresolved) > 0 { + log.Warning("Can't collect coverage for dependencies of %s; not in build graph: %s", target.Label, unresolved) + } + for _, dep := range extDeps { collectAllFiles(state, dep, coverageFiles, includeAllFiles, deps, doneTargets) } } diff --git a/src/test/surefire.go b/src/test/surefire.go index dac7275552..5c574f613e 100644 --- a/src/test/surefire.go +++ b/src/test/surefire.go @@ -11,7 +11,10 @@ import ( // CopySurefireXMLFilesToDir copies all the XML test results files into the given directory. func CopySurefireXMLFilesToDir(state *core.BuildState, surefireDir string) { for _, label := range state.ExpandOriginalLabels() { - target := state.Graph.TargetOrDie(label) + target := state.Graph.Target(label) + if target == nil { + continue // The target failed to parse, so there's nothing to copy for it. + } if state.ShouldInclude(target) && target.IsTest() && !target.Test.NoOutput { copySurefireXMLtoDir(target.TestResultsFile(), surefireDir) } diff --git a/src/watch/watch.go b/src/watch/watch.go index 3d7112f595..01ac403475 100644 --- a/src/watch/watch.go +++ b/src/watch/watch.go @@ -103,7 +103,9 @@ func startWatching(watcher *fsnotify.Watcher, state *core.BuildState, labels []c addSource(watcher, state, datum, dirs, files) } } - for _, dep := range target.Dependencies() { + // Anything unresolved just doesn't get watched; it's not worth failing the whole watch for. + deps, _ := target.Dependencies(state.Graph) + for _, dep := range deps { startWatch(dep) } pkg := state.Graph.PackageOrDie(target.Label) diff --git a/third_party/python/BUILD b/third_party/python/BUILD index 2e85c368d6..2d16ec3fc0 100644 --- a/third_party/python/BUILD +++ b/third_party/python/BUILD @@ -65,12 +65,13 @@ python_wheel( python_wheel( name = "absl", package_name = "absl_py", - hashes = ["c106f6ef0ae86c1273b0858b40ee15b99fad1c223838387b9d11446a033bbcb1"], - version = "0.9.0", + hashes = ["0f17b89f2a4eaaedc4f28c622998aa690564b3012a396a4ffad0821007fe03ba"], + version = "2.5.0", deps = [":six"], ) pip_library( name = "progress", version = "1.5", + licences = ["ISC"], ) diff --git a/tools/build_langserver/lsp/lsp_test.go b/tools/build_langserver/lsp/lsp_test.go index 8d05240f8c..c2f51521f2 100644 --- a/tools/build_langserver/lsp/lsp_test.go +++ b/tools/build_langserver/lsp/lsp_test.go @@ -15,7 +15,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/thought-machine/please/src/cli" - "github.com/thought-machine/please/src/core" ) func init() { @@ -629,10 +628,10 @@ func (h *Handler) CurrentContent(doc string) string { // WaitForPackage blocks until the given package has been parsed. func (h *Handler) WaitForPackage(pkg string) { - // this can only be described as 'grotty', but it is test code - h.state.Graph.WaitForTarget(core.BuildLabel{PackageName: pkg}) - if h.state.Graph.Package(pkg, "") == nil { - log.Fatalf("package %s doesn't exist", pkg) + // As with WaitForPackageTree below, polling is a bit yucky, but it's the only way of syncing + // up to this without interfering with the parse we're waiting on. + for h.state.Graph.Package(pkg, "") == nil { + time.Sleep(5 * time.Millisecond) } } diff --git a/tools/performance/gen_parse_tree.py b/tools/performance/gen_parse_tree.py index 028d526e1d..9cefbfa094 100644 --- a/tools/performance/gen_parse_tree.py +++ b/tools/performance/gen_parse_tree.py @@ -61,7 +61,7 @@ """ GO_SRC_TEMPLATE = """ -package {dir} +package {pkg} """ LANGUAGE_SRC_TEMPLATES = { @@ -102,10 +102,10 @@ def main(argv): ext = LANGUAGE_EXTENSIONS[lang] library_filename = os.path.join(dir, f'{basedir}.{ext}') with open(library_filename, 'w') as f: - f.write(src_template.format(dir = relative_dir)) + f.write(src_template.format(dir = relative_dir, pkg = os.path.basename(relative_dir))) test_filename = os.path.join(dir, f'{basedir}_test.{ext}') with open(test_filename, 'w') as f: - f.write(src_template.format(dir = relative_dir)) + f.write(src_template.format(dir = relative_dir, pkg = os.path.basename(relative_dir))) packages.append(dir) pkgset.add(dir) filenames.append(filename) @@ -124,7 +124,6 @@ def main(argv): [Plugin "cc"] Target = //plugins:cc -TestMain = ///pleasings//cc:unittest_main [Plugin "go"] Target = //plugins:go From c25121e566dbb591c38eb2a7ff51713fd2af2b04 Mon Sep 17 00:00:00 2001 From: Peter Ebden Date: Sun, 9 Aug 2026 14:51:28 +0100 Subject: [PATCH 02/16] Don't run non-test targets in :all path --- src/plz/plz.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plz/plz.go b/src/plz/plz.go index 09eac34f84..471605cd02 100644 --- a/src/plz/plz.go +++ b/src/plz/plz.go @@ -492,7 +492,7 @@ func (r *runner) Test(ctx context.Context, label, dependent core.BuildLabel) err } g, ctx := r.group(ctx) for _, target := range pkg.AllTargets() { - if r.state.ShouldInclude(target) { + if r.state.ShouldInclude(target) && (target.IsTest() || r.state.NeedCoverage) && !target.AddedPostBuild { g.Go(func() error { return r.testOne(ctx, target, dependent) }) From c644f00d0fcf9001c8d087bdcff0dd999abc3c26 Mon Sep 17 00:00:00 2001 From: Peter Ebden Date: Mon, 10 Aug 2026 10:10:51 +0100 Subject: [PATCH 03/16] rm log, is too noisy --- src/parse/parse_step.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/parse/parse_step.go b/src/parse/parse_step.go index 3f1082c94f..0a591e3058 100644 --- a/src/parse/parse_step.go +++ b/src/parse/parse_step.go @@ -98,7 +98,6 @@ func parsePackage(state *core.BuildState, label, dependent core.BuildLabel, subr filename, dir := buildFileName(state, subrepo, fileSystem, label.PackageName) if filename != "" { pkg.Filename = filename - log.Debug("Parsing build file %s %s", label, filename) if err := state.Parser.ParseFile(pkg, &label, &dependent, fileSystem, filename); err != nil { return nil, err } From 08bf87248f8c3443771cf09e42c1962ad385430d Mon Sep 17 00:00:00 2001 From: Peter Ebden Date: Mon, 10 Aug 2026 15:42:09 +0100 Subject: [PATCH 04/16] Ensure things are downloaded, I _think_ this will fix the rex test --- src/plz/plz.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/plz/plz.go b/src/plz/plz.go index 471605cd02..d30c16af67 100644 --- a/src/plz/plz.go +++ b/src/plz/plz.go @@ -81,7 +81,12 @@ func Run(targets, preTargets []core.BuildLabel, state *core.BuildState, progress // We don't have context as an argument to this, because they're not fully plumbed through (but probably should be) state.Build = func(label, dependent core.BuildLabel) (*core.BuildTarget, error) { - return r.Build(ctx, label, dependent) + target, err := r.Build(ctx, label, dependent) + if err != nil { + return nil, err + } + // Anything calling this will likely want this thing to end up being downloaded (it's mostly for subincludes) + return target, state.EnsureDownloaded(target) } state.Parse = func(label, dependent core.BuildLabel) (*core.Package, error) { return r.Parse(ctx, label, dependent) From 489e4a212c07d37e9b06ba5ae6231e535b693349 Mon Sep 17 00:00:00 2001 From: Peter Ebden Date: Mon, 10 Aug 2026 17:13:08 +0100 Subject: [PATCH 05/16] Revert Python changes, looks like darwin workers can't cope --- third_party/python/BUILD | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/third_party/python/BUILD b/third_party/python/BUILD index 2d16ec3fc0..2e85c368d6 100644 --- a/third_party/python/BUILD +++ b/third_party/python/BUILD @@ -65,13 +65,12 @@ python_wheel( python_wheel( name = "absl", package_name = "absl_py", - hashes = ["0f17b89f2a4eaaedc4f28c622998aa690564b3012a396a4ffad0821007fe03ba"], - version = "2.5.0", + hashes = ["c106f6ef0ae86c1273b0858b40ee15b99fad1c223838387b9d11446a033bbcb1"], + version = "0.9.0", deps = [":six"], ) pip_library( name = "progress", version = "1.5", - licences = ["ISC"], ) From 86aeb98790f398c15509448c40b662d97bfb3f17 Mon Sep 17 00:00:00 2001 From: Peter Ebden Date: Tue, 11 Aug 2026 12:39:48 +0100 Subject: [PATCH 06/16] Pass preload through fully --- src/parse/asp/builtins.go | 2 +- src/parse/asp/interpreter.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/parse/asp/builtins.go b/src/parse/asp/builtins.go index fb7359b38b..1e88134dcf 100644 --- a/src/parse/asp/builtins.go +++ b/src/parse/asp/builtins.go @@ -353,7 +353,7 @@ func subinclude(s *scope, args []pyObject) pyObject { outs = t.Outputs(s.state.Graph) } for _, out := range outs { - s.SetAll(s.interpreter.Subinclude(s, filepath.Join(t.OutDir(), out), t.Label, false), false) + s.SetAll(s.interpreter.Subinclude(s, filepath.Join(t.OutDir(), out), t.Label, s.Preload), false) } } return None diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index f4a526f882..4e6b6d68e7 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -225,6 +225,7 @@ func (i *interpreter) Subinclude(pkgScope *scope, path string, label core.BuildL } s := i.scope.NewScope(path) + s.Preload = preload s.state = pkgScope.state // Scope needs a local version of CONFIG @@ -304,6 +305,8 @@ type scope struct { globber *fs.Globber // True if this scope is for a pre- or post-build callback. Callback bool + // True if this scope is from a preloaded subinclude + Preload bool } // parseAnnotatedLabelInPackage similarly to parseLabelInPackage, parses the label contextualising it to the provided @@ -419,6 +422,7 @@ func (s *scope) newScope(pkg *core.Package, filename string, hint int) *scope { locals: make(pyDict, hint), config: s.config, Callback: s.Callback, + Preload: s.Preload, } if pkg != nil && pkg.Subrepo != nil && pkg.Subrepo.State != nil { s2.state = pkg.Subrepo.State From e5ea789d9c58db6d8108a4b5fc272b650a0db191 Mon Sep 17 00:00:00 2001 From: Peter Ebden Date: Wed, 12 Aug 2026 08:50:54 +0100 Subject: [PATCH 07/16] Keep source ordering the same as master --- src/core/utils.go | 28 +++++++++++++++++++++++++--- src/core/utils_test.go | 2 +- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/core/utils.go b/src/core/utils.go index e59f4f9cdf..f217f74d2c 100644 --- a/src/core/utils.go +++ b/src/core/utils.go @@ -7,6 +7,7 @@ import ( "iter" "os" "path/filepath" + "slices" "strings" "github.com/thought-machine/please/src/fs" @@ -171,7 +172,19 @@ func IterInputs(state *BuildState, graph *BuildGraph, target *BuildTarget, inclu done[dependency.Label] = true if target == dependency || (target.NeedsTransitiveDependencies && !dependency.OutputIsComplete) { - for dep := range dependency.BuildDependencies() { + // TODO(peterebden): We are maintaining a particular ordering here to avoid causing rebuilds. + // At some point when we are happy to do so, we should push this up to places that care + // (e.g. incrementality.go). + deps := slices.SortedFunc(func(yield func(BuildLabel) bool) { + for dep := range dependency.BuildDependencies() { + for _, provided := range graph.TargetOrDie(dep).ProvideFor(dependency) { + if !yield(provided) { + return + } + } + } + }, BuildLabel.Compare) + for _, dep := range deps { for dep2 := range recursivelyProvideFor(graph, target, dependency, dep) { if !done[dep2] && !dependency.IsTool(dep2) { if !inner(graph.TargetOrDie(dep2)) { @@ -430,8 +443,17 @@ func IterInputPaths(graph *BuildGraph, target *BuildTarget) iter.Seq[string] { } } - // Finally recurse for all the deps of this rule. - for dep := range target.DeclaredDependencies() { + // Finally recurse for all the deps of this rule, again in a deterministic order. + deps := slices.SortedFunc(func(yield func(BuildLabel) bool) { + for dep := range target.DeclaredDependencies() { + for _, provided := range graph.TargetOrDie(dep).ProvideFor(target) { + if !yield(provided) { + return + } + } + } + }, BuildLabel.Compare) + for _, dep := range deps { t := graph.TargetOrDie(dep) for d := range recursivelyProvideFor(graph, target, t, t.Label) { if !inner(graph.TargetOrDie(d)) { diff --git a/src/core/utils_test.go b/src/core/utils_test.go index 831c12fb9a..79b61493b6 100644 --- a/src/core/utils_test.go +++ b/src/core/utils_test.go @@ -72,8 +72,8 @@ func TestIterSources(t *testing.T) { assert.Equal(t, []SourcePair{ {"src/output/output2.go", "plz-out/tmp/src/output/output2._build/src/output/output2.go"}, - {"plz-out/gen/src/output/output1.a", "plz-out/tmp/src/output/output2._build/src/output/output1.a"}, {"plz-out/gen/src/core/target2.a", "plz-out/tmp/src/output/output2._build/src/core/target2.a"}, + {"plz-out/gen/src/output/output1.a", "plz-out/tmp/src/output/output2._build/src/output/output1.a"}, }, iterSources("//src/output:output2")) assert.Equal(t, []SourcePair{ From 9155c07282886425fb67a44077c14ba651681c0c Mon Sep 17 00:00:00 2001 From: Peter Ebden Date: Wed, 12 Aug 2026 09:23:41 +0100 Subject: [PATCH 08/16] Fix some tests --- src/core/graph.go | 2 +- src/core/state.go | 17 ++++++++++++----- src/please.go | 2 ++ src/plz/plz.go | 10 ++++++++++ 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/core/graph.go b/src/core/graph.go index c30824c9f6..a5be17cad2 100644 --- a/src/core/graph.go +++ b/src/core/graph.go @@ -66,7 +66,7 @@ func (graph *BuildGraph) Target(label BuildLabel) *BuildTarget { func (graph *BuildGraph) TargetOrDie(label BuildLabel) *BuildTarget { target := graph.Target(label) if target == nil { - panic(fmt.Sprintf("Target %s not found in build graph\n", label)) + log.Fatalf("Target %s not found in build graph\n", label) } return target } diff --git a/src/core/state.go b/src/core/state.go index 22fa8733e3..f07de4bf10 100644 --- a/src/core/state.go +++ b/src/core/state.go @@ -35,6 +35,9 @@ var startTime = time.Now() // cycleCheckDuration is the length of time we allow inactivity for before we trigger cycle detection. const cycleCheckDuration = 5 * time.Second +// resultsChanSize is the buffer size of the channel we report build results on. +const resultsChanSize = 1000 + // ParseTask is the type for the parse task queue type ParseTask struct { Label, Dependent BuildLabel @@ -318,11 +321,15 @@ func (state *BuildState) CloseResults() { state.progress.cycleDetector.Stop() state.progress.mutex.Lock() defer state.progress.mutex.Unlock() - if state.progress.results != nil { - state.progress.resultOnce.Do(func() { - close(state.progress.results) - }) + // N.B. We create the channel if nobody has asked for it yet, rather than doing nothing; otherwise + // anyone calling Results() after this would get a fresh channel that is never closed and would + // wait on it forever. + if state.progress.results == nil { + state.progress.results = make(chan *BuildResult, resultsChanSize) } + state.progress.resultOnce.Do(func() { + close(state.progress.results) + }) } // AddOriginalTarget adds an original target to this state @@ -550,7 +557,7 @@ func (state *BuildState) Results() <-chan *BuildResult { state.progress.mutex.Lock() defer state.progress.mutex.Unlock() if state.progress.results == nil { - state.progress.results = make(chan *BuildResult, 1000) + state.progress.results = make(chan *BuildResult, resultsChanSize) } return state.progress.results } diff --git a/src/please.go b/src/please.go index 4c54e845ab..441ad1c4d3 100644 --- a/src/please.go +++ b/src/please.go @@ -1223,6 +1223,8 @@ func runPlease(state *core.BuildState, targets []core.BuildLabel) { state.Cache = cache.NewCache(state) // Run the display + // TODO(peterebden): Refactor out the results stuff from state in a future PR to avoid this kind of race condition. + state.Results() // important this is called now, don't ask... var progress plz.Progress var wg sync.WaitGroup wg.Add(1) diff --git a/src/plz/plz.go b/src/plz/plz.go index d30c16af67..221fa72dd1 100644 --- a/src/plz/plz.go +++ b/src/plz/plz.go @@ -359,6 +359,16 @@ func (r *runner) buildOne(ctx context.Context, target *core.BuildTarget) error { } } + // The target's own build command can refer to its run-time & data dependencies via shell replacements, so we must at + // least have them resolved here (they don't have to be built yet though) + for dep := range target.RuntimeAndDataDependencies() { + for _, err := range r.resolveTarget(ctx, dep, target) { + if err != nil { + return err + } + } + } + // Okay, now the runtime dependencies can happen in parallel with the target itself. // N.B. Even when there are none we can't just build the target and return; its own callbacks // can add some, which we won't know about until it's built. From 548f78f7e72575172ac84cb51dff08f6cc30a8e1 Mon Sep 17 00:00:00 2001 From: Peter Ebden Date: Mon, 10 Aug 2026 17:38:58 +0100 Subject: [PATCH 09/16] Plumb context through --- src/build/build_step_stress_test.go | 5 +++-- src/build/build_step_test.go | 5 +++-- src/core/state.go | 8 ++++---- src/parse/asp/builtins.go | 17 +++++++++++++---- src/parse/asp/interpreter.go | 3 ++- src/parse/asp/interpreter_test.go | 3 ++- src/parse/asp/logging_test.go | 3 ++- src/parse/asp/parser.go | 9 +++++---- src/parse/init.go | 16 +++++++++------- src/parse/parse_step.go | 13 +++++++------ src/plz/plz.go | 7 +++---- 11 files changed, 53 insertions(+), 36 deletions(-) diff --git a/src/build/build_step_stress_test.go b/src/build/build_step_stress_test.go index 045db96d91..67744e055b 100644 --- a/src/build/build_step_stress_test.go +++ b/src/build/build_step_stress_test.go @@ -4,6 +4,7 @@ package build_test import ( + "context" "fmt" "io" iofs "io/fs" @@ -98,7 +99,7 @@ type fakeParser struct { } // ParseFile stub -func (fake *fakeParser) ParseFile(pkg *core.Package, label, dependent *core.BuildLabel, fs iofs.FS, filename string) error { +func (fake *fakeParser) ParseFile(_ context.Context, pkg *core.Package, label, dependent *core.BuildLabel, fs iofs.FS, filename string) error { return nil } @@ -117,7 +118,7 @@ func (fake *fakeParser) Init(state *core.BuildState) { } // ParseReader stub -func (fake *fakeParser) ParseReader(pkg *core.Package, r io.ReadSeeker, label, dependent *core.BuildLabel) error { +func (fake *fakeParser) ParseReader(_ context.Context, pkg *core.Package, r io.ReadSeeker, label, dependent *core.BuildLabel) error { return nil } diff --git a/src/build/build_step_test.go b/src/build/build_step_test.go index ba0a4a36c4..c4266596c4 100644 --- a/src/build/build_step_test.go +++ b/src/build/build_step_test.go @@ -8,6 +8,7 @@ package build import ( + "context" "encoding/hex" "fmt" "io" @@ -610,7 +611,7 @@ type fakeParser struct { } // ParseFile stub -func (fake *fakeParser) ParseFile(pkg *core.Package, label, dependent *core.BuildLabel, fs iofs.FS, filename string) error { +func (fake *fakeParser) ParseFile(_ context.Context, pkg *core.Package, label, dependent *core.BuildLabel, fs iofs.FS, filename string) error { return nil } @@ -628,7 +629,7 @@ func (fake *fakeParser) NewParser(state *core.BuildState) { } // ParseReader stub -func (fake *fakeParser) ParseReader(pkg *core.Package, r io.ReadSeeker, label, dependent *core.BuildLabel) error { +func (fake *fakeParser) ParseReader(_ context.Context, pkg *core.Package, r io.ReadSeeker, label, dependent *core.BuildLabel) error { return nil } diff --git a/src/core/state.go b/src/core/state.go index f07de4bf10..15fb88835a 100644 --- a/src/core/state.go +++ b/src/core/state.go @@ -73,9 +73,9 @@ const ( // A Parser is the interface to reading and interacting with BUILD files. type Parser interface { // ParseFile parses a single BUILD file into the given package. - ParseFile(pkg *Package, forLabel, dependent *BuildLabel, fs iofs.FS, filename string) error + ParseFile(ctx context.Context, pkg *Package, forLabel, dependent *BuildLabel, fs iofs.FS, filename string) error // ParseReader parses a single BUILD file into the given package. - ParseReader(pkg *Package, reader io.ReadSeeker, forLabel, dependent *BuildLabel) error + ParseReader(ctx context.Context, pkg *Package, reader io.ReadSeeker, forLabel, dependent *BuildLabel) error // RunPreBuildFunction runs a pre-build function for a target. RunPreBuildFunction(state *BuildState, target *BuildTarget) error // RunPostBuildFunction runs a post-build function for a target. @@ -224,10 +224,10 @@ type BuildState struct { // Build is a callback to build a single target. It's set from outside here. // TODO(peter): can we find a way of moving these off this struct? it feels weird here // The second label is the dependent, i.e. whatever is asking for this to be built. - Build func(label, dependent BuildLabel) (*BuildTarget, error) + Build func(ctx context.Context, label, dependent BuildLabel) (*BuildTarget, error) // Parse is a callback to parse a single package. It's also set from outside. // The second label is the dependent, i.e. whatever is asking for this to be parsed. - Parse func(label, dependent BuildLabel) (*Package, error) + Parse func(ctx context.Context, label, dependent BuildLabel) (*Package, error) // Cancel is a cancel function called when the state detects a cycle. Cancel func() diff --git a/src/parse/asp/builtins.go b/src/parse/asp/builtins.go index 1e88134dcf..2637d73a48 100644 --- a/src/parse/asp/builtins.go +++ b/src/parse/asp/builtins.go @@ -1,6 +1,7 @@ package asp import ( + "context" "encoding/json" "errors" "fmt" @@ -8,6 +9,7 @@ import ( "path/filepath" "reflect" "regexp" + "runtime/pprof" "slices" "sort" "strconv" @@ -302,10 +304,10 @@ func bazelLoad(s *scope, args []pyObject) pyObject { // WaitForSubincludedTarget drops the interpreter lock and waits for the subincluded target to be built. This is // important to keep us from deadlocking all available parser threads (easy to happen if they're all waiting on a // single target which now can't start) -func (s *scope) WaitForSubincludedTarget(l, dependent core.BuildLabel) (*core.BuildTarget, error) { +func (s *scope) WaitForSubincludedTarget(ctx context.Context, l, dependent core.BuildLabel) (*core.BuildTarget, error) { s.interpreter.limiter.Release() defer s.interpreter.limiter.Acquire() - return s.state.Build(l, dependent) + return s.state.Build(ctx, l, dependent) } // builtinFail raises an immediate error that can't be intercepted. @@ -381,11 +383,18 @@ func subincludeTarget(s *scope, l core.BuildLabel) *core.BuildTarget { Subrepo: subrepoLabel.Subrepo, Name: "all", } - if _, err := s.state.Parse(subrepoPackageLabel, pkgLabel); err != nil { + ctx := pprof.WithLabels(s.ctx, pprof.Labels("subinclude "+subrepoPackageLabel.String(), pkgLabel.String())) + pprof.SetGoroutineLabels(ctx) + defer pprof.SetGoroutineLabels(s.ctx) + if _, err := s.state.Parse(ctx, subrepoPackageLabel, pkgLabel); err != nil { s.Error("Failed to parse subrepo target: %w", err) } } + ctx := pprof.WithLabels(s.ctx, pprof.Labels("subinclude "+l.String(), pkgLabel.String())) + pprof.SetGoroutineLabels(ctx) + defer pprof.SetGoroutineLabels(s.ctx) + // isLocal is true when this subinclude target in the current package being parsed isLocal := s.pkg != nil && l.Subrepo == s.pkg.Label().Subrepo && l.PackageName == s.pkg.Name @@ -396,7 +405,7 @@ func subincludeTarget(s *scope, l core.BuildLabel) *core.BuildTarget { if t == nil && isLocal { s.Error("Target :%s is not defined in this package; it has to be defined before the subinclude() call", l.Name) } - t, err := s.WaitForSubincludedTarget(l, pkgLabel) + t, err := s.WaitForSubincludedTarget(ctx, l, pkgLabel) if err != nil { s.Error("Failed to build subincluded target: %w", err) } else if s.pkg != nil { diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index 4e6b6d68e7..be4449235b 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -164,8 +164,9 @@ func (i *interpreter) preloadSubinclude(s *scope, label core.BuildLabel) (err er // interpretAll runs a series of statements in the scope of the given package. // The first return value is for testing only. -func (i *interpreter) interpretAll(pkg *core.Package, forLabel, dependent *core.BuildLabel, statements []*Statement) (*scope, error) { +func (i *interpreter) interpretAll(ctx context.Context, pkg *core.Package, forLabel, dependent *core.BuildLabel, statements []*Statement) (*scope, error) { s := i.scope.NewPackagedScope(pkg, 1) + s.ctx = ctx s.config = i.getConfig(s.state).Copy() // Config needs a little separate tweaking. diff --git a/src/parse/asp/interpreter_test.go b/src/parse/asp/interpreter_test.go index 11c720b05a..f3c2db1f13 100644 --- a/src/parse/asp/interpreter_test.go +++ b/src/parse/asp/interpreter_test.go @@ -4,6 +4,7 @@ package asp import ( + "context" "fmt" "testing" @@ -34,7 +35,7 @@ func parseFileToStatementsInPkg(filename string, pkg *core.Package) (*scope, []* } statements = parser.optimise(statements) parser.interpreter.optimiseExpressions(statements) - s, err := parser.interpreter.interpretAll(pkg, nil, nil, statements) + s, err := parser.interpreter.interpretAll(context.Background(), pkg, nil, nil, statements) return s, statements, err } diff --git a/src/parse/asp/logging_test.go b/src/parse/asp/logging_test.go index 3ac63c070e..ec9e2d2815 100644 --- a/src/parse/asp/logging_test.go +++ b/src/parse/asp/logging_test.go @@ -5,6 +5,7 @@ package asp import ( + "context" "testing" "github.com/stretchr/testify/assert" @@ -34,7 +35,7 @@ func parseFile2(filename string) (*scope, error) { if err != nil { panic(err) } - return parser.interpreter.interpretAll(pkg, nil, nil, statements) + return parser.interpreter.interpretAll(context.Background(), pkg, nil, nil, statements) } // assertRecords asserts equality of a series of logging records. diff --git a/src/parse/asp/parser.go b/src/parse/asp/parser.go index 4d52615ad8..f82c09fb1b 100644 --- a/src/parse/asp/parser.go +++ b/src/parse/asp/parser.go @@ -5,6 +5,7 @@ package asp import ( "bytes" + "context" "fmt" "io" iofs "io/fs" @@ -73,7 +74,7 @@ func (p *Parser) MustLoadBuiltins(filename string, contents []byte) { // ParseFile parses the contents of a single file in the BUILD language. // It returns true if the call was deferred at some point awaiting target to build, // along with any error encountered. -func (p *Parser) ParseFile(pkg *core.Package, label, dependent *core.BuildLabel, fs iofs.FS, filename string) error { +func (p *Parser) ParseFile(ctx context.Context, pkg *core.Package, label, dependent *core.BuildLabel, fs iofs.FS, filename string) error { p.limiter.Acquire() defer p.limiter.Release() @@ -81,7 +82,7 @@ func (p *Parser) ParseFile(pkg *core.Package, label, dependent *core.BuildLabel, if err != nil { return err } - _, err = p.interpreter.interpretAll(pkg, label, dependent, statements) + _, err = p.interpreter.interpretAll(ctx, pkg, label, dependent, statements) if err != nil { f, _ := p.open(fs, filename) p.annotate(err, f) @@ -109,7 +110,7 @@ func (p *Parser) RegisterPreloads(labels []core.BuildLabel) { // ParseReader parses the contents of the given ReadSeeker as a BUILD file. // The first return value is true if parsing succeeds - if the error is still non-nil // that indicates that interpretation failed. -func (p *Parser) ParseReader(pkg *core.Package, r io.ReadSeeker, forLabel, dependent *core.BuildLabel) (bool, error) { +func (p *Parser) ParseReader(ctx context.Context, pkg *core.Package, r io.ReadSeeker, forLabel, dependent *core.BuildLabel) (bool, error) { p.limiter.Acquire() defer p.limiter.Release() @@ -117,7 +118,7 @@ func (p *Parser) ParseReader(pkg *core.Package, r io.ReadSeeker, forLabel, depen if err != nil { return false, err } - _, err = p.interpreter.interpretAll(pkg, forLabel, dependent, stmts) + _, err = p.interpreter.interpretAll(ctx, pkg, forLabel, dependent, stmts) return true, err } diff --git a/src/parse/init.go b/src/parse/init.go index 8bef84d787..a98ce36f33 100644 --- a/src/parse/init.go +++ b/src/parse/init.go @@ -1,6 +1,7 @@ package parse import ( + "context" "fmt" "io" iofs "io/fs" @@ -67,12 +68,12 @@ func newAspParser(state *core.BuildState) *asp.Parser { return p } -func (p *aspParser) ParseFile(pkg *core.Package, forLabel, dependent *core.BuildLabel, fs iofs.FS, filename string) error { - return p.parser.ParseFile(pkg, forLabel, dependent, fs, filename) +func (p *aspParser) ParseFile(ctx context.Context, pkg *core.Package, forLabel, dependent *core.BuildLabel, fs iofs.FS, filename string) error { + return p.parser.ParseFile(ctx, pkg, forLabel, dependent, fs, filename) } -func (p *aspParser) ParseReader(pkg *core.Package, reader io.ReadSeeker, forLabel, dependent *core.BuildLabel) error { - _, err := p.parser.ParseReader(pkg, reader, forLabel, dependent) +func (p *aspParser) ParseReader(ctx context.Context, pkg *core.Package, reader io.ReadSeeker, forLabel, dependent *core.BuildLabel) error { + _, err := p.parser.ParseReader(ctx, pkg, reader, forLabel, dependent) return err } @@ -92,9 +93,10 @@ func (p *aspParser) RunPostBuildFunction(state *core.BuildState, target *core.Bu // runBuildFunction runs either the pre- or post-build function. func (p *aspParser) runBuildFunction(state *core.BuildState, target *core.BuildTarget, callbackType string, f func() error) error { state.LogBuildResult(target, core.PackageParsing, fmt.Sprintf("Running %s-build function for %s", callbackType, target.Label)) - if _, err := state.Parse(target.Label, target.Label); err != nil { - return err - } + // TODO(peterebden): What is this here for? Why do we need to parse again - by definition we should already have done so + // if _, err := state.Parse(target.Label, target.Label); err != nil { + // return err + // } if err := f(); err != nil { state.LogBuildError(target.Label, core.ParseFailed, err, "Failed %s-build function for %s", callbackType, target.Label) return err diff --git a/src/parse/parse_step.go b/src/parse/parse_step.go index 0a591e3058..190e97e46a 100644 --- a/src/parse/parse_step.go +++ b/src/parse/parse_step.go @@ -6,6 +6,7 @@ package parse import ( + "context" "errors" "fmt" iofs "io/fs" @@ -22,7 +23,7 @@ var log = logging.Log var ErrMissingBuildFile = errors.New("build file not found") // Parse parses the package corresponding to a single build label. The label can be :all to add all targets in a package. -func Parse(state *core.BuildState, label, dependent core.BuildLabel) (*core.Package, error) { +func Parse(ctx context.Context, state *core.BuildState, label, dependent core.BuildLabel) (*core.Package, error) { subrepo, err := checkSubrepo(state, label) if err != nil { return nil, err @@ -36,7 +37,7 @@ func Parse(state *core.BuildState, label, dependent core.BuildLabel) (*core.Pack if subrepo != nil && subrepo.Target != nil { // We have got the definition of the subrepo, but it depends on something, make sure that has been built. - if _, err := state.Build(subrepo.Target.Label, dependent); err != nil { + if _, err := state.Build(ctx, subrepo.Target.Label, dependent); err != nil { return nil, err } if err := subrepo.State.Initialise(subrepo); err != nil { @@ -49,7 +50,7 @@ func Parse(state *core.BuildState, label, dependent core.BuildLabel) (*core.Pack // TODO(peter): is this relevant still? return nil, nil } - pkg, err := parsePackage(state, label, dependent, subrepo) + pkg, err := parsePackage(ctx, state, label, dependent, subrepo) if err != nil { return nil, err } @@ -76,7 +77,7 @@ func checkSubrepo(state *core.BuildState, label core.BuildLabel) (*core.Subrepo, } // parsePackage parses a BUILD file and adds the package to the build graph -func parsePackage(state *core.BuildState, label, dependent core.BuildLabel, subrepo *core.Subrepo) (*core.Package, error) { +func parsePackage(ctx context.Context, state *core.BuildState, label, dependent core.BuildLabel, subrepo *core.Subrepo) (*core.Package, error) { packageName := label.PackageName pkg := core.NewPackage(packageName) pkg.Subrepo = subrepo @@ -91,14 +92,14 @@ func parsePackage(state *core.BuildState, label, dependent core.BuildLabel, subr if err != nil { return nil, fmt.Errorf("failed to generate internal package: %w", err) } - if err := state.Parser.ParseReader(pkg, strings.NewReader(pkgStr), &label, &dependent); err != nil { + if err := state.Parser.ParseReader(ctx, pkg, strings.NewReader(pkgStr), &label, &dependent); err != nil { return nil, fmt.Errorf("failed to parse internal package: %w", err) } } else { filename, dir := buildFileName(state, subrepo, fileSystem, label.PackageName) if filename != "" { pkg.Filename = filename - if err := state.Parser.ParseFile(pkg, &label, &dependent, fileSystem, filename); err != nil { + if err := state.Parser.ParseFile(ctx, pkg, &label, &dependent, fileSystem, filename); err != nil { return nil, err } } else { diff --git a/src/plz/plz.go b/src/plz/plz.go index 221fa72dd1..e716020528 100644 --- a/src/plz/plz.go +++ b/src/plz/plz.go @@ -79,8 +79,7 @@ func Run(targets, preTargets []core.BuildLabel, state *core.BuildState, progress g, ctx := r.group(ctx) r.tasks = g - // We don't have context as an argument to this, because they're not fully plumbed through (but probably should be) - state.Build = func(label, dependent core.BuildLabel) (*core.BuildTarget, error) { + state.Build = func(ctx context.Context, label, dependent core.BuildLabel) (*core.BuildTarget, error) { target, err := r.Build(ctx, label, dependent) if err != nil { return nil, err @@ -88,7 +87,7 @@ func Run(targets, preTargets []core.BuildLabel, state *core.BuildState, progress // Anything calling this will likely want this thing to end up being downloaded (it's mostly for subincludes) return target, state.EnsureDownloaded(target) } - state.Parse = func(label, dependent core.BuildLabel) (*core.Package, error) { + state.Parse = func(ctx context.Context, label, dependent core.BuildLabel) (*core.Package, error) { return r.Parse(ctx, label, dependent) } @@ -176,7 +175,7 @@ func (r *runner) parse(ctx context.Context, label, dependent core.BuildLabel, qu return nil, err } } - return parse.Parse(r.state, label, dependent) + return parse.Parse(ctx, r.state, label, dependent) }() if err != nil && !quiet { r.state.LogBuildError(label, core.ParseFailed, err, "Failed to parse package") From 59285794ae6e3286e3520af2afa364040f60f72b Mon Sep 17 00:00:00 2001 From: Peter Ebden Date: Wed, 12 Aug 2026 11:10:37 +0100 Subject: [PATCH 10/16] Bunch of preload fixes --- src/core/graph.go | 1 - src/core/state.go | 64 +++++++++++++++-- src/parse/asp/interpreter.go | 46 +++++++----- src/parse/asp/main/main.go | 3 +- src/parse/asp/objects.go | 4 +- src/parse/asp/parser.go | 11 ++- src/parse/asp/targets.go | 9 ++- src/parse/parse_step.go | 41 +---------- src/plz/plz.go | 134 ++++++++++++++++++++++++----------- 9 files changed, 198 insertions(+), 115 deletions(-) diff --git a/src/core/graph.go b/src/core/graph.go index a5be17cad2..667878da90 100644 --- a/src/core/graph.go +++ b/src/core/graph.go @@ -6,7 +6,6 @@ package core import ( "context" - "fmt" "maps" "slices" "sort" diff --git a/src/core/state.go b/src/core/state.go index 15fb88835a..bbd3fccee0 100644 --- a/src/core/state.go +++ b/src/core/state.go @@ -233,9 +233,65 @@ type BuildState struct { // initOnce is used to control loading the subrepo .plzconfig initOnce *sync.Once +} + +// A PreloadRun tracks a single attempt at resolving a repo's preloaded subincludes. +// It exists so that a marker left behind on a context outlives its usefulness harmlessly; see +// IsPreloading for why that matters. +type PreloadRun struct { + done atomic.Bool +} + +// NewPreloadRun returns a new PreloadRun. +func NewPreloadRun() *PreloadRun { + return &PreloadRun{} +} + +// Done marks this resolution as complete, after which it no longer suppresses anything. +func (run *PreloadRun) Done() { + run.done.Store(true) +} + +// A preloadChain records the preload resolutions in progress on this call chain. It's a linked list +// rather than a single entry because resolving one repo's preloads can require resolving another's +// (e.g. when a preload lives in a subrepo), so they nest. +type preloadChain struct { + run *PreloadRun + parent *preloadChain +} + +type preloadChainKey struct{} + +// WithPreloading returns a context noting that we are resolving the given repo's preloads. Anything +// parsed on behalf of that resolution must not try to apply those same preloads; they aren't built +// yet, and waiting for them would mean waiting on the thing that is waiting on us. +func WithPreloading(ctx context.Context, run *PreloadRun) context.Context { + parent, _ := ctx.Value(preloadChainKey{}).(*preloadChain) + return context.WithValue(ctx, preloadChainKey{}, &preloadChain{run: run, parent: parent}) +} - // preloadDownloadOnce is used - preloadDownloadOnce *sync.Once +// IsPreloading returns true if we are currently resolving preloaded subincludes. +// +// N.B. This is deliberately not "resolving the preloads of repo X". Resolving one repo's preloads +// routinely means parsing packages in another (a preload usually lives in a plugin subrepo), and those +// repos preload each other in turn; making this per-repo lets two resolutions wait on each other. +// Whilst we are resolving any preloads, nothing we parse on the way gets preloads applied - which also +// matches what a preload can actually rely on. Such a package picks them up the next time it is parsed +// by something that isn't preload resolution. +// +// It also only reports runs that haven't finished. Contexts get captured by things that outlive the +// chain they came from - most notably a pyFunc holds the scope it was defined in, so a function defined +// in a preloaded build_defs carries the resolver's context to every later call of it. Checking the run +// means such a leftover marker stops suppressing anything the moment its resolution completes, so it +// can't make an unrelated parse skip its preloads. +func IsPreloading(ctx context.Context) bool { + c, _ := ctx.Value(preloadChainKey{}).(*preloadChain) + for ; c != nil; c = c.parent { + if !c.run.done.Load() { + return true + } + } + return false } // Copy creates a copy of this state object @@ -244,7 +300,6 @@ func (state *BuildState) Copy() *BuildState { *ret = *state ret.initOnce = new(sync.Once) - ret.preloadDownloadOnce = new(sync.Once) return ret } @@ -976,8 +1031,7 @@ func NewBuildState(config *Configuration) *BuildState { cycleDetector: cycleDetector{graph: graph}, originalTargets: NewTargetSet(), }, - initOnce: new(sync.Once), - preloadDownloadOnce: new(sync.Once), + initOnce: new(sync.Once), } state.PathHasher = state.Hasher(config.Build.HashFunction) diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index be4449235b..8a290d30a9 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -33,10 +33,6 @@ type interpreter struct { stringMethods, dictMethods, configMethods map[string]*pyFunc regexCache *cmap.Map[string, *regexp.Regexp] - - // TODO(peter): rethink what we can do here, we don't really need build labels for this, - // we should be able to store a preloaded set of symbols or smthn that we can wang into scopes as needed. - preloads []core.BuildLabel } // newInterpreter creates and returns a new interpreter instance. @@ -132,7 +128,10 @@ func (i *interpreter) loadBuiltinStatements(s *scope, statements []*Statement, e } func (i *interpreter) preloadSubincludes(s *scope) error { - for _, label := range i.preloads { + // N.B. These come from the scope's state, not ours; a package in a subrepo preloads whatever that + // subrepo's config asks for, which isn't the same set as the host repo's. + // The driver has ensured these are built before letting us start on this package. + for _, label := range s.state.GetPreloadedSubincludes() { if err := i.preloadSubinclude(s, label); err != nil { return err } @@ -165,8 +164,7 @@ func (i *interpreter) preloadSubinclude(s *scope, label core.BuildLabel) (err er // interpretAll runs a series of statements in the scope of the given package. // The first return value is for testing only. func (i *interpreter) interpretAll(ctx context.Context, pkg *core.Package, forLabel, dependent *core.BuildLabel, statements []*Statement) (*scope, error) { - s := i.scope.NewPackagedScope(pkg, 1) - s.ctx = ctx + s := i.scope.NewPackagedScope(ctx, pkg, 1) s.config = i.getConfig(s.state).Copy() // Config needs a little separate tweaking. @@ -183,8 +181,14 @@ func (i *interpreter) interpretAll(ctx context.Context, pkg *core.Package, forLa defer pprof.SetGoroutineLabels(old) } - if err := i.preloadSubincludes(s); err != nil { - return nil, err + // If we're being parsed on behalf of resolving preloads then we mustn't apply them; they aren't built + // yet, and waiting for them would mean waiting on the thing that's waiting on us. + // N.B. We check the argument rather than s.ctx; scopes get captured by things that outlive the chain + // they were created on, so only the context we were handed describes what we're actually doing. + if !core.IsPreloading(ctx) { + if err := i.preloadSubincludes(s); err != nil { + return nil, err + } } s.Set("CONFIG", s.config) @@ -225,8 +229,14 @@ func (i *interpreter) Subinclude(pkgScope *scope, path string, label core.BuildL return nil, err } - s := i.scope.NewScope(path) + // N.B. This hangs off the interpreter's root scope so it sees the builtins rather than the + // caller's locals, but the work belongs to the caller's chain, so it takes their context. + s := i.scope.newScope(pkgScope.ctx, nil, path, 0) s.Preload = preload + // Whether this file gets preloads applied to it. Not if it is itself a preload, and not if we're + // resolving preloads at all - they aren't available yet, and reaching for one here would wait on + // a subinclude that the resolution we're part of is itself waiting to finish. + applyPreloads := !preload && !core.IsPreloading(pkgScope.ctx) s.state = pkgScope.state // Scope needs a local version of CONFIG @@ -234,7 +244,7 @@ func (i *interpreter) Subinclude(pkgScope *scope, path string, label core.BuildL s.Set("CONFIG", s.config) s.subincludeLabel = &label - if !preload { + if applyPreloads { if err := i.preloadSubincludes(s); err != nil { return nil, err } @@ -400,20 +410,22 @@ func (s *scope) subincludePackage() *core.Package { return nil } -// NewScope creates a new child scope of this one. +// NewScope creates a new child scope of this one, continuing on the same context. +// Use newScope directly if the new scope belongs to a different chain of work to this one; the context +// describes what we are currently doing, which isn't always the same as where a scope sits lexically. func (s *scope) NewScope(filename string) *scope { - return s.newScope(s.pkg, filename, 0) + return s.newScope(s.ctx, s.pkg, filename, 0) } // NewPackagedScope creates a new child scope of this one pointing to the given package. // hint is a size hint for the new set of locals. -func (s *scope) NewPackagedScope(pkg *core.Package, hint int) *scope { - return s.newScope(pkg, pkg.Filename, hint) +func (s *scope) NewPackagedScope(ctx context.Context, pkg *core.Package, hint int) *scope { + return s.newScope(ctx, pkg, pkg.Filename, hint) } -func (s *scope) newScope(pkg *core.Package, filename string, hint int) *scope { +func (s *scope) newScope(ctx context.Context, pkg *core.Package, filename string, hint int) *scope { s2 := &scope{ - ctx: s.ctx, + ctx: ctx, filename: filename, interpreter: s.interpreter, state: s.state, diff --git a/src/parse/asp/main/main.go b/src/parse/asp/main/main.go index 336dc6faab..b7b0d91001 100644 --- a/src/parse/asp/main/main.go +++ b/src/parse/asp/main/main.go @@ -4,6 +4,7 @@ package main import ( + "context" "fmt" iofs "io/fs" "os" @@ -63,7 +64,7 @@ func parseFile(pkg *core.Package, p *asp.Parser, filename string) error { } return err } - return p.ParseFile(pkg, nil, nil, nil, filename) + return p.ParseFile(context.Background(), pkg, nil, nil, nil, filename) } type assignment struct { diff --git a/src/parse/asp/objects.go b/src/parse/asp/objects.go index b7d69612ce..8947322f34 100644 --- a/src/parse/asp/objects.go +++ b/src/parse/asp/objects.go @@ -698,7 +698,9 @@ func (f *pyFunc) Call(s *scope, c *Call) pyObject { } return f.callNative(s, c) } - s2 := f.scope.newScope(s.pkg, f.scope.filename, len(f.args)+1) + // N.B. The scope hangs off the one the function was defined in, so it sees that file's globals, but + // anything about what we are currently doing comes from the calling scope - including its context. + s2 := f.scope.newScope(s.ctx, s.pkg, f.scope.filename, len(f.args)+1) s2.config = s.config s2.Set("CONFIG", s.config) // This needs to be copied across too :( s2.Callback = s.Callback diff --git a/src/parse/asp/parser.go b/src/parse/asp/parser.go index f82c09fb1b..e03d35ae4b 100644 --- a/src/parse/asp/parser.go +++ b/src/parse/asp/parser.go @@ -91,22 +91,19 @@ func (p *Parser) ParseFile(ctx context.Context, pkg *core.Package, label, depend } // PreloadSubinclude pre-registers a preload, forcing us to build any transitive preloads before we move on -func (p *Parser) PreloadSubinclude(label core.BuildLabel) error { +func (p *Parser) PreloadSubinclude(ctx context.Context, label core.BuildLabel) error { p.limiter.Acquire() defer p.limiter.Release() // This is a throw away scope. We're just doing this to avoid race conditions setting this on the main scope. - s := p.interpreter.scope.newScope(nil, "", 0) + // It takes the caller's context, not the interpreter's; this is the chain that is resolving the preloads, + // and anything it goes on to parse must know that. + s := p.interpreter.scope.newScope(ctx, nil, "", 0) s.config = p.interpreter.scope.config.Copy() s.Set("CONFIG", s.config) return p.interpreter.preloadSubinclude(s, label) } -// RegisterPreloads registers the set of preloaded subincludes. -func (p *Parser) RegisterPreloads(labels []core.BuildLabel) { - p.interpreter.preloads = labels -} - // ParseReader parses the contents of the given ReadSeeker as a BUILD file. // The first return value is true if parsing succeeds - if the error is still non-nil // that indicates that interpretation failed. diff --git a/src/parse/asp/targets.go b/src/parse/asp/targets.go index e92d1588e3..a36e60751b 100644 --- a/src/parse/asp/targets.go +++ b/src/parse/asp/targets.go @@ -1,6 +1,7 @@ package asp import ( + "context" "fmt" "os" "path/filepath" @@ -623,7 +624,9 @@ type preBuildFunction struct { } func (f *preBuildFunction) Call(target *core.BuildTarget) error { - s := f.f.scope.NewPackagedScope(f.f.scope.state.Graph.PackageOrDie(target.Label), 1) + // Callbacks run during the build, long after the parse that defined them; there is no parse in + // flight to inherit a context from. + s := f.f.scope.NewPackagedScope(context.Background(), f.f.scope.state.Graph.PackageOrDie(target.Label), 1) s.config = f.s.config s.Set("CONFIG", f.s.config) s.Callback = true @@ -643,7 +646,9 @@ type postBuildFunction struct { } func (f *postBuildFunction) Call(target *core.BuildTarget, output string) error { - s := f.f.scope.NewPackagedScope(f.f.scope.state.Graph.PackageOrDie(target.Label), 2) + // Callbacks run during the build, long after the parse that defined them; there is no parse in + // flight to inherit a context from. + s := f.f.scope.NewPackagedScope(context.Background(), f.f.scope.state.Graph.PackageOrDie(target.Label), 2) s.config = f.s.config s.Set("CONFIG", f.s.config) s.Callback = true diff --git a/src/parse/parse_step.go b/src/parse/parse_step.go index 190e97e46a..d27a526e07 100644 --- a/src/parse/parse_step.go +++ b/src/parse/parse_step.go @@ -23,28 +23,11 @@ var log = logging.Log var ErrMissingBuildFile = errors.New("build file not found") // Parse parses the package corresponding to a single build label. The label can be :all to add all targets in a package. -func Parse(ctx context.Context, state *core.BuildState, label, dependent core.BuildLabel) (*core.Package, error) { - subrepo, err := checkSubrepo(state, label) - if err != nil { - return nil, err - } - - if subrepo != nil { - state = subrepo.State - } - +// The state and subrepo must be the ones the label belongs to; the caller resolves those (and makes sure +// the subrepo's preloads are ready) before claiming the package, since neither can be done from in here. +func Parse(ctx context.Context, state *core.BuildState, label, dependent core.BuildLabel, subrepo *core.Subrepo) (*core.Package, error) { state.LogParseResult(label, core.PackageParsing, "Parsing...") - if subrepo != nil && subrepo.Target != nil { - // We have got the definition of the subrepo, but it depends on something, make sure that has been built. - if _, err := state.Build(ctx, subrepo.Target.Label, dependent); err != nil { - return nil, err - } - if err := subrepo.State.Initialise(subrepo); err != nil { - return nil, err - } - } - // Subrepo & nothing else means we just want to ensure that subrepo is present. if label.Subrepo != "" && label.PackageName == "" && label.Name == "" { // TODO(peter): is this relevant still? @@ -58,24 +41,6 @@ func Parse(ctx context.Context, state *core.BuildState, label, dependent core.Bu return pkg, nil } -// checkSubrepo checks if the label we're parsing is within a subrepo, returning that subrepo, if present in the label. -// -// The subrepo target can be inferred from the subrepo name using convention i.e. ///foo/bar//:baz has a subrepo label -// //foo:bar. checkSubrepo parses package foo, expecting a call to `subrepo()` that registers a subrepo named foo/bar, -// so it can return it. -func checkSubrepo(state *core.BuildState, label core.BuildLabel) (*core.Subrepo, error) { - if label.Subrepo == "" { - return nil, nil - } - - // Check if we already have it (we expect the higher-level driver code to have arranged for it to be parsed - // before we get here) - if subrepo := state.Graph.Subrepo(label.Subrepo); subrepo != nil { - return subrepo, nil - } - return nil, fmt.Errorf("Subrepo %s is not defined", label.Subrepo) -} - // parsePackage parses a BUILD file and adds the package to the build graph func parsePackage(ctx context.Context, state *core.BuildState, label, dependent core.BuildLabel, subrepo *core.Subrepo) (*core.Package, error) { packageName := label.PackageName diff --git a/src/plz/plz.go b/src/plz/plz.go index e716020528..e4175651df 100644 --- a/src/plz/plz.go +++ b/src/plz/plz.go @@ -64,6 +64,7 @@ func Run(targets, preTargets []core.BuildLabel, state *core.BuildState, progress r := runner{ state: state, + parser: parser, arch: arch, progress: progress, buildOnce: cmap.NewErrMap[core.BuildLabel, *core.BuildTarget](cmap.DefaultShardCount, func(l core.BuildLabel) uint64 { @@ -72,6 +73,7 @@ func Run(targets, preTargets []core.BuildLabel, state *core.BuildState, progress parseOnce: cmap.New[core.BuildLabel, struct{}](cmap.DefaultShardCount, func(l core.BuildLabel) uint64 { return cmap.XXHashes(l.Subrepo, l.PackageName, l.Name) }), + preloadOnce: cmap.NewErrMap[string, struct{}](cmap.SmallShardCount, cmap.XXHash, nil), localLimiter: make(limiter, state.Config.Please.NumThreads), remoteLimiter: make(limiter, state.Config.NumRemoteExecutors()), anyRemote: state.Config.NumRemoteExecutors() > 0, @@ -91,11 +93,6 @@ func Run(targets, preTargets []core.BuildLabel, state *core.BuildState, progress return r.Parse(ctx, label, dependent) } - // Register the preloaded targets with the parser - if err := r.RegisterPreloads(ctx, state, parser); err != nil { - return err - } - if state.Config.Bazel.Compatibility && fs.FileExists("WORKSPACE") { // We have to parse the WORKSPACE file before anything else to understand subrepos. // This is a bit crap really since it inhibits parallelism for the first step. @@ -140,10 +137,12 @@ func RunHost(targets []core.BuildLabel, state *core.BuildState) { type runner struct { tasks *errgroup.Group state *core.BuildState + parser *asp.Parser arch cli.Arch progress *Progress buildOnce *cmap.ErrMap[core.BuildLabel, *core.BuildTarget] parseOnce *cmap.Map[core.BuildLabel, struct{}] + preloadOnce *cmap.ErrMap[string, struct{}] localLimiter limiter remoteLimiter limiter anyRemote bool @@ -164,19 +163,25 @@ func (r *runner) tryParse(ctx context.Context, label, dependent core.BuildLabel) } func (r *runner) parse(ctx context.Context, label, dependent core.BuildLabel, quiet bool) (*core.Package, error) { + // Work out which repo this package belongs to, and make sure that repo's preloaded subincludes are + // resolved, before we claim the package below. + // Both of these can need to parse other packages - and preload resolution routinely parses packages in + // the very repo we're about to claim. Anything we do while holding the claim can't wait on this package, + // so it all has to happen out here. + state, subrepo, err := r.repoFor(ctx, label, dependent) + if err != nil { + if !quiet { + r.state.LogBuildError(label, core.ParseFailed, err, "Failed to parse package") + } + return nil, err + } + if err := r.ensurePreloads(ctx, state); err != nil { + return nil, err + } return r.state.Graph.GetOrSetPackage(ctx, label, func() (*core.Package, error) { r.progress.numParsing.Add(1) defer r.progress.numParsing.Add(-1) - pkg, err := func() (*core.Package, error) { - // If the target is in a subrepo that we don't know about yet, we must make sure that is defined first. - // If we already have it there's nothing to do here; it's been registered by whatever parse defined it. - if label.Subrepo != "" && r.state.Graph.Subrepo(label.Subrepo) == nil { - if err := r.ensureSubrepo(ctx, label, dependent); err != nil { - return nil, err - } - } - return parse.Parse(ctx, r.state, label, dependent) - }() + pkg, err := parse.Parse(ctx, state, label, dependent, subrepo) if err != nil && !quiet { r.state.LogBuildError(label, core.ParseFailed, err, "Failed to parse package") } @@ -184,6 +189,77 @@ func (r *runner) parse(ctx context.Context, label, dependent core.BuildLabel, qu }) } +// repoFor returns the state and subrepo that the given label should be parsed against, defining the +// subrepo if we don't know about it yet. The returned subrepo is nil for the host repo. +func (r *runner) repoFor(ctx context.Context, label, dependent core.BuildLabel) (*core.BuildState, *core.Subrepo, error) { + if label.Subrepo == "" { + return r.state, nil, nil + } + // If we already have it there's nothing to do here; it's been registered by whatever parse defined it. + if r.state.Graph.Subrepo(label.Subrepo) == nil { + if err := r.ensureSubrepo(ctx, label, dependent); err != nil { + return nil, nil, err + } + } + subrepo := r.state.Graph.Subrepo(label.Subrepo) + if subrepo == nil { + return nil, nil, fmt.Errorf("Subrepo %s is not defined", label.Subrepo) + } + if subrepo.Target != nil { + // We have the definition of the subrepo, but it depends on something; that has to be built before + // we can read its config off disk. + if _, err := r.Build(ctx, subrepo.Target.Label, dependent); err != nil { + return nil, nil, err + } + } + // This is what reads the subrepo's .plzconfig, and hence what tells us its preloads. + if err := subrepo.State.Initialise(subrepo); err != nil { + return nil, nil, err + } + return subrepo.State, subrepo, nil +} + +// ensurePreloads makes sure the preloaded subincludes of the given repo have been built and registered +// with the parser, which must happen before we parse any package in it. +func (r *runner) ensurePreloads(ctx context.Context, state *core.BuildState) error { + // If we're already resolving preloads then this parse is part of that resolution; it has to go ahead + // without them rather than wait for work that is waiting on us. + if core.IsPreloading(ctx) { + return nil + } + _, err := r.preloadOnce.GetOrSetCtx(ctx, state.CurrentSubrepo, func() (struct{}, error) { + return struct{}{}, r.registerPreloads(ctx, state) + }) + return err +} + +// registerPreloads builds each of a repo's preloaded subinclude targets and registers it with the parser. +// We have to actually register them, otherwise this would return before we build any transitive subincludes. +func (r *runner) registerPreloads(ctx context.Context, state *core.BuildState) error { + preloads := state.GetPreloadedSubincludes() + if len(preloads) == 0 { + return nil + } + run := core.NewPreloadRun() + defer run.Done() + g, gctx := r.group(core.WithPreloading(ctx, run)) + for _, inc := range preloads { + if inc.IsPseudoTarget() { + return fmt.Errorf("Can't preload pseudotarget %v", inc) + } + // Queue them up asynchronously to feed the queues as quickly as possible + g.Go(func() error { + if _, err := r.Build(gctx, inc, core.OriginalTarget); err != nil { + return err + } + return r.parser.PreloadSubinclude(gctx, inc) + }) + } + // We must wait for all the subinclude targets to be built otherwise updating the locals might race with + // parsing a package + return g.Wait() +} + // ensureSubrepo makes sure that the subrepo the given label is in has been defined. // // A name like `linux_amd64` is ambiguous: it could be a subrepo defined by a target somewhere, or one @@ -620,34 +696,6 @@ func (r *runner) queueTask(ctx context.Context, target core.BuildLabel, needTest }) } -// RegisterPreloads waits for all preloaded subinclude targets to be built, downloads them, and then registers them with -// the interpreter. We have to actually register them otherwise this will return before we build any -// transitive subincludes. -func (r *runner) RegisterPreloads(ctx context.Context, state *core.BuildState, parser *asp.Parser) error { - g, ctx := r.group(ctx) - preloads := state.GetPreloadedSubincludes() - for _, inc := range preloads { - if inc.IsPseudoTarget() { - return fmt.Errorf("Can't preload pseudotarget %v", inc) - } - - // Queue them up asynchronously to feed the queues as quickly as possible - g.Go(func() error { - if _, err := r.Build(ctx, inc, core.OriginalTarget); err != nil { - return err - } - return parser.PreloadSubinclude(inc) - }) - } - // We must wait for all the subinclude targets to be built otherwise updating the locals might race with parsing - // a package - if err := g.Wait(); err != nil { - return err - } - parser.RegisterPreloads(preloads) - return nil -} - // FindAllBuildFiles finds all BUILD files under a particular path. // Used to implement rules with ... where we need to know all possible packages // under that location. From 7c9126f05bb0eea40af3fbd83bf4c7130f93fdf3 Mon Sep 17 00:00:00 2001 From: Peter Ebden Date: Wed, 12 Aug 2026 12:09:45 +0100 Subject: [PATCH 11/16] Linter --- src/plz/plz.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/plz/plz.go b/src/plz/plz.go index e4175651df..4f9dc19771 100644 --- a/src/plz/plz.go +++ b/src/plz/plz.go @@ -89,9 +89,7 @@ func Run(targets, preTargets []core.BuildLabel, state *core.BuildState, progress // Anything calling this will likely want this thing to end up being downloaded (it's mostly for subincludes) return target, state.EnsureDownloaded(target) } - state.Parse = func(ctx context.Context, label, dependent core.BuildLabel) (*core.Package, error) { - return r.Parse(ctx, label, dependent) - } + state.Parse = r.Parse if state.Config.Bazel.Compatibility && fs.FileExists("WORKSPACE") { // We have to parse the WORKSPACE file before anything else to understand subrepos. From 809211893586b88719cb9aefc462690cdd660189 Mon Sep 17 00:00:00 2001 From: Peter Ebden Date: Wed, 12 Aug 2026 13:08:58 +0100 Subject: [PATCH 12/16] Don't log the full error, we get it all anyway --- src/please.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/please.go b/src/please.go index 441ad1c4d3..0bbdfe6514 100644 --- a/src/please.go +++ b/src/please.go @@ -1227,14 +1227,12 @@ func runPlease(state *core.BuildState, targets []core.BuildLabel) { state.Results() // important this is called now, don't ask... var progress plz.Progress var wg sync.WaitGroup - wg.Add(1) - go func() { + wg.Go(func() { output.MonitorState(state, &progress, !pretty, detailedTests, streamTests, shell, shellRun, string(opts.OutputFlags.TraceFile)) - wg.Done() - }() + }) if err := plz.Run(targets, opts.BuildFlags.PreTargets, state, &progress, state.TargetArch); err != nil { - // TODO(peter): we might want to do something else with this - log.Error("%s", err) + // Failures to build are logged through the central error reporting mechanism and will be printed by MonitorState above. + log.Debug("Build error: %s", err) } wg.Wait() } From 187f1d6078e487d870d6e20735c1aaeccd08734f Mon Sep 17 00:00:00 2001 From: Peter Ebden Date: Wed, 12 Aug 2026 13:44:09 +0100 Subject: [PATCH 13/16] Ensure we wait for parses to finish before running post-build functions --- src/parse/init.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/parse/init.go b/src/parse/init.go index a98ce36f33..acf7588e47 100644 --- a/src/parse/init.go +++ b/src/parse/init.go @@ -93,10 +93,14 @@ func (p *aspParser) RunPostBuildFunction(state *core.BuildState, target *core.Bu // runBuildFunction runs either the pre- or post-build function. func (p *aspParser) runBuildFunction(state *core.BuildState, target *core.BuildTarget, callbackType string, f func() error) error { state.LogBuildResult(target, core.PackageParsing, fmt.Sprintf("Running %s-build function for %s", callbackType, target.Label)) - // TODO(peterebden): What is this here for? Why do we need to parse again - by definition we should already have done so - // if _, err := state.Parse(target.Label, target.Label); err != nil { - // return err - // } + // This doesn't re-parse anything; it waits for the parse of this target's package to complete if + // one is still in flight. Targets are added to the graph as each rule is created, so a target can + // be picked up and built before the file that defines it has been fully interpreted - but the + // callback both reads the package out of the graph and mutates it, so it can't run until then. + // There's no parse in flight for us to inherit a context from, hence Background. + if _, err := state.Parse(context.Background(), target.Label, target.Label); err != nil { + return err + } if err := f(); err != nil { state.LogBuildError(target.Label, core.ParseFailed, err, "Failed %s-build function for %s", callbackType, target.Label) return err From 3f961cb34a2481d7847ea0570c393f0d091b8a40 Mon Sep 17 00:00:00 2001 From: Peter Ebden Date: Sun, 16 Aug 2026 10:03:44 +0100 Subject: [PATCH 14/16] Some further refactors / cleanups / bugfixes --- src/core/build_target.go | 43 ++++--- src/core/state.go | 143 +++++------------------ src/gc/gc.go | 2 +- src/hashes/rewrite_hashes.go | 2 +- src/help/help.go | 2 +- src/help/rules.go | 4 +- src/output/shell_output.go | 46 +------- src/parse/asp/builtins.go | 4 +- src/parse/asp/interpreter.go | 5 +- src/parse/asp/interpreter_test.go | 6 +- src/parse/asp/logging_test.go | 2 +- src/parse/asp/main/main.go | 2 +- src/parse/asp/parser.go | 15 ++- src/parse/init.go | 37 +++--- src/parse/parse_step.go | 6 - src/please.go | 7 +- src/plz/BUILD | 5 +- src/{core => plz}/cycle_detector.go | 83 +++++++++---- src/{core => plz}/cycle_detector_test.go | 23 ++-- src/plz/plz.go | 85 +++++++++----- src/query/deps.go | 5 +- src/query/print.go | 8 +- src/query/somepath.go | 3 +- tools/build_langserver/lsp/lsp.go | 6 +- 24 files changed, 240 insertions(+), 304 deletions(-) rename src/{core => plz}/cycle_detector.go (53%) rename src/{core => plz}/cycle_detector_test.go (68%) diff --git a/src/core/build_target.go b/src/core/build_target.go index 7d5e1b5f7b..966885ae76 100644 --- a/src/core/build_target.go +++ b/src/core/build_target.go @@ -340,17 +340,16 @@ type BuildTargetState uint8 // The available states for a target. const ( - Inactive BuildTargetState = iota // Target isn't used in current build - Building // Target is currently being built - Stopped // We stopped building the target because we'd gone as far as needed. - Built // Target has been successfully built - Cached // Target has been retrieved from the cache - Unchanged // Target has been built but hasn't changed since last build - Reused // Outputs of previous build have been reused. - BuiltRemotely // Target has been built but outputs are not necessarily local. - ReusedRemotely // Outputs of previous remote action have been reused. - DependencyFailed // At least one dependency of this target has failed. - Failed // Target failed for some reason + Inactive BuildTargetState = iota // Target isn't used in current build + Building // Target is currently being built + Stopped // We stopped building the target because we'd gone as far as needed. + Built // Target has been successfully built + Cached // Target has been retrieved from the cache + Unchanged // Target has been built but hasn't changed since last build + Reused // Outputs of previous build have been reused. + BuiltRemotely // Target has been built but outputs are not necessarily local. + ReusedRemotely // Outputs of previous remote action have been reused. + Failed // Target failed for some reason ) // String implements the fmt.Stringer interface. @@ -370,8 +369,6 @@ func (s BuildTargetState) String() string { return "Unchanged" case Reused: return "Reused" - case DependencyFailed: - return "Dependency Failed" case Failed: return "Failed" case BuiltRemotely: @@ -384,7 +381,7 @@ func (s BuildTargetState) String() string { } func (s BuildTargetState) IsBuilt() bool { - return Built <= s && s < DependencyFailed + return Built <= s && s < Failed } // NewBuildTarget constructs & returns a new BuildTarget. @@ -1019,15 +1016,15 @@ func (target *BuildTarget) CanSee(state *BuildState, dep *BuildTarget) bool { // Returns an error if not, or nil if all's well. func (target *BuildTarget) CheckDependencyVisibility(state *BuildState) error { for _, d := range target.dependencies { - if dep := state.Graph.Target(d.Label); dep != nil { - if !target.CanSee(state, dep) { - return fmt.Errorf("Target %s isn't visible to %s", dep.Label, target.Label) - } else if dep.TestOnly && !target.IsTest() && !target.TestOnly { - if target.Label.isExperimental(state) { - log.Info("Test-only restrictions suppressed for %s since %s is in the experimental tree", dep.Label, target.Label) - } else { - return fmt.Errorf("Target %s can't depend on %s, it's marked test_only", target.Label, dep.Label) - } + if dep := state.Graph.Target(d.Label); dep == nil { + return fmt.Errorf("target %s (dependency of %s) not defined in build graph", d.Label, target) + } else if !target.CanSee(state, dep) { + return fmt.Errorf("Target %s isn't visible to %s", dep.Label, target.Label) + } else if dep.TestOnly && !target.IsTest() && !target.TestOnly { + if target.Label.isExperimental(state) { + log.Info("Test-only restrictions suppressed for %s since %s is in the experimental tree", dep.Label, target.Label) + } else { + return fmt.Errorf("Target %s can't depend on %s, it's marked test_only", target.Label, dep.Label) } } } diff --git a/src/core/state.go b/src/core/state.go index bbd3fccee0..3e44d7c46a 100644 --- a/src/core/state.go +++ b/src/core/state.go @@ -1,7 +1,6 @@ package core import ( - "bytes" "context" "crypto/sha1" "crypto/sha256" @@ -14,7 +13,6 @@ import ( iofs "io/fs" "iter" "path/filepath" - "runtime/pprof" "sort" "strings" "sync" @@ -32,9 +30,6 @@ import ( // startTime is as close as we can conveniently get to process start time. var startTime = time.Now() -// cycleCheckDuration is the length of time we allow inactivity for before we trigger cycle detection. -const cycleCheckDuration = 5 * time.Second - // resultsChanSize is the buffer size of the channel we report build results on. const resultsChanSize = 1000 @@ -221,16 +216,6 @@ type BuildState struct { // NeedDebugDeps is true if we're doing a `plz debug` and we need to build the debug tools and data NeedDebugDeps bool - // Build is a callback to build a single target. It's set from outside here. - // TODO(peter): can we find a way of moving these off this struct? it feels weird here - // The second label is the dependent, i.e. whatever is asking for this to be built. - Build func(ctx context.Context, label, dependent BuildLabel) (*BuildTarget, error) - // Parse is a callback to parse a single package. It's also set from outside. - // The second label is the dependent, i.e. whatever is asking for this to be parsed. - Parse func(ctx context.Context, label, dependent BuildLabel) (*Package, error) - // Cancel is a cancel function called when the state detects a cycle. - Cancel func() - // initOnce is used to control loading the subrepo .plzconfig initOnce *sync.Once } @@ -325,8 +310,7 @@ func (state *BuildState) Initialise(subrepo *Subrepo) (err error) { // A stateProgress records various points of progress for a State. // This is split out from above so we can share it between multiple instances. type stateProgress struct { - mutex sync.Mutex - resultOnce sync.Once + mutex sync.Mutex // The set of known states allStates []*BuildState // Targets that we were originally requested to build @@ -337,12 +321,8 @@ type stateProgress struct { buildFailed atomic.Bool // True if >= 1 target has failed test cases testFailed atomic.Bool - // Stream of results from the build - results chan *BuildResult - // Internal result stream, used to intermediate them for the cycle checker. - internalResults chan *BuildResult - // The cycle checker itself. - cycleDetector cycleDetector + // Streams of results from the build + results []chan *BuildResult } // SystemStats stores information about the system. @@ -373,18 +353,12 @@ type lockedStats struct { // CloseResults closes the result channels. func (state *BuildState) CloseResults() { - state.progress.cycleDetector.Stop() state.progress.mutex.Lock() defer state.progress.mutex.Unlock() - // N.B. We create the channel if nobody has asked for it yet, rather than doing nothing; otherwise - // anyone calling Results() after this would get a fresh channel that is never closed and would - // wait on it forever. - if state.progress.results == nil { - state.progress.results = make(chan *BuildResult, resultsChanSize) + for _, ch := range state.progress.results { + close(ch) } - state.progress.resultOnce.Do(func() { - close(state.progress.results) - }) + state.progress.results = nil } // AddOriginalTarget adds an original target to this state @@ -433,13 +407,23 @@ func (state *BuildState) SetIncludeAndExclude(include, exclude []string) { } } -// ShouldInclude returns true if the given target is included by the include/exclude flags. -func (state *BuildState) ShouldInclude(target *BuildTarget) bool { +// IsExcluded returns true if the given label has been excluded by a build label passed to --exclude. +// Note that this is only about labels; --exclude can also take rule labels, which only make sense +// once we have a target to look at and hence are handled by ShouldInclude. +func (state *BuildState) IsExcluded(label BuildLabel) bool { for _, e := range state.ExcludeTargets { - if e.Includes(target.Label) { - return false + if e.Includes(label) { + return true } } + return false +} + +// ShouldInclude returns true if the given target is included by the include/exclude flags. +func (state *BuildState) ShouldInclude(target *BuildTarget) bool { + if state.IsExcluded(target.Label) { + return false + } return target.ShouldInclude(state.Include, state.Exclude) } @@ -479,7 +463,7 @@ func (state *BuildState) LogParseResult(label BuildLabel, status BuildResultStat func (state *BuildState) LogBuildResult(target *BuildTarget, status BuildResultStatus, description string) { state.logResult(&BuildResult{ Label: target.Label, - target: target, + Target: target, Status: status, Err: nil, Description: description, @@ -494,7 +478,7 @@ func (state *BuildState) LogTestRunning(target *BuildTarget, run int, status Bui } state.logResult(&BuildResult{ Label: target.Label, - target: target, + Target: target, Run: run, Status: status, Description: message, @@ -505,7 +489,7 @@ func (state *BuildState) LogTestRunning(target *BuildTarget, run int, status Bui func (state *BuildState) LogTestResult(target *BuildTarget, run int, status BuildResultStatus, results *TestSuite, coverage *TestCoverage, err error, format string, args ...interface{}) { state.logResult(&BuildResult{ Label: target.Label, - target: target, + Target: target, Run: run, Status: status, Err: err, @@ -533,7 +517,9 @@ func (state *BuildState) logResult(result *BuildResult) { return } result.Time = time.Now() - state.progress.internalResults <- result + for _, ch := range state.progress.results { + ch <- result + } if result.Status.IsFailure() { state.progress.failed.Store(true) switch result.Status { @@ -545,76 +531,19 @@ func (state *BuildState) logResult(result *BuildResult) { } } -// forwardResults runs indefinitely, forwarding results from the internal -// channel to the external one. On the way it checks if we need to do -// cycle detection. -func (state *BuildState) forwardResults() { - defer func() { - if r := recover(); r != nil { - // Ensure we don't get a "send on closed channel" when the - // outward results channel is closed. - log.Debug("%s", r) - } - }() - activeTargets := map[*BuildTarget]struct{}{} - // Persist this one timer throughout so we don't generate bazillions of them. - t := time.NewTimer(cycleCheckDuration) - t.Stop() - var result *BuildResult - for { - if len(activeTargets) == 0 { - t.Reset(cycleCheckDuration) - select { - case result = <-state.progress.internalResults: - // This has to be properly managed to prevent hangs. - if !t.Stop() { - <-t.C - } - case <-t.C: - go state.checkForCycles() - go dumpGoroutineInfo() - // Still need to get a result! - result = <-state.progress.internalResults - } - } else { - result = <-state.progress.internalResults - } - if target := result.target; target != nil { - if result.Status.IsActive() { - activeTargets[target] = struct{}{} - } else { - delete(activeTargets, target) - } - } - state.progress.mutex.Lock() - if state.progress.results != nil { - state.progress.results <- result - } - state.progress.mutex.Unlock() - } -} - -// checkForCycles is run to detect a cycle in the graph. It converts any returned error into an async error. -func (state *BuildState) checkForCycles() { - if err := state.progress.cycleDetector.Check(); err != nil { - state.LogBuildError(err.Cycle[0].Label, TargetBuildFailed, err, "") - state.Cancel() - } -} - // Failures returns anything that has failed about the current build. func (state *BuildState) Failures() (anything, build, test bool) { return state.progress.failed.Load(), state.progress.buildFailed.Load(), state.progress.testFailed.Load() } // Results returns a channel on which the caller can listen for results. +// After calling this, the caller is honour-bound to consume the channel, or eventually things will block. func (state *BuildState) Results() <-chan *BuildResult { state.progress.mutex.Lock() defer state.progress.mutex.Unlock() - if state.progress.results == nil { - state.progress.results = make(chan *BuildResult, resultsChanSize) - } - return state.progress.results + ch := make(chan *BuildResult, resultsChanSize) + state.progress.results = append(state.progress.results, ch) + return ch } // ExpandOriginalLabels expands any pseudo-labels (ie. :all, ... has already been resolved to a bunch :all targets) @@ -1027,8 +956,6 @@ func NewBuildState(config *Configuration) *BuildState { Arch: cli.HostArch(), stats: &lockedStats{}, progress: &stateProgress{ - internalResults: make(chan *BuildResult, 1000), - cycleDetector: cycleDetector{graph: graph}, originalTargets: NewTargetSet(), }, initOnce: new(sync.Once), @@ -1040,7 +967,6 @@ func NewBuildState(config *Configuration) *BuildState { for _, exp := range config.Parse.ExperimentalDir { state.experimentalLabels = append(state.experimentalLabels, BuildLabel{PackageName: exp, Name: "..."}) } - go state.forwardResults() return state } @@ -1058,7 +984,7 @@ type BuildResult struct { // Target which has just changed Label BuildLabel // Target which has changed. Nil if it's a parse action. - target *BuildTarget + Target *BuildTarget // Test run index. 0 if not a test. Run int // Its current status @@ -1118,10 +1044,3 @@ func (s BuildResultStatus) IsFailure() bool { func (s BuildResultStatus) IsActive() bool { return s == PackageParsing || s == TargetBuilding || s == TargetTesting } - -// dumpGoroutineInfo logs out the goroutine stacks when we believe we might have hung. -func dumpGoroutineInfo() { - var buf bytes.Buffer - pprof.Lookup("goroutine").WriteTo(&buf, 1) - log.Debug("Current stacks: %s", buf.String()) -} diff --git a/src/gc/gc.go b/src/gc/gc.go index 66e7859a6c..896f9040fa 100644 --- a/src/gc/gc.go +++ b/src/gc/gc.go @@ -218,7 +218,7 @@ func publicDependencies(graph *core.BuildGraph, target *core.BuildTarget) []*cor // RewriteFile rewrites a BUILD file to exclude a set of targets. func RewriteFile(state *core.BuildState, filename string, targets []string) error { - p := asp.NewParser(state) + p := asp.NewParser(state, nil) stmts, err := p.ParseFileOnly(filename) if err != nil { return err diff --git a/src/hashes/rewrite_hashes.go b/src/hashes/rewrite_hashes.go index 7569c40431..3f1997cafc 100644 --- a/src/hashes/rewrite_hashes.go +++ b/src/hashes/rewrite_hashes.go @@ -50,7 +50,7 @@ func RewriteHashes(state *core.BuildState, labels []core.BuildLabel) { // rewriteHashes rewrites hashes in a single file. func rewriteHashes(state *core.BuildState, filename, platform string, hashes map[string]string) error { log.Notice("Rewriting hashes in %s...", filename) - p := asp.NewParser(state) + p := asp.NewParser(state, nil) stmts, err := p.ParseFileOnly(filename) if err != nil { return err diff --git a/src/help/help.go b/src/help/help.go index b105ccd5b9..050ffd966e 100644 --- a/src/help/help.go +++ b/src/help/help.go @@ -251,7 +251,7 @@ func getPluginBuildDefs(subrepo *core.Subrepo) map[string]*asp.Statement { dirs = append(dirs, "build_defs") } - p := asp.NewParser(subrepo.State) + p := asp.NewParser(subrepo.State, nil) ret := make(map[string]*asp.Statement) for _, dir := range dirs { fs := subrepo.FS() diff --git a/src/help/rules.go b/src/help/rules.go index 2054ac61cd..8e241d6e8c 100644 --- a/src/help/rules.go +++ b/src/help/rules.go @@ -54,7 +54,7 @@ func newState() *core.BuildState { // by the config (e.g. PreloadBuildDefs or BuildDefsDir). // You are guaranteed that every statement in the returned map is a FuncDef. func AllBuiltinFunctions(state *core.BuildState) map[string]*asp.Statement { - p := asp.NewParser(state) + p := asp.NewParser(state, nil) m := map[string]*asp.Statement{} dir, _ := rules.AllAssets() sort.Strings(dir) @@ -124,7 +124,7 @@ func getFunctionsFromState(state *core.BuildState) map[string]*asp.Statement { func getFunctionsFromFiles(files cli.StdinStrings) map[string]*asp.Statement { state := newState() - p := asp.NewParser(state) + p := asp.NewParser(state, nil) return parseFilesForFunctions(p, files) } diff --git a/src/output/shell_output.go b/src/output/shell_output.go index 46850dd0e3..7479954ec6 100644 --- a/src/output/shell_output.go +++ b/src/output/shell_output.go @@ -33,7 +33,7 @@ type Progress interface { // MonitorState monitors the build while it's running and prints output until the results // channel of state has completed. -func MonitorState(state *core.BuildState, progress Progress, plainOutput, detailedTests, streamTestResults, shell, shellRun bool, traceFile string) { +func MonitorState(state *core.BuildState, progress Progress, results <-chan *core.BuildResult, plainOutput, detailedTests, streamTestResults, shell, shellRun bool, traceFile string) { initPrintf(state.Config) if len(state.Config.Please.Motd) != 0 { @@ -50,7 +50,6 @@ func MonitorState(state *core.BuildState, progress Progress, plainOutput, detail displayer := setupDisplayer(state, progress, plainOutput) t := time.NewTicker(displayer.Frequency()) defer t.Stop() - results := state.Results() bt := newBuildingTargets(state, progress, plainOutput) displayer.Update(bt.Targets()) loop: @@ -78,19 +77,7 @@ loop: printFailedBuildResults(bt.FailedNonTests, bt.FailedTargets, duration) return } - if state.NeedBuild { - // Check all the targets we wanted to build actually have been built. - for _, label := range state.ExpandOriginalLabels() { - if target := state.Graph.Target(label); target == nil { - log.Fatalf("Target %s doesn't exist in build graph", label) - } else if (state.NeedHashesOnly || state.PrepareOnly || shell) && target.State() == core.Stopped { - // Do nothing, we will output about this shortly. - } else if target.State() < core.Built && len(bt.FailedTargets) == 0 && !target.AddedPostBuild { - log.Fatalf("Target %s hasn't built but we have no pending tasks left.\n%s", label, unbuiltDepsMessage(state.Graph, target)) - } - } - } - if state.NeedBuild && len(bt.FailedNonTests) == 0 { + if state.NeedBuild { // N.B. We've returned above if anything failed in the build step. if state.PrepareOnly || shell { printTempDirs(state, duration, shell, shellRun) } else if state.NeedTests { // Got to the test phase, report their results. @@ -643,35 +630,6 @@ func colouriseError(err error) error { // errorMessageRe is a regex to find lines that look like they're specifying a file. var errorMessageRe = deferredregex.DeferredRegex{Re: `^([^ ]+\.[^: /]+):([0-9]+):(?:([0-9]+):)? *(?:([a-z-_ ]+):)? (.*)$`} -// unbuiltDepsMessage returns a message describing why the given target hasn't built, by listing -// any of its transitive dependencies that aren't built either. -func unbuiltDepsMessage(graph *core.BuildGraph, target *core.BuildTarget) string { - var b strings.Builder - seen := map[*core.BuildTarget]bool{} - var walk func(*core.BuildTarget) - walk = func(t *core.BuildTarget) { - if seen[t] { - return - } - seen[t] = true - deps, unresolved := t.Dependencies(graph) - for _, l := range unresolved { - fmt.Fprintf(&b, " %s (not in the build graph)\n", l) - } - for _, dep := range deps { - if !dep.State().IsBuilt() { - fmt.Fprintf(&b, " %s (%s)\n", dep.Label, dep.State()) - walk(dep) - } - } - } - walk(target) - if b.Len() == 0 { - return "" - } - return "\nThe following dependencies have not built:\n" + b.String() -} - // shortError returns the message for an error, shortening it if the error supports that. func shortError(err error) string { if se, ok := err.(shortenableError); ok { diff --git a/src/parse/asp/builtins.go b/src/parse/asp/builtins.go index 2637d73a48..545a084281 100644 --- a/src/parse/asp/builtins.go +++ b/src/parse/asp/builtins.go @@ -307,7 +307,7 @@ func bazelLoad(s *scope, args []pyObject) pyObject { func (s *scope) WaitForSubincludedTarget(ctx context.Context, l, dependent core.BuildLabel) (*core.BuildTarget, error) { s.interpreter.limiter.Release() defer s.interpreter.limiter.Acquire() - return s.state.Build(ctx, l, dependent) + return s.interpreter.callbacks.BuildAndDownload(ctx, l, dependent) } // builtinFail raises an immediate error that can't be intercepted. @@ -386,7 +386,7 @@ func subincludeTarget(s *scope, l core.BuildLabel) *core.BuildTarget { ctx := pprof.WithLabels(s.ctx, pprof.Labels("subinclude "+subrepoPackageLabel.String(), pkgLabel.String())) pprof.SetGoroutineLabels(ctx) defer pprof.SetGoroutineLabels(s.ctx) - if _, err := s.state.Parse(ctx, subrepoPackageLabel, pkgLabel); err != nil { + if _, err := s.interpreter.callbacks.Parse(ctx, subrepoPackageLabel, pkgLabel); err != nil { s.Error("Failed to parse subrepo target: %w", err) } } diff --git a/src/parse/asp/interpreter.go b/src/parse/asp/interpreter.go index 8a290d30a9..5037f46044 100644 --- a/src/parse/asp/interpreter.go +++ b/src/parse/asp/interpreter.go @@ -33,11 +33,13 @@ type interpreter struct { stringMethods, dictMethods, configMethods map[string]*pyFunc regexCache *cmap.Map[string, *regexp.Regexp] + + callbacks Callbacks } // newInterpreter creates and returns a new interpreter instance. // It loads all the builtin rules at this point. -func newInterpreter(state *core.BuildState, p *Parser) *interpreter { +func newInterpreter(state *core.BuildState, p *Parser, callbacks Callbacks) *interpreter { s := &scope{ ctx: context.Background(), state: state, @@ -49,6 +51,7 @@ func newInterpreter(state *core.BuildState, p *Parser) *interpreter { configs: map[*core.BuildState]*pyConfig{}, limiter: make(semaphore, state.Config.Parse.NumThreads), regexCache: cmap.New[string, *regexp.Regexp](cmap.SmallShardCount, cmap.XXHash), + callbacks: callbacks, } // If we're creating an interpreter for a subrepo, we should share the subinclude cache. if p.interpreter != nil { diff --git a/src/parse/asp/interpreter_test.go b/src/parse/asp/interpreter_test.go index f3c2db1f13..bd390acdbf 100644 --- a/src/parse/asp/interpreter_test.go +++ b/src/parse/asp/interpreter_test.go @@ -22,7 +22,7 @@ func parseFileToStatements(filename string) (*scope, []*Statement, error) { func parseFileToStatementsInPkg(filename string, pkg *core.Package) (*scope, []*Statement, error) { state := core.NewDefaultBuildState() state.Config.BuildConfig = map[string]string{"parser-engine": "python27"} - parser := NewParser(state) + parser := NewParser(state, nil) src, err := rules.ReadAsset("builtins.build_defs") if err != nil { @@ -594,7 +594,7 @@ func TestIsSemver(t *testing.T) { func TestJSON(t *testing.T) { state := core.NewDefaultBuildState() - parser := NewParser(state) + parser := NewParser(state, nil) src, err := rules.ReadAsset("builtins.build_defs") if err != nil { @@ -659,7 +659,7 @@ func TestSemverCheck(t *testing.T) { func TestLogConfigVariable(t *testing.T) { state := core.NewDefaultBuildState() - parser := NewParser(state) + parser := NewParser(state, nil) src, err := rules.ReadAsset("builtins.build_defs") if err != nil { diff --git a/src/parse/asp/logging_test.go b/src/parse/asp/logging_test.go index ec9e2d2815..23a45b60a1 100644 --- a/src/parse/asp/logging_test.go +++ b/src/parse/asp/logging_test.go @@ -25,7 +25,7 @@ func parseFile2(filename string) (*scope, error) { state := core.NewDefaultBuildState() pkg := core.NewPackage("test/package") pkg.Filename = "test/package/BUILD" - parser := NewParser(state) + parser := NewParser(state, nil) src, err := rules.ReadAsset("builtins.build_defs") if err != nil { panic(err) diff --git a/src/parse/asp/main/main.go b/src/parse/asp/main/main.go index b7b0d91001..c36898a685 100644 --- a/src/parse/asp/main/main.go +++ b/src/parse/asp/main/main.go @@ -233,7 +233,7 @@ func main() { var wg sync.WaitGroup wg.Add(opts.NumThreads) total := len(opts.Args.BuildFiles) - p := asp.NewParser(state) + p := asp.NewParser(state, nil, nil) log.Debug("Loading built-in build rules...") dir, _ := rules.AllAssets() diff --git a/src/parse/asp/parser.go b/src/parse/asp/parser.go index e03d35ae4b..99d55d87c7 100644 --- a/src/parse/asp/parser.go +++ b/src/parse/asp/parser.go @@ -24,6 +24,12 @@ type semaphore chan struct{} func (s semaphore) Acquire() { s <- struct{}{} } func (s semaphore) Release() { <-s } +// Callbacks is the interface we require from something that we can call back to for builds / parses. +type Callbacks interface { + Parse(context.Context, core.BuildLabel, core.BuildLabel) (*core.Package, error) + BuildAndDownload(context.Context, core.BuildLabel, core.BuildLabel) (*core.BuildTarget, error) +} + // A Parser implements parsing of BUILD files. type Parser struct { interpreter *interpreter @@ -35,9 +41,9 @@ type Parser struct { } // NewParser creates a new parser instance. One is normally sufficient for a process lifetime. -func NewParser(state *core.BuildState) *Parser { +func NewParser(state *core.BuildState, callbacks Callbacks) *Parser { p := newParser() - p.interpreter = newInterpreter(state, p) + p.interpreter = newInterpreter(state, p, callbacks) p.limiter = p.interpreter.limiter return p } @@ -50,6 +56,11 @@ func newParser() *Parser { } } +// SetCallbacks sets the callback functions on an existing parser instance. +func (p *Parser) SetCallbacks(callbacks Callbacks) { + p.interpreter.callbacks = callbacks +} + // LoadBuiltins instructs the parser to load rules from this file as built-ins. // Optionally the file contents can be supplied directly. func (p *Parser) LoadBuiltins(filename string, contents []byte) error { diff --git a/src/parse/init.go b/src/parse/init.go index acf7588e47..da10bbebaa 100644 --- a/src/parse/init.go +++ b/src/parse/init.go @@ -18,33 +18,28 @@ import ( ) // InitParser initialises the parser engine. -func InitParser(state *core.BuildState) *asp.Parser { - p := newAspParser(state) - state.Parser = &aspParser{parser: p} - return p -} - -// GetAspParser returns the underlying asp.Parser from the state's parser. -// This is useful for tools like the language server that need direct access to AST information. -// Returns nil if the state's parser is not set or is not an aspParser. -func GetAspParser(state *core.BuildState) *asp.Parser { - if state.Parser == nil { - return nil +func InitParser(state *core.BuildState, callbacks asp.Callbacks) *asp.Parser { + // There is some awkward coupling here for the benefit of the language server, which wants to get its + // hands on the parser, but it cannot create a fully functional one any more. + if p, ok := state.Parser.(*aspParser); ok { + p.callbacks = callbacks + p.parser.SetCallbacks(callbacks) + return p.parser } - if ap, ok := state.Parser.(*aspParser); ok { - return ap.parser - } - return nil + p := newAspParser(state, callbacks) + state.Parser = &aspParser{parser: p, callbacks: callbacks} + return p } // aspParser implements the core.Parser interface around our parser package. type aspParser struct { - parser *asp.Parser + parser *asp.Parser + callbacks asp.Callbacks } // newAspParser returns a asp.Parser object with all the builtins loaded -func newAspParser(state *core.BuildState) *asp.Parser { - p := asp.NewParser(state) +func newAspParser(state *core.BuildState, callbacks asp.Callbacks) *asp.Parser { + p := asp.NewParser(state, callbacks) log.Debug("Loading built-in build rules...") dir, _ := rules.AllAssets() sort.Strings(dir) @@ -98,7 +93,7 @@ func (p *aspParser) runBuildFunction(state *core.BuildState, target *core.BuildT // be picked up and built before the file that defines it has been fully interpreted - but the // callback both reads the package out of the graph and mutates it, so it can't run until then. // There's no parse in flight for us to inherit a context from, hence Background. - if _, err := state.Parse(context.Background(), target.Label, target.Label); err != nil { + if _, err := p.callbacks.Parse(context.Background(), target.Label, target.Label); err != nil { return err } if err := f(); err != nil { @@ -130,7 +125,7 @@ func createBazelSubrepo(state *core.BuildState) { // BuildRuleArgOrder returns a map of the arguments to build rule and the order they appear in the source file func BuildRuleArgOrder(state *core.BuildState) map[string]int { - p := asp.NewParser(state) + p := asp.NewParser(state, nil) b, _ := rules.ReadAsset("builtins.build_defs") stmts, _ := p.ParseData(b, "builtins.build_defs") m := map[string]int{} diff --git a/src/parse/parse_step.go b/src/parse/parse_step.go index d27a526e07..489bd57a32 100644 --- a/src/parse/parse_step.go +++ b/src/parse/parse_step.go @@ -27,12 +27,6 @@ var ErrMissingBuildFile = errors.New("build file not found") // the subrepo's preloads are ready) before claiming the package, since neither can be done from in here. func Parse(ctx context.Context, state *core.BuildState, label, dependent core.BuildLabel, subrepo *core.Subrepo) (*core.Package, error) { state.LogParseResult(label, core.PackageParsing, "Parsing...") - - // Subrepo & nothing else means we just want to ensure that subrepo is present. - if label.Subrepo != "" && label.PackageName == "" && label.Name == "" { - // TODO(peter): is this relevant still? - return nil, nil - } pkg, err := parsePackage(ctx, state, label, dependent, subrepo) if err != nil { return nil, err diff --git a/src/please.go b/src/please.go index 0bbdfe6514..9cbd089281 100644 --- a/src/please.go +++ b/src/please.go @@ -1222,13 +1222,12 @@ func runPlease(state *core.BuildState, targets []core.BuildLabel) { pretty := prettyOutput(opts.OutputFlags.InteractiveOutput, opts.OutputFlags.PlainOutput || opts.BehaviorFlags.Debug, opts.OutputFlags.Verbosity) && state.NeedBuild && !streamTests state.Cache = cache.NewCache(state) - // Run the display - // TODO(peterebden): Refactor out the results stuff from state in a future PR to avoid this kind of race condition. - state.Results() // important this is called now, don't ask... + // Run the display & build simultaneously + results := state.Results() var progress plz.Progress var wg sync.WaitGroup wg.Go(func() { - output.MonitorState(state, &progress, !pretty, detailedTests, streamTests, shell, shellRun, string(opts.OutputFlags.TraceFile)) + output.MonitorState(state, &progress, results, !pretty, detailedTests, streamTests, shell, shellRun, string(opts.OutputFlags.TraceFile)) }) if err := plz.Run(targets, opts.BuildFlags.PreTargets, state, &progress, state.TargetArch); err != nil { // Failures to build are logged through the central error reporting mechanism and will be printed by MonitorState above. diff --git a/src/plz/BUILD b/src/plz/BUILD index c70633f2f8..2cc2c6690c 100644 --- a/src/plz/BUILD +++ b/src/plz/BUILD @@ -1,6 +1,6 @@ go_library( name = "plz", - srcs = ["plz.go"], + srcs = glob(["*.go"], exclude=["*_test.go"]), pgo_file = "//:pgo", visibility = ["PUBLIC"], deps = [ @@ -22,10 +22,11 @@ go_library( go_test( name = "plz_test", - srcs = ["plz_test.go"], + srcs = glob(["*_test.go"]), deps = [ ":plz", "///third_party/go/github.com_stretchr_testify//assert", + "///third_party/go/github.com_stretchr_testify//require", "//src/cli", "//src/core", ], diff --git a/src/core/cycle_detector.go b/src/plz/cycle_detector.go similarity index 53% rename from src/core/cycle_detector.go rename to src/plz/cycle_detector.go index 12d76aff79..a64b96717e 100644 --- a/src/core/cycle_detector.go +++ b/src/plz/cycle_detector.go @@ -1,38 +1,41 @@ -package core +package plz import ( + "bytes" + "context" "fmt" + "runtime/pprof" "strings" + "time" + + "github.com/thought-machine/please/src/core" ) +// cycleCheckDuration is the length of time we allow inactivity for before we trigger cycle detection. +const cycleCheckDuration = 5 * time.Second + type cycleDetector struct { - graph *BuildGraph - stopped bool + graph *core.BuildGraph } // Check runs a single check of the build graph to see if any cycles can be detected. // If it finds one an errCycle is returned. func (c *cycleDetector) Check() *errCycle { - if c.stopped { - return nil - } log.Debug("Running cycle detection...") - complete := map[*BuildTarget]struct{}{} - partial := map[*BuildTarget]struct{}{} + complete := map[*core.BuildTarget]struct{}{} + partial := map[*core.BuildTarget]struct{}{} // visit visits a target and all its transitive dependencies. As each is visited they are marked as // partially visited; when we bottom out a tree successfully we mark it as completely visited (this // saves us from revisiting any node we've successfully visited before). // If a cycle is found it returns a slice of the targets in that cycle, and a bool indicating if the // cycle is complete or not (if not the caller will need to add its node to it as well). - var visit func(target *BuildTarget) ([]*BuildTarget, bool) - visit = func(target *BuildTarget) ([]*BuildTarget, bool) { - if c.stopped { - return nil, false - } else if _, present := complete[target]; present { + var visit func(target *core.BuildTarget) ([]*core.BuildTarget, bool) + visit = func(target *core.BuildTarget) ([]*core.BuildTarget, bool) { + if _, present := complete[target]; present { return nil, false } else if _, present := partial[target]; present { - return []*BuildTarget{target}, false + return []*core.BuildTarget{target}, false } partial[target] = struct{}{} // Ignore anything we can't resolve; we run while the build is still going on so it's @@ -43,7 +46,7 @@ func (c *cycleDetector) Check() *errCycle { if done || target == cycle[len(cycle)-1] { return cycle, true // This target is already in the cycle } - return append([]*BuildTarget{target}, cycle...), false + return append([]*core.BuildTarget{target}, cycle...), false } } delete(partial, target) @@ -52,10 +55,6 @@ func (c *cycleDetector) Check() *errCycle { } for _, target := range c.graph.AllTargets() { - if c.stopped { - log.Debug("Cycle detection terminated") - return nil - } if _, present := complete[target]; !present { if cycle, _ := visit(target); cycle != nil { log.Debug("Cycle detection complete, cycle found: %s", cycle) @@ -64,17 +63,16 @@ func (c *cycleDetector) Check() *errCycle { } } log.Debug("Cycle detection complete, no cycles found") + // Dump the goroutine info in case that helps to shed light + var buf bytes.Buffer + pprof.Lookup("goroutine").WriteTo(&buf, 1) + log.Debug("Current stacks: %s", buf.String()) return nil } -// Stop stops any existing run of the cycle detector. -func (c *cycleDetector) Stop() { - c.stopped = true -} - // An errCycle is emitted when a graph cycle is detected. type errCycle struct { - Cycle []*BuildTarget + Cycle []*core.BuildTarget } func (err *errCycle) Error() string { @@ -85,3 +83,38 @@ func (err *errCycle) Error() string { labels[len(labels)-1] = labels[0] return fmt.Sprintf("Dependency cycle found:\n%s\nSorry, but you'll have to refactor your build files to avoid this cycle", strings.Join(labels, "\n -> ")) } + +// checkForCycles consumes a stream of build results and triggers cycle detection when appropriate +func checkForCycles(state *core.BuildState, results <-chan *core.BuildResult, cancel context.CancelCauseFunc) { + checker := cycleDetector{graph: state.Graph} + active := map[*core.BuildTarget]struct{}{} + t := time.NewTimer(cycleCheckDuration) + defer t.Stop() + for { + select { + case result, ok := <-results: + if !ok { + return // results channel closed means the build is complete + } + t.Reset(cycleCheckDuration) + if target := result.Target; target != nil { + if result.Status.IsActive() { + active[target] = struct{}{} + } else { + delete(active, target) + } + } + case <-t.C: + t.Reset(cycleCheckDuration) + if len(active) > 0 { + continue + } + go func() { + if err := checker.Check(); err != nil { + state.LogBuildError(err.Cycle[0].Label, core.TargetBuildFailed, err, "") + cancel(err) + } + }() + } + } +} diff --git a/src/core/cycle_detector_test.go b/src/plz/cycle_detector_test.go similarity index 68% rename from src/core/cycle_detector_test.go rename to src/plz/cycle_detector_test.go index 58a9ca8cf8..158f23afb1 100644 --- a/src/core/cycle_detector_test.go +++ b/src/plz/cycle_detector_test.go @@ -1,24 +1,27 @@ -package core +package plz import ( + "errors" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/thought-machine/please/src/core" ) func TestCycleDetector(t *testing.T) { - newTarget := func(state *BuildState, label string, deps ...string) *BuildTarget { - target := NewBuildTarget(ParseBuildLabel(label, "")) + newTarget := func(state *core.BuildState, label string, deps ...string) *core.BuildTarget { + target := core.NewBuildTarget(core.ParseBuildLabel(label, "")) for _, dep := range deps { - target.AddDependency(ParseBuildLabel(dep, "")) + target.AddDependency(core.ParseBuildLabel(dep, "")) } state.Graph.AddTarget(target) return target } t.Run("NoCycle", func(t *testing.T) { - state := NewDefaultBuildState() + state := core.NewDefaultBuildState() newTarget(state, "//src:a", "//src:b", "//src:c") newTarget(state, "//src:b", "//src:d", "//src:e") newTarget(state, "//src:c", "//src:b", "//src:f") @@ -32,7 +35,7 @@ func TestCycleDetector(t *testing.T) { }) t.Run("Cycle", func(t *testing.T) { - state := NewDefaultBuildState() + state := core.NewDefaultBuildState() newTarget(state, "//src:a", "//src:b", "//src:c") newTarget(state, "//src:b", "//src:d", "//src:e") newTarget(state, "//src:c", "//src:b", "//src:f") @@ -43,9 +46,9 @@ func TestCycleDetector(t *testing.T) { detector := cycleDetector{graph: state.Graph} err := detector.Check() - require.NotNil(t, err) - require.Equal(t, 3, len(err.Cycle)) - log.Warning("%s", err) - assert.Equal(t, []*BuildTarget{g, e, f}, err.Cycle) + require.Error(t, err) + cerr, ok := errors.AsType[*errCycle](err) + require.True(t, ok) + assert.Equal(t, []*core.BuildTarget{g, e, f}, cerr.Cycle) }) } diff --git a/src/plz/plz.go b/src/plz/plz.go index 4f9dc19771..a58472400d 100644 --- a/src/plz/plz.go +++ b/src/plz/plz.go @@ -43,8 +43,6 @@ func Run(targets, preTargets []core.BuildLabel, state *core.BuildState, progress go state.UpdateResources() } - parser := parse.InitParser(state) - // This must happen however we exit; anything reading state.Results() (e.g. the display) // waits for that channel to be closed, so it would hang forever if we returned an error first. defer func() { @@ -59,12 +57,9 @@ func Run(targets, preTargets []core.BuildLabel, state *core.BuildState, progress metrics.Push(state.Config.Metrics, state.Config.IsRemoteExecution()) }() - ctx, cancel := context.WithCancel(context.Background()) - state.Cancel = cancel - + topctx, cancel := context.WithCancelCause(context.Background()) r := runner{ state: state, - parser: parser, arch: arch, progress: progress, buildOnce: cmap.NewErrMap[core.BuildLabel, *core.BuildTarget](cmap.DefaultShardCount, func(l core.BuildLabel) uint64 { @@ -78,18 +73,11 @@ func Run(targets, preTargets []core.BuildLabel, state *core.BuildState, progress remoteLimiter: make(limiter, state.Config.NumRemoteExecutors()), anyRemote: state.Config.NumRemoteExecutors() > 0, } - g, ctx := r.group(ctx) + g, ctx := r.group(topctx) r.tasks = g - - state.Build = func(ctx context.Context, label, dependent core.BuildLabel) (*core.BuildTarget, error) { - target, err := r.Build(ctx, label, dependent) - if err != nil { - return nil, err - } - // Anything calling this will likely want this thing to end up being downloaded (it's mostly for subincludes) - return target, state.EnsureDownloaded(target) - } - state.Parse = r.Parse + r.parser = parse.InitParser(state, &r) + results := state.Results() + go checkForCycles(state, results, cancel) if state.Config.Bazel.Compatibility && fs.FileExists("WORKSPACE") { // We have to parse the WORKSPACE file before anything else to understand subrepos. @@ -107,10 +95,8 @@ func Run(targets, preTargets []core.BuildLabel, state *core.BuildState, progress if err := g.Wait(); err != nil { return err } - // Reset the group & context for next time - ctx, cancel = context.WithCancel(context.Background()) - g, ctx = r.group(ctx) - state.Cancel = cancel + // Reset the group & context for next time (the context is now expired because the group is done) + g, ctx = r.group(topctx) r.tasks = g } r.FindOriginalTaskSet(ctx, targets, r.state.NeedTests, r.state.NeedBuild) @@ -126,7 +112,7 @@ func Run(targets, preTargets []core.BuildLabel, state *core.BuildState, progress return g.Wait() } -// RunHost is a convenience function that uses the host architecture, the given state's +// RunHostAsync is a convenience function that uses the host architecture, the given state's // configuration and no pre targets. It is otherwise identical to Run. func RunHost(targets []core.BuildLabel, state *core.BuildState) { Run(targets, nil, state, &Progress{}, cli.HostArch()) @@ -281,6 +267,18 @@ func (r *runner) ensureSubrepo(ctx context.Context, label, dependent core.BuildL if r.state.Graph.Subrepo(label.Subrepo) != nil { return nil // The parse above defined it, we're done. } + if sl.Subrepo != dependent.Subrepo { + nested := sl + nested.Subrepo = dependent.Subrepo + if !inSamePackage(nested, dependent) { + if _, err := r.tryParse(ctx, nested, label); err != nil && !errors.Is(err, parse.ErrMissingBuildFile) { + return err + } + if r.state.Graph.Subrepo(label.Subrepo) != nil { + return nil + } + } + } // Nothing defines it, so the only remaining possibility is an architecture subrepo. if arch, ok := couldBeArch(label.Subrepo); ok { r.state.Graph.MaybeAddSubrepo(core.SubrepoForArch(r.state, arch)) @@ -446,13 +444,13 @@ func (r *runner) buildOne(ctx context.Context, target *core.BuildTarget) error { // N.B. Even when there are none we can't just build the target and return; its own callbacks // can add some, which we won't know about until it's built. if deps := slices.Collect(target.RuntimeAndDataDependencies()); len(deps) == 0 { - if err := r.buildJustOne(target); err != nil { + if err := r.buildJustOne(ctx, target); err != nil { return err } } else { g, gctx = r.group(ctx) g.Go(func() error { - return r.buildJustOne(target) + return r.buildJustOne(gctx, target) }) for _, dep := range deps { g.Go(func() error { @@ -479,11 +477,14 @@ func (r *runner) buildOne(ctx context.Context, target *core.BuildTarget) error { } // buildJustOne calls the build for a single target. -func (r *runner) buildJustOne(target *core.BuildTarget) error { +func (r *runner) buildJustOne(ctx context.Context, target *core.BuildTarget) error { remote := r.anyRemote && !target.Local limiter := r.limiter(remote) limiter.Acquire() defer limiter.Release() + if err := ctx.Err(); err != nil { + return err + } return build.Build(r.state, target, remote) } @@ -528,6 +529,15 @@ func (r *runner) Build(ctx context.Context, label, dependent core.BuildLabel) (* }) } +func (r *runner) BuildAndDownload(ctx context.Context, label, dependent core.BuildLabel) (*core.BuildTarget, error) { + target, err := r.Build(ctx, label, dependent) + if err != nil { + return nil, err + } + // Anything calling this will likely want this thing to end up being downloaded (it's mostly for subincludes) + return target, r.state.EnsureDownloaded(target) +} + // testOne tests one single target func (r *runner) testOne(ctx context.Context, target *core.BuildTarget, dependent core.BuildLabel) error { if target.IsTest() { @@ -543,13 +553,14 @@ func (r *runner) testOne(ctx context.Context, target *core.BuildTarget, dependen // TODO(peter): Is it okay for none of these to return errors? I _think_ so and we will capture it later? remote := r.anyRemote && !target.Local limiter := r.limiter(remote) - if r.state.TestSequentially || r.state.NumTestRuns == 1 { // minor optimisation to avoid creating unnecessary goroutines + if r.state.TestSequentially || r.state.NumTestRuns == 1 { limiter.Acquire() defer limiter.Release() - for run := range int(r.state.NumTestRuns) { - test.Test(r.state, target, remote, run+1) - r.progress.numDone.Add(1) + if err := ctx.Err(); err != nil { + return err } + test.Test(r.state, target, remote, 1) + r.progress.numDone.Add(int64(r.state.NumTestRuns)) return nil } var wg sync.WaitGroup @@ -557,6 +568,9 @@ func (r *runner) testOne(ctx context.Context, target *core.BuildTarget, dependen wg.Go(func() { limiter.Acquire() defer limiter.Release() + if ctx.Err() != nil { + return + } test.Test(r.state, target, remote, run+1) r.progress.numDone.Add(1) }) @@ -681,6 +695,9 @@ func (r *runner) findOriginalTask(ctx context.Context, target core.BuildLabel, n } func (r *runner) queueTask(ctx context.Context, target core.BuildLabel, needTest, needBuild bool) { + if r.state.IsExcluded(target) { + return + } r.state.AddOriginalTarget(target) r.tasks.Go(func() error { if needTest { @@ -690,6 +707,16 @@ func (r *runner) queueTask(ctx context.Context, target core.BuildLabel, needTest // TODO(peter): Ensure this gets downloaded if needed return err } + if r.state.ParsePackageOnly { + // Some kinds of query don't need a recursive parse. A named target still has to exist + // though, so those go via parseTarget for the error (and suggestions) that produces. + if target.IsAllTargets() { + _, err := r.Parse(ctx, target, core.OriginalTarget) + return err + } + _, err := r.parseTarget(ctx, target, core.OriginalTarget) + return err + } return r.RecursiveParse(ctx, target, core.OriginalTarget) }) } diff --git a/src/query/deps.go b/src/query/deps.go index 559174a5a3..7af293167c 100644 --- a/src/query/deps.go +++ b/src/query/deps.go @@ -4,7 +4,6 @@ import ( "fmt" "io" "slices" - "sort" "strings" "github.com/thought-machine/please/src/core" @@ -34,9 +33,7 @@ func deps(out io.Writer, state *core.BuildState, target *core.BuildTarget, done return } // Sort so output is stable and readable; DeclaredDependencies yields in declaration order. - declaredDeps := core.BuildLabels(slices.Collect(target.DeclaredDependencies())) - sort.Sort(declaredDeps) - for _, l := range declaredDeps { + for _, l := range slices.SortedFunc(target.DeclaredDependencies(), core.BuildLabel.Compare) { dep := state.Graph.TargetOrDie(l) for _, l := range dep.ProvideFor(target) { if !state.ShouldInclude(dep) || done[l] { diff --git a/src/query/print.go b/src/query/print.go index f06b6b170d..3955901c65 100644 --- a/src/query/print.go +++ b/src/query/print.go @@ -132,14 +132,16 @@ func specialFields() specialFieldsMap { } return "" }, + // These all yield in declaration order; we sort them so the printed rule is canonical + // regardless of how the original was written. "deps": func(target *core.BuildTarget) interface{} { - return slices.Collect(target.DeclaredDependenciesStrict()) + return slices.SortedFunc(target.DeclaredDependenciesStrict(), core.BuildLabel.Compare) }, "exported_deps": func(target *core.BuildTarget) interface{} { - return slices.Collect(target.ExportedDependencies()) + return slices.SortedFunc(target.ExportedDependencies(), core.BuildLabel.Compare) }, "runtime_deps": func(target *core.BuildTarget) interface{} { - return slices.Collect(target.RuntimeDependencies()) + return slices.SortedFunc(target.RuntimeDependencies(), core.BuildLabel.Compare) }, "visibility": func(target *core.BuildTarget) interface{} { if len(target.Visibility) == 1 && target.Visibility[0] == core.WholeGraph[0] { diff --git a/src/query/somepath.go b/src/query/somepath.go index dc0f12cce1..07235e8248 100644 --- a/src/query/somepath.go +++ b/src/query/somepath.go @@ -92,7 +92,8 @@ func somePath(graph *core.BuildGraph, target1, target2 *core.BuildTarget, seen, return nil } seen[target1.Label] = struct{}{} - for dep := range target1.DeclaredDependencies() { + // Sorted so we always report the same path back; DeclaredDependencies yields in declaration order. + for _, dep := range slices.SortedFunc(target1.DeclaredDependencies(), core.BuildLabel.Compare) { if t := graph.Target(dep); t != nil { if _, present := except[t.Label]; present { continue diff --git a/tools/build_langserver/lsp/lsp.go b/tools/build_langserver/lsp/lsp.go index fbdbdcceb0..ec1882c9bd 100644 --- a/tools/build_langserver/lsp/lsp.go +++ b/tools/build_langserver/lsp/lsp.go @@ -198,11 +198,7 @@ func (h *Handler) initialize(params *lsp.InitializeParams) (*lsp.InitializeResul h.state.NeedBuild = false // Initialize the parser on state first, so that plz.RunHost uses the same parser. // This ensures plugin subincludes are stored in the same AST cache we use. - parse.InitParser(h.state) - h.parser = parse.GetAspParser(h.state) - if h.parser == nil { - return nil, fmt.Errorf("failed to get asp parser from state") - } + h.parser = parse.InitParser(h.state, nil) // Parse everything in the repo up front. // This is a lot easier than trying to do clever partial parses later on, although // eventually we may want that if we start dealing with truly large repos. From 37c1914ceebcfa3b455e243ba098ce73a9a2e422 Mon Sep 17 00:00:00 2001 From: Peter Ebden Date: Sun, 16 Aug 2026 10:44:57 +0100 Subject: [PATCH 15/16] Fix compile --- src/parse/asp/main/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parse/asp/main/main.go b/src/parse/asp/main/main.go index c36898a685..d325197640 100644 --- a/src/parse/asp/main/main.go +++ b/src/parse/asp/main/main.go @@ -233,7 +233,7 @@ func main() { var wg sync.WaitGroup wg.Add(opts.NumThreads) total := len(opts.Args.BuildFiles) - p := asp.NewParser(state, nil, nil) + p := asp.NewParser(state, nil) log.Debug("Loading built-in build rules...") dir, _ := rules.AllAssets() From d7b78da71f4738576b80cb63e2ed9aef15cdf32e Mon Sep 17 00:00:00 2001 From: Peter Ebden Date: Mon, 17 Aug 2026 09:29:44 +0100 Subject: [PATCH 16/16] Drop test post rebase --- src/core/state_test.go | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/src/core/state_test.go b/src/core/state_test.go index 50885deae4..581cf09834 100644 --- a/src/core/state_test.go +++ b/src/core/state_test.go @@ -2,9 +2,7 @@ package core import ( "strings" - "sync" "testing" - "time" "github.com/stretchr/testify/assert" ) @@ -141,39 +139,3 @@ func TestCopyPlugin(t *testing.T) { assert.NotEqual(t, plugin.ExtraValues["foo"], newPlugin.ExtraValues["foo"]) } - -func TestWaitForPackageConcurrent(t *testing.T) { - // Regression test for a lost-wakeup race: concurrent callers waiting on - // the same unparsed package could overwrite each other's wait channel in - // packageWaits, so the channel one of them waited on was never closed and - // that caller blocked forever. - dependent := BuildLabel{PackageName: "other", Name: "all"} - for i := 0; i < 200; i++ { - state := NewDefaultBuildState() - label := BuildLabel{PackageName: "pkg", Name: "all"} - const n = 32 - var wg sync.WaitGroup - wg.Add(n) - start := make(chan struct{}) - for j := 0; j < n; j++ { - go func() { - defer wg.Done() - <-start - state.WaitForPackage(label, dependent, ParseModeNormal) - }() - } - close(start) - // Let the waiters register against the unparsed package first, then - // complete the parse the way LogParseResult does for real parses. - time.Sleep(time.Millisecond) - state.Graph.AddPackage(NewPackage("pkg")) - state.LogParseResult(label, PackageParsed, "parsed") - done := make(chan struct{}) - go func() { wg.Wait(); close(done) }() - select { - case <-done: - case <-time.After(10 * time.Second): - t.Fatalf("iteration %d: a WaitForPackage caller never woke", i) - } - } -}