diff --git a/tsc/internal/compiler/fileInclude.go b/tsc/internal/compiler/fileInclude.go index e07123ad5e1ca..6fb9458b710f3 100644 --- a/tsc/internal/compiler/fileInclude.go +++ b/tsc/internal/compiler/fileInclude.go @@ -11,7 +11,7 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/tspath" ) -type fileIncludeKind int +type fileIncludeKind uint8 const ( // References from file @@ -28,8 +28,8 @@ const ( type FileIncludeReason struct { kind fileIncludeKind - index int isDefaultLib bool + index int referencedFile *referencedFileData automaticTypeDirective *automaticTypeDirectiveFileData canonicalSourceFile tspath.PathKey @@ -41,6 +41,18 @@ type referencedFileData struct { synthetic *ast.Node } +func newReferencedFileIncludeReason(kind fileIncludeKind, file tspath.PathKey, index int, synthetic *ast.Node) *FileIncludeReason { + r := &struct { + reason FileIncludeReason + data referencedFileData + }{ + reason: FileIncludeReason{kind: kind}, + data: referencedFileData{file: file, index: index, synthetic: synthetic}, + } + r.reason.referencedFile = &r.data + return &r.reason +} + type referenceFileLocation struct { file *ast.SourceFile node *ast.Node diff --git a/tsc/internal/compiler/fileloader.go b/tsc/internal/compiler/fileloader.go index 0346cf84b0ecb..cf67af89b4169 100644 --- a/tsc/internal/compiler/fileloader.go +++ b/tsc/internal/compiler/fileloader.go @@ -262,6 +262,7 @@ func (p *fileLoader) addRootFileTask(fileName tspath.RootedFilePath, referenceTe rootTask.normalizedFilePath = fileName rootTask.path = p.caseSensitivity.PathKey(tspath.RootedPath(fileName)) rootTask.failedLookup = true + rootTask.parseTaskResult = &parseTaskResult{} rootTask.processingDiagnostics = []*processingDiagnostic{{ kind: processingDiagnosticKindExplainingFileInclude, explanation: &includeExplainingDiagnostic{ @@ -314,7 +315,7 @@ func (p *fileLoader) resolveAutomaticTypeDirectives(containingFileName tspath.Ro kind: fileIncludeKindAutomaticTypeDirectiveFile, automaticTypeDirective: &automaticTypeDirectiveFileData{name, resolved.PackageId}, }, - packageId: resolved.PackageId, + packageId: packageIdRef(&resolved.PackageId), }) } else { pDiagnostics = append(pDiagnostics, &processingDiagnostic{ @@ -723,13 +724,7 @@ func (p *fileLoader) getSourceFileFromReference( func (p *fileLoader) resolveTripleslashPathReference(moduleName string, containingFile tspath.RootedFilePath, index int) (*resolvedRef, *processingDiagnostic) { normalizedFileName := containingFile.Directory().ResolveFile(moduleName) containingPath := p.caseSensitivity.PathKey(tspath.RootedPath(containingFile)) - includeReason := &FileIncludeReason{ - kind: fileIncludeKindReferenceFile, - referencedFile: &referencedFileData{ - file: containingPath, - index: index, - }, - } + includeReason := newReferencedFileIncludeReason(fileIncludeKindReferenceFile, containingPath, index, nil) resolvedFileName, resolvedPath, diagnostic := p.getSourceFileFromReference( normalizedFileName, @@ -783,13 +778,7 @@ func (p *fileLoader) resolveTypeReferenceDirectives(t *parseTask) { traceDone = p.tracing.Push(tracing.PhaseProgram, "processTypeReferenceDirective", map[string]any{"directive": ref.FileName, "hasResolved": resolved.IsResolved(), "refKind": int(fileIncludeKindTypeReferenceDirective), "refPath": string(t.path)}, false) } typeResolutionsInFile[module.ModeAwareCacheKey{Name: ref.FileName, Mode: resolutionMode}] = resolved - includeReason := &FileIncludeReason{ - kind: fileIncludeKindTypeReferenceDirective, - referencedFile: &referencedFileData{ - file: t.path, - index: index, - }, - } + includeReason := newReferencedFileIncludeReason(fileIncludeKindTypeReferenceDirective, t.path, index, nil) typeResolutionsTrace = append(typeResolutionsTrace, trace...) if resolved.IsResolved() { @@ -799,7 +788,7 @@ func (p *fileLoader) resolveTypeReferenceDirectives(t *parseTask) { increaseDepth: resolved.IsExternalLibraryImport, elideOnDepth: false, includeReason: includeReason, - packageId: resolved.PackageId, + packageId: packageIdRef(&resolved.PackageId), }, nil) } else { t.processingDiagnostics = append(t.processingDiagnostics, &processingDiagnostic{ @@ -922,15 +911,8 @@ func (p *fileLoader) resolveImportsAndModuleAugmentations(t *parseTask) { path: resolvedModule.ResolvedPath, increaseDepth: resolvedModule.IsExternalLibraryImport, elideOnDepth: isJsFileFromNodeModules, - includeReason: &FileIncludeReason{ - kind: fileIncludeKindImport, - referencedFile: &referencedFileData{ - file: t.path, - index: importIndex, - synthetic: core.IfElse(importIndex < 0, entry, nil), - }, - }, - packageId: resolvedModule.PackageId, + includeReason: newReferencedFileIncludeReason(fileIncludeKindImport, t.path, importIndex, core.IfElse(importIndex < 0, entry, nil)), + packageId: packageIdRef(&resolvedModule.PackageId), }, nil) } } diff --git a/tsc/internal/compiler/filesparser.go b/tsc/internal/compiler/filesparser.go index 5947c5fbb3a40..3076cb62fa43a 100644 --- a/tsc/internal/compiler/filesparser.go +++ b/tsc/internal/compiler/filesparser.go @@ -29,23 +29,26 @@ type parseTask struct { isContentMapperSupplemental bool failedLookup bool includeReason *FileIncludeReason - packageId module.PackageId + packageId *module.PackageId + increaseDepth bool + elideOnDepth bool + loadedTask *parseTask + + // Most tasks are duplicate references to an already-loaded file, so + // state produced by loading lives in a separately allocated record that + // is only created when the task is loaded. + *parseTaskResult +} +type parseTaskResult struct { metadata ast.SourceFileMetaData resolutionsInFile module.ModeAwareCache[*module.ResolvedModule] resolutionsTrace []module.DiagAndArgs typeResolutionsInFile module.ModeAwareCache[*module.ResolvedTypeReferenceDirective] typeResolutionsTrace []module.DiagAndArgs - resolutionDiagnostics []*ast.Diagnostic processingDiagnostics []*processingDiagnostic importHelpersImportSpecifier *ast.StringLiteralNode jsxRuntimeImportSpecifier *jsxRuntimeImportSpecifier - - increaseDepth bool - elideOnDepth bool - - loadedTask *parseTask - allIncludeReasons []*FileIncludeReason } func (t *parseTask) FileName() tspath.RootedFilePath { @@ -58,6 +61,9 @@ func (t *parseTask) PathKey() tspath.PathKey { func (t *parseTask) load(loader *fileLoader) { t.loaded = true + if t.parseTaskResult == nil { + t.parseTaskResult = &parseTaskResult{} + } if t.isForAutomaticTypeDirective { t.loadAutomaticTypeDirectives(loader) return @@ -146,13 +152,7 @@ func (t *parseTask) load(loader *fileLoader) { if compilerOptions.NoLib != core.TSTrue && !loader.opts.SkipModuleResolution { for index, lib := range file.LibReferenceDirectives { - includeReason := &FileIncludeReason{ - kind: fileIncludeKindLibReferenceDirective, - referencedFile: &referencedFileData{ - file: t.path, - index: index, - }, - } + includeReason := newReferencedFileIncludeReason(fileIncludeKindLibReferenceDirective, t.path, index, nil) if name, ok := tsoptions.GetLibFileName(lib.FileName); ok { libFile := loader.pathForLibFile(name) t.addSubTask(resolvedRef{ @@ -214,7 +214,16 @@ type resolvedRef struct { increaseDepth bool elideOnDepth bool includeReason *FileIncludeReason - packageId module.PackageId + packageId *module.PackageId +} + +// packageIdRef returns id, or nil if id is empty. Pointing into an existing +// resolution keeps the commonly empty ID out of every parse task. +func packageIdRef(id *module.PackageId) *module.PackageId { + if id.Name == "" { + return nil + } + return id } func (t *parseTask) addSubTask(ref resolvedRef, libFile *LibFile) { @@ -297,8 +306,8 @@ func (w *filesParser) start(loader *fileLoader, tasks []*parseTask, depth int) { } // Propagate packageId to data if we have one and data doesn't yet - if data.packageId.Name == "" && task.packageId.Name != "" { - data.packageId = task.packageId + if data.packageId.Name == "" && task.packageId != nil { + data.packageId = *task.packageId } currentDepth := core.IfElse(task.increaseDepth, depth+1, depth) diff --git a/tsc/internal/core/workgroup.go b/tsc/internal/core/workgroup.go index 73618a1a19fc1..0a66c73375846 100644 --- a/tsc/internal/core/workgroup.go +++ b/tsc/internal/core/workgroup.go @@ -36,9 +36,7 @@ func (w *parallelWorkGroup) Queue(fn func()) { panic("Queue called after RunAndWait returned") } - w.wg.Go(func() { - fn() - }) + w.wg.Go(fn) } func (w *parallelWorkGroup) RunAndWait() { diff --git a/tsc/internal/ls/autoimport/export.go b/tsc/internal/ls/autoimport/export.go index 4ed8512b05327..123e0d4c22b3a 100644 --- a/tsc/internal/ls/autoimport/export.go +++ b/tsc/internal/ls/autoimport/export.go @@ -19,29 +19,22 @@ const ( ) // ModuleID uniquely identifies either a file module or an ambient module. +// For file modules, value is a tspath.PathKey; for ambient modules, a tspath.ModuleSpecifier. type ModuleID struct { - path tspath.PathKey - specifier tspath.ModuleSpecifier - kind moduleIDKind + value string + kind moduleIDKind } func fileModuleID(path tspath.PathKey) ModuleID { - return ModuleID{path: path, kind: moduleIDKindFile} + return ModuleID{value: string(path), kind: moduleIDKindFile} } func ambientModuleID(specifier string) ModuleID { - return ModuleID{specifier: tspath.ToModuleSpecifier(specifier), kind: moduleIDKindAmbient} + return ModuleID{value: string(tspath.ToModuleSpecifier(specifier)), kind: moduleIDKindAmbient} } func (m ModuleID) AsString() string { - switch m.kind { - case moduleIDKindFile: - return string(m.path) - case moduleIDKindAmbient: - return m.specifier.AsString() - default: - return "" - } + return m.value } func (m ModuleID) IsAmbient() bool { @@ -52,14 +45,19 @@ func (m ModuleID) AsPathKey() (tspath.PathKey, bool) { if m.kind != moduleIDKindFile { return "", false } - return m.path, true + return tspath.PathKeyFromCanonical(m.value), true +} + +// IsFile reports whether m identifies the file module at path. +func (m ModuleID) IsFile(path tspath.PathKey) bool { + return m.kind == moduleIDKindFile && m.value == path.AsString() } func (m ModuleID) AsModuleSpecifier() (tspath.ModuleSpecifier, bool) { if m.kind != moduleIDKindAmbient { return "", false } - return m.specifier, true + return tspath.ToModuleSpecifier(m.value), true } type ExportID struct { @@ -67,7 +65,7 @@ type ExportID struct { ExportName string } -type ExportSyntax int +type ExportSyntax uint8 const ( ExportSyntaxNone ExportSyntax = iota @@ -95,8 +93,8 @@ type Export struct { ExportID ModuleFileName tspath.RootedFilePath UnresolvedModuleSpecifier tspath.ModuleSpecifier - Syntax ExportSyntax Flags ast.SymbolFlags + Syntax ExportSyntax localName string // through is the name of the module symbol's export that this export was found on, // either 'export=', InternalSymbolNameExportStar, or empty string. @@ -105,9 +103,9 @@ type Export struct { // Checker-set fields Target ExportID - IsTypeOnly bool - ScriptElementKind lsutil.ScriptElementKind ScriptElementKindModifiers lsutil.ScriptElementKindModifier + ScriptElementKind lsutil.ScriptElementKind + IsTypeOnly bool // The file where the export was found. Path tspath.PathKey diff --git a/tsc/internal/ls/autoimport/view.go b/tsc/internal/ls/autoimport/view.go index b743a46b1aa02..9a6046c746d53 100644 --- a/tsc/internal/ls/autoimport/view.go +++ b/tsc/internal/ls/autoimport/view.go @@ -111,7 +111,7 @@ func (v *View) search(searchFn func(*RegistryBucket) []*Export) []*Export { exports := searchFn(bucket) results = slices.Grow(results, len(exports)) for _, e := range exports { - if modulePath, ok := e.ModuleID.AsPathKey(); ok && modulePath == v.importingFile.PathKey() { + if e.ModuleID.IsFile(v.importingFile.PathKey()) { // Don't auto-import from the importing file itself continue } diff --git a/tsc/internal/ls/lsutil/symbol_display.go b/tsc/internal/ls/lsutil/symbol_display.go index 58c9f01b70760..42f055b2a76fb 100644 --- a/tsc/internal/ls/lsutil/symbol_display.go +++ b/tsc/internal/ls/lsutil/symbol_display.go @@ -7,7 +7,7 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/core" ) -type ScriptElementKind int +type ScriptElementKind uint8 const ( ScriptElementKindUnknown ScriptElementKind = iota diff --git a/tsc/internal/module/resolver.go b/tsc/internal/module/resolver.go index 4b59f0582f927..2fb21ab2a4907 100644 --- a/tsc/internal/module/resolver.go +++ b/tsc/internal/module/resolver.go @@ -111,6 +111,13 @@ func resolutionCandidateFromDiskLogicalPath(directory tspath.RootedDirectoryPath } func resolutionCandidateFromEncodedPaths(directory tspath.RootedDirectoryPath, filePath string, directoryPath string, directoryOnly bool) resolutionCandidate { + if filePath == directoryPath { + // File and directory resolution of the same input are identical. + return resolutionCandidate{ + path: directory.ResolveFile(filePath).AsPath(), + directoryOnly: directoryOnly, + } + } resolvedDirectoryPath := directory.ResolveDirectory(directoryPath) if directoryOnly { return resolutionCandidate{ diff --git a/tsc/internal/packagejson/cache.go b/tsc/internal/packagejson/cache.go index 01004398d3238..94d3f4b54dfbd 100644 --- a/tsc/internal/packagejson/cache.go +++ b/tsc/internal/packagejson/cache.go @@ -1,6 +1,7 @@ package packagejson import ( + "maps" "sync" "github.com/microsoft/TypeScript/tsc/internal/collections" @@ -216,8 +217,13 @@ func (p *InfoCacheEntry) WithPackageDirectory(packageDirectory PackageDirectory) } } +// InfoCache reads from an immutable base map without locking (contention on a +// shared lock is costly during parallel module resolution) and records new +// entries in a concurrent overlay. Clone flattens both into a new base map, +// which is much cheaper than copying a sync.Map entry by entry. type InfoCache struct { - cache collections.SyncMap[tspath.PathKey, *InfoCacheEntry] + base map[tspath.PathKey]*InfoCacheEntry + added collections.SyncMap[tspath.PathKey, *InfoCacheEntry] caseSensitivity tspath.CaseSensitivity } @@ -228,12 +234,18 @@ func NewInfoCache(caseSensitivity tspath.CaseSensitivity) *InfoCache { } func (p *InfoCache) Clone() *InfoCache { - clone := NewInfoCache(p.caseSensitivity) - p.cache.Range(func(key tspath.PathKey, value *InfoCacheEntry) bool { - clone.cache.Store(key, value) + base := maps.Clone(p.base) + p.added.Range(func(key tspath.PathKey, value *InfoCacheEntry) bool { + if base == nil { + base = make(map[tspath.PathKey]*InfoCacheEntry) + } + base[key] = value return true }) - return clone + return &InfoCache{ + base: base, + caseSensitivity: p.caseSensitivity, + } } func (p *InfoCache) PackageDirectory(directory tspath.RootedDirectoryPath) PackageDirectory { @@ -245,17 +257,28 @@ func (p *InfoCache) CaseSensitivity() tspath.CaseSensitivity { } func (p *InfoCache) Get(directory PackageDirectory) *InfoCacheEntry { - if value, ok := p.cache.Load(directory.PathKey()); ok { + key := directory.PathKey() + if value, ok := p.base[key]; ok { return value } - return nil + value, _ := p.added.Load(key) + return value } func (p *InfoCache) Set(directory PackageDirectory, info *InfoCacheEntry) *InfoCacheEntry { - actual, _ := p.cache.LoadOrStore(directory.PathKey(), info) + key := directory.PathKey() + if existing, ok := p.base[key]; ok { + return existing + } + actual, _ := p.added.LoadOrStore(key, info) return actual } func (p *InfoCache) Range(f func(key tspath.PathKey, value *InfoCacheEntry) bool) { - p.cache.Range(f) + for key, value := range p.base { + if !f(key, value) { + return + } + } + p.added.Range(f) } diff --git a/tsc/internal/packagejson/cache_test.go b/tsc/internal/packagejson/cache_test.go new file mode 100644 index 0000000000000..879adfffa788d --- /dev/null +++ b/tsc/internal/packagejson/cache_test.go @@ -0,0 +1,155 @@ +package packagejson_test + +import ( + "fmt" + "maps" + "sync" + "testing" + + "github.com/microsoft/TypeScript/tsc/internal/packagejson" + "github.com/microsoft/TypeScript/tsc/internal/tspath" + "gotest.tools/v3/assert" +) + +func infoCacheDirectory(name string) packagejson.PackageDirectory { + return packagejson.NewPackageDirectory(tspath.RootedDirectoryPathFromAbsolute("/"+name), tspath.CaseSensitive) +} + +func newInfoCacheEntry(name string, exists bool) *packagejson.InfoCacheEntry { + entry := &packagejson.InfoCacheEntry{PackageDirectory: infoCacheDirectory(name), DirectoryExists: exists} + if exists { + entry.Contents = &packagejson.PackageJson{} + } + return entry +} + +func infoCacheKeys(cache *packagejson.InfoCache) map[string]*packagejson.InfoCacheEntry { + result := map[string]*packagejson.InfoCacheEntry{} + cache.Range(func(key tspath.PathKey, value *packagejson.InfoCacheEntry) bool { + result[key.AsString()] = value + return true + }) + return result +} + +func TestInfoCacheCloneIsolation(t *testing.T) { + t.Parallel() + + a := newInfoCacheEntry("a", true) + missing := newInfoCacheEntry("missing", false) + b := newInfoCacheEntry("b", true) + c := newInfoCacheEntry("c", true) + + gen1 := packagejson.NewInfoCache(tspath.CaseSensitive) + assert.Equal(t, gen1.Set(a.PackageDirectory, a), a) + assert.Equal(t, gen1.Set(missing.PackageDirectory, missing), missing) + + gen2 := gen1.Clone() + assert.Equal(t, gen2.Set(b.PackageDirectory, b), b) + gen3 := gen2.Clone() + assert.Equal(t, gen1.Set(c.PackageDirectory, c), c) + + assert.Assert(t, maps.Equal(infoCacheKeys(gen1), map[string]*packagejson.InfoCacheEntry{"/a": a, "/missing": missing, "/c": c})) + assert.Assert(t, maps.Equal(infoCacheKeys(gen2), map[string]*packagejson.InfoCacheEntry{"/a": a, "/missing": missing, "/b": b})) + assert.Assert(t, maps.Equal(infoCacheKeys(gen3), map[string]*packagejson.InfoCacheEntry{"/a": a, "/missing": missing, "/b": b})) + + assert.Assert(t, gen2.Get(c.PackageDirectory) == nil) + assert.Assert(t, gen1.Get(b.PackageDirectory) == nil) + + // Negative entries are cached like any other entry. + got := gen3.Get(missing.PackageDirectory) + assert.Equal(t, got, missing) + assert.Assert(t, !got.Exists()) +} + +func TestInfoCacheFirstWriterWins(t *testing.T) { + t.Parallel() + + first := newInfoCacheEntry("a", true) + second := newInfoCacheEntry("a", false) + + cache := packagejson.NewInfoCache(tspath.CaseSensitive) + assert.Equal(t, cache.Set(first.PackageDirectory, first), first) + assert.Equal(t, cache.Set(second.PackageDirectory, second), first, "overlay entry must win") + + clone := cache.Clone() + assert.Equal(t, clone.Set(second.PackageDirectory, second), first, "base entry must win") + assert.Equal(t, clone.Get(second.PackageDirectory), first) +} + +func TestInfoCacheRangeStopsEarly(t *testing.T) { + t.Parallel() + + cache := packagejson.NewInfoCache(tspath.CaseSensitive) + for _, name := range []string{"a", "b"} { + entry := newInfoCacheEntry(name, true) + cache.Set(entry.PackageDirectory, entry) + } + cache = cache.Clone() + for _, name := range []string{"c", "d"} { + entry := newInfoCacheEntry(name, true) + cache.Set(entry.PackageDirectory, entry) + } + + for stopAfter := 1; stopAfter <= 4; stopAfter++ { + visited := 0 + cache.Range(func(tspath.PathKey, *packagejson.InfoCacheEntry) bool { + visited++ + return visited < stopAfter + }) + assert.Equal(t, visited, stopAfter) + } +} + +func TestInfoCacheConcurrentUse(t *testing.T) { + t.Parallel() + + const writers = 8 + const keys = 200 + + base := packagejson.NewInfoCache(tspath.CaseSensitive) + for i := range keys / 2 { + entry := newInfoCacheEntry(fmt.Sprintf("k%d", i), true) + base.Set(entry.PackageDirectory, entry) + } + cache := base.Clone() + + winners := make([][]*packagejson.InfoCacheEntry, writers) + var clones sync.Map + var wg sync.WaitGroup + for w := range writers { + winners[w] = make([]*packagejson.InfoCacheEntry, keys) + wg.Go(func() { + for i := range keys { + entry := newInfoCacheEntry(fmt.Sprintf("k%d", i), w%2 == 0) + winners[w][i] = cache.Set(entry.PackageDirectory, entry) + assert.Equal(t, cache.Get(entry.PackageDirectory), winners[w][i]) + if i%50 == 0 { + clones.Store(fmt.Sprintf("%d-%d", w, i), cache.Clone()) + } + } + }) + } + wg.Wait() + + final := cache.Clone() + for i := range keys { + directory := infoCacheDirectory(fmt.Sprintf("k%d", i)) + winner := final.Get(directory) + assert.Assert(t, winner != nil) + for w := range writers { + assert.Equal(t, winners[w][i], winner, "key %d writer %d", i, w) + } + if i < keys/2 { + assert.Equal(t, winner, base.Get(directory)) + } + } + // Every snapshot taken during the writes agrees with the final winners. + clones.Range(func(_, value any) bool { + value.(*packagejson.InfoCache).Range(func(key tspath.PathKey, entry *packagejson.InfoCacheEntry) bool { + assert.Equal(t, entry, final.Get(entry.PackageDirectory), "key %s", key) + return true + }) + return true + }) +} diff --git a/tsc/internal/project/dirty/syncmap.go b/tsc/internal/project/dirty/syncmap.go index 6d75259d935f6..aa2f62d13bec2 100644 --- a/tsc/internal/project/dirty/syncmap.go +++ b/tsc/internal/project/dirty/syncmap.go @@ -223,6 +223,17 @@ func (m *SyncMap[K, V]) Load(key K) (*SyncMapEntry[K, V], bool) { return nil, false } +// LoadClean returns the base value for key without allocating an entry, +// provided the key has not been touched in the dirty map. Callers must +// fall back to Load when ok is false. +func (m *SyncMap[K, V]) LoadClean(key K) (value V, ok bool) { + if _, inDirty := m.dirty.Load(key); inDirty { + return value, false + } + value, ok = m.base[key] + return value, ok +} + func (m *SyncMap[K, V]) LoadOrStore(key K, value V) (*SyncMapEntry[K, V], bool) { // Check for existence in the base map first so the sync map access is atomic. if baseValue, ok := m.base[key]; ok { diff --git a/tsc/internal/project/dirty/syncmap_test.go b/tsc/internal/project/dirty/syncmap_test.go index bb2c5171088d1..ac9acda7ecac5 100644 --- a/tsc/internal/project/dirty/syncmap_test.go +++ b/tsc/internal/project/dirty/syncmap_test.go @@ -243,3 +243,38 @@ func TestSyncMapProxyFor(t *testing.T) { assert.Equal(t, "changed", entry.Value().data) }) } + +func TestSyncMapLoadClean(t *testing.T) { + t.Parallel() + + base := map[string]*testValue{"a": {data: "a"}, "b": {data: "b"}, "c": {data: "c"}} + m := NewSyncMap(base) + + value, ok := m.LoadClean("a") + assert.Assert(t, ok) + assert.Equal(t, value, base["a"]) + _, ok = m.LoadClean("missing") + assert.Assert(t, !ok) + + entry, _ := m.Load("a") + entry.Change(func(v *testValue) { v.data = "changed" }) + _, ok = m.LoadClean("a") + assert.Assert(t, !ok, "changed keys must fall back to Load") + entry, _ = m.Load("a") + assert.Equal(t, entry.Value().data, "changed") + assert.Equal(t, base["a"].data, "a") + + m.Delete("b") + _, ok = m.LoadClean("b") + assert.Assert(t, !ok, "deleted keys must fall back to Load") + _, ok = m.Load("b") + assert.Assert(t, !ok) + + m.LoadOrStore("d", &testValue{data: "d"}) + _, ok = m.LoadClean("d") + assert.Assert(t, !ok, "stored keys must fall back to Load") + + value, ok = m.LoadClean("c") + assert.Assert(t, ok) + assert.Equal(t, value, base["c"]) +} diff --git a/tsc/internal/project/overlayfs.go b/tsc/internal/project/overlayfs.go index 458b94d0ee831..9851115534724 100644 --- a/tsc/internal/project/overlayfs.go +++ b/tsc/internal/project/overlayfs.go @@ -319,6 +319,9 @@ func (fs *overlayFS) GetAccessibleEntries(directoryName tspath.RootedDirectoryPa return vfs.Entries{} } hostEntries := fs.host.GetAccessibleEntries(directoryName) + if len(directory) == 0 { + return hostEntries + } entries := vfs.Entries{ Files: slices.Clone(hostEntries.Files), Directories: slices.Clone(hostEntries.Directories), diff --git a/tsc/internal/project/snapshotfs.go b/tsc/internal/project/snapshotfs.go index 3e95fb3709231..9697950ad4e32 100644 --- a/tsc/internal/project/snapshotfs.go +++ b/tsc/internal/project/snapshotfs.go @@ -321,6 +321,9 @@ func (s *snapshotFSBuilder) deleteCacheEntry(entry *dirty.SyncMapEntry[tspath.Pa } func (s *snapshotFSBuilder) FileExists(fileName tspath.RootedFilePath, path tspath.PathKey) bool { + if file, ok := s.cacheFiles.LoadClean(path); ok && file != nil && file.MatchesDiskText() { + return true + } if entry, ok := s.cacheFiles.Load(path); ok { val := entry.Value() if val == nil { @@ -334,6 +337,9 @@ func (s *snapshotFSBuilder) FileExists(fileName tspath.RootedFilePath, path tspa } func (s *snapshotFSBuilder) GetFileByPath(fileName tspath.RootedFilePath, path tspath.PathKey) FileHandle { + if file, ok := s.cacheFiles.LoadClean(path); ok && file != nil && file.MatchesDiskText() { + return file + } if entry, ok := s.cacheFiles.Load(path); ok { return s.reloadEntryIfNeeded(entry) } @@ -351,6 +357,9 @@ func (s *snapshotFSBuilder) GetAccessibleEntries(path tspath.RootedDirectoryPath return lowerEntries } return mergeCachedDirectoryEntries(lowerEntries, directory.Value(), func(key tspath.PathKey, childName string) bool { + if file, clean := s.cacheFiles.LoadClean(key); clean { + return file != nil || s.fs.FileExists(path.ResolveFile(childName)) + } entry, cached := s.cacheFiles.Load(key) return cached && entry.Value() != nil || s.fs.FileExists(path.ResolveFile(childName)) }, s.fs.CaseSensitivity()) @@ -803,7 +812,16 @@ func (fs *sourceFS) Track(fileName tspath.RootedFilePath) { if !fs.tracking { return } - fs.seenFiles.Store(fs.caseSensitivity.PathKey(fileName.AsPath()), fileName) + fs.track(fileName, fs.caseSensitivity.PathKey(fileName.AsPath())) +} + +func (fs *sourceFS) track(fileName tspath.RootedFilePath, path tspath.PathKey) { + if !fs.tracking { + return + } + if _, ok := fs.seenFiles.Load(path); !ok { + fs.seenFiles.Store(path, fileName) + } } func (fs *sourceFS) SeenFile(path tspath.PathKey) bool { @@ -842,7 +860,7 @@ func (fs *sourceFS) GetFile(fileName tspath.RootedFilePath) FileHandle { } func (fs *sourceFS) GetFileByPath(fileName tspath.RootedFilePath, path tspath.PathKey) FileHandle { - fs.Track(fileName) + fs.track(fileName, path) return fs.source.GetFileByPath(fileName, path) } @@ -857,8 +875,9 @@ func (fs *sourceFS) DirectoryExists(path tspath.RootedDirectoryPath) bool { // FileExists implements vfs.FS. func (fs *sourceFS) FileExists(path tspath.RootedFilePath) bool { - fs.Track(path) - return fs.source.FileExists(path, fs.caseSensitivity.PathKey(path.AsPath())) + pathKey := fs.caseSensitivity.PathKey(path.AsPath()) + fs.track(path, pathKey) + return fs.source.FileExists(path, pathKey) } // GetAccessibleEntries implements vfs.FS. diff --git a/tsc/internal/tsoptions/extensionsiblings_test.go b/tsc/internal/tsoptions/extensionsiblings_test.go new file mode 100644 index 0000000000000..e2bbace2d7a7a --- /dev/null +++ b/tsc/internal/tsoptions/extensionsiblings_test.go @@ -0,0 +1,210 @@ +package tsoptions + +import ( + "math/rand/v2" + "slices" + "testing" + + "github.com/microsoft/TypeScript/tsc/internal/collections" + "github.com/microsoft/TypeScript/tsc/internal/core" + "github.com/microsoft/TypeScript/tsc/internal/tspath" + "github.com/microsoft/TypeScript/tsc/internal/vfs" + "github.com/microsoft/TypeScript/tsc/internal/vfs/vfsmatch" + "github.com/microsoft/TypeScript/tsc/internal/vfs/vfstest" + "gotest.tools/v3/assert" +) + +func TestExtensionSiblingKey(t *testing.T) { + t.Parallel() + + files := []string{ + "/p/a.ts", "/p/A.TS", "/p/Foo.d.ts", "/p/Foo.D.TS", "/p/x.d.mts", "/p/x.d.cts", + "/p/x.d.json.ts", "/p/Comp.vue", "/p/Comp.d.vue.ts", "/p/noext", "/p/dir.v1/a.js", + "/p/İstanbul.tsx", "c:/P/a.mjs", "//server/share/A.cjs", "file:///c:/P/a.ts", + "^/untitled/ts-nul-authority/Untitled-1.ts", "^/~ts-uri~/custom/authority/A.ts", + } + exts := slices.Concat(core.Flatten(tspath.AllSupportedExtensionsWithJson), []string{".vue", ".d.vue.ts"}) + for _, caseSensitivity := range []tspath.CaseSensitivity{tspath.CaseSensitive, tspath.CaseInsensitive} { + keyMapper := func(f tspath.RootedFilePath) tspath.PathKey { return caseSensitivity.PathKey(f.AsPath()) } + for _, name := range files { + file := tspath.RootedFilePathFromNormalized(name) + siblings := newExtensionSiblings(file, keyMapper(file), keyMapper) + for _, ext := range exts { + assert.Equal(t, siblings.siblingKey(ext), keyMapper(file.ChangeExtension(ext)), "%v %s -> %s", caseSensitivity, name, ext) + } + } + } + + insensitive := func(f tspath.RootedFilePath) tspath.PathKey { return tspath.CaseInsensitive.PathKey(f.AsPath()) } + for name, derived := range map[string]bool{ + "/p/A.ts": true, + "/p/A.TS": false, // canonical key's extension differs from the file's + "/p/noext": false, + "^/~ts-uri~/custom/authority/a.ts": false, + } { + file := tspath.RootedFilePathFromNormalized(name) + siblings := newExtensionSiblings(file, insensitive(file), insensitive) + assert.Equal(t, siblings.keyMapper == nil, derived, name) + } +} + +// TestGetFileNamesExtensionPriority compares wildcard expansion against a +// reference that computes every sibling with keyMapper(file.ChangeExtension(ext)). +func TestGetFileNamesExtensionPriority(t *testing.T) { + t.Parallel() + + basePath := tspath.RootedDirectoryPath("/p") + pattern := func(s string) tspath.PathPattern { return tspath.ToPathPattern("/p/" + s) } + + t.Run("examples", func(t *testing.T) { + t.Parallel() + cases := []struct { + files []string + include []string + literals []string + allowJs bool + insens bool + expected []string + }{ + {files: []string{"a.ts", "a.d.ts"}, include: []string{"**/*"}, expected: []string{"/p/a.ts"}}, + {files: []string{"a.ts", "a.tsx"}, include: []string{"**/*"}, expected: []string{"/p/a.ts"}}, + {files: []string{"a.ts", "a.js"}, include: []string{"**/*"}, allowJs: true, expected: []string{"/p/a.ts"}}, + {files: []string{"a.js", "a.ts"}, include: []string{"**/*.js", "**/*.ts"}, allowJs: true, expected: []string{"/p/a.ts"}}, + {files: []string{"a.d.ts", "a.js"}, include: []string{"**/*"}, allowJs: true, expected: []string{"/p/a.d.ts", "/p/a.js"}}, + {files: []string{"a.mts", "a.d.mts", "a.cts", "a.d.cts"}, include: []string{"**/*"}, expected: []string{"/p/a.cts", "/p/a.d.cts", "/p/a.mts"}}, + {files: []string{"A.ts", "a.d.ts"}, include: []string{"**/*"}, insens: true, expected: []string{"/p/A.ts"}}, + {files: []string{"A.ts", "a.d.ts"}, include: []string{"**/*"}, expected: []string{"/p/A.ts", "/p/a.d.ts"}}, + {files: []string{"a.d.ts", "a.ts"}, include: []string{"**/*"}, literals: []string{"a.d.ts"}, expected: []string{"/p/a.d.ts", "/p/a.ts"}}, + {files: []string{"a.ts", "a.d.ts"}, include: []string{"**/*"}, literals: []string{"a.ts"}, expected: []string{"/p/a.ts"}}, + } + for _, c := range cases { + got, want := runGetFileNames(c.files, c.include, c.literals, c.allowJs, c.insens, nil, pattern, basePath) + slices.Sort(got) + slices.Sort(want) + assert.DeepEqual(t, got, c.expected) + assert.DeepEqual(t, want, c.expected) + } + }) + + t.Run("differential", func(t *testing.T) { + t.Parallel() + stems := []string{"a", "A", "sub/a", "b.d", "İ"} + exts := []string{".ts", ".tsx", ".d.ts", ".js", ".jsx", ".mts", ".d.mts", ".cts", ".d.cts", ".mjs", ".cjs", ".vue", ".d.vue.ts", ".TS", ".D.TS"} + includes := [][]string{{"**/*"}, {"**/*.js", "**/*.ts"}, {"**/*.d.ts", "**/*.ts", "**/*.tsx"}, {"**/*.vue", "**/*"}, {"**/*.mjs", "**/*.mts", "**/*.d.mts"}} + rng := rand.New(rand.NewPCG(1, 2)) + for i := range 3000 { + var files []string + for _, stem := range stems { + for _, ext := range exts { + if rng.IntN(4) == 0 { + files = append(files, stem+ext) + } + } + } + var literals []string + if len(files) > 0 && rng.IntN(3) == 0 { + literals = []string{files[rng.IntN(len(files))]} + } + var extra []string + if i%2 == 0 { + extra = []string{".vue"} + } + got, want := runGetFileNames(files, includes[i%len(includes)], literals, i%3 != 0, i%4 < 2, extra, pattern, basePath) + assert.DeepEqual(t, got, want) + } + }) +} + +func runGetFileNames( + files, include, literals []string, + allowJs, insensitive bool, + extra []string, + pattern func(string) tspath.PathPattern, + basePath tspath.RootedDirectoryPath, +) (got []string, want []string) { + caseSensitivity := tspath.CaseSensitive + if insensitive { + caseSensitivity = tspath.CaseInsensitive + } + contents := map[string]string{} + seen := map[tspath.PathKey]bool{} + for _, f := range files { + key := caseSensitivity.PathKey(tspath.RootedPathFromNormalized("/p/" + f)) + if !seen[key] { + seen[key] = true + contents["/p/"+f] = "" + } + } + host := vfstest.FromMap(contents, caseSensitivity) + specs := configFileSpecs{} + for _, l := range literals { + specs.validatedFilesSpec = append(specs.validatedFilesSpec, tspath.ToFileSpec(l)) + specs.validatedFileNames = append(specs.validatedFileNames, basePath.ResolveFile(l)) + } + for _, i := range include { + specs.validatedIncludeSpecs = append(specs.validatedIncludeSpecs, pattern(i)) + } + options := &core.CompilerOptions{AllowJs: core.BoolToTristate(allowJs)} + result, _ := getFileNamesFromConfigSpecs(specs, basePath, options, host, extra) + for _, f := range result { + got = append(got, f.AsString()) + } + for _, f := range referenceGetFileNames(specs, basePath, options, host, extra) { + want = append(want, f.AsString()) + } + return got, want +} + +// referenceGetFileNames is getFileNamesFromConfigSpecs without JSON handling, +// computing sibling keys by changing the extension and remapping. +func referenceGetFileNames(specs configFileSpecs, basePath tspath.RootedDirectoryPath, options *core.CompilerOptions, host vfs.FS, extra []string) []tspath.RootedFilePath { + keyMapper := func(f tspath.RootedFilePath) tspath.PathKey { return host.CaseSensitivity().PathKey(f.AsPath()) } + var literalFileMap, wildcardFileMap collections.OrderedMap[tspath.PathKey, tspath.RootedFilePath] + supported := GetSupportedExtensions(options, extra) + for _, f := range specs.validatedFileNames { + literalFileMap.Set(keyMapper(f), f) + } + hasFile := func(f tspath.RootedFilePath) bool { + k := keyMapper(f) + return literalFileMap.Has(k) || wildcardFileMap.Has(k) + } + matchingGroups := func(file tspath.RootedFilePath) []string { + var group []string + for _, g := range supported { + if file.ExtensionIsOneOf(g) { + group = append(group, g...) + } + } + return group + } + for _, file := range vfsmatch.ReadDirectory(host, basePath, core.Flatten(supported), specs.validatedExcludeSpecs, specs.validatedIncludeSpecs, vfsmatch.UnlimitedDepth) { + group := matchingGroups(file) + higher := false + for _, ext := range group { + if file.ExtensionIs(ext) && (ext != tspath.ExtensionTs || !file.ExtensionIs(tspath.ExtensionDts)) { + break + } + if hasFile(file.ChangeExtension(ext)) { + if ext == tspath.ExtensionDts && (file.ExtensionIs(tspath.ExtensionJs) || file.ExtensionIs(tspath.ExtensionJsx)) { + continue + } + higher = true + break + } + } + if higher { + continue + } + for _, ext := range slices.Backward(group) { + if file.ExtensionIs(ext) { + break + } + wildcardFileMap.Delete(keyMapper(file.ChangeExtension(ext))) + } + key := keyMapper(file) + if !literalFileMap.Has(key) && !wildcardFileMap.Has(key) { + wildcardFileMap.Set(key, file) + } + } + return slices.Concat(slices.Collect(literalFileMap.Values()), slices.Collect(wildcardFileMap.Values())) +} diff --git a/tsc/internal/tsoptions/tsconfigparsing.go b/tsc/internal/tsoptions/tsconfigparsing.go index 4241f27529dd2..67afe06db9b41 100644 --- a/tsc/internal/tsoptions/tsconfigparsing.go +++ b/tsc/internal/tsoptions/tsconfigparsing.go @@ -1723,57 +1723,71 @@ func getSubstitutedStringArrayWithConfigDirTemplate[T ~string](list []T, basePat return nil } +// extensionSiblings computes keys of files that differ from file only by extension. +type extensionSiblings struct { + file tspath.RootedFilePath + key tspath.PathKey + ext string + // keyMapper is set when sibling keys cannot be derived from the file's key. + keyMapper func(tspath.RootedFilePath) tspath.PathKey +} + +func newExtensionSiblings(file tspath.RootedFilePath, key tspath.PathKey, keyMapper func(tspath.RootedFilePath) tspath.PathKey) extensionSiblings { + ext := file.Extension() + if ext == "" || tspath.IsEncodedDynamicFileName(string(file)) || !strings.HasSuffix(key.AsString(), ext) { + return extensionSiblings{file: file, keyMapper: keyMapper} + } + return extensionSiblings{file: file, key: key, ext: ext} +} + +func (s *extensionSiblings) siblingKey(ext string) tspath.PathKey { + if s.keyMapper != nil { + return s.keyMapper(s.file.ChangeExtension(ext)) + } + return s.key.ReplaceCanonicalExtension(s.ext, ext) +} + // hasFileWithHigherPriorityExtension determines whether a literal or wildcard file has already been included that has a higher extension priority. -// file is the path to the file. -func hasFileWithHigherPriorityExtension(file tspath.RootedFilePath, extensions [][]string, hasFile func(fileName tspath.RootedFilePath) bool) bool { - var extensionGroup []string +func hasFileWithHigherPriorityExtension(siblings *extensionSiblings, extensions [][]string, hasFile func(key tspath.PathKey) bool) bool { + file := siblings.file for _, group := range extensions { - if file.ExtensionIsOneOf(group) { - extensionGroup = append(extensionGroup, group...) + if !file.ExtensionIsOneOf(group) { + continue } - } - if len(extensionGroup) == 0 { - return false - } - for _, ext := range extensionGroup { - // d.ts files match with .ts extension and with case sensitive sorting the file order for same files with ts tsx and dts extension is - // d.ts, .ts, .tsx in that order so we need to handle tsx and dts of same same name case here and in remove files with same extensions - // So dont match .d.ts files with .ts extension - if file.ExtensionIs(ext) && (ext != tspath.ExtensionTs || !file.ExtensionIs(tspath.ExtensionDts)) { - return false - } - if hasFile(file.ChangeExtension(ext)) { - if ext == tspath.ExtensionDts && (file.ExtensionIs(tspath.ExtensionJs) || file.ExtensionIs(tspath.ExtensionJsx)) { - // LEGACY BEHAVIOR: An off-by-one bug somewhere in the extension priority system for wildcard module loading allowed declaration - // files to be loaded alongside their js(x) counterparts. We regard this as generally undesirable, but retain the behavior to - // prevent breakage. - continue + for _, ext := range group { + // d.ts files match with .ts extension and with case sensitive sorting the file order for same files with ts tsx and dts extension is + // d.ts, .ts, .tsx in that order so we need to handle tsx and dts of same same name case here and in remove files with same extensions + // So dont match .d.ts files with .ts extension + if file.ExtensionIs(ext) && (ext != tspath.ExtensionTs || !file.ExtensionIs(tspath.ExtensionDts)) { + return false + } + if hasFile(siblings.siblingKey(ext)) { + if ext == tspath.ExtensionDts && (file.ExtensionIs(tspath.ExtensionJs) || file.ExtensionIs(tspath.ExtensionJsx)) { + // LEGACY BEHAVIOR: An off-by-one bug somewhere in the extension priority system for wildcard module loading allowed declaration + // files to be loaded alongside their js(x) counterparts. We regard this as generally undesirable, but retain the behavior to + // prevent breakage. + continue + } + return true } - return true } } return false } // Removes files included via wildcard expansion with a lower extension priority that have already been included. -// file is the path to the file. -func removeWildcardFilesWithLowerPriorityExtension[V any](file tspath.RootedFilePath, wildcardFiles *collections.OrderedMap[tspath.PathKey, V], extensions [][]string, keyMapper func(value tspath.RootedFilePath) tspath.PathKey) { - var extensionGroup []string - for _, group := range extensions { - if file.ExtensionIsOneOf(group) { - extensionGroup = append(extensionGroup, group...) +func removeWildcardFilesWithLowerPriorityExtension[V any](siblings *extensionSiblings, wildcardFiles *collections.OrderedMap[tspath.PathKey, V], extensions [][]string) { + file := siblings.file + for _, group := range slices.Backward(extensions) { + if !file.ExtensionIsOneOf(group) { + continue } - } - if extensionGroup == nil { - return - } - for _, ext := range slices.Backward(extensionGroup) { - - if file.ExtensionIs(ext) { - return + for _, ext := range slices.Backward(group) { + if file.ExtensionIs(ext) { + return + } + wildcardFiles.Delete(siblings.siblingKey(ext)) } - lowerPriorityPath := keyMapper(file.ChangeExtension(ext)) - wildcardFiles.Delete(lowerPriorityPath) } } @@ -1820,9 +1834,17 @@ func getFileNamesFromConfigSpecs( literalFileMap.Set(keyMapper(validatedFileNames[i]), validatedFileNames[i]) } + hasFile := func(key tspath.PathKey) bool { + return literalFileMap.Has(key) || wildcardFileMap.Has(key) + } + var jsonOnlyIncludeMatchers *vfsmatch.SpecMatcher if len(validatedIncludeSpecs) > 0 { - files := vfsmatch.ReadDirectory(host, basePath, core.Flatten(supportedExtensionsWithJsonIfResolveJsonModule), validatedExcludeSpecs, validatedIncludeSpecs, vfsmatch.UnlimitedDepth) + readDirectory := vfsmatch.ReadDirectory[tspath.PathPattern] + if options.SingleThreaded.IsTrue() { + readDirectory = vfsmatch.ReadDirectorySingleThreaded[tspath.PathPattern] + } + files := readDirectory(host, basePath, core.Flatten(supportedExtensionsWithJsonIfResolveJsonModule), validatedExcludeSpecs, validatedIncludeSpecs, vfsmatch.UnlimitedDepth) for _, file := range files { if file.ExtensionIs(tspath.ExtensionJson) { if jsonOnlyIncludeMatchers == nil { @@ -1849,18 +1871,16 @@ func getFileNamesFromConfigSpecs( // This handles cases where we may encounter both .ts and // .d.ts (or .js if "allowJs" is enabled) in the same // directory when they are compilation outputs. - if hasFileWithHigherPriorityExtension(file, supportedExtensions, func(fileName tspath.RootedFilePath) bool { - canonicalFileName := keyMapper(fileName) - return literalFileMap.Has(canonicalFileName) || wildcardFileMap.Has(canonicalFileName) - }) { + key := keyMapper(file) + siblings := newExtensionSiblings(file, key, keyMapper) + if hasFileWithHigherPriorityExtension(&siblings, supportedExtensions, hasFile) { continue } // We may have included a wildcard path with a lower priority // extension due to the user-defined order of entries in the // "include" array. If there is a lower priority extension in the // same directory, we should remove it. - removeWildcardFilesWithLowerPriorityExtension(file, &wildcardFileMap, supportedExtensions, keyMapper) - key := keyMapper(file) + removeWildcardFilesWithLowerPriorityExtension(&siblings, &wildcardFileMap, supportedExtensions) if !literalFileMap.Has(key) && !wildcardFileMap.Has(key) { wildcardFileMap.Set(key, file) } diff --git a/tsc/internal/tsoptions/tsconfigparsing_test.go b/tsc/internal/tsoptions/tsconfigparsing_test.go index 586902b5a24c5..b25c40dc98f52 100644 --- a/tsc/internal/tsoptions/tsconfigparsing_test.go +++ b/tsc/internal/tsoptions/tsconfigparsing_test.go @@ -7,7 +7,9 @@ import ( "path/filepath" "slices" "strings" + "sync/atomic" "testing" + "time" "github.com/microsoft/TypeScript/tsc/internal/ast" "github.com/microsoft/TypeScript/tsc/internal/collections" @@ -25,6 +27,7 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/tspath" "github.com/microsoft/TypeScript/tsc/internal/vfs" "github.com/microsoft/TypeScript/tsc/internal/vfs/osvfs" + "github.com/microsoft/TypeScript/tsc/internal/vfs/wrapvfs" "gotest.tools/v3/assert" ) @@ -1886,3 +1889,42 @@ func TestExtendedConfigConfigDirPathsAreNotCached(t *testing.T) { paths := parseConfig("/packages/b/tsconfig.json").CompilerOptions().Paths assert.DeepEqual(t, paths.GetOrZero("@pkg/*"), []string{"/packages/b/src/*"}) } + +func TestParseJsonConfigFileContentSingleThreadedReadsDirectoriesSerially(t *testing.T) { + t.Parallel() + + files := map[string]string{} + for i := range 16 { + files[fmt.Sprintf("/project/dir%d/index.ts", i)] = "" + } + base := tsoptionstest.NewVFS(files, tspath.CaseSensitive) + var inFlight, maxInFlight atomic.Int32 + fs := wrapvfs.Wrap(base, wrapvfs.Replacements{ + GetAccessibleEntries: func(path tspath.RootedDirectoryPath) vfs.Entries { + n := inFlight.Add(1) + for { + current := maxInFlight.Load() + if n <= current || maxInFlight.CompareAndSwap(current, n) { + break + } + } + // Give concurrent readers time to overlap. + time.Sleep(time.Millisecond) + defer inFlight.Add(-1) + return base.GetAccessibleEntries(path) + }, + }) + + parsed := tsoptions.ParseJsonConfigFileContent( + map[string]any{}, + fs, + "/project", + &core.CompilerOptions{SingleThreaded: core.TSTrue}, + "/project/tsconfig.json", + nil, /*resolutionStack*/ + nil, /*extendedConfigCache*/ + ) + + assert.Equal(t, len(parsed.FileNames()), 16) + assert.Equal(t, maxInFlight.Load(), int32(1)) +} diff --git a/tsc/internal/tspath/leading_dot_segments_test.go b/tsc/internal/tspath/leading_dot_segments_test.go new file mode 100644 index 0000000000000..1c1408fb70a45 --- /dev/null +++ b/tsc/internal/tspath/leading_dot_segments_test.go @@ -0,0 +1,96 @@ +package tspath + +import ( + "strings" + "testing" + + "gotest.tools/v3/assert" +) + +// TestResolveLeadingDotSegments checks the leading "./" and "../" fast path in +// ResolveFile and ResolveDirectory against concatenation followed by full +// normalization, over every combination of a few segment kinds and roots. +func TestResolveLeadingDotSegments(t *testing.T) { + t.Parallel() + + directories := []string{ + "/", + "/a", + "/a/b/c", + "c:/", + "c:/a/b", + "//server/share", + "//server/share/a/b", + "file:///c:/a/b", + "http://server/base/a", + "^/untitled/ts-nul-authority/a/b", + } + segments := []string{".", "..", "x", "y.ts", ""} + + var specs []string + var build func(prefix []string) + build = func(prefix []string) { + if len(prefix) > 0 { + specs = append(specs, strings.Join(prefix, "/")) + } + if len(prefix) == 5 { + return + } + for _, segment := range segments { + build(append(prefix, segment)) + } + } + build(nil) + + fastPathHits := 0 + for _, directory := range directories { + d := RootedDirectoryPathFromNormalized(directory) + for _, spec := range specs { + if !isNormalizedSlashesRelativePath(spec) { + continue + } + if _, ok := appendPathWithLeadingDotSegments(d, spec); ok { + fastPathHits++ + } + expected, expectedPanic := capturePanic(func() string { + return string(rootedFilePathFromResolved(getNormalizedAbsolutePathFromNormalizedSlashes(appendPathToDirectory(d, spec)))) + }) + file, filePanic := capturePanic(func() string { return d.ResolveFile(spec).AsString() }) + dir, dirPanic := capturePanic(func() string { return d.ResolveDirectory(spec).AsString() }) + assert.Equal(t, file, expected, "ResolveFile(%q, %q)", directory, spec) + assert.Equal(t, filePanic, expectedPanic, "ResolveFile(%q, %q) panic", directory, spec) + assert.Equal(t, dir, expected, "ResolveDirectory(%q, %q)", directory, spec) + assert.Equal(t, dirPanic, expectedPanic, "ResolveDirectory(%q, %q) panic", directory, spec) + } + } + assert.Assert(t, fastPathHits > 1000, "fast path hits: %d", fastPathHits) + + root := RootedDirectoryPathFromNormalized("/a/b") + for spec, ok := range map[string]bool{ + "./x/y.ts": true, + "../x": true, + "./../.././x": true, + "../../x": true, + "../../../x": false, // ascends past the root + "./x/../y": false, // dot segment after the leading ones + "x/./y": false, + "./": false, + "..": false, + "./x//y": false, + "./x/y/": false, // trailing separator needs normalization + "../x/y/./": false, + "../x/y/../z/": false, + } { + _, got := appendPathWithLeadingDotSegments(root, spec) + assert.Equal(t, got, ok, "%q", spec) + } +} + +func capturePanic(f func() string) (result string, panicked bool) { + defer func() { + if recover() != nil { + panicked = true + } + }() + return f(), false +} diff --git a/tsc/internal/tspath/pathkey.go b/tsc/internal/tspath/pathkey.go index 11b7e23f9f6a4..9b85d5bfacf0d 100644 --- a/tsc/internal/tspath/pathkey.go +++ b/tsc/internal/tspath/pathkey.go @@ -117,6 +117,19 @@ func (p PathKey) AppendCanonicalSuffix(suffix string) PathKey { return PathKeyFromCanonical(string(p) + suffix) } +// ReplaceCanonicalExtension replaces the suffix oldExt, which p must end with, +// with newExt. newExt must already be canonical under the CaseSensitivity used +// to construct p. +func (p PathKey) ReplaceCanonicalExtension(oldExt string, newExt string) PathKey { + if !strings.HasSuffix(string(p), oldExt) || len(oldExt) == len(p) { + panic("path key must end with the extension being replaced") + } + if strings.ContainsAny(newExt, `/\`) { + panic("extension must not contain a directory separator") + } + return PathKey(string(p)[:len(p)-len(oldExt)] + newExt) +} + // SplitAtCanonicalComponent finds component and returns the path before it and // the path through it. Both results retain the key's canonical casing. func (p PathKey) SplitAtCanonicalComponent(component string) (before PathKey, through PathKey, ok bool) { diff --git a/tsc/internal/tspath/rooted_path.go b/tsc/internal/tspath/rooted_path.go index b61a2d73cb983..9eda86dbf0619 100644 --- a/tsc/internal/tspath/rooted_path.go +++ b/tsc/internal/tspath/rooted_path.go @@ -738,6 +738,9 @@ func (d RootedDirectoryPath) ResolveFile(path string) RootedFilePath { return rootedFilePathFromResolved(appendPathToDirectory(d, path)) } if isNormalizedSlashesRelativePath(path) { + if resolved, ok := appendPathWithLeadingDotSegments(d, path); ok { + return rootedFilePathFromResolved(resolved) + } return rootedFilePathFromResolved(getNormalizedAbsolutePathFromNormalizedSlashes(appendPathToDirectory(d, path))) } return ToRootedFilePath(path, d) @@ -795,6 +798,9 @@ func (d RootedDirectoryPath) ResolveDirectory(path string) RootedDirectoryPath { return RootedDirectoryPath(rootedFilePathFromResolved(appendPathToDirectory(d, path))) } if isNormalizedSlashesRelativePath(path) { + if resolved, ok := appendPathWithLeadingDotSegments(d, path); ok { + return RootedDirectoryPath(rootedFilePathFromResolved(resolved)) + } return RootedDirectoryPath(rootedFilePathFromResolved(getNormalizedAbsolutePathFromNormalizedSlashes(appendPathToDirectory(d, path)))) } return ToRootedDirectoryPath(path, d) @@ -850,6 +856,33 @@ func isNormalizedSlashesRelativePath(path string) bool { strings.IndexByte(path, '\\') < 0 } +// appendPathWithLeadingDotSegments resolves paths like "./a/b" or "../../a/b" +// by walking up directory before appending, avoiding an intermediate string. +// It reports false if path has dot segments beyond its leading ones or +// would ascend past the root. +func appendPathWithLeadingDotSegments(directory RootedDirectoryPath, path string) (string, bool) { + dir := directory.AsString() + rest := path + for { + if strings.HasPrefix(rest, "./") { + rest = rest[2:] + } else if strings.HasPrefix(rest, "../") { + parent := getDirectoryPathFromNormalized(dir) + if parent == dir { + return "", false + } + dir = parent + rest = rest[3:] + } else { + break + } + } + if len(rest) == len(path) || !canAppendPathWithoutNormalization(rest) { + return "", false + } + return appendPathToDirectory(RootedDirectoryPath(dir), rest), true +} + func appendPathToDirectory(directory RootedDirectoryPath, path string) string { if HasTrailingDirectorySeparator(directory.AsString()) { return directory.AsString() + path diff --git a/tsc/internal/vfs/vfsmatch/prefetch.go b/tsc/internal/vfs/vfsmatch/prefetch.go new file mode 100644 index 0000000000000..ebf33d43eb1cc --- /dev/null +++ b/tsc/internal/vfs/vfsmatch/prefetch.go @@ -0,0 +1,205 @@ +package vfsmatch + +import ( + "sync" + + "github.com/microsoft/TypeScript/tsc/internal/tspath" + "github.com/microsoft/TypeScript/tsc/internal/vfs" +) + +type prefetchedDirectory struct { + realPath tspath.RootedDirectoryPath + entries vfs.Entries +} + +// prefetchDirectories reads the directories that globVisitor.visit is expected +// to walk using up to maxWorkers concurrent readers, so the subsequent ordered +// walk does not wait on sequential directory reads. The walk remains the source +// of truth and reads anything missing from the result itself. +// +// The calling goroutine is the first reader; additional workers start only +// while queued directories outnumber idle workers, so scanning a leaf directory +// starts none. Each directory is queued at most once. Base paths are queued +// first with the full depth, so a walk from an ancestor base path stops at a +// descendant base path rather than reading its subtree again. +// +// Symlinked directories are not followed. Every prefetched directory is then +// reached through its only non-aliased spelling below a base path, so the +// prefetch cannot cycle and its result does not depend on scheduling. The walk +// reads symlinked subtrees itself, exactly as it would without a prefetch. +// +// If the host panics, the remaining work is abandoned and the panic is +// re-raised on the calling goroutine. +func prefetchDirectories( + host vfs.FS, + directoryMatcher *globMatcher, + caseSensitivity tspath.CaseSensitivity, + basePaths []tspath.RootedDirectoryPath, + depth int, + maxWorkers int, +) map[tspath.RootedDirectoryPath]prefetchedDirectory { + p := &directoryPrefetcher{ + host: host, + directoryMatcher: directoryMatcher, + caseSensitivity: caseSensitivity, + scheduled: make(map[tspath.RootedDirectoryPath]struct{}), + results: make(map[tspath.RootedDirectoryPath]prefetchedDirectory), + } + p.cond.L = &p.mu + p.maxWorkers = maxWorkers + // The calling goroutine is the first worker. + p.started = 1 + p.idle = 1 + for _, basePath := range basePaths { + p.schedule(prefetchItem{path: basePath, depth: depth}) + } + p.mu.Lock() + p.startWorkers() + p.mu.Unlock() + + p.work() + p.wg.Wait() + + if p.panicked { + panic(p.panicValue) + } + return p.results +} + +type prefetchItem struct { + path tspath.RootedDirectoryPath + // realPath is empty when it must be resolved with the host. + realPath tspath.RootedDirectoryPath + depth int +} + +type directoryPrefetcher struct { + host vfs.FS + directoryMatcher *globMatcher + caseSensitivity tspath.CaseSensitivity + + mu sync.Mutex + cond sync.Cond + wg sync.WaitGroup + queue []prefetchItem + // pending counts queued and in-progress items. + pending int + scheduled map[tspath.RootedDirectoryPath]struct{} + maxWorkers int + started int + idle int + panicked bool + panicValue any + results map[tspath.RootedDirectoryPath]prefetchedDirectory +} + +// schedule queues item unless its directory has already been queued. +// Callers must hold p.mu or be the only goroutine. +func (p *directoryPrefetcher) schedule(item prefetchItem) { + if _, ok := p.scheduled[item.path]; ok { + return + } + p.scheduled[item.path] = struct{}{} + p.queue = append(p.queue, item) + p.pending++ +} + +// startWorkers starts workers for queued items that idle workers cannot take. +// Workers count as idle from when they are started until they take an item. +// Callers must hold p.mu. +func (p *directoryPrefetcher) startWorkers() { + for p.started < p.maxWorkers && len(p.queue) > p.idle { + p.started++ + p.idle++ + p.wg.Go(p.work) + } +} + +func (p *directoryPrefetcher) work() { + p.mu.Lock() + defer p.mu.Unlock() + for { + for len(p.queue) == 0 && p.pending > 0 && !p.panicked { + p.cond.Wait() + } + if p.pending == 0 || p.panicked { + p.idle-- + p.cond.Broadcast() + return + } + item := p.queue[len(p.queue)-1] + p.queue = p.queue[:len(p.queue)-1] + p.idle-- + + p.mu.Unlock() + dir, children, panicked, panicValue := p.fetch(item) + p.mu.Lock() + + p.idle++ + p.pending-- + if panicked { + if !p.panicked { + p.panicked = true + p.panicValue = panicValue + } + p.cond.Broadcast() + continue + } + p.results[item.path] = dir + for _, child := range children { + p.schedule(child) + } + if len(p.queue) > 0 { + p.startWorkers() + p.cond.Broadcast() + } else if p.pending == 0 { + p.cond.Broadcast() + } + } +} + +func (p *directoryPrefetcher) fetch(item prefetchItem) (dir prefetchedDirectory, children []prefetchItem, panicked bool, panicValue any) { + defer func() { + if r := recover(); r != nil { + panicked, panicValue = true, r + } + }() + + realPath := item.realPath + if realPath == "" { + realPath = tspath.RootedDirectoryPathFromPath(p.host.Realpath(item.path.AsPath())) + } + entries := p.host.GetAccessibleEntries(item.path) + dir = prefetchedDirectory{realPath: realPath, entries: entries} + + depth := item.depth + if depth != UnlimitedDepth { + depth-- + if depth == 0 { + return dir, nil, false, nil + } + } + + absPrefix := tspath.EnsureTrailingDirectorySeparator(item.path.AsString()) + for _, name := range entries.Directories { + if !p.directoryMatcher.matchesDirectoryParts(absPrefix, name) { + continue + } + childPath := item.path.ResolveDirectory(name) + childRealPath := realPath.ResolveDirectory(name) + if entries.Symlinks != nil { + if _, isSymlink := entries.Symlinks[name]; isSymlink { + continue + } + } else { + // The host does not report symlinks; resolve the child to find out. + resolved := tspath.RootedDirectoryPathFromPath(p.host.Realpath(childPath.AsPath())) + if p.caseSensitivity.PathKey(resolved.AsPath()) != p.caseSensitivity.PathKey(childRealPath.AsPath()) { + continue + } + childRealPath = resolved + } + children = append(children, prefetchItem{path: childPath, realPath: childRealPath, depth: depth}) + } + return dir, children, false, nil +} diff --git a/tsc/internal/vfs/vfsmatch/prefetch_test.go b/tsc/internal/vfs/vfsmatch/prefetch_test.go new file mode 100644 index 0000000000000..73d400347fd20 --- /dev/null +++ b/tsc/internal/vfs/vfsmatch/prefetch_test.go @@ -0,0 +1,307 @@ +package vfsmatch + +import ( + "fmt" + "maps" + "runtime" + "sync" + "sync/atomic" + "testing" + + "github.com/microsoft/TypeScript/tsc/internal/tspath" + "github.com/microsoft/TypeScript/tsc/internal/vfs" + "github.com/microsoft/TypeScript/tsc/internal/vfs/vfstest" + "github.com/microsoft/TypeScript/tsc/internal/vfs/wrapvfs" + "gotest.tools/v3/assert" +) + +// countingFS counts directory reads per path. When symlinks is non-nil, entries +// report which child directories are symlinks, like the OS file system does; +// otherwise Symlinks is nil and callers must use Realpath. +type countingFS struct { + vfs.FS + mu sync.Mutex + reads map[string]int + symlinks map[string]bool +} + +func newCountingFS(files map[string]any, reportSymlinks bool) *countingFS { + c := &countingFS{reads: map[string]int{}} + if reportSymlinks { + c.symlinks = map[string]bool{} + } + base := vfstest.FromMap(toMapFiles(files, c.symlinks), tspath.CaseSensitive) + c.FS = wrapvfs.Wrap(base, wrapvfs.Replacements{ + GetAccessibleEntries: func(path tspath.RootedDirectoryPath) vfs.Entries { + c.mu.Lock() + c.reads[path.AsString()]++ + c.mu.Unlock() + entries := base.GetAccessibleEntries(path) + entries.Symlinks = nil + if c.symlinks != nil { + entries.Symlinks = map[string]struct{}{} + for _, name := range entries.Directories { + if c.symlinks[tspath.CombinePaths(path.AsString(), name)] { + entries.Symlinks[name] = struct{}{} + } + } + } + return entries + }, + }) + return c +} + +// symlinkMarker declares a directory symlink in a test file map. +type symlinkMarker struct{ target string } + +func toMapFiles(files map[string]any, symlinks map[string]bool) map[string]any { + result := make(map[string]any, len(files)) + for path, file := range files { + if link, ok := file.(symlinkMarker); ok { + result[path] = vfstest.Symlink(link.target) + if symlinks != nil { + symlinks[path] = true + } + continue + } + result[path] = file + } + return result +} + +func (c *countingFS) readCounts() map[string]int { + c.mu.Lock() + defer c.mu.Unlock() + return maps.Clone(c.reads) +} + +func TestPrefetchMatchesSerialWalk(t *testing.T) { + t.Parallel() + + type testCase struct { + name string + // path defaults to /root. + path tspath.RootedDirectoryPath + files map[string]any + includes []string + excludes []string + depth int + // reads are the expected directory reads with the prefetch enabled; each + // directory spelling must be read at most once. + reads map[string]int + // prefetchOnly lists entries of reads that the serial walk alone does not read. + prefetchOnly []string + } + cases := []testCase{ + { + name: "symlink cycle", + files: map[string]any{ + "/root/file.ts": "", + "/root/a/file.ts": "", + "/root/a/b": symlinkMarker{"/root/a"}, + }, + includes: []string{"**/*"}, + depth: UnlimitedDepth, + reads: map[string]int{"/root": 1, "/root/a": 1}, + }, + { + // The walk reaches z through the alias a first, so it reads a's subtree + // itself and skips z. The prefetch does not follow symlinks and has read + // z's subtree under its real spelling; no spelling is read twice. + name: "alias sorts before target", + files: map[string]any{ + "/root/a": symlinkMarker{"/root/z"}, + "/root/z/f.ts": "", + "/root/z/sub/g.ts": "", + "/root/z/sub/x/h.ts": "", + }, + includes: []string{"**/*"}, + depth: UnlimitedDepth, + reads: map[string]int{ + "/root": 1, + "/root/a": 1, "/root/a/sub": 1, "/root/a/sub/x": 1, + "/root/z": 1, "/root/z/sub": 1, "/root/z/sub/x": 1, + }, + prefetchOnly: []string{"/root/z", "/root/z/sub", "/root/z/sub/x"}, + }, + { + name: "alias sorts after target", + files: map[string]any{ + "/root/a/f.ts": "", + "/root/a/sub/g.ts": "", + "/root/b": symlinkMarker{"/root/a"}, + }, + includes: []string{"**/*"}, + depth: UnlimitedDepth, + reads: map[string]int{"/root": 1, "/root/a": 1, "/root/a/sub": 1}, + }, + { + name: "depth, excludes, and multiple base paths", + files: map[string]any{ + "/root/src/a/b/c/deep.ts": "", + "/root/src/a/shallow.ts": "", + "/root/src/skip/x.ts": "", + "/root/src/node_modules/m.ts": "", + "/other/lib/l.ts": "", + "/other/lib/inner/i.ts": "", + }, + includes: []string{"src/**/*", "../other/lib/**/*"}, + excludes: []string{"src/skip"}, + depth: 3, + reads: map[string]int{ + "/root": 1, "/root/src": 1, "/root/src/a": 1, "/root/src/skip": 1, + "/other/lib": 1, "/other/lib/inner": 1, + }, + }, + { + // The base path /root/src is also below the include base path /root. + name: "include in ancestor directory", + path: "/root/src", + files: map[string]any{ + "/root/top.ts": "", + "/root/src/a/f.ts": "", + "/root/src/a/b/g.ts": "", + "/root/lib/sub/h.ts": "", + }, + includes: []string{"../**/*"}, + depth: UnlimitedDepth, + reads: map[string]int{ + "/root": 1, "/root/lib": 1, "/root/lib/sub": 1, + "/root/src": 1, "/root/src/a": 1, "/root/src/a/b": 1, + }, + }, + { + // Base /root/src is read with the full depth even though base /root + // reaches it with less remaining depth. + name: "include in ancestor directory with depth", + path: "/root/src", + files: map[string]any{ + "/root/src/a/f.ts": "", + "/root/src/a/b/g.ts": "", + "/root/src/a/b/c/h.ts": "", + }, + includes: []string{"**/*", "../**/*"}, + depth: 3, + reads: map[string]int{ + "/root": 1, "/root/src": 1, "/root/src/a": 1, "/root/src/a/b": 1, + }, + }, + } + + for _, tc := range cases { + if tc.path == "" { + tc.path = "/root" + } + for _, reportSymlinks := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/reportSymlinks=%v", tc.name, reportSymlinks), func(t *testing.T) { + t.Parallel() + serialFS := newCountingFS(tc.files, reportSymlinks) + expected := matchFileNamesWithPrefetch(tc.path, []string{".ts"}, tc.excludes, tc.includes, tc.depth, serialFS, 0) + serialReads := maps.Clone(tc.reads) + for _, path := range tc.prefetchOnly { + delete(serialReads, path) + } + assert.DeepEqual(t, serialFS.readCounts(), serialReads) + + for _, workers := range []int{1, 4} { + fs := newCountingFS(tc.files, reportSymlinks) + got := matchFileNamesWithPrefetch(tc.path, []string{".ts"}, tc.excludes, tc.includes, tc.depth, fs, workers) + assert.DeepEqual(t, got, expected) + assert.DeepEqual(t, fs.readCounts(), tc.reads) + } + }) + } + } +} + +func TestPrefetchBoundsConcurrency(t *testing.T) { //nolint:paralleltest // Measures process-wide goroutine counts + const width = 2000 + const workers = 4 + + files := map[string]any{} + for i := range width { + files[fmt.Sprintf("/root/d%d/f.ts", i)] = "" + } + base := vfstest.FromMap(files, tspath.CaseSensitive) + + release := make(chan struct{}) + var inFlight, maxInFlight, blocked atomic.Int32 + host := wrapvfs.Wrap(base, wrapvfs.Replacements{ + GetAccessibleEntries: func(path tspath.RootedDirectoryPath) vfs.Entries { + n := inFlight.Add(1) + for { + m := maxInFlight.Load() + if n <= m || maxInFlight.CompareAndSwap(m, n) { + break + } + } + defer inFlight.Add(-1) + if path != "/root" { + blocked.Add(1) + <-release + } + return base.GetAccessibleEntries(path) + }, + }) + + before := runtime.NumGoroutine() + done := make(chan []tspath.RootedFilePath) + go func() { + done <- matchFileNamesWithPrefetch("/root", []string{".ts"}, nil, []string{"**/*"}, UnlimitedDepth, host, workers) + }() + + for blocked.Load() < workers { + runtime.Gosched() + } + // All workers are blocked on the host; nothing else may have been started. + assert.Equal(t, inFlight.Load(), int32(workers)) + assert.Assert(t, runtime.NumGoroutine()-before <= workers+1, "goroutines: %d", runtime.NumGoroutine()-before) + + close(release) + got := <-done + assert.Equal(t, len(got), width) + assert.Equal(t, maxInFlight.Load(), int32(workers)) +} + +func TestPrefetchLeafScanStartsNoWorkers(t *testing.T) { //nolint:paralleltest // Measures process-wide goroutine counts + base := vfstest.FromMap(map[string]any{"/root/f.ts": "", "/root/g.ts": ""}, tspath.CaseSensitive) + var goroutinesDuringRead int + host := wrapvfs.Wrap(base, wrapvfs.Replacements{ + GetAccessibleEntries: func(path tspath.RootedDirectoryPath) vfs.Entries { + goroutinesDuringRead = runtime.NumGoroutine() + return base.GetAccessibleEntries(path) + }, + }) + before := runtime.NumGoroutine() + got := matchFileNamesWithPrefetch("/root", []string{".ts"}, nil, []string{"**/*"}, UnlimitedDepth, host, 64) + assert.Equal(t, len(got), 2) + assert.Equal(t, goroutinesDuringRead, before) +} + +func TestPrefetchRethrowsHostPanic(t *testing.T) { + t.Parallel() + + base := vfstest.FromMap(map[string]any{ + "/root/a/f.ts": "", + "/root/b/g.ts": "", + }, tspath.CaseSensitive) + var panicked atomic.Bool + host := wrapvfs.Wrap(base, wrapvfs.Replacements{ + GetAccessibleEntries: func(path tspath.RootedDirectoryPath) vfs.Entries { + if path == "/root/a" && panicked.CompareAndSwap(false, true) { + panic("host failure") + } + return base.GetAccessibleEntries(path) + }, + }) + + // The host fails only once, so a walk that retried the directory after + // swallowing the prefetch panic would succeed. + var recovered any + func() { + defer func() { recovered = recover() }() + matchFileNamesWithPrefetch("/root", []string{".ts"}, nil, []string{"**/*"}, UnlimitedDepth, host, 4) + }() + assert.Equal(t, recovered, "host failure") +} diff --git a/tsc/internal/vfs/vfsmatch/vfsmatch.go b/tsc/internal/vfs/vfsmatch/vfsmatch.go index 882d83bb0bede..967d1f6fcba30 100644 --- a/tsc/internal/vfs/vfsmatch/vfsmatch.go +++ b/tsc/internal/vfs/vfsmatch/vfsmatch.go @@ -2,6 +2,7 @@ package vfsmatch import ( "math" + "runtime" "slices" "strings" "unicode/utf8" @@ -31,10 +32,18 @@ type PathPatternInput interface { string | tspath.PathPattern } +// ReadDirectory returns the files below path matching the include and exclude +// patterns. Directories may be read concurrently. func ReadDirectory[T PathPatternInput](host vfs.FS, path tspath.RootedDirectoryPath, extensions []string, excludes []T, includes []T, depth int) []tspath.RootedFilePath { return matchFileNames(path, extensions, excludes, includes, depth, host) } +// ReadDirectorySingleThreaded is ReadDirectory with all directory reads on the +// calling goroutine. +func ReadDirectorySingleThreaded[T PathPatternInput](host vfs.FS, path tspath.RootedDirectoryPath, extensions []string, excludes []T, includes []T, depth int) []tspath.RootedFilePath { + return matchFileNamesWithPrefetch(path, extensions, excludes, includes, depth, host, 0) +} + // IsImplicitGlob checks if a path component is implicitly a glob. // An "includes" path "foo" is implicitly a glob "foo/** /*" (without the space) if its last component has no extension, // and does not contain any glob characters itself. @@ -587,6 +596,7 @@ type globVisitor struct { caseSensitivity tspath.CaseSensitivity visited collections.Set[tspath.PathKey] results [][]tspath.RootedFilePath + prefetched map[tspath.RootedDirectoryPath]prefetchedDirectory } // visit walks a directory tree, collecting files that match the glob patterns. @@ -595,9 +605,12 @@ type globVisitor struct { // called to resolve symlinks. func (v *globVisitor) visit(absolutePath tspath.RootedDirectoryPath, depth int, resolvedRealPath tspath.RootedDirectoryPath) { // Detect symlink cycles + prefetched, hasPrefetched := v.prefetched[absolutePath] var realPath tspath.RootedDirectoryPath if resolvedRealPath != "" { realPath = resolvedRealPath + } else if hasPrefetched { + realPath = prefetched.realPath } else { realPath = tspath.RootedDirectoryPathFromPath(v.host.Realpath(absolutePath.AsPath())) } @@ -607,7 +620,12 @@ func (v *globVisitor) visit(absolutePath tspath.RootedDirectoryPath, depth int, } v.visited.Add(canonicalPath) - entries := v.host.GetAccessibleEntries(absolutePath) + var entries vfs.Entries + if hasPrefetched { + entries = prefetched.entries + } else { + entries = v.host.GetAccessibleEntries(absolutePath) + } absPrefix := tspath.EnsureTrailingDirectorySeparator(absolutePath.AsString()) @@ -647,6 +665,12 @@ func (v *globVisitor) visit(absolutePath tspath.RootedDirectoryPath, depth int, } func matchFileNames[T PathPatternInput](path tspath.RootedDirectoryPath, extensions []string, excludes, includes []T, depth int, host vfs.FS) []tspath.RootedFilePath { + return matchFileNamesWithPrefetch(path, extensions, excludes, includes, depth, host, runtime.GOMAXPROCS(0)) +} + +// matchFileNamesWithPrefetch is matchFileNames with prefetchWorkers concurrent +// directory readers; zero disables the prefetch. +func matchFileNamesWithPrefetch[T PathPatternInput](path tspath.RootedDirectoryPath, extensions []string, excludes, includes []T, depth int, host vfs.FS, prefetchWorkers int) []tspath.RootedFilePath { caseSensitivity := host.CaseSensitivity() fileMatcher := newGlobMatcher(includes, excludes, path, caseSensitivity, UsageFiles) @@ -661,7 +685,11 @@ func matchFileNames[T PathPatternInput](path tspath.RootedDirectoryPath, extensi results: make([][]tspath.RootedFilePath, max(len(fileMatcher.includes), 1)), } - for _, basePath := range getBasePaths(path, includes, caseSensitivity) { + basePaths := getBasePaths(path, includes, caseSensitivity) + if prefetchWorkers > 0 { + v.prefetched = prefetchDirectories(host, directoryMatcher, caseSensitivity, basePaths, depth, prefetchWorkers) + } + for _, basePath := range basePaths { v.visit(basePath, depth, "") }