C#: Improve package download logic.

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs

@@ -24,22 +24,79 @@ internal sealed partial class FeedManager : IDisposable
         private readonly FileProvider fileProvider;
         private readonly DependabotProxy? dependabotProxy;
         private readonly DependencyDirectory emptyPackageDirectory;
+        private readonly ImmutableHashSet<string> privateRegistryFeeds;
 
-        public ImmutableHashSet<string> PrivateRegistryFeeds { get; }
         public bool HasPrivateRegistryFeeds { get; }
         public bool CheckNugetFeedResponsiveness { get; } = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.CheckNugetFeedResponsiveness);
 
+        private readonly Lazy<ImmutableHashSet<string>> lazyExplicitFeeds;
+
+        /// <summary>
+        /// Gets the list of NuGet feeds that are explicitly configured
+        /// - NuGet configuration files.
+        /// - Private package registries that are configured for C#.
+        /// </summary>
+        public ImmutableHashSet<string> ExplicitFeeds => lazyExplicitFeeds.Value;
+
+        private readonly Lazy<ImmutableHashSet<string>> lazyAllFeeds;
+
+        /// <summary>
+        /// Gets the list of all NuGet feeds that are configured in the environment. That is
+        /// - Explicit feeds
+        /// - Inherited feeds from the machine and environment (if not explicitly disabled by a
+        /// root directory NuGet configuration).
+        /// </summary>
+        public ImmutableHashSet<string> AllFeeds => lazyAllFeeds.Value;
+
+        /// <summary>
+        /// Gets the list of inherited NuGet feeds that are configured in the environment.
+        /// </summary>
+        public ImmutableHashSet<string> InheritedFeeds => AllFeeds.Except(ExplicitFeeds).ToImmutableHashSet();
+
+        private readonly Lazy<(bool, ImmutableHashSet<string>)> lazyReachableExplicitFeeds;
+
+        /// <summary>
+        /// Gets whether there was a timeout when checking the reachability of the explicitly configured NuGet feeds.
+        /// </summary>
+        public bool ExplicitFeedTimeout => lazyReachableExplicitFeeds.Value.Item1;
+
+        /// <summary>
+        /// Gets the list of reachable NuGet feeds that are explicitly configured.
+        /// </summary>
+        public ImmutableHashSet<string> ReachableExplicitFeeds => lazyReachableExplicitFeeds.Value.Item2;
+
+        private readonly Lazy<ImmutableHashSet<string>> lazyReachableFeeds;
+        /// <summary>
+        /// Gets the list of reachable NuGet feeds that are configured in the environment.
+        /// </summary>
+        public ImmutableHashSet<string> ReachableFeeds => lazyReachableFeeds.Value;
+
         public FeedManager(ILogger logger, IDotNet dotnet, DependabotProxy? dependabotProxy, FileProvider fileProvider)
         {
             this.logger = logger;
             this.dotnet = dotnet;
             this.dependabotProxy = dependabotProxy;
             this.fileProvider = fileProvider;
-            PrivateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? [];
-            HasPrivateRegistryFeeds = PrivateRegistryFeeds.Count > 0;
+            privateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? [];
+            HasPrivateRegistryFeeds = privateRegistryFeeds.Count > 0;
             emptyPackageDirectory = new DependencyDirectory("empty", "empty package", logger);
+
+            lazyExplicitFeeds = new Lazy<ImmutableHashSet<string>>(GetExplicitFeeds);
+            lazyAllFeeds = new Lazy<ImmutableHashSet<string>>(GetAllFeeds);
+            lazyReachableExplicitFeeds = new Lazy<(bool, ImmutableHashSet<string>)>(() =>
+            {
+                var timeout = CheckSpecifiedFeeds(ExplicitFeeds, out var reachableFeeds);
+                return (timeout, reachableFeeds);
+            });
+            lazyReachableFeeds = new Lazy<ImmutableHashSet<string>>(() =>
+            {
+                // Inherited feeds should only be used, if they are indeed reachable (as they may be environment specific).
+                CheckSpecifiedFeeds(InheritedFeeds, out var reachableInheritedFeeds);
+                return ReachableExplicitFeeds.Union(reachableInheritedFeeds).ToImmutableHashSet();
+            });
         }
 
+
         private string? GetDirectoryName(string path)
         {
             try
@@ -88,20 +145,20 @@ private IEnumerable<string> GetFeedsFromFolder(string folderPath) =>
         private IEnumerable<string> GetFeedsFromNugetConfig(string nugetConfigPath) =>
             GetFeeds(() => dotnet.GetNugetFeeds(nugetConfigPath));
 
-        private string FeedsToRestoreArgument(IEnumerable<string> feeds)
+        public string FeedsToRestoreArgument(IEnumerable<string> feeds, string sourceArgumentPrefix)
         {
-            // If there are no feeds, we want to override any default feeds that `dotnet restore` would use by passing a dummy source argument.
+            // If there are no feeds, we want to override any default feeds that `restore` would use by passing a dummy source argument.
             if (!feeds.Any())
             {
-                return $" -s \"{emptyPackageDirectory.DirInfo.FullName}\"";
+                return $" {sourceArgumentPrefix} \"{emptyPackageDirectory.DirInfo.FullName}\"";
             }
 
             // Add package sources. If any are present, they override all sources specified in
             // the configuration file(s).
             var feedArgs = new StringBuilder();
             foreach (var feed in feeds)
             {
-                feedArgs.Append($" -s \"{feed}\"");
+                feedArgs.Append($" {sourceArgumentPrefix} \"{feed}\"");
             }
 
             return feedArgs.ToString();
@@ -112,31 +169,44 @@ private string FeedsToRestoreArgument(IEnumerable<string> feeds)
         /// (1) Use the feeds we get from `dotnet nuget list source`
         /// (2) Use private registries, if they are configured
         /// </summary>
-        /// <param name="path">Path to project/solution</param>
-        /// <param name="reachableFeeds">The set of reachable NuGet feeds.</param>
-        /// <returns>A string representing the NuGet sources argument for the restore command.</returns>
-        public string? MakeRestoreSourcesArgument(string path, HashSet<string> reachableFeeds)
+        /// <param name="path">Path to project/solution/packages.config</param>
+        /// <returns>The list of NuGet feeds to use for this restore.</returns>
+        public IEnumerable<string> FeedsToUse(string path)
         {
-            // Do not construct a set of explicit NuGet sources to use for restore.
-            if (!CheckNugetFeedResponsiveness && !HasPrivateRegistryFeeds)
-            {
-                return null;
-            }
-
             // Find the path specific feeds.
             var folder = GetDirectoryName(path);
             var feedsToConsider = folder is not null ? GetFeedsFromFolder(folder).ToHashSet() : new HashSet<string>();
 
             if (HasPrivateRegistryFeeds)
             {
-                feedsToConsider.UnionWith(PrivateRegistryFeeds);
+                feedsToConsider.UnionWith(privateRegistryFeeds);
             }
 
             var feedsToUse = CheckNugetFeedResponsiveness
-                ? feedsToConsider.Where(reachableFeeds.Contains)
+                ? feedsToConsider.Where(ReachableFeeds.Contains)
                 : feedsToConsider;
 
-            return FeedsToRestoreArgument(feedsToUse);
+            return feedsToUse;
+        }
+
+        /// <summary>
+        /// Constructs the list of NuGet sources to use for dotnet restore.
+        /// (1) Use the feeds we get from `dotnet nuget list source`
+        /// (2) Use private registries, if they are configured
+        /// </summary>
+        /// <param name="path">Path to project/solution</param>
+        /// <returns>A string representing the NuGet sources argument for the restore command.</returns>
+        public string? MakeDotnetRestoreSourcesArgument(string path)
+        {
+            // Do not construct a set of explicit NuGet sources to use for restore.
+            if (!CheckNugetFeedResponsiveness && !HasPrivateRegistryFeeds)
+            {
+                return null;
+            }
+
+            var feedsToUse = FeedsToUse(path);
+
+            return FeedsToRestoreArgument(feedsToUse, "-s");
         }
 
         private (int initialTimeout, int tryCount) GetFeedRequestSettings(bool isFallback)
@@ -256,7 +326,7 @@ private HashSet<string> GetExcludedFeeds()
         /// True if there is a timeout when trying to reach the feeds (excluding any feeds that are configured
         /// to be excluded from the check) or false otherwise.
         /// </returns>
-        public bool CheckSpecifiedFeeds(HashSet<string> feeds, out HashSet<string> reachableFeeds)
+        private bool CheckSpecifiedFeeds(ImmutableHashSet<string> feeds, out ImmutableHashSet<string> reachableFeeds)
         {
             // Exclude any feeds from the feed check that are configured by the corresponding environment variable.
             // These feeds are always assumed to be reachable.
@@ -272,10 +342,10 @@ public bool CheckSpecifiedFeeds(HashSet<string> feeds, out HashSet<string> reach
                 return true;
             }).ToHashSet();
 
-            reachableFeeds = GetReachableNuGetFeeds(feedsToCheck, isFallback: false, out var isTimeout).ToHashSet();
+            var reachable = GetReachableNuGetFeeds(feedsToCheck, isFallback: false, out var isTimeout);
 
             // Always consider feeds excluded for the reachability check as reachable.
-            reachableFeeds.UnionWith(feeds.Where(feed => excludedFeeds.Contains(feed)));
+            reachableFeeds = reachable.Union(feeds.Where(feed => excludedFeeds.Contains(feed))).ToImmutableHashSet();
 
             return isTimeout;
         }
@@ -327,7 +397,7 @@ private List<string> GetReachableNuGetFeeds(HashSet<string> feedsToCheck, bool i
             return reachableFeeds;
         }
 
-        public List<string> GetReachableFallbackNugetFeeds(HashSet<string>? feedsFromNugetConfigs)
+        public List<string> GetReachableFallbackNugetFeeds(ImmutableHashSet<string>? feedsFromNugetConfigs)
         {
             var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet();
             if (fallbackFeeds.Count == 0)
@@ -350,7 +420,7 @@ public List<string> GetReachableFallbackNugetFeeds(HashSet<string>? feedsFromNug
             return GetReachableNuGetFeeds(fallbackFeeds, isFallback: true, out var _);
         }
 
-        public (HashSet<string> explicitFeeds, HashSet<string> allFeeds) GetAllFeeds()
+        private ImmutableHashSet<string> GetExplicitFeeds()
         {
             var nugetConfigs = fileProvider.NugetConfigs;
 
@@ -372,14 +442,21 @@ public List<string> GetReachableFallbackNugetFeeds(HashSet<string>? feedsFromNug
             // in addition to the ones that are configured in `nuget.config` files.
             if (HasPrivateRegistryFeeds)
             {
-                logger.LogInfo($"Found {PrivateRegistryFeeds.Count} private registry feeds configured for C#: {string.Join(", ", PrivateRegistryFeeds.OrderBy(f => f))}");
-                explicitFeeds.UnionWith(PrivateRegistryFeeds);
+                logger.LogInfo($"Found {privateRegistryFeeds.Count} private registry feeds configured for C#: {string.Join(", ", privateRegistryFeeds.OrderBy(f => f))}");
+                explicitFeeds.UnionWith(privateRegistryFeeds);
             }
 
+            return explicitFeeds.ToImmutableHashSet();
+        }
+
+        private ImmutableHashSet<string> GetAllFeeds()
+        {
+            var nugetConfigs = fileProvider.NugetConfigs;
+
             HashSet<string> allFeeds = [];
 
             // Add all explicitFeeds to the set of all feeds.
-            allFeeds.UnionWith(explicitFeeds);
+            allFeeds.UnionWith(ExplicitFeeds);
 
             // Obtain the list of feeds from the root source directory.
             // If a NuGet file is present it will be respected, otherwise we will just get the machine/environment specific feeds.
@@ -399,7 +476,7 @@ public List<string> GetReachableFallbackNugetFeeds(HashSet<string>? feedsFromNug
 
             logger.LogInfo($"Found {allFeeds.Count} NuGet feeds (with inherited ones) in nuget.config files: {string.Join(", ", allFeeds.OrderBy(f => f))}");
 
-            return (explicitFeeds, allFeeds);
+            return allFeeds.ToImmutableHashSet();
         }
 
         [GeneratedRegex(@"^E\s(.*)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]