C#: Use the RegistryUrls from the dependabot proxy as default feeds is replaces-base is set, otherwise use nuget.org.

Author: michaelnebel

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs

@@ -17,6 +17,7 @@ internal sealed partial class FeedManager : IDisposable
         private readonly IFileProvider fileProvider;
         private readonly DependencyDirectory emptyPackageDirectory;
         private readonly ImmutableHashSet<string> privateRegistryFeeds;
+        private readonly ImmutableHashSet<string> defaultFeeds;
         private readonly IFeedManagerIO feedManagerIo;
 
         /// <summary>
@@ -72,6 +73,13 @@ internal sealed partial class FeedManager : IDisposable
         /// </summary>
         public ImmutableHashSet<string> ReachableFallbackFeeds => lazyReachableFallbackFeeds.Value;
 
+        private readonly Lazy<ImmutableHashSet<string>> lazyReachableDefaultFeeds;
+
+        /// <summary>
+        /// Gets the list of reachable default NuGet feeds.
+        /// </summary>
+        public ImmutableHashSet<string> ReachableDefaultFeeds => lazyReachableDefaultFeeds.Value;
+
         public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotProxy, IFileProvider fileProvider, IFeedManagerIO feedManagerIo)
         {
             this.logger = logger;
@@ -80,6 +88,9 @@ public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotP
             this.feedManagerIo = feedManagerIo;
             privateRegistryFeeds = dependabotProxy?.RegistryURLs ?? [];
             HasPrivateRegistryFeeds = privateRegistryFeeds.Count > 0;
+            defaultFeeds = dependabotProxy?.RegistryBaseURLs.Any() == true
+                ? dependabotProxy.RegistryBaseURLs
+                : [PublicNugetOrgFeed];
             emptyPackageDirectory = new DependencyDirectory("empty", "empty package", logger);
 
             lazyExplicitFeeds = new Lazy<ImmutableHashSet<string>>(GetExplicitFeeds);
@@ -96,6 +107,7 @@ public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotP
                 var reachableFallbackFeeds = GetReachableFallbackNugetFeeds();
                 return reachableFallbackFeeds.ToImmutableHashSet();
             });
+            lazyReachableDefaultFeeds = new Lazy<ImmutableHashSet<string>>(() => CheckSpecifiedFeeds(defaultFeeds));
         }
 
         public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotProxy, IFileProvider fileProvider)
@@ -266,22 +278,6 @@ private ImmutableHashSet<string> CheckSpecifiedFeeds(ImmutableHashSet<string> fe
             return reachable.Union(feeds.Where(feed => excludedFeeds.Contains(feed))).ToImmutableHashSet();
         }
 
-        /// <summary>
-        /// Return true if the default NuGet feed is reachable, false otherwise.
-        /// If the reachability check is disabled, this method will always return true.
-        /// </summary>
-        /// <returns>True if the default NuGet feed is reachable, false otherwise.</returns>
-        public bool IsDefaultFeedReachable()
-        {
-            if (CheckNugetFeedResponsiveness)
-            {
-                var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false);
-                return feedManagerIo.IsFeedReachable(PublicNugetOrgFeed, initialTimeout, tryCount);
-            }
-
-            return true;
-        }
-
         /// <summary>
         /// Tests which of the feeds given by <paramref name="feedsToCheck"/> are reachable.
         /// </summary>
@@ -315,8 +311,8 @@ private List<string> GetReachableFallbackNugetFeeds()
             var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet();
             if (fallbackFeeds.Count == 0)
             {
-                fallbackFeeds.Add(PublicNugetOrgFeed);
-                logger.LogInfo($"No fallback NuGet feeds specified. Adding default feed: {PublicNugetOrgFeed}");
+                fallbackFeeds.UnionWith(defaultFeeds);
+                logger.LogInfo($"No fallback NuGet feeds specified. Adding default feeds: {string.Join(", ", defaultFeeds.OrderBy(f => f))}");
 
                 var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback);
                 logger.LogInfo($"Adding feeds from nuget.config to fallback restore: {shouldAddNugetConfigFeeds}");

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs

@@ -67,10 +67,6 @@ private class NugetExeWrapper : IPackagesConfigRestore
 
             private bool IsWindows => SystemBuildActions.Instance.IsWindows();
 
-            private bool? isDefaultFeedReachable;
-            private bool IsDefaultFeedReachable =>
-                isDefaultFeedReachable ??= feedManager.IsDefaultFeedReachable();
-
             /// <summary>
             /// Create the package manager for a specified source tree.
             /// </summary>
@@ -169,15 +165,15 @@ private bool TryRestoreNugetPackage(string packagesConfig)
 
                 List<string> sourcesArgument = [];
                 var feedsToUse = feedManager.FeedsToUse(packagesConfig).ToList();
-                var useDefaultFeed = feedsToUse.Count == 0 && IsDefaultFeedReachable;
+                var useDefaultFeeds = feedsToUse.Count == 0 && feedManager.ReachableDefaultFeeds.Count > 0;
 
                 // Explicitly construct the sources to be used for the restore command when checking feed
-                // responsiveness, using private registries, or falling back to nuget.org.
-                if (feedManager.CheckNugetFeedResponsiveness || feedManager.HasPrivateRegistryFeeds || useDefaultFeed)
+                // responsiveness, using private registries, or falling back to default feeds.
+                if (feedManager.CheckNugetFeedResponsiveness || feedManager.HasPrivateRegistryFeeds || useDefaultFeeds)
                 {
-                    if (useDefaultFeed)
+                    if (useDefaultFeeds)
                     {
-                        feedsToUse.Add(FeedManager.PublicNugetOrgFeed);
+                        feedsToUse.AddRange(feedManager.ReachableDefaultFeeds);
                     }
                     var restoreFeeds = feedManager.RestoreFeeds(feedsToUse);
                     sourcesArgument = restoreFeeds.SelectMany<string, string>(feed => ["-Source", feed]).ToList();

csharp/extractor/Semmle.Extraction.Tests/FeedManager.cs

@@ -1,10 +1,11 @@
 using Xunit;
 using System;
 using System.Collections.Generic;
+using System.Collections.Immutable;
 using System.IO;
 using System.Linq;
+using System.Security.Cryptography.X509Certificates;
 using Semmle.Extraction.CSharp.DependencyFetching;
-using System.Collections.Immutable;
 
 namespace Semmle.Extraction.Tests
 {
@@ -14,7 +15,18 @@ public class DependabotProxyStub : IDependabotProxy
         public ImmutableHashSet<string> RegistryURLs { get; } = ["https://example.com/registry1", "https://example.com/registry2"];
         public ImmutableHashSet<string> RegistryBaseURLs { get; } = [];
         public string? CertificatePath { get; } = null;
-        public System.Security.Cryptography.X509Certificates.X509Certificate2? Certificate { get; } = null;
+        public X509Certificate2? Certificate { get; } = null;
+
+        public void Dispose() { }
+    }
+
+    public class DependabotProxyStubWithBaseUrls : IDependabotProxy
+    {
+        public string Address { get; } = "";
+        public ImmutableHashSet<string> RegistryURLs { get; } = ["https://example.com/registry1", "https://example.com/registry2", "https://example.com/base1", "https://example.com/base2"];
+        public ImmutableHashSet<string> RegistryBaseURLs { get; } = ["https://example.com/base1", "https://example.com/base2"];
+        public string? CertificatePath { get; } = null;
+        public X509Certificate2? Certificate { get; } = null;
 
         public void Dispose() { }
     }
@@ -185,5 +197,45 @@ public void TestFeedsToUse()
                 "https://feed.from/folder1"
             ], feedsToUse);
         }
+
+        [Fact]
+        public void TestDefaultFeeds1()
+        {
+            // Setup
+            var feedManager = MakeFeedManager();
+
+            // Execute
+            var reachableDefault = feedManager.ReachableDefaultFeeds;
+
+            // Verify
+            Assert.Equal([
+                "https://api.nuget.org/v3/index.json"
+            ], reachableDefault);
+        }
+
+        [Fact]
+        public void TestDefaultFeeds2()
+        {
+            // Setup
+            var logger = new LoggerStub();
+            var dotnet = new DotNetStub([], [], [], []);
+            var dependabotProxy = new DependabotProxyStubWithBaseUrls();
+            var fileProvider = new FileProviderStub();
+            var feedManagerIo = new FeedManagerIOStub(["https://example.com/registry2", "https://example.com/base1"]);
+            var feedManager = new FeedManager(logger, dotnet, dependabotProxy, fileProvider, feedManagerIo);
+
+            // Execute
+            var reachableDefault = feedManager.ReachableDefaultFeeds;
+            var reachableFallback = feedManager.ReachableFallbackFeeds;
+
+            // Verify
+            Assert.Equal([
+                "https://example.com/base2"
+            ], reachableDefault);
+            Assert.Equal([
+                "https://example.com/registry1",
+                "https://example.com/base2"
+            ], reachableFallback);
+        }
     }
 }