From f562c9ae08a88486d828b0c35fc14c5b80040420 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enes=20Ya=C4=9F=C4=B1z?= Date: Mon, 19 Oct 2026 21:10:34 +0300 Subject: [PATCH] fix(packages): keep and find the GitHub repository of NuGet and pub.dev packages 14 of 325 cached packages had no GitHub repository, so "View on GitHub" was missing and their notes were never looked for on GitHub. Three causes: - A metadata refresh overwrote the stored repository with the registry's value. NuGet's metadata never carries one, so a repository found while scraping was wiped on the next refresh (Newtonsoft.Json: source "GitHub", repository empty). - NuGet metadata ignored the nuspec. It now reads from the flat container (Newtonsoft.Json, System.Text.Json, uuid) and falls back to a GitHub project URL (Bogus, Duende). - A pub.dev page links no repository when the pubspec names GitHub only as its homepage (hive). The pub.dev API's pubspec repository or homepage is used then. Co-Authored-By: Claude Opus 5.5 --- .../Services/PackageCacheService.cs | 3 +- .../Services/NugetService.cs | 38 ++++++++++++++++++- .../Services/PubDevService.cs | 27 +++++++++++++ .../Smoke/LiveRegistrySmokeTests.cs | 3 +- 4 files changed, 67 insertions(+), 4 deletions(-) diff --git a/src/backend/PkgDiff.Application/Services/PackageCacheService.cs b/src/backend/PkgDiff.Application/Services/PackageCacheService.cs index a605a55..894ffbb 100644 --- a/src/backend/PkgDiff.Application/Services/PackageCacheService.cs +++ b/src/backend/PkgDiff.Application/Services/PackageCacheService.cs @@ -263,7 +263,8 @@ private async Task SaveFreshMetadataToCache( cached.Authors = metadata.Authors; cached.IconUrl = metadata.IconUrl; cached.ProjectUrl = metadata.ProjectUrl; - cached.GitHubRepositoryUrl = metadata.GitHubRepositoryUrl; + // A repository found while scraping release notes is kept when the registry names none + cached.GitHubRepositoryUrl = metadata.GitHubRepositoryUrl ?? cached.GitHubRepositoryUrl; cached.AllVersionsJson = JsonSerializer.Serialize(metadata.Versions.Select(v => v.Version)); cached.CachedAt = DateTime.UtcNow; cached.ExpiresAt = DateTime.UtcNow.AddDays(14); // Extended from 7 to 14 days diff --git a/src/backend/PkgDiff.Infrastructure/Services/NugetService.cs b/src/backend/PkgDiff.Infrastructure/Services/NugetService.cs index d5387b5..5870517 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/NugetService.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/NugetService.cs @@ -8,6 +8,9 @@ using PkgDiff.Application.Services; using PkgDiff.Domain.Interfaces; using PkgDiff.Shared.DTOs; +using PkgDiff.Shared.Validation; +using System.Xml; +using System.Xml.Linq; namespace PkgDiff.Infrastructure.Services; @@ -16,14 +19,19 @@ public class NugetService : INugetService private readonly SourceRepository _repository; private readonly SourceCacheContext _cache; private readonly IVersionFilterService _versionFilterService; + private readonly IHttpClientFactory _httpClientFactory; private readonly ILogger _logger; - public NugetService(IVersionFilterService versionFilterService, ILogger logger) + public NugetService( + IVersionFilterService versionFilterService, + IHttpClientFactory httpClientFactory, + ILogger logger) { var packageSource = new PackageSource("https://api.nuget.org/v3/index.json"); _repository = Repository.Factory.GetCoreV3(packageSource); _cache = new SourceCacheContext(); _versionFilterService = versionFilterService; + _httpClientFactory = httpClientFactory; _logger = logger; } @@ -148,6 +156,7 @@ public async Task GetPackageMetadataAsync(string packageId) // Use filtered metadata for latest version var latest = filteredMetadata.OrderByDescending(m => m.Identity.Version).First(); + var projectUrl = latest.ProjectUrl?.ToString(); return new PackageMetadata { @@ -156,7 +165,9 @@ public async Task GetPackageMetadataAsync(string packageId) Description = latest.Description, Authors = latest.Authors, IconUrl = latest.IconUrl?.ToString(), - ProjectUrl = latest.ProjectUrl?.ToString(), + ProjectUrl = projectUrl, + GitHubRepositoryUrl = await GetNuspecRepositoryUrlAsync(packageId, latest.Identity.Version) + ?? GitHubRepositoryUrl(projectUrl), Versions = filteredMetadata .Select(m => new { m.Identity.Version, m.Published }) .DistinctBy(x => x.Version.ToString()) @@ -170,4 +181,27 @@ public async Task GetPackageMetadataAsync(string packageId) .ToList() }; } + + // The nuspec's , which the metadata API leaves out + private async Task GetNuspecRepositoryUrlAsync(string packageId, NuGetVersion version) + { + var id = PackageUrlEncoder.EscapeSegment(packageId.ToLowerInvariant()); + var normalized = PackageUrlEncoder.EscapeSegment(version.ToNormalizedString().ToLowerInvariant()); + try + { + var nuspec = await _httpClientFactory.CreateClient() + .GetStringAsync($"https://api.nuget.org/v3-flatcontainer/{id}/{normalized}/{id}.nuspec"); + var repository = XDocument.Parse(nuspec).Descendants().FirstOrDefault(e => e.Name.LocalName == "repository"); + return GitHubRepositoryUrl(repository?.Attribute("url")?.Value); + } + catch (Exception ex) when (ex is HttpRequestException or XmlException + || (ex is TaskCanceledException && !WorkCancellation.Token.IsCancellationRequested)) + { + _logger.LogWarning(ex, "Could not read the nuspec of {PackageId} {Version}", packageId, version); + return null; + } + } + + private static string? GitHubRepositoryUrl(string? url) => + GitHubUrls.IsGitHub(url) ? GitHubUrls.FromRepositoryField(url) : null; } diff --git a/src/backend/PkgDiff.Infrastructure/Services/PubDevService.cs b/src/backend/PkgDiff.Infrastructure/Services/PubDevService.cs index 569c95c..cb2e768 100644 --- a/src/backend/PkgDiff.Infrastructure/Services/PubDevService.cs +++ b/src/backend/PkgDiff.Infrastructure/Services/PubDevService.cs @@ -166,6 +166,8 @@ public async Task GetReleaseNotesAsync(string packageId, string version) } } } + + repositoryUrl ??= await GetPubspecGitHubUrlAsync(packageId); } catch (Exception ex) { @@ -175,6 +177,31 @@ public async Task GetReleaseNotesAsync(string packageId, string version) return (publisher, description, repositoryUrl); } + // The page links no repository when the pubspec names its GitHub repository only as the homepage + private async Task GetPubspecGitHubUrlAsync(string packageId) + { + try + { + var json = await _httpClient.GetStringAsync($"https://pub.dev/api/packages/{PackageUrlEncoder.EscapeSegment(packageId)}"); + using var doc = JsonDocument.Parse(json); + if (!doc.RootElement.TryGetProperty("latest", out var latest) || !latest.TryGetProperty("pubspec", out var pubspec)) + return null; + + foreach (var field in new[] { "repository", "homepage" }) + { + if (pubspec.TryGetProperty(field, out var value) && value.ValueKind == JsonValueKind.String + && GitHubUrls.IsGitHub(value.GetString())) + return value.GetString(); + } + return null; + } + catch (Exception ex) when (ex is HttpRequestException or JsonException) + { + _logger.LogWarning(ex, "Could not read the pubspec of {PackageId} from the pub.dev API", packageId); + return null; + } + } + public async Task> GetAllVersionsAsync(string packageId) { var versions = new List<(string, DateTime?)>(); diff --git a/src/backend/PkgDiff.Tests/Smoke/LiveRegistrySmokeTests.cs b/src/backend/PkgDiff.Tests/Smoke/LiveRegistrySmokeTests.cs index ba5a913..9286b46 100644 --- a/src/backend/PkgDiff.Tests/Smoke/LiveRegistrySmokeTests.cs +++ b/src/backend/PkgDiff.Tests/Smoke/LiveRegistrySmokeTests.cs @@ -80,7 +80,8 @@ public async Task NuGet_LiveSmoke() { SkipUnlessOptedIn(); var filter = await RegistrySmokeTests.CreateSeededVersionFilterAsync(); - var service = new NugetService(filter, NullLogger.Instance); + await using var provider = CreateProvider(); + var service = new NugetService(filter, provider.GetRequiredService(), NullLogger.Instance); var versions = await service.GetVersionsInRangeAsync( "Newtonsoft.Json",