fix(seo): resolve the findings of the SEO audit

.env.example

@@ -62,6 +62,14 @@ FORWARDED_TRUSTED_NETWORKS=172.28.0.0/16
 # Empty = unauthenticated GitHub API calls (60 requests/hour limit).
 GITHUB_PAT=
 
+# ---------------------------------------------------------------------------
+# Server rendering
+# ---------------------------------------------------------------------------
+
+# Optional. Gives API calls from the render server their own rate limit.
+# Empty = they count against the visitor's limit. Generate: openssl rand -hex 32
+RENDER_SERVER_KEY=
+
 # ---------------------------------------------------------------------------
 # Monitoring
 # ---------------------------------------------------------------------------

docker-compose.yml

@@ -22,6 +22,7 @@ services:
       - AdminSettings__ViewerPassword=${VIEWER_PASSWORD:-}
       - ForwardedHeaders__TrustedNetworks=${FORWARDED_TRUSTED_NETWORKS:-172.28.0.0/16}
       - GitHubSettings__PersonalAccessToken=${GITHUB_PAT:-}
+      - RateLimiting__RenderServerKey=${RENDER_SERVER_KEY:-}
     cap_drop:
       - ALL
     security_opt:
@@ -72,6 +73,7 @@ services:
       - NODE_ENV=production
       - PORT=4000
       - SSR_INTERNAL_ORIGIN=http://frontend
+      - RENDER_SERVER_KEY=${RENDER_SERVER_KEY:-}
     mem_limit: 512m
     cpus: 1.0
     pids_limit: 256

src/backend/PkgDiff.API/Controllers/PackageController.cs

@@ -15,6 +15,7 @@ namespace PkgDiff.API.Controllers;
 public class PackageController : ControllerBase
 {
     private const int MaxConcurrentAnalyses = 10;
+    private const int MaxVersionLimit = 100;
 
     private const int MaxConcurrentAnalysesPerAddress = 2;
     private static readonly System.Collections.Concurrent.ConcurrentDictionary<string, int> AnalysesPerAddress = new();
@@ -198,7 +199,11 @@ private async Task<bool> MayQueueScrapeAsync()
 
     [HttpGet("{registry}/{packageId}")]
     [ValidatePackageRoute]
-    public async Task<ActionResult<PackageMetadata>> GetPackageMetadata(string registry, string packageId)
+    public async Task<ActionResult<PackageMetadata>> GetPackageMetadata(
+        string registry,
+        string packageId,
+        [FromQuery] int? versionLimit = null,
+        [FromQuery] string? includeVersion = null)
     {
         try
         {
@@ -222,6 +227,16 @@ public async Task<ActionResult<PackageMetadata>> GetPackageMetadata(string regis
                 }
             }
 
+            if (versionLimit.HasValue)
+            {
+                var limit = Math.Clamp(versionLimit.Value, 1, MaxVersionLimit);
+                metadata.Versions = metadata.Versions
+                    .Where((version, index) => index < limit
+                        || version.Version == metadata.LatestVersion
+                        || string.Equals(version.Version, includeVersion, StringComparison.OrdinalIgnoreCase))
+                    .ToList();
+            }
+
             return Ok(metadata);
         }
         catch (ArgumentException ex)
@@ -379,6 +394,7 @@ public async Task<IActionResult> ExportDiffToPdf([FromBody] AnalyzeRequest reque
         ProjectUrl = metadata.ProjectUrl,
         GitHubRepositoryUrl = metadata.GitHubRepositoryUrl,
         Versions = metadata.Versions,
+        TotalVersions = metadata.Versions.Count,
         Trust = trust
     };
 }

src/backend/PkgDiff.API/Controllers/SitemapController.cs

@@ -1,18 +1,41 @@
+using System.Security;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Caching.Memory;
 using PkgDiff.Application.Interfaces;
+using PkgDiff.Shared.Constants;
 
 namespace PkgDiff.API.Controllers;
 
 [ApiController]
 [Route("api")]
 public class SitemapController : ControllerBase
 {
+    private const string BaseUrl = "https://pkgdiff.dev";
+    private const string CacheKey = "sitemap-xml";
+
+    private static readonly string[] StaticPaths =
+    {
+        "/",
+        "/packages",
+        "/audit",
+        "/about",
+        "/contributors",
+        "/terms",
+        "/privacy",
+        "/cookies",
+    };
+
     private readonly IAdminPackageService _adminService;
+    private readonly IMemoryCache _memory;
     private readonly ILogger<SitemapController> _logger;
 
-    public SitemapController(IAdminPackageService adminService, ILogger<SitemapController> logger)
+    public SitemapController(
+        IAdminPackageService adminService,
+        IMemoryCache memory,
+        ILogger<SitemapController> logger)
     {
         _adminService = adminService;
+        _memory = memory;
         _logger = logger;
     }
 
@@ -22,76 +45,57 @@ public async Task<IActionResult> GetDynamicSitemap()
     {
         try
         {
-            var packages = await _adminService.GetPackagesForSitemapAsync();
-
-            var xml = new System.Text.StringBuilder();
-            xml.AppendLine("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
-            xml.AppendLine("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">");
-
-            xml.AppendLine("  <url>");
-            xml.AppendLine("    <loc>https://pkgdiff.dev/</loc>");
-            xml.AppendLine("    <changefreq>daily</changefreq>");
-            xml.AppendLine("    <priority>1.0</priority>");
-            xml.AppendLine($"    <lastmod>{DateTime.UtcNow:yyyy-MM-dd}</lastmod>");
-            xml.AppendLine("  </url>");
-
-            xml.AppendLine("  <url>");
-            xml.AppendLine("    <loc>https://pkgdiff.dev/packages</loc>");
-            xml.AppendLine("    <changefreq>daily</changefreq>");
-            xml.AppendLine("    <priority>0.9</priority>");
-            xml.AppendLine($"    <lastmod>{DateTime.UtcNow:yyyy-MM-dd}</lastmod>");
-            xml.AppendLine("  </url>");
+            if (!_memory.TryGetValue(CacheKey, out string? sitemap) || sitemap is null)
+            {
+                sitemap = await BuildSitemapAsync();
+                _memory.Set(CacheKey, sitemap, TimeSpan.FromHours(1));
+            }
 
-            xml.AppendLine("  <url>");
-            xml.AppendLine("    <loc>https://pkgdiff.dev/audit</loc>");
-            xml.AppendLine("    <changefreq>monthly</changefreq>");
-            xml.AppendLine("    <priority>0.8</priority>");
-            xml.AppendLine("  </url>");
+            return Content(sitemap, "application/xml");
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Error generating dynamic sitemap");
+            return StatusCode(500, new { error = "An error occurred while generating sitemap" });
+        }
+    }
 
+    private async Task<string> BuildSitemapAsync()
+    {
+        var packages = await _adminService.GetPackagesForSitemapAsync();
 
-            xml.AppendLine("  <url>");
-            xml.AppendLine("    <loc>https://pkgdiff.dev/terms</loc>");
-            xml.AppendLine("    <changefreq>yearly</changefreq>");
-            xml.AppendLine("    <priority>0.3</priority>");
-            xml.AppendLine($"    <lastmod>{DateTime.UtcNow:yyyy-MM-dd}</lastmod>");
-            xml.AppendLine("  </url>");
+        var xml = new System.Text.StringBuilder();
+        xml.AppendLine("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+        xml.AppendLine("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">");
 
-            xml.AppendLine("  <url>");
-            xml.AppendLine("    <loc>https://pkgdiff.dev/privacy</loc>");
-            xml.AppendLine("    <changefreq>yearly</changefreq>");
-            xml.AppendLine("    <priority>0.3</priority>");
-            xml.AppendLine($"    <lastmod>{DateTime.UtcNow:yyyy-MM-dd}</lastmod>");
-            xml.AppendLine("  </url>");
+        foreach (var path in StaticPaths)
+        {
+            AppendUrl(xml, path, null);
+        }
 
-            xml.AppendLine("  <url>");
-            xml.AppendLine("    <loc>https://pkgdiff.dev/cookies</loc>");
-            xml.AppendLine("    <changefreq>yearly</changefreq>");
-            xml.AppendLine("    <priority>0.3</priority>");
-            xml.AppendLine($"    <lastmod>{DateTime.UtcNow:yyyy-MM-dd}</lastmod>");
-            xml.AppendLine("  </url>");
+        foreach (var registry in RegistryNames.All)
+        {
+            AppendUrl(xml, $"/packages?registry={registry}", null);
+        }
 
-            foreach (var package in packages)
-            {
-                var encodedPackageId = Uri.EscapeDataString(package.PackageId);
-                xml.AppendLine("  <url>");
-                xml.AppendLine($"    <loc>https://pkgdiff.dev/package/{package.Registry}/{encodedPackageId}</loc>");
-                xml.AppendLine($"    <changefreq>{package.ChangeFreq}</changefreq>");
-                xml.AppendLine($"    <priority>{package.Priority:0.0}</priority>");
-                if (package.LastModified.HasValue)
-                {
-                    xml.AppendLine($"    <lastmod>{package.LastModified.Value:yyyy-MM-dd}</lastmod>");
-                }
-                xml.AppendLine("  </url>");
-            }
+        foreach (var package in packages)
+        {
+            AppendUrl(xml, $"/package/{package.Registry}/{package.PackageId}", package.LastModified);
+        }
 
-            xml.AppendLine("</urlset>");
+        xml.AppendLine("</urlset>");
+        return xml.ToString();
+    }
 
-            return Content(xml.ToString(), "application/xml");
-        }
-        catch (Exception ex)
+    // Must match the page's canonical URL, so the path is written as is and only escaped for XML
+    private static void AppendUrl(System.Text.StringBuilder xml, string path, DateTime? lastModified)
+    {
+        xml.AppendLine("  <url>");
+        xml.AppendLine($"    <loc>{SecurityElement.Escape(BaseUrl + path)}</loc>");
+        if (lastModified.HasValue)
         {
-            _logger.LogError(ex, "Error generating dynamic sitemap");
-            return StatusCode(500, new { error = "An error occurred while generating sitemap" });
+            xml.AppendLine($"    <lastmod>{lastModified.Value:yyyy-MM-dd}</lastmod>");
         }
+        xml.AppendLine("  </url>");
     }
 }

src/backend/PkgDiff.API/Middleware/RateLimitingMiddleware.cs

@@ -3,14 +3,19 @@
 using PkgDiff.API.Options;
 using PkgDiff.API.Services;
 using System.Net;
+using System.Security.Cryptography;
+using System.Text;
 
 namespace PkgDiff.API.Middleware;
 
 public class RateLimitingMiddleware
 {
+    private const string RenderServerHeader = "X-Render-Server-Key";
+
     private readonly RequestDelegate _next;
     private readonly IRateLimitCounterStore _counters;
     private readonly InboundRateLimitOptions _options;
+    private readonly byte[]? _renderServerKeyHash;
 
     public RateLimitingMiddleware(
         RequestDelegate next,
@@ -20,13 +25,25 @@ public RateLimitingMiddleware(
         _next = next;
         _counters = counters;
         _options = options.Value;
+        _renderServerKeyHash = string.IsNullOrEmpty(_options.RenderServerKey)
+            ? null
+            : SHA256.HashData(Encoding.UTF8.GetBytes(_options.RenderServerKey));
     }
 
     public async Task InvokeAsync(HttpContext context)
     {
-        var ip = context.GetClientIpAddress();
         var path = context.Request.Path;
 
+        // Served from a short cache, so a crawler polling it costs next to nothing
+        if (path.StartsWithSegments("/api/sitemap", StringComparison.OrdinalIgnoreCase))
+        {
+            await _next(context);
+            return;
+        }
+
+        var ip = context.GetClientIpAddress();
+        var isRenderServer = IsRenderServer(context);
+
         // Runs before authentication: only callers with credentials get the admin budget
         if (IsAdminPath(path) && HasCredentials(context))
         {
@@ -36,7 +53,9 @@ public async Task InvokeAsync(HttpContext context)
                 return;
             }
         }
-        else if (await IsOverLimitAsync("rl", ip, _options.General))
+        else if (isRenderServer
+            ? await IsOverLimitAsync("rl:render", ip, _options.RenderServer)
+            : await IsOverLimitAsync("rl", ip, _options.General))
         {
             await ReturnRateLimitError(context, "Too many requests. Please slow down.");
             return;
@@ -61,6 +80,16 @@ public async Task InvokeAsync(HttpContext context)
         await _next(context);
     }
 
+    private bool IsRenderServer(HttpContext context)
+    {
+        if (_renderServerKeyHash is null)
+            return false;
+
+        var sent = context.Request.Headers[RenderServerHeader].ToString();
+        return sent.Length > 0
+            && CryptographicOperations.FixedTimeEquals(SHA256.HashData(Encoding.UTF8.GetBytes(sent)), _renderServerKeyHash);
+    }
+
     private static bool IsAdminPath(PathString path) =>
         path.StartsWithSegments("/api/admin", StringComparison.OrdinalIgnoreCase)
         || path.StartsWithSegments("/hangfire", StringComparison.OrdinalIgnoreCase);

src/backend/PkgDiff.API/Options/InboundRateLimitOptions.cs

@@ -15,8 +15,15 @@ public sealed class InboundRateLimitOptions
 
     public RateLimitTier AuditStart { get; set; } = new() { PerMinute = 3, PerFiveMinutes = 6 };
 
+    // One rendered page makes several API calls, so a crawler would otherwise run out of the visitor budget
+    public RateLimitTier RenderServer { get; set; } = new() { PerMinute = 600, PerFiveMinutes = 2000 };
+
+    // Empty: render server calls count as ordinary visitor calls
+    public string RenderServerKey { get; set; } = string.Empty;
+
     public bool IsValid() =>
-        General.IsValid() && PdfExport.IsValid() && Admin.IsValid() && Login.IsValid() && AuditStart.IsValid();
+        General.IsValid() && PdfExport.IsValid() && Admin.IsValid() && Login.IsValid() && AuditStart.IsValid()
+        && RenderServer.IsValid();
 }
 
 public sealed class RateLimitTier

src/backend/PkgDiff.API/appsettings.json

@@ -45,6 +45,10 @@
     "AuditStart": {
       "PerMinute": 3,
       "PerFiveMinutes": 6
+    },
+    "RenderServer": {
+      "PerMinute": 600,
+      "PerFiveMinutes": 2000
     }
   },
   "RateLimitSettings": {

src/backend/PkgDiff.Shared/DTOs/AnalysisModels.cs

@@ -69,6 +69,7 @@ public class PackageMetadata
     public string? GitHubRepositoryUrl { get; set; }
     public string? SourceDirectory { get; set; }
     public List<VersionSummary> Versions { get; set; } = new();
+    public int TotalVersions { get; set; }
     public PackageTrust? Trust { get; set; }
 }