Skip to content

Commit d1361d7

Browse files
fleizeanfleizean
authored
Merge pull request #29 from pkgdiff-dev/gsd/quick-261102-seo-audit-fixes
fix(seo): resolve the findings of the SEO audit
2 parents 1276dde + 7d31ffd commit d1361d7

37 files changed

Lines changed: 562 additions & 254 deletions

‎.env.example‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,14 @@ FORWARDED_TRUSTED_NETWORKS=172.28.0.0/16
6262
# Empty = unauthenticated GitHub API calls (60 requests/hour limit).
6363
GITHUB_PAT=
6464

65+
# ---------------------------------------------------------------------------
66+
# Server rendering
67+
# ---------------------------------------------------------------------------
68+
69+
# Optional. Gives API calls from the render server their own rate limit.
70+
# Empty = they count against the visitor's limit. Generate: openssl rand -hex 32
71+
RENDER_SERVER_KEY=
72+
6573
# ---------------------------------------------------------------------------
6674
# Monitoring
6775
# ---------------------------------------------------------------------------

‎docker-compose.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ services:
2222
- AdminSettings__ViewerPassword=${VIEWER_PASSWORD:-}
2323
- ForwardedHeaders__TrustedNetworks=${FORWARDED_TRUSTED_NETWORKS:-172.28.0.0/16}
2424
- GitHubSettings__PersonalAccessToken=${GITHUB_PAT:-}
25+
- RateLimiting__RenderServerKey=${RENDER_SERVER_KEY:-}
2526
cap_drop:
2627
- ALL
2728
security_opt:
@@ -72,6 +73,7 @@ services:
7273
- NODE_ENV=production
7374
- PORT=4000
7475
- SSR_INTERNAL_ORIGIN=http://frontend
76+
- RENDER_SERVER_KEY=${RENDER_SERVER_KEY:-}
7577
mem_limit: 512m
7678
cpus: 1.0
7779
pids_limit: 256

‎src/backend/PkgDiff.API/Controllers/PackageController.cs‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ namespace PkgDiff.API.Controllers;
1515
public class PackageController : ControllerBase
1616
{
1717
private const int MaxConcurrentAnalyses = 10;
18+
private const int MaxVersionLimit = 100;
1819

1920
private const int MaxConcurrentAnalysesPerAddress = 2;
2021
private static readonly System.Collections.Concurrent.ConcurrentDictionary<string, int> AnalysesPerAddress = new();
@@ -198,7 +199,11 @@ private async Task<bool> MayQueueScrapeAsync()
198199

199200
[HttpGet("{registry}/{packageId}")]
200201
[ValidatePackageRoute]
201-
public async Task<ActionResult<PackageMetadata>> GetPackageMetadata(string registry, string packageId)
202+
public async Task<ActionResult<PackageMetadata>> GetPackageMetadata(
203+
string registry,
204+
string packageId,
205+
[FromQuery] int? versionLimit = null,
206+
[FromQuery] string? includeVersion = null)
202207
{
203208
try
204209
{
@@ -222,6 +227,16 @@ public async Task<ActionResult<PackageMetadata>> GetPackageMetadata(string regis
222227
}
223228
}
224229

230+
if (versionLimit.HasValue)
231+
{
232+
var limit = Math.Clamp(versionLimit.Value, 1, MaxVersionLimit);
233+
metadata.Versions = metadata.Versions
234+
.Where((version, index) => index < limit
235+
|| version.Version == metadata.LatestVersion
236+
|| string.Equals(version.Version, includeVersion, StringComparison.OrdinalIgnoreCase))
237+
.ToList();
238+
}
239+
225240
return Ok(metadata);
226241
}
227242
catch (ArgumentException ex)
@@ -379,6 +394,7 @@ public async Task<IActionResult> ExportDiffToPdf([FromBody] AnalyzeRequest reque
379394
ProjectUrl = metadata.ProjectUrl,
380395
GitHubRepositoryUrl = metadata.GitHubRepositoryUrl,
381396
Versions = metadata.Versions,
397+
TotalVersions = metadata.Versions.Count,
382398
Trust = trust
383399
};
384400
}

‎src/backend/PkgDiff.API/Controllers/SitemapController.cs‎

Lines changed: 66 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,41 @@
1+
using System.Security;
12
using Microsoft.AspNetCore.Mvc;
3+
using Microsoft.Extensions.Caching.Memory;
24
using PkgDiff.Application.Interfaces;
5+
using PkgDiff.Shared.Constants;
36

47
namespace PkgDiff.API.Controllers;
58

69
[ApiController]
710
[Route("api")]
811
public class SitemapController : ControllerBase
912
{
13+
private const string BaseUrl = "https://pkgdiff.dev";
14+
private const string CacheKey = "sitemap-xml";
15+
16+
private static readonly string[] StaticPaths =
17+
{
18+
"/",
19+
"/packages",
20+
"/audit",
21+
"/about",
22+
"/contributors",
23+
"/terms",
24+
"/privacy",
25+
"/cookies",
26+
};
27+
1028
private readonly IAdminPackageService _adminService;
29+
private readonly IMemoryCache _memory;
1130
private readonly ILogger<SitemapController> _logger;
1231

13-
public SitemapController(IAdminPackageService adminService, ILogger<SitemapController> logger)
32+
public SitemapController(
33+
IAdminPackageService adminService,
34+
IMemoryCache memory,
35+
ILogger<SitemapController> logger)
1436
{
1537
_adminService = adminService;
38+
_memory = memory;
1639
_logger = logger;
1740
}
1841

@@ -22,76 +45,57 @@ public async Task<IActionResult> GetDynamicSitemap()
2245
{
2346
try
2447
{
25-
var packages = await _adminService.GetPackagesForSitemapAsync();
26-
27-
var xml = new System.Text.StringBuilder();
28-
xml.AppendLine("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
29-
xml.AppendLine("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">");
30-
31-
xml.AppendLine(" <url>");
32-
xml.AppendLine(" <loc>https://pkgdiff.dev/</loc>");
33-
xml.AppendLine(" <changefreq>daily</changefreq>");
34-
xml.AppendLine(" <priority>1.0</priority>");
35-
xml.AppendLine($" <lastmod>{DateTime.UtcNow:yyyy-MM-dd}</lastmod>");
36-
xml.AppendLine(" </url>");
37-
38-
xml.AppendLine(" <url>");
39-
xml.AppendLine(" <loc>https://pkgdiff.dev/packages</loc>");
40-
xml.AppendLine(" <changefreq>daily</changefreq>");
41-
xml.AppendLine(" <priority>0.9</priority>");
42-
xml.AppendLine($" <lastmod>{DateTime.UtcNow:yyyy-MM-dd}</lastmod>");
43-
xml.AppendLine(" </url>");
48+
if (!_memory.TryGetValue(CacheKey, out string? sitemap) || sitemap is null)
49+
{
50+
sitemap = await BuildSitemapAsync();
51+
_memory.Set(CacheKey, sitemap, TimeSpan.FromHours(1));
52+
}
4453

45-
xml.AppendLine(" <url>");
46-
xml.AppendLine(" <loc>https://pkgdiff.dev/audit</loc>");
47-
xml.AppendLine(" <changefreq>monthly</changefreq>");
48-
xml.AppendLine(" <priority>0.8</priority>");
49-
xml.AppendLine(" </url>");
54+
return Content(sitemap, "application/xml");
55+
}
56+
catch (Exception ex)
57+
{
58+
_logger.LogError(ex, "Error generating dynamic sitemap");
59+
return StatusCode(500, new { error = "An error occurred while generating sitemap" });
60+
}
61+
}
5062

63+
private async Task<string> BuildSitemapAsync()
64+
{
65+
var packages = await _adminService.GetPackagesForSitemapAsync();
5166

52-
xml.AppendLine(" <url>");
53-
xml.AppendLine(" <loc>https://pkgdiff.dev/terms</loc>");
54-
xml.AppendLine(" <changefreq>yearly</changefreq>");
55-
xml.AppendLine(" <priority>0.3</priority>");
56-
xml.AppendLine($" <lastmod>{DateTime.UtcNow:yyyy-MM-dd}</lastmod>");
57-
xml.AppendLine(" </url>");
67+
var xml = new System.Text.StringBuilder();
68+
xml.AppendLine("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
69+
xml.AppendLine("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">");
5870

59-
xml.AppendLine(" <url>");
60-
xml.AppendLine(" <loc>https://pkgdiff.dev/privacy</loc>");
61-
xml.AppendLine(" <changefreq>yearly</changefreq>");
62-
xml.AppendLine(" <priority>0.3</priority>");
63-
xml.AppendLine($" <lastmod>{DateTime.UtcNow:yyyy-MM-dd}</lastmod>");
64-
xml.AppendLine(" </url>");
71+
foreach (var path in StaticPaths)
72+
{
73+
AppendUrl(xml, path, null);
74+
}
6575

66-
xml.AppendLine(" <url>");
67-
xml.AppendLine(" <loc>https://pkgdiff.dev/cookies</loc>");
68-
xml.AppendLine(" <changefreq>yearly</changefreq>");
69-
xml.AppendLine(" <priority>0.3</priority>");
70-
xml.AppendLine($" <lastmod>{DateTime.UtcNow:yyyy-MM-dd}</lastmod>");
71-
xml.AppendLine(" </url>");
76+
foreach (var registry in RegistryNames.All)
77+
{
78+
AppendUrl(xml, $"/packages?registry={registry}", null);
79+
}
7280

73-
foreach (var package in packages)
74-
{
75-
var encodedPackageId = Uri.EscapeDataString(package.PackageId);
76-
xml.AppendLine(" <url>");
77-
xml.AppendLine($" <loc>https://pkgdiff.dev/package/{package.Registry}/{encodedPackageId}</loc>");
78-
xml.AppendLine($" <changefreq>{package.ChangeFreq}</changefreq>");
79-
xml.AppendLine($" <priority>{package.Priority:0.0}</priority>");
80-
if (package.LastModified.HasValue)
81-
{
82-
xml.AppendLine($" <lastmod>{package.LastModified.Value:yyyy-MM-dd}</lastmod>");
83-
}
84-
xml.AppendLine(" </url>");
85-
}
81+
foreach (var package in packages)
82+
{
83+
AppendUrl(xml, $"/package/{package.Registry}/{package.PackageId}", package.LastModified);
84+
}
8685

87-
xml.AppendLine("</urlset>");
86+
xml.AppendLine("</urlset>");
87+
return xml.ToString();
88+
}
8889

89-
return Content(xml.ToString(), "application/xml");
90-
}
91-
catch (Exception ex)
90+
// Must match the page's canonical URL, so the path is written as is and only escaped for XML
91+
private static void AppendUrl(System.Text.StringBuilder xml, string path, DateTime? lastModified)
92+
{
93+
xml.AppendLine(" <url>");
94+
xml.AppendLine($" <loc>{SecurityElement.Escape(BaseUrl + path)}</loc>");
95+
if (lastModified.HasValue)
9296
{
93-
_logger.LogError(ex, "Error generating dynamic sitemap");
94-
return StatusCode(500, new { error = "An error occurred while generating sitemap" });
97+
xml.AppendLine($" <lastmod>{lastModified.Value:yyyy-MM-dd}</lastmod>");
9598
}
99+
xml.AppendLine(" </url>");
96100
}
97101
}

‎src/backend/PkgDiff.API/Middleware/RateLimitingMiddleware.cs‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,19 @@
33
using PkgDiff.API.Options;
44
using PkgDiff.API.Services;
55
using System.Net;
6+
using System.Security.Cryptography;
7+
using System.Text;
68

79
namespace PkgDiff.API.Middleware;
810

911
public class RateLimitingMiddleware
1012
{
13+
private const string RenderServerHeader = "X-Render-Server-Key";
14+
1115
private readonly RequestDelegate _next;
1216
private readonly IRateLimitCounterStore _counters;
1317
private readonly InboundRateLimitOptions _options;
18+
private readonly byte[]? _renderServerKeyHash;
1419

1520
public RateLimitingMiddleware(
1621
RequestDelegate next,
@@ -20,13 +25,25 @@ public RateLimitingMiddleware(
2025
_next = next;
2126
_counters = counters;
2227
_options = options.Value;
28+
_renderServerKeyHash = string.IsNullOrEmpty(_options.RenderServerKey)
29+
? null
30+
: SHA256.HashData(Encoding.UTF8.GetBytes(_options.RenderServerKey));
2331
}
2432

2533
public async Task InvokeAsync(HttpContext context)
2634
{
27-
var ip = context.GetClientIpAddress();
2835
var path = context.Request.Path;
2936

37+
// Served from a short cache, so a crawler polling it costs next to nothing
38+
if (path.StartsWithSegments("/api/sitemap", StringComparison.OrdinalIgnoreCase))
39+
{
40+
await _next(context);
41+
return;
42+
}
43+
44+
var ip = context.GetClientIpAddress();
45+
var isRenderServer = IsRenderServer(context);
46+
3047
// Runs before authentication: only callers with credentials get the admin budget
3148
if (IsAdminPath(path) && HasCredentials(context))
3249
{
@@ -36,7 +53,9 @@ public async Task InvokeAsync(HttpContext context)
3653
return;
3754
}
3855
}
39-
else if (await IsOverLimitAsync("rl", ip, _options.General))
56+
else if (isRenderServer
57+
? await IsOverLimitAsync("rl:render", ip, _options.RenderServer)
58+
: await IsOverLimitAsync("rl", ip, _options.General))
4059
{
4160
await ReturnRateLimitError(context, "Too many requests. Please slow down.");
4261
return;
@@ -61,6 +80,16 @@ public async Task InvokeAsync(HttpContext context)
6180
await _next(context);
6281
}
6382

83+
private bool IsRenderServer(HttpContext context)
84+
{
85+
if (_renderServerKeyHash is null)
86+
return false;
87+
88+
var sent = context.Request.Headers[RenderServerHeader].ToString();
89+
return sent.Length > 0
90+
&& CryptographicOperations.FixedTimeEquals(SHA256.HashData(Encoding.UTF8.GetBytes(sent)), _renderServerKeyHash);
91+
}
92+
6493
private static bool IsAdminPath(PathString path) =>
6594
path.StartsWithSegments("/api/admin", StringComparison.OrdinalIgnoreCase)
6695
|| path.StartsWithSegments("/hangfire", StringComparison.OrdinalIgnoreCase);

‎src/backend/PkgDiff.API/Options/InboundRateLimitOptions.cs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,15 @@ public sealed class InboundRateLimitOptions
1515

1616
public RateLimitTier AuditStart { get; set; } = new() { PerMinute = 3, PerFiveMinutes = 6 };
1717

18+
// One rendered page makes several API calls, so a crawler would otherwise run out of the visitor budget
19+
public RateLimitTier RenderServer { get; set; } = new() { PerMinute = 600, PerFiveMinutes = 2000 };
20+
21+
// Empty: render server calls count as ordinary visitor calls
22+
public string RenderServerKey { get; set; } = string.Empty;
23+
1824
public bool IsValid() =>
19-
General.IsValid() && PdfExport.IsValid() && Admin.IsValid() && Login.IsValid() && AuditStart.IsValid();
25+
General.IsValid() && PdfExport.IsValid() && Admin.IsValid() && Login.IsValid() && AuditStart.IsValid()
26+
&& RenderServer.IsValid();
2027
}
2128

2229
public sealed class RateLimitTier

‎src/backend/PkgDiff.API/appsettings.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,10 @@
4545
"AuditStart": {
4646
"PerMinute": 3,
4747
"PerFiveMinutes": 6
48+
},
49+
"RenderServer": {
50+
"PerMinute": 600,
51+
"PerFiveMinutes": 2000
4852
}
4953
},
5054
"RateLimitSettings": {

‎src/backend/PkgDiff.Shared/DTOs/AnalysisModels.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ public class PackageMetadata
6969
public string? GitHubRepositoryUrl { get; set; }
7070
public string? SourceDirectory { get; set; }
7171
public List<VersionSummary> Versions { get; set; } = new();
72+
public int TotalVersions { get; set; }
7273
public PackageTrust? Trust { get; set; }
7374
}
7475

0 commit comments

Comments
 (0)