Skip to content

Commit 51e5e19

Browse files
Enes Yağızclaude
Enes Yağız
and
claude
committed
fix(seo): give server rendering its own rate limit and correct the sitemap
The sitemap no longer counts against any limit and is cached for an hour. Its URLs now match the canonicals, and static pages no longer claim to change every day. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 1276dde commit 51e5e19

8 files changed

Lines changed: 135 additions & 69 deletions

File tree

‎.env.example‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,14 @@ FORWARDED_TRUSTED_NETWORKS=172.28.0.0/16
6262
# Empty = unauthenticated GitHub API calls (60 requests/hour limit).
6363
GITHUB_PAT=
6464

65+
# ---------------------------------------------------------------------------
66+
# Server rendering
67+
# ---------------------------------------------------------------------------
68+
69+
# Optional. Gives API calls from the render server their own rate limit.
70+
# Empty = they count against the visitor's limit. Generate: openssl rand -hex 32
71+
RENDER_SERVER_KEY=
72+
6573
# ---------------------------------------------------------------------------
6674
# Monitoring
6775
# ---------------------------------------------------------------------------

‎docker-compose.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ services:
2222
- AdminSettings__ViewerPassword=${VIEWER_PASSWORD:-}
2323
- ForwardedHeaders__TrustedNetworks=${FORWARDED_TRUSTED_NETWORKS:-172.28.0.0/16}
2424
- GitHubSettings__PersonalAccessToken=${GITHUB_PAT:-}
25+
- RateLimiting__RenderServerKey=${RENDER_SERVER_KEY:-}
2526
cap_drop:
2627
- ALL
2728
security_opt:
@@ -72,6 +73,7 @@ services:
7273
- NODE_ENV=production
7374
- PORT=4000
7475
- SSR_INTERNAL_ORIGIN=http://frontend
76+
- RENDER_SERVER_KEY=${RENDER_SERVER_KEY:-}
7577
mem_limit: 512m
7678
cpus: 1.0
7779
pids_limit: 256

‎src/backend/PkgDiff.API/Controllers/SitemapController.cs‎

Lines changed: 66 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,41 @@
1+
using System.Security;
12
using Microsoft.AspNetCore.Mvc;
3+
using Microsoft.Extensions.Caching.Memory;
24
using PkgDiff.Application.Interfaces;
5+
using PkgDiff.Shared.Constants;
36

47
namespace PkgDiff.API.Controllers;
58

69
[ApiController]
710
[Route("api")]
811
public class SitemapController : ControllerBase
912
{
13+
private const string BaseUrl = "https://pkgdiff.dev";
14+
private const string CacheKey = "sitemap-xml";
15+
16+
private static readonly string[] StaticPaths =
17+
{
18+
"/",
19+
"/packages",
20+
"/audit",
21+
"/about",
22+
"/contributors",
23+
"/terms",
24+
"/privacy",
25+
"/cookies",
26+
};
27+
1028
private readonly IAdminPackageService _adminService;
29+
private readonly IMemoryCache _memory;
1130
private readonly ILogger<SitemapController> _logger;
1231

13-
public SitemapController(IAdminPackageService adminService, ILogger<SitemapController> logger)
32+
public SitemapController(
33+
IAdminPackageService adminService,
34+
IMemoryCache memory,
35+
ILogger<SitemapController> logger)
1436
{
1537
_adminService = adminService;
38+
_memory = memory;
1639
_logger = logger;
1740
}
1841

@@ -22,76 +45,57 @@ public async Task<IActionResult> GetDynamicSitemap()
2245
{
2346
try
2447
{
25-
var packages = await _adminService.GetPackagesForSitemapAsync();
26-
27-
var xml = new System.Text.StringBuilder();
28-
xml.AppendLine("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
29-
xml.AppendLine("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">");
30-
31-
xml.AppendLine(" <url>");
32-
xml.AppendLine(" <loc>https://pkgdiff.dev/</loc>");
33-
xml.AppendLine(" <changefreq>daily</changefreq>");
34-
xml.AppendLine(" <priority>1.0</priority>");
35-
xml.AppendLine($" <lastmod>{DateTime.UtcNow:yyyy-MM-dd}</lastmod>");
36-
xml.AppendLine(" </url>");
37-
38-
xml.AppendLine(" <url>");
39-
xml.AppendLine(" <loc>https://pkgdiff.dev/packages</loc>");
40-
xml.AppendLine(" <changefreq>daily</changefreq>");
41-
xml.AppendLine(" <priority>0.9</priority>");
42-
xml.AppendLine($" <lastmod>{DateTime.UtcNow:yyyy-MM-dd}</lastmod>");
43-
xml.AppendLine(" </url>");
48+
if (!_memory.TryGetValue(CacheKey, out string? sitemap) || sitemap is null)
49+
{
50+
sitemap = await BuildSitemapAsync();
51+
_memory.Set(CacheKey, sitemap, TimeSpan.FromHours(1));
52+
}
4453

45-
xml.AppendLine(" <url>");
46-
xml.AppendLine(" <loc>https://pkgdiff.dev/audit</loc>");
47-
xml.AppendLine(" <changefreq>monthly</changefreq>");
48-
xml.AppendLine(" <priority>0.8</priority>");
49-
xml.AppendLine(" </url>");
54+
return Content(sitemap, "application/xml");
55+
}
56+
catch (Exception ex)
57+
{
58+
_logger.LogError(ex, "Error generating dynamic sitemap");
59+
return StatusCode(500, new { error = "An error occurred while generating sitemap" });
60+
}
61+
}
5062

63+
private async Task<string> BuildSitemapAsync()
64+
{
65+
var packages = await _adminService.GetPackagesForSitemapAsync();
5166

52-
xml.AppendLine(" <url>");
53-
xml.AppendLine(" <loc>https://pkgdiff.dev/terms</loc>");
54-
xml.AppendLine(" <changefreq>yearly</changefreq>");
55-
xml.AppendLine(" <priority>0.3</priority>");
56-
xml.AppendLine($" <lastmod>{DateTime.UtcNow:yyyy-MM-dd}</lastmod>");
57-
xml.AppendLine(" </url>");
67+
var xml = new System.Text.StringBuilder();
68+
xml.AppendLine("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
69+
xml.AppendLine("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">");
5870

59-
xml.AppendLine(" <url>");
60-
xml.AppendLine(" <loc>https://pkgdiff.dev/privacy</loc>");
61-
xml.AppendLine(" <changefreq>yearly</changefreq>");
62-
xml.AppendLine(" <priority>0.3</priority>");
63-
xml.AppendLine($" <lastmod>{DateTime.UtcNow:yyyy-MM-dd}</lastmod>");
64-
xml.AppendLine(" </url>");
71+
foreach (var path in StaticPaths)
72+
{
73+
AppendUrl(xml, path, null);
74+
}
6575

66-
xml.AppendLine(" <url>");
67-
xml.AppendLine(" <loc>https://pkgdiff.dev/cookies</loc>");
68-
xml.AppendLine(" <changefreq>yearly</changefreq>");
69-
xml.AppendLine(" <priority>0.3</priority>");
70-
xml.AppendLine($" <lastmod>{DateTime.UtcNow:yyyy-MM-dd}</lastmod>");
71-
xml.AppendLine(" </url>");
76+
foreach (var registry in RegistryNames.All)
77+
{
78+
AppendUrl(xml, $"/packages?registry={registry}", null);
79+
}
7280

73-
foreach (var package in packages)
74-
{
75-
var encodedPackageId = Uri.EscapeDataString(package.PackageId);
76-
xml.AppendLine(" <url>");
77-
xml.AppendLine($" <loc>https://pkgdiff.dev/package/{package.Registry}/{encodedPackageId}</loc>");
78-
xml.AppendLine($" <changefreq>{package.ChangeFreq}</changefreq>");
79-
xml.AppendLine($" <priority>{package.Priority:0.0}</priority>");
80-
if (package.LastModified.HasValue)
81-
{
82-
xml.AppendLine($" <lastmod>{package.LastModified.Value:yyyy-MM-dd}</lastmod>");
83-
}
84-
xml.AppendLine(" </url>");
85-
}
81+
foreach (var package in packages)
82+
{
83+
AppendUrl(xml, $"/package/{package.Registry}/{package.PackageId}", package.LastModified);
84+
}
8685

87-
xml.AppendLine("</urlset>");
86+
xml.AppendLine("</urlset>");
87+
return xml.ToString();
88+
}
8889

89-
return Content(xml.ToString(), "application/xml");
90-
}
91-
catch (Exception ex)
90+
// Must match the page's canonical URL, so the path is written as is and only escaped for XML
91+
private static void AppendUrl(System.Text.StringBuilder xml, string path, DateTime? lastModified)
92+
{
93+
xml.AppendLine(" <url>");
94+
xml.AppendLine($" <loc>{SecurityElement.Escape(BaseUrl + path)}</loc>");
95+
if (lastModified.HasValue)
9296
{
93-
_logger.LogError(ex, "Error generating dynamic sitemap");
94-
return StatusCode(500, new { error = "An error occurred while generating sitemap" });
97+
xml.AppendLine($" <lastmod>{lastModified.Value:yyyy-MM-dd}</lastmod>");
9598
}
99+
xml.AppendLine(" </url>");
96100
}
97101
}

‎src/backend/PkgDiff.API/Middleware/RateLimitingMiddleware.cs‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,19 @@
33
using PkgDiff.API.Options;
44
using PkgDiff.API.Services;
55
using System.Net;
6+
using System.Security.Cryptography;
7+
using System.Text;
68

79
namespace PkgDiff.API.Middleware;
810

911
public class RateLimitingMiddleware
1012
{
13+
private const string RenderServerHeader = "X-Render-Server-Key";
14+
1115
private readonly RequestDelegate _next;
1216
private readonly IRateLimitCounterStore _counters;
1317
private readonly InboundRateLimitOptions _options;
18+
private readonly byte[]? _renderServerKeyHash;
1419

1520
public RateLimitingMiddleware(
1621
RequestDelegate next,
@@ -20,13 +25,25 @@ public RateLimitingMiddleware(
2025
_next = next;
2126
_counters = counters;
2227
_options = options.Value;
28+
_renderServerKeyHash = string.IsNullOrEmpty(_options.RenderServerKey)
29+
? null
30+
: SHA256.HashData(Encoding.UTF8.GetBytes(_options.RenderServerKey));
2331
}
2432

2533
public async Task InvokeAsync(HttpContext context)
2634
{
27-
var ip = context.GetClientIpAddress();
2835
var path = context.Request.Path;
2936

37+
// Served from a short cache, so a crawler polling it costs next to nothing
38+
if (path.StartsWithSegments("/api/sitemap", StringComparison.OrdinalIgnoreCase))
39+
{
40+
await _next(context);
41+
return;
42+
}
43+
44+
var ip = context.GetClientIpAddress();
45+
var isRenderServer = IsRenderServer(context);
46+
3047
// Runs before authentication: only callers with credentials get the admin budget
3148
if (IsAdminPath(path) && HasCredentials(context))
3249
{
@@ -36,7 +53,9 @@ public async Task InvokeAsync(HttpContext context)
3653
return;
3754
}
3855
}
39-
else if (await IsOverLimitAsync("rl", ip, _options.General))
56+
else if (isRenderServer
57+
? await IsOverLimitAsync("rl:render", ip, _options.RenderServer)
58+
: await IsOverLimitAsync("rl", ip, _options.General))
4059
{
4160
await ReturnRateLimitError(context, "Too many requests. Please slow down.");
4261
return;
@@ -61,6 +80,16 @@ public async Task InvokeAsync(HttpContext context)
6180
await _next(context);
6281
}
6382

83+
private bool IsRenderServer(HttpContext context)
84+
{
85+
if (_renderServerKeyHash is null)
86+
return false;
87+
88+
var sent = context.Request.Headers[RenderServerHeader].ToString();
89+
return sent.Length > 0
90+
&& CryptographicOperations.FixedTimeEquals(SHA256.HashData(Encoding.UTF8.GetBytes(sent)), _renderServerKeyHash);
91+
}
92+
6493
private static bool IsAdminPath(PathString path) =>
6594
path.StartsWithSegments("/api/admin", StringComparison.OrdinalIgnoreCase)
6695
|| path.StartsWithSegments("/hangfire", StringComparison.OrdinalIgnoreCase);

‎src/backend/PkgDiff.API/Options/InboundRateLimitOptions.cs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,15 @@ public sealed class InboundRateLimitOptions
1515

1616
public RateLimitTier AuditStart { get; set; } = new() { PerMinute = 3, PerFiveMinutes = 6 };
1717

18+
// One rendered page makes several API calls, so a crawler would otherwise run out of the visitor budget
19+
public RateLimitTier RenderServer { get; set; } = new() { PerMinute = 600, PerFiveMinutes = 2000 };
20+
21+
// Empty: render server calls count as ordinary visitor calls
22+
public string RenderServerKey { get; set; } = string.Empty;
23+
1824
public bool IsValid() =>
19-
General.IsValid() && PdfExport.IsValid() && Admin.IsValid() && Login.IsValid() && AuditStart.IsValid();
25+
General.IsValid() && PdfExport.IsValid() && Admin.IsValid() && Login.IsValid() && AuditStart.IsValid()
26+
&& RenderServer.IsValid();
2027
}
2128

2229
public sealed class RateLimitTier

‎src/backend/PkgDiff.API/appsettings.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,10 @@
4545
"AuditStart": {
4646
"PerMinute": 3,
4747
"PerFiveMinutes": 6
48+
},
49+
"RenderServer": {
50+
"PerMinute": 600,
51+
"PerFiveMinutes": 2000
4852
}
4953
},
5054
"RateLimitSettings": {

‎src/frontend/src/app/app.config.server.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,13 @@ import { ApplicationConfig, mergeApplicationConfig } from '@angular/core';
22
import { provideServerRendering, withRoutes } from '@angular/ssr';
33
import { appConfig } from './app.config';
44
import { serverRoutes } from './app.routes.server';
5+
import { RENDER_SERVER_KEY } from './core/interceptors/forwarded-for.interceptor';
56

67
const serverConfig: ApplicationConfig = {
7-
providers: [provideServerRendering(withRoutes(serverRoutes))],
8+
providers: [
9+
provideServerRendering(withRoutes(serverRoutes)),
10+
{ provide: RENDER_SERVER_KEY, useFactory: () => process.env['RENDER_SERVER_KEY'] ?? '' },
11+
],
812
};
913

1014
export const config = mergeApplicationConfig(appConfig, serverConfig);

‎src/frontend/src/app/core/interceptors/forwarded-for.interceptor.ts‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,22 @@
11
import { HttpInterceptorFn } from '@angular/common/http';
2-
import { REQUEST, inject } from '@angular/core';
2+
import { InjectionToken, REQUEST, inject } from '@angular/core';
33
import { environment } from '../../../environments/environment';
44

5+
// Provided only on the render server, so the key never reaches a browser bundle
6+
export const RENDER_SERVER_KEY = new InjectionToken<string>('RENDER_SERVER_KEY');
7+
58
// Keeps rate limits and request logs per visitor when the render server calls the API
69
export const forwardedForInterceptor: HttpInterceptorFn = (req, next) => {
710
const visitor = inject(REQUEST, { optional: true })?.headers.get('x-forwarded-for');
11+
const renderServerKey = inject(RENDER_SERVER_KEY, { optional: true });
812

9-
if (!visitor || !req.url.startsWith(environment.apiUrl)) {
13+
if (!req.url.startsWith(environment.apiUrl)) {
1014
return next(req);
1115
}
1216

13-
return next(req.clone({ setHeaders: { 'X-Forwarded-For': visitor } }));
17+
const headers: Record<string, string> = {};
18+
if (visitor) headers['X-Forwarded-For'] = visitor;
19+
if (renderServerKey) headers['X-Render-Server-Key'] = renderServerKey;
20+
21+
return Object.keys(headers).length > 0 ? next(req.clone({ setHeaders: headers })) : next(req);
1422
};

0 commit comments

Comments
 (0)