Deploys that explain themselves: one list of startup problems

SW.Bitween.Api/Data/SecretColumnCipher.cs

@@ -27,6 +27,11 @@ namespace SW.Bitween;
 /// change the key, move the old one to <c>Bitween:PreviousSettingsEncryptionKey</c>: values are read
 /// with either and written with the new one.
 /// </para>
+/// <para>
+/// A key that is gone for good leaves nothing to read them with. <c>Bitween:SettingsEncryptionKeyLost</c>
+/// then reads such a value as empty, so it can be entered again, and leaves what is stored in place
+/// until it is: should the key turn up after all, whatever wasn't entered again reads once more.
+/// </para>
 /// </remarks>
 public static class SecretColumnCipher
 {
@@ -41,14 +46,19 @@ public static class SecretColumnCipher
 
     private static byte[] _key;
     private static byte[] _previousKey;
+    private static bool _keyLost;
 
     /// <summary>Whether values are being encrypted as they are written.</summary>
     public static bool Enabled => _key is not null;
 
-    public static void Configure(string passphrase, string previousPassphrase = null)
+    /// <summary>Whether a value no key here can read is taken as empty rather than refused.</summary>
+    public static bool KeyLost => _keyLost;
+
+    public static void Configure(string passphrase, string previousPassphrase = null, bool keyLost = false)
     {
         _key = string.IsNullOrWhiteSpace(passphrase) ? null : Derive(passphrase);
         _previousKey = string.IsNullOrWhiteSpace(previousPassphrase) ? null : Derive(previousPassphrase);
+        _keyLost = keyLost;
     }
 
     public static string Protect(string plaintext)
@@ -66,7 +76,12 @@ public static string Protect(string plaintext)
         return Prefix + Convert.ToBase64String(output);
     }
 
-    public static string Unprotect(string stored)
+    public static string Unprotect(string stored) => Unprotect(stored, unreadable: null);
+
+    /// <summary>The same for JSON, where a value that can't be read is JSON's own empty value.</summary>
+    private static string UnprotectJson(string stored) => Unprotect(stored, unreadable: "null");
+
+    private static string Unprotect(string stored, string unreadable)
     {
         if (stored is null || !stored.StartsWith(Prefix, StringComparison.Ordinal))
             return stored;
@@ -85,10 +100,8 @@ public static string Unprotect(string stored)
             }
         }
 
-        throw new InvalidOperationException(
-            "A stored value is encrypted with a key this instance does not have. Set " +
-            "Bitween:SettingsEncryptionKey to the key it was written with, or put that key in " +
-            "Bitween:PreviousSettingsEncryptionKey while moving to a new one.");
+        if (_keyLost) return unreadable;
+        throw new UnreadableSecretException();
     }
 
     private static string Decrypt(byte[] key, byte[] input)
@@ -115,7 +128,7 @@ public static PropertyBuilder<T> Encrypted<T>(this PropertyBuilder<T> builder)
                         $"{builder.Metadata.Name} has no converter to encrypt the output of; call StoreAsJson first.");
         var comparer = builder.Metadata.GetValueComparer();
 
-        var cipher = new ValueConverter<string, string>(v => Protect(v), v => Unprotect(v));
+        var cipher = new ValueConverter<string, string>(v => Protect(v), v => UnprotectJson(v));
         builder.Metadata.SetValueConverter(inner.ComposeWith(cipher));
         if (comparer is not null) builder.Metadata.SetValueComparer(comparer);
         return builder;
@@ -153,7 +166,7 @@ private static T FromJsonb<T>(string stored) where T : class
     {
         if (stored is null) return null;
         var json = stored.TrimStart().StartsWith("\"" + Prefix, StringComparison.Ordinal)
-            ? Unprotect(JsonConvert.DeserializeObject<string>(stored))
+            ? UnprotectJson(JsonConvert.DeserializeObject<string>(stored))
             : stored;
         return JsonConvert.DeserializeObject<T>(json);
     }
@@ -184,3 +197,9 @@ private static System.Collections.Generic.Dictionary<string, string> MapValues(
         return result;
     }
 }
+
+/// <summary>A stored credential that neither of this instance's keys can decrypt.</summary>
+public class UnreadableSecretException() : InvalidOperationException(
+    "A stored value is encrypted with a key this instance does not have. Set " +
+    "Bitween:SettingsEncryptionKey to the key it was written with, or put that key in " +
+    "Bitween:PreviousSettingsEncryptionKey while moving to a new one.");

SW.Bitween.Api/Resources/Ops/LaneResolver.cs

@@ -25,7 +25,7 @@ public enum QueueLane
     /// <summary>A work group's <c>-Result</c> lane. Evaluates notifiers.</summary>
     Notifications,
 
-    /// <summary>A legacy event type, consumed only while that setting is on.</summary>
+    /// <summary>A legacy event type, consumed only while its queue still holds something.</summary>
     Legacy,
 
     /// <summary>Bookkeeping. No integration traffic.</summary>
@@ -50,7 +50,7 @@ public class LaneResolver(BitweenDbContext dbContext, BitweenOptions options)
         ["SubscriptionUnpausedEvent"] = "Resumed integrations",
     };
 
-    /// <summary>Only declared while <c>Bitween.ConsumeLegacyEventMessages</c> is on.</summary>
+    /// <summary>Only read while their queues still hold something; see <see cref="SW.Bitween.Services.LegacyEventQueues"/>.</summary>
     private static readonly Dictionary<string, string> Legacy = new(System.StringComparer.OrdinalIgnoreCase)
     {
         ["ApiXchangeCreatedEvent"] = "API exchange created",

SW.Bitween.Api/Services/BitweenOptions.cs

@@ -324,7 +324,6 @@ public BitweenOptions()
         /// <c>partnerkey</c> keeps working either way. Read per request.
         /// </summary>
         public string PartnerKeyHeader { get; set; }
-        public bool ConsumeLegacyEventMessages { get; set; }
         public string QueuePrefix { get; set; }
 
         /// <summary>

SW.Bitween.Api/Services/BrokerQueues.cs

@@ -0,0 +1,70 @@
+using System;
+using System.Threading.Tasks;
+using RabbitMQ.Client;
+using RabbitMQ.Client.Exceptions;
+using SW.Bus;
+
+namespace SW.Bitween.Services;
+
+/// <summary>
+/// Bitween's own queues on RabbitMQ, looked at before the bus starts: whether one exists, what is in
+/// it, and removing one nothing uses any more. The bus itself only ever declares, which would create
+/// the very queue being asked about.
+/// </summary>
+public sealed class BrokerQueues : IAsyncDisposable
+{
+    private readonly IConnection _connection;
+
+    private BrokerQueues(IConnection connection) => _connection = connection;
+
+    public static async Task<BrokerQueues> ConnectAsync(string connectionString)
+    {
+        var factory = new ConnectionFactory
+        {
+            Uri = new Uri(connectionString),
+            AutomaticRecoveryEnabled = false,
+            RequestedConnectionTimeout = TimeSpan.FromSeconds(15)
+        };
+        return new BrokerQueues(await factory.CreateConnectionAsync("bitween-startup"));
+    }
+
+    /// <summary>
+    /// The name the bus gives a consumer's queue: <c>v3.{environment}.{queue prefix}.{consumer}.{message}</c>.
+    /// </summary>
+    public static string QueueName(string environment, string queuePrefix, string consumer, string message) =>
+        $"{new BusOptions(environment).ProcessExchange}.{queuePrefix}.{consumer}.{message}".ToLowerInvariant();
+
+    /// <returns>The queue's message and consumer counts, or null when there is no such queue.</returns>
+    public async Task<QueueDeclareOk> InspectAsync(string queue)
+    {
+        // Asking about a missing queue closes the channel it was asked on, so each question gets its own.
+        await using var channel = await _connection.CreateChannelAsync();
+        try
+        {
+            return await channel.QueueDeclarePassiveAsync(queue);
+        }
+        catch (OperationInterruptedException ex) when (ex.ShutdownReason?.ReplyCode == 404)
+        {
+            return null;
+        }
+    }
+
+    /// <summary>Deletes the queue only while it is empty and nobody consumes it.</summary>
+    /// <returns>Whether it was deleted.</returns>
+    public async Task<bool> DeleteIfIdleAsync(string queue)
+    {
+        await using var channel = await _connection.CreateChannelAsync();
+        try
+        {
+            await channel.QueueDeleteAsync(queue, ifUnused: true, ifEmpty: true);
+            return true;
+        }
+        catch (OperationInterruptedException)
+        {
+            // Not empty, or consumed, by now: it waits for the next start.
+            return false;
+        }
+    }
+
+    public ValueTask DisposeAsync() => _connection.DisposeAsync();
+}

SW.Bitween.Api/Services/BusEnvironment.cs

@@ -0,0 +1,125 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using SW.Bitween.Domain;
+
+namespace SW.Bitween.Services;
+
+/// <summary>
+/// The bus names its exchange and every queue after ASPNETCORE_ENVIRONMENT. A new name moves Bitween
+/// to new queues while the other applications on the same RabbitMQ keep sending to the old ones, and
+/// what was waiting in them stays there. So a name Bitween didn't run with before is refused until
+/// it is confirmed.
+/// </summary>
+/// <remarks>
+/// Not a hypothetical: the chart's default environment changed from Development to Production, so a
+/// deployment that never chose one changed it on upgrade without anyone deciding to.
+/// </remarks>
+public static class BusEnvironment
+{
+    /// <summary>
+    /// The name Bitween last ran with. A Settings row nobody edits and the settings page doesn't list,
+    /// as <see cref="SettingsService.LegacyDocumentPrefixKey"/>.
+    /// </summary>
+    public const string RecordedKey = "Bitween.BusEnvironment";
+
+    public const string ConfirmSetting = "Bitween:ConfirmBusEnvironment";
+
+    private static readonly string[] UsualNames = ["Development", "Staging", "Production"];
+
+    public static IHost CheckBusEnvironment(this IHost host)
+    {
+        using var scope = host.Services.CreateScope();
+        var services = scope.ServiceProvider;
+        var configuration = services.GetRequiredService<IConfiguration>();
+        var current = services.GetRequiredService<IHostEnvironment>().EnvironmentName;
+        var queuePrefix = services.GetRequiredService<BitweenOptions>().QueuePrefix;
+
+        var problem = Check(services.GetRequiredService<BitweenDbContext>(), current, configuration[ConfirmSetting],
+                () => FormerName(configuration.GetConnectionString("RabbitMQ"), queuePrefix, current),
+                services.GetRequiredService<ILogger<BitweenDbContext>>())
+            .GetAwaiter().GetResult();
+        if (problem is not null) host.Services.GetRequiredService<StartupProblems>().Add(problem);
+        return host;
+    }
+
+    /// <param name="formerName">
+    /// For a Bitween that ran before names were recorded: the name its queues were made under.
+    /// </param>
+    /// <returns>The startup problem, or null when this name may run.</returns>
+    public static async Task<string> Check(BitweenDbContext db, string current, string confirmed,
+        Func<Task<string>> formerName, ILogger logger)
+    {
+        var row = await db.Set<Setting>().SingleOrDefaultAsync(s => s.Id == RecordedKey);
+        var recorded = row?.Value;
+
+        // A new installation has nothing to strand. One already in use tells by its queues.
+        if (recorded is null && await db.Set<Subscription>().AnyAsync())
+        {
+            try
+            {
+                recorded = await formerName();
+            }
+            catch (Exception ex)
+            {
+                // Not left to a later start: by then the bus has made queues under the new name, and they
+                // would be all that start finds. Without the broker the bus can't run anyway.
+                logger.LogWarning(ex, "Could not reach RabbitMQ to tell which environment this Bitween ran as before.");
+                return "RabbitMQ could not be reached to tell which ASPNETCORE_ENVIRONMENT this Bitween's queues were " +
+                       $"made under ({ex.GetBaseException().Message}). Check ConnectionStrings:RabbitMQ and that the " +
+                       "broker is up.";
+            }
+        }
+
+        if (recorded is not null && !Same(recorded, current) && !Same(confirmed, current))
+            return $"ASPNETCORE_ENVIRONMENT is {current}, but this Bitween's RabbitMQ queues are named after " +
+                   $"{recorded}, which it ran with before. With the new name it would read from new queues, " +
+                   "while other applications on this RabbitMQ keep sending to the old ones. If the change is a " +
+                   $"mistake, set ASPNETCORE_ENVIRONMENT (the chart's global.environment) back to {recorded}. If " +
+                   $"every application on this RabbitMQ moved to {current} on purpose, set " +
+                   $"Bitween__ConfirmBusEnvironment={current}.";
+
+        if (Same(row?.Value, current)) return null;
+
+        if (row is null) db.Add(new Setting { Id = RecordedKey, Value = current });
+        else row.Value = current;
+        try
+        {
+            await db.SaveChangesAsync();
+            logger.LogInformation("Bitween's RabbitMQ queues are named after the environment {Environment}.", current);
+        }
+        catch (DbUpdateException ex)
+        {
+            // Another node starting at the same moment recorded it first, with the same name.
+            logger.LogWarning(ex, "The bus environment was recorded by another node.");
+        }
+
+        return null;
+    }
+
+    private static async Task<string> FormerName(string connectionString, string queuePrefix, string current)
+    {
+        await using var broker = await BrokerQueues.ConnectAsync(connectionString);
+        foreach (var name in UsualNames.Prepend(current).Distinct(StringComparer.OrdinalIgnoreCase))
+        {
+            // The lane of exchanges without a work group, which every Bitween since work groups has;
+            // before them, the queue of exchanges from API calls.
+            if (await broker.InspectAsync(BrokerQueues.QueueName(name, queuePrefix, nameof(XchangeService),
+                    WorkGroup.None.GetBusMessageName())) is not null ||
+                await broker.InspectAsync(BrokerQueues.QueueName(name, queuePrefix, nameof(XchangeService),
+                    nameof(ApiXchangeCreatedEvent))) is not null)
+                return name;
+        }
+
+        // Nothing under any of them, so nothing to leave behind.
+        return current;
+    }
+
+    // The bus lowercases the name in every exchange and queue.
+    private static bool Same(string a, string b) => string.Equals(a, b, StringComparison.OrdinalIgnoreCase);
+}