From de2a6f4ee11d26d338a870b5b5fc2f9fefaedc3e Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Mon, 19 Oct 2026 11:42:08 +0300 Subject: [PATCH 01/10] feat: list every startup problem in one log entry --- .../Services/SeededAdministrator.cs | 10 ++-- SW.Bitween.Api/Services/StartupProblems.cs | 41 ++++++++++++++++ .../Tests/SeededAdministratorTests.cs | 4 +- SW.Bitween.UnitTests/StartupProblemsTests.cs | 33 +++++++++++++ SW.Bitween.Web/Program.cs | 34 ++++++++++++- SW.Bitween.Web/Startup.cs | 48 ++++++++++++------- docs/deployment.md | 7 +++ docs/operations.md | 1 + 8 files changed, 152 insertions(+), 26 deletions(-) create mode 100644 SW.Bitween.Api/Services/StartupProblems.cs create mode 100644 SW.Bitween.UnitTests/StartupProblemsTests.cs diff --git a/SW.Bitween.Api/Services/SeededAdministrator.cs b/SW.Bitween.Api/Services/SeededAdministrator.cs index bc6010a9..cc1fc895 100644 --- a/SW.Bitween.Api/Services/SeededAdministrator.cs +++ b/SW.Bitween.Api/Services/SeededAdministrator.cs @@ -26,11 +26,11 @@ public static class SeededAdministrator public static IHost SecureSeededAdministrator(this IHost host) { using var scope = host.Services.CreateScope(); - Secure( + host.Services.GetRequiredService().Check(() => Secure( scope.ServiceProvider.GetRequiredService(), scope.ServiceProvider.GetRequiredService()["Bitween:InitialAdminPassword"], scope.ServiceProvider.GetRequiredService>()) - .GetAwaiter().GetResult(); + .GetAwaiter().GetResult()); return host; } @@ -54,14 +54,14 @@ public static async Task Secure(BitweenDbContext dbContext, string initialPasswo if (await dbContext.Set().AnyAsync(a => a.Id != Id && !a.Deleted)) return; - throw new InvalidOperationException( + throw new StartupProblemException( "This is a new installation and Bitween:InitialAdminPassword is not configured. Set it " + $"(env: Bitween__InitialAdminPassword) to the password for the administrator, {admin.Email}. " + "It is read on startup only while that account still has the default password."); } if (initialPassword == PublishedPassword) - throw new InvalidOperationException( + throw new StartupProblemException( "Bitween:InitialAdminPassword is the default password, which is published in a public " + "repository. Choose a different one."); @@ -69,7 +69,7 @@ public static async Task Secure(BitweenDbContext dbContext, string initialPasswo policy.RuleFor(p => p).Password().OverridePropertyName("InitialAdminPassword"); var result = policy.Validate(initialPassword); if (!result.IsValid) - throw new InvalidOperationException( + throw new StartupProblemException( "Bitween:InitialAdminPassword does not meet the password policy: " + string.Join(" ", result.Errors.Select(e => e.ErrorMessage))); diff --git a/SW.Bitween.Api/Services/StartupProblems.cs b/SW.Bitween.Api/Services/StartupProblems.cs new file mode 100644 index 00000000..f560d3bc --- /dev/null +++ b/SW.Bitween.Api/Services/StartupProblems.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace SW.Bitween.Services; + +/// +/// A reason Bitween must not start, worded as what to change. A check throws it and +/// collects it, so the checks after it still run. +/// +public class StartupProblemException(string message) : InvalidOperationException(message); + +/// +/// Every reason this start is refused. They are logged together, so whoever deploys fixes all of +/// them in one redeploy rather than meeting them one restart at a time. +/// +public class StartupProblems +{ + private readonly List _all = []; + + public bool Any => _all.Count > 0; + + public void Add(string problem) => _all.Add(problem); + + /// Runs , recording the problem it reports instead of stopping. + public void Check(Action check) + { + try + { + check(); + } + catch (StartupProblemException ex) + { + Add(ex.Message); + } + } + + public override string ToString() => + "Bitween can't start until these are fixed:" + + string.Concat(_all.Select((problem, i) => $"{Environment.NewLine} {i + 1}. {problem}")); +} diff --git a/SW.Bitween.IntegrationTests/Tests/SeededAdministratorTests.cs b/SW.Bitween.IntegrationTests/Tests/SeededAdministratorTests.cs index 8e0fcb69..390583ae 100644 --- a/SW.Bitween.IntegrationTests/Tests/SeededAdministratorTests.cs +++ b/SW.Bitween.IntegrationTests/Tests/SeededAdministratorTests.cs @@ -56,7 +56,7 @@ public async Task Without_one_a_new_installation_refuses_to_start() await db.Set().Where(a => a.Id != SeededAdministrator.Id) .ExecuteUpdateAsync(s => s.SetProperty(a => a.Deleted, true)); - var refused = await Assert.ThrowsAsync(() => + var refused = await Assert.ThrowsAsync(() => SeededAdministrator.Secure(db, null, NullLogger.Instance)); Assert.Contains("Bitween__InitialAdminPassword", refused.Message); } @@ -83,7 +83,7 @@ public async Task The_published_password_is_refused_as_the_configured_one() var db = scope.ServiceProvider.GetRequiredService(); await using var transaction = await db.Database.BeginTransactionAsync(); - await Assert.ThrowsAsync(() => + await Assert.ThrowsAsync(() => SeededAdministrator.Secure(db, SeededAdministrator.PublishedPassword, NullLogger.Instance)); } diff --git a/SW.Bitween.UnitTests/StartupProblemsTests.cs b/SW.Bitween.UnitTests/StartupProblemsTests.cs new file mode 100644 index 00000000..5fd6febf --- /dev/null +++ b/SW.Bitween.UnitTests/StartupProblemsTests.cs @@ -0,0 +1,33 @@ +using System; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Bitween.Services; + +namespace SW.Bitween.UnitTests; + +[TestClass] +public class StartupProblemsTests +{ + [TestMethod] + public void Every_problem_is_listed_together() + { + var problems = new StartupProblems(); + problems.Check(() => throw new StartupProblemException("Token:Key is not configured.")); + problems.Add("ConnectionStrings:RabbitMQ is not configured."); + + Assert.IsTrue(problems.Any); + Assert.AreEqual( + "Bitween can't start until these are fixed:" + + $"{Environment.NewLine} 1. Token:Key is not configured." + + $"{Environment.NewLine} 2. ConnectionStrings:RabbitMQ is not configured.", + problems.ToString()); + } + + [TestMethod] + public void Other_failures_are_not_mistaken_for_a_problem() + { + var problems = new StartupProblems(); + Assert.ThrowsException(() => + problems.Check(() => throw new InvalidOperationException("the database went away"))); + Assert.IsFalse(problems.Any); + } +} diff --git a/SW.Bitween.Web/Program.cs b/SW.Bitween.Web/Program.cs index acfb4c15..b01d6065 100644 --- a/SW.Bitween.Web/Program.cs +++ b/SW.Bitween.Web/Program.cs @@ -20,7 +20,18 @@ public class Program public static void Main(string[] args) { //var id = (long)(DateTime.UtcNow.Subtract(new DateTime(2010, 1, 1)).TotalMilliseconds * 1000); - var host = CreateHostBuilder(args).Build(); + IHost host; + try + { + host = CreateHostBuilder(args).Build(); + } + catch (StartupProblemException ex) + { + // No database to check further, and no logger yet: the list is all there is to say. + Console.Error.WriteLine(ex.Message); + Environment.ExitCode = 1; + return; + } // Startup migration failures otherwise surface only as a bare unhandled exception with // no indication of which database was targeted, which makes an environment-specific @@ -41,7 +52,26 @@ public static void Main(string[] args) } host.SecureSeededAdministrator().SecureSystemPartnerKey().HashStoredPartnerKeys() - .MoveRetryAlertsToChannels().ApplyStoredSettings().Run(); + .MoveRetryAlertsToChannels().ApplyStoredSettings(); + + if (RefusesToStart(host)) return; + host.Run(); + } + + /// + /// Logs every problem found while starting as one list, configuration and database alike, and + /// exits without serving. + /// + private static bool RefusesToStart(IHost host) + { + var problems = host.Services.GetRequiredService(); + if (!problems.Any) return false; + + host.Services.GetRequiredService>().LogCritical("{Problems}", problems.ToString()); + // Flushes the log before the process ends. + host.Dispose(); + Environment.ExitCode = 1; + return true; } /// The longest one migration statement may run, such as an index build on a big table. diff --git a/SW.Bitween.Web/Startup.cs b/SW.Bitween.Web/Startup.cs index 40e64221..3c197d1b 100644 --- a/SW.Bitween.Web/Startup.cs +++ b/SW.Bitween.Web/Startup.cs @@ -74,6 +74,29 @@ public void ConfigureServices(IServiceCollection services) var themeOptions = new ThemeOptions(); Configuration.GetSection(BitweenOptions.ConfigurationSection).Bind(bitweenOptions); Configuration.GetSection(ThemeOptions.ConfigurationSection).Bind(themeOptions); + + // Configuration mistakes are collected here, not thrown one at a time; Program logs them + // with the ones only the database can show, and refuses to start. + var problems = new StartupProblems(); + services.AddSingleton(problems); + problems.Check(RejectSampleSigningKey); + if (string.Equals(bitweenOptions.StorageProvider, "Local", StringComparison.OrdinalIgnoreCase) && + !Environment.IsDevelopment()) + problems.Add("StorageProvider 'Local' stores files on the local filesystem and is only allowed when " + + "ASPNETCORE_ENVIRONMENT is 'Development'."); + var unregistrable = false; + foreach (var (name, example) in new[] + { + (BitweenDbContext.ConnectionString, "the database's connection string"), + ("RabbitMQ", "the broker's address, as amqp://user:password@rabbitmq:5672/") + }) + { + if (!string.IsNullOrWhiteSpace(Configuration.GetConnectionString(name))) continue; + problems.Add($"ConnectionStrings:{name} is not configured. Set it (env: ConnectionStrings__{name}) to {example}."); + unregistrable = true; + } + // Nothing past this point can be registered without them: report them with the rest now. + if (unregistrable) throw new StartupProblemException(problems.ToString()); services.AddSingleton(themeOptions); services.AddSingleton(bitweenOptions); // The built-in mappers cannot see these options; the shared-parse switch reaches them here. @@ -225,9 +248,6 @@ public void ConfigureServices(IServiceCollection services) services.AddS3CloudFiles(); break; case "LOCAL": - if (!Environment.IsDevelopment()) - throw new InvalidOperationException( - "StorageProvider 'Local' stores files on the local filesystem and is only allowed when ASPNETCORE_ENVIRONMENT is 'Development'."); services.AddLocalTestsCloudFiles(); break; default: @@ -280,14 +300,8 @@ public void ConfigureServices(IServiceCollection services) } services.AddScoped(); - // Get and validate connection string + // Present: checked with the rest of the configuration at the top. var connectionString = Configuration.GetConnectionString(BitweenDbContext.ConnectionString); - if (string.IsNullOrWhiteSpace(connectionString)) - { - throw new InvalidOperationException( - $"Connection string '{BitweenDbContext.ConnectionString}' is not configured. " + - "Please check your appsettings.json or environment configuration."); - } // MySQL connections are encrypted unless the connection string chooses otherwise: the driver's // default, Preferred, falls back to plain text against a server without TLS. Set here so the @@ -357,10 +371,11 @@ public void ConfigureServices(IServiceCollection services) if (!connectionString.Contains("Host=", StringComparison.OrdinalIgnoreCase) && !connectionString.Contains("Server=", StringComparison.OrdinalIgnoreCase)) { - throw new InvalidOperationException( + problems.Add( $"PostgreSQL connection string is missing 'Host=' or 'Server=' parameter. " + // Never the connection string itself: it carries the password, and this goes to the log. "Please check your ConnectionStrings:BitweenDb configuration."); + throw new StartupProblemException(problems.ToString()); } // Configure connection with Azure Managed Identity for PostgreSQL if enabled @@ -486,7 +501,6 @@ public void ConfigureServices(IServiceCollection services) // { // config.DefaultApiClientFactory = sp => sp.GetService(); // }); - RejectSampleSigningKey(); AddRateLimiting(services); services.AddJwtTokenParameters(); services.AddAuthorization(); @@ -683,19 +697,19 @@ private void RejectSampleSigningKey() var key = Configuration["Token:Key"]; if (string.IsNullOrWhiteSpace(key)) - throw new InvalidOperationException( - "Token:Key is not configured. Generate a random secret unique to this " + - "deployment - tokens cannot be trusted without one."); + throw new StartupProblemException( + "Token:Key is not configured. Set it (env: Token__Key) to a random secret unique to " + + "this deployment - tokens cannot be trusted without one."); // The length our own configuration documents. A short key is brute-forceable offline // against any token the holder has seen, which is every token they were ever issued. if (key.Length < MinimumSigningKeyLength) - throw new InvalidOperationException( + throw new StartupProblemException( $"Token:Key is {key.Length} characters; it must be at least " + $"{MinimumSigningKeyLength}. Generate a random secret rather than choosing one."); if (key == SampleSigningKey) - throw new InvalidOperationException( + throw new StartupProblemException( "Token:Key is still the sample value that ships as the default in the " + "deployment chart, and it is published in a public repository - anyone can " + "mint a valid token with it. Set Token:Key (env: Token__Key) to a random " + diff --git a/docs/deployment.md b/docs/deployment.md index fd8483e9..4f6f1728 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -119,6 +119,13 @@ See [Data sources](data-sources.md). Run several replicas behind a load balancer for throughput and availability, and tune prefetch, parallel exchanges and priority per work group. Read the [replica notes](architecture.md#running-more-than-one-replica) about the scheduler first. +## When Bitween won't start + +Bitween checks its configuration and its database before it serves anything. When something is wrong it logs one entry, `Bitween can't start until these are fixed:`, with a numbered list of what to change, and exits. Fix everything on the list and redeploy. + +- Without `ConnectionStrings:BitweenDb` or `ConnectionStrings:RabbitMQ`, the list is written to the console and holds only the configuration problems; the database is checked on the next start. +- A failed migration is logged on its own, as `Database migration failed on startup against {Host}:{Port}/{Database}`. + ## Upgrading - Migrations apply at startup, so back up the database before deploying a new version. diff --git a/docs/operations.md b/docs/operations.md index 3d98d306..d3c683a5 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -119,6 +119,7 @@ Bitween logs to the console. SQL statements are logged without their parameter v Messages worth watching for: +- `Bitween can't start until these are fixed:`, followed by what to change. See [When Bitween won't start](deployment.md#when-bitween-wont-start) - `Database migration failed on startup against {Host}:{Port}/{Database}` - A startup warning that the Microsoft sign-in redirect URI is not the sign-in landing page - `Auto-retry evaluation failed for xchange …` From c099deb2305572b900a7d07ce0219757c7bcdd3e Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Mon, 19 Oct 2026 11:42:08 +0300 Subject: [PATCH 02/10] feat: refuse to start when the encryption key can't read stored credentials --- SW.Bitween.Api/Data/SecretColumnCipher.cs | 35 ++++++-- .../Services/SecretColumnEncryptionPass.cs | 4 + SW.Bitween.Api/Services/StoredSecretsKey.cs | 84 +++++++++++++++++++ .../Tests/SecretColumnEncryptionTests.cs | 46 +++++++++- .../SecretColumnCipherTests.cs | 13 ++- SW.Bitween.Web/Program.cs | 7 +- SW.Bitween.Web/Startup.cs | 3 +- docs/configuration.md | 1 + docs/security.md | 1 + 9 files changed, 179 insertions(+), 15 deletions(-) create mode 100644 SW.Bitween.Api/Services/StoredSecretsKey.cs diff --git a/SW.Bitween.Api/Data/SecretColumnCipher.cs b/SW.Bitween.Api/Data/SecretColumnCipher.cs index 45f98d8b..afe9bd09 100644 --- a/SW.Bitween.Api/Data/SecretColumnCipher.cs +++ b/SW.Bitween.Api/Data/SecretColumnCipher.cs @@ -27,6 +27,11 @@ namespace SW.Bitween; /// change the key, move the old one to Bitween:PreviousSettingsEncryptionKey: values are read /// with either and written with the new one. /// +/// +/// A key that is gone for good leaves nothing to read them with. Bitween:SettingsEncryptionKeyLost +/// then reads such a value as empty, so it can be entered again, and leaves what is stored in place +/// until it is: should the key turn up after all, whatever wasn't entered again reads once more. +/// /// public static class SecretColumnCipher { @@ -41,14 +46,19 @@ public static class SecretColumnCipher private static byte[] _key; private static byte[] _previousKey; + private static bool _keyLost; /// Whether values are being encrypted as they are written. public static bool Enabled => _key is not null; - public static void Configure(string passphrase, string previousPassphrase = null) + /// Whether a value no key here can read is taken as empty rather than refused. + public static bool KeyLost => _keyLost; + + public static void Configure(string passphrase, string previousPassphrase = null, bool keyLost = false) { _key = string.IsNullOrWhiteSpace(passphrase) ? null : Derive(passphrase); _previousKey = string.IsNullOrWhiteSpace(previousPassphrase) ? null : Derive(previousPassphrase); + _keyLost = keyLost; } public static string Protect(string plaintext) @@ -66,7 +76,12 @@ public static string Protect(string plaintext) return Prefix + Convert.ToBase64String(output); } - public static string Unprotect(string stored) + public static string Unprotect(string stored) => Unprotect(stored, unreadable: null); + + /// The same for JSON, where a value that can't be read is JSON's own empty value. + private static string UnprotectJson(string stored) => Unprotect(stored, unreadable: "null"); + + private static string Unprotect(string stored, string unreadable) { if (stored is null || !stored.StartsWith(Prefix, StringComparison.Ordinal)) return stored; @@ -85,10 +100,8 @@ public static string Unprotect(string stored) } } - throw new InvalidOperationException( - "A stored value is encrypted with a key this instance does not have. Set " + - "Bitween:SettingsEncryptionKey to the key it was written with, or put that key in " + - "Bitween:PreviousSettingsEncryptionKey while moving to a new one."); + if (_keyLost) return unreadable; + throw new UnreadableSecretException(); } private static string Decrypt(byte[] key, byte[] input) @@ -115,7 +128,7 @@ public static PropertyBuilder Encrypted(this PropertyBuilder builder) $"{builder.Metadata.Name} has no converter to encrypt the output of; call StoreAsJson first."); var comparer = builder.Metadata.GetValueComparer(); - var cipher = new ValueConverter(v => Protect(v), v => Unprotect(v)); + var cipher = new ValueConverter(v => Protect(v), v => UnprotectJson(v)); builder.Metadata.SetValueConverter(inner.ComposeWith(cipher)); if (comparer is not null) builder.Metadata.SetValueComparer(comparer); return builder; @@ -153,7 +166,7 @@ private static T FromJsonb(string stored) where T : class { if (stored is null) return null; var json = stored.TrimStart().StartsWith("\"" + Prefix, StringComparison.Ordinal) - ? Unprotect(JsonConvert.DeserializeObject(stored)) + ? UnprotectJson(JsonConvert.DeserializeObject(stored)) : stored; return JsonConvert.DeserializeObject(json); } @@ -184,3 +197,9 @@ private static System.Collections.Generic.Dictionary MapValues( return result; } } + +/// A stored credential that neither of this instance's keys can decrypt. +public class UnreadableSecretException() : InvalidOperationException( + "A stored value is encrypted with a key this instance does not have. Set " + + "Bitween:SettingsEncryptionKey to the key it was written with, or put that key in " + + "Bitween:PreviousSettingsEncryptionKey while moving to a new one."); diff --git a/SW.Bitween.Api/Services/SecretColumnEncryptionPass.cs b/SW.Bitween.Api/Services/SecretColumnEncryptionPass.cs index 154cead7..8dc4dc57 100644 --- a/SW.Bitween.Api/Services/SecretColumnEncryptionPass.cs +++ b/SW.Bitween.Api/Services/SecretColumnEncryptionPass.cs @@ -41,6 +41,10 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) return; } + // Re-saving a row writes what it read, and with the key lost that is empty: the values the + // lost key encrypted would be gone for good, rather than readable should it be found. + if (SecretColumnCipher.KeyLost) return; + try { await Task.Delay(Delay, stoppingToken); diff --git a/SW.Bitween.Api/Services/StoredSecretsKey.cs b/SW.Bitween.Api/Services/StoredSecretsKey.cs new file mode 100644 index 00000000..755c84fb --- /dev/null +++ b/SW.Bitween.Api/Services/StoredSecretsKey.cs @@ -0,0 +1,84 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using SW.Bitween.Domain; +using SW.Bitween.Domain.DataSources; + +namespace SW.Bitween.Services; + +/// +/// Whether the credentials this database holds can be read with the configured +/// Bitween:SettingsEncryptionKey. A missing or wrong key used to start without complaint and +/// then fail every exchange that needed a partner's or a subscription's settings. +/// +public static class StoredSecretsKey +{ + public const string KeyLostSetting = "Bitween:SettingsEncryptionKeyLost"; + + // Every value is written with the same key, so a table's newest rows tell, being the likeliest + // written since encryption was turned on; the encryption pass reads all of them minutes later. + private const int Sample = 100; + + /// + /// Records a startup problem when they can't be read. Runs after the migrations, before anything + /// that reads those tables. + /// + /// Whether they can be read, so the startup steps that read them can run. + public static bool CanReadStoredSecrets(this IHost host) + { + using var scope = host.Services.CreateScope(); + var logger = scope.ServiceProvider.GetRequiredService>(); + + if (SecretColumnCipher.KeyLost) + { + logger.LogWarning("{Setting} is on: credentials that can't be read with the current key show as " + + "empty until they are entered again. Remove it once they are.", KeyLostSetting); + return true; + } + + if (AllReadable(scope.ServiceProvider.GetRequiredService()).GetAwaiter().GetResult()) + return true; + + host.Services.GetRequiredService().Add( + (SecretColumnCipher.Enabled + ? "Bitween:SettingsEncryptionKey is not the key this database's credentials were encrypted with. " + : "Bitween:SettingsEncryptionKey is not set, but this database holds credentials encrypted with one. ") + + "Set it (env: Bitween__SettingsEncryptionKey) to the key used before: a Helm deployment still has " + + "it in an earlier release, shown by `helm history ` and `helm get values " + + "--revision `. Only if it is lost for good, set Bitween__SettingsEncryptionKeyLost=true: " + + "partner, subscription and data source settings it encrypted then show as empty until " + + "they are entered again."); + return false; + } + + public static async Task AllReadable(BitweenDbContext db) => + await Readable(db) && await Readable(db) && + await Readable(db) && await Readable(db) && + await Readable(db) && await Readable(db) && + await Readable(db) && await Readable(db) && + await Readable(db); + + private static async Task Readable(BitweenDbContext db) where T : class + { + var key = db.Model.FindEntityType(typeof(T))!.FindPrimaryKey()!.Properties[0].Name; + try + { + await db.Set().AsNoTracking() + .OrderByDescending(e => EF.Property(e, key)) + .Take(Sample).ToListAsync(); + return true; + } + catch (Exception ex) when (IsUnreadable(ex)) + { + return false; + } + } + + // EF wraps what a value converter throws. + private static bool IsUnreadable(Exception ex) => + ex is UnreadableSecretException || ex.InnerException is not null && IsUnreadable(ex.InnerException); +} diff --git a/SW.Bitween.IntegrationTests/Tests/SecretColumnEncryptionTests.cs b/SW.Bitween.IntegrationTests/Tests/SecretColumnEncryptionTests.cs index 4d3b5877..30baff35 100644 --- a/SW.Bitween.IntegrationTests/Tests/SecretColumnEncryptionTests.cs +++ b/SW.Bitween.IntegrationTests/Tests/SecretColumnEncryptionTests.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.DependencyInjection; using SW.Bitween.Domain; using SW.Bitween.IntegrationTests.Fixtures; +using SW.Bitween.Services; using Xunit; namespace SW.Bitween.IntegrationTests.Tests; @@ -24,15 +25,16 @@ public class SecretColumnEncryptionTests(BitweenFixture fixture) public async Task A_partner_credential_is_stored_encrypted_and_read_back_plain() { SecretColumnCipher.Configure("integration test passphrase"); + var partnerId = await CreatePartnerAsync(); try { - var partnerId = await CreatePartnerAsync(); - Assert.DoesNotContain(Secret, await RawAdapterPropertiesAsync(partnerId)); Assert.Equal(Secret, (await ReadPartnerAsync(partnerId)).AdapterProperties["password"]); } finally { + // Unreadable to the rest of the collection once the key is gone. + await DeletePartnerAsync(partnerId); SecretColumnCipher.Configure(null); } } @@ -64,6 +66,46 @@ public async Task A_credential_saved_before_encryption_is_encrypted_by_the_pass( } } + [Fact] + public async Task A_key_that_did_not_encrypt_them_is_found_before_startup() + { + SecretColumnCipher.Configure("the key it was written with"); + var partnerId = await CreatePartnerAsync(); + try + { + Assert.True(await AllReadableAsync()); + + SecretColumnCipher.Configure("a different key"); + Assert.False(await AllReadableAsync()); + + SecretColumnCipher.Configure(null); + Assert.False(await AllReadableAsync()); + + // Lost for good: what it encrypted reads as empty, so it can be entered again. + SecretColumnCipher.Configure("a different key", keyLost: true); + Assert.True(await AllReadableAsync()); + Assert.Null((await ReadPartnerAsync(partnerId)).AdapterProperties); + } + finally + { + await DeletePartnerAsync(partnerId); + SecretColumnCipher.Configure(null); + } + } + + private async Task DeletePartnerAsync(int id) + { + await using var scope = fixture.CreateScope(); + await scope.ServiceProvider.GetRequiredService() + .Set().Where(p => p.Id == id).ExecuteDeleteAsync(); + } + + private async Task AllReadableAsync() + { + await using var scope = fixture.CreateScope(); + return await StoredSecretsKey.AllReadable(scope.ServiceProvider.GetRequiredService()); + } + private async Task CreatePartnerAsync() { await using var scope = fixture.CreateScope(); diff --git a/SW.Bitween.UnitTests/SecretColumnCipherTests.cs b/SW.Bitween.UnitTests/SecretColumnCipherTests.cs index 993d4187..776ae9e7 100644 --- a/SW.Bitween.UnitTests/SecretColumnCipherTests.cs +++ b/SW.Bitween.UnitTests/SecretColumnCipherTests.cs @@ -1,4 +1,3 @@ -using System; using Microsoft.VisualStudio.TestTools.UnitTesting; namespace SW.Bitween.UnitTests; @@ -51,7 +50,17 @@ public void A_value_written_with_an_unknown_key_says_so() var stored = SecretColumnCipher.Protect("secret"); SecretColumnCipher.Configure("a different key"); - var ex = Assert.ThrowsException(() => SecretColumnCipher.Unprotect(stored)); + var ex = Assert.ThrowsException(() => SecretColumnCipher.Unprotect(stored)); StringAssert.Contains(ex.Message, "PreviousSettingsEncryptionKey"); } + + [TestMethod] + public void With_the_key_lost_a_value_it_encrypted_reads_as_empty() + { + SecretColumnCipher.Configure("the lost key"); + var stored = SecretColumnCipher.Protect("secret"); + + SecretColumnCipher.Configure("a new key", keyLost: true); + Assert.IsNull(SecretColumnCipher.Unprotect(stored)); + } } diff --git a/SW.Bitween.Web/Program.cs b/SW.Bitween.Web/Program.cs index b01d6065..ff7a24b7 100644 --- a/SW.Bitween.Web/Program.cs +++ b/SW.Bitween.Web/Program.cs @@ -51,8 +51,11 @@ public static void Main(string[] args) throw; } - host.SecureSeededAdministrator().SecureSystemPartnerKey().HashStoredPartnerKeys() - .MoveRetryAlertsToChannels().ApplyStoredSettings(); + host.SecureSeededAdministrator(); + // These read partners and alerts, whose stored credentials need the encryption key. + if (host.CanReadStoredSecrets()) + host.SecureSystemPartnerKey().HashStoredPartnerKeys().MoveRetryAlertsToChannels(); + host.ApplyStoredSettings(); if (RefusesToStart(host)) return; host.Run(); diff --git a/SW.Bitween.Web/Startup.cs b/SW.Bitween.Web/Startup.cs index 3c197d1b..a5652cad 100644 --- a/SW.Bitween.Web/Startup.cs +++ b/SW.Bitween.Web/Startup.cs @@ -104,7 +104,8 @@ public void ConfigureServices(IServiceCollection services) // The same passphrase the settings secrets use, now also for the credential columns. SecretColumnCipher.Configure(bitweenOptions.SettingsEncryptionKey, - Configuration["Bitween:PreviousSettingsEncryptionKey"]); + Configuration["Bitween:PreviousSettingsEncryptionKey"], + Configuration.GetValue(StoredSecretsKey.KeyLostSetting)); // Keeps both option objects in sync with the Settings table, which owns every editable // setting; Program hands configuration over to it at boot. The protector encrypts // secret settings before they're stored. diff --git a/docs/configuration.md b/docs/configuration.md index afa770b8..b9753ca1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -103,6 +103,7 @@ applications the same `Token` values. Until they match, `Bus:OnInvalidUserToken` | `ConsumeLegacyEventMessages` | `false` | Also drain the queues that versions before work groups published to. | | `CorsOrigins` | empty | Browser origins allowed to call the API with credentials. Set as `Bitween__CorsOrigins__0`, `Bitween__CorsOrigins__1` and so on. | | `SettingsEncryptionKey` | | Passphrase that encrypts secret settings and stored credentials in the database. See [Security](security.md#secrets). | +| `SettingsEncryptionKeyLost` | `false` | Only when the key that encrypted the stored credentials is gone for good: start anyway, and show what it encrypted as empty until entered again. See [Security](security.md#secrets). | | `ReceiveAttemptRetentionDays` | `30` | Days to keep receive attempts. | | `AuditRetentionDays` | `0` | Days to keep audit entries; the daily clean-up deletes older ones. 0 keeps them indefinitely. | | `LogSqlParameterValues` | `false` | Write SQL parameter values into the logs. They hold passwords, partner keys and payloads, so it is off in every environment unless set; the repository's `launchSettings.json` turns it on for a developer's own machine. | diff --git a/docs/security.md b/docs/security.md index 6077c872..a1951b3d 100644 --- a/docs/security.md +++ b/docs/security.md @@ -139,6 +139,7 @@ The seeded **SYSTEM** partner has a key named `default` whose value is fixed in - Teams and Slack webhook addresses are secret, because anyone who has one can post to the channel. - A bus gateway's endpoint settings are masked as its data source's settings are: those the data source names as secret, and any with a credential's name (password, token, key and so on). Sending `__private__` back keeps the stored value. - Partner properties, global values, adapter properties, data source settings, notification channel, notifier and retry alert properties, including passwords, are encrypted in the database with AES-256-GCM when `Bitween:SettingsEncryptionKey` is set, and stored in plain text when it isn't. Values stored before the key was set are encrypted a few minutes after the next start. `Bitween:PreviousSettingsEncryptionKey` still decrypts values while the key is rotated. +- Bitween [won't start](deployment.md#when-bitween-wont-start) when it can't read these with the key it has: a key that isn't set, or isn't the one they were encrypted with. A Helm deployment still has the key in an earlier release: `helm history ` lists them and `helm get values --revision ` shows one's values. A database backup doesn't help without it, as it is encrypted with the same key. When the key is lost for good, `Bitween:SettingsEncryptionKeyLost=true` lets Bitween start: every value the lost key encrypted shows as empty until it is entered again, and stays stored until then, so it reads again if the key turns up. Remove the setting once they are entered again. ## Exchange files From 5da9ba9a3b64b7e02e1d1d34ba7dd02c86eff483 Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Mon, 19 Oct 2026 11:42:09 +0300 Subject: [PATCH 03/10] feat: refuse to start when the bus environment name changes --- SW.Bitween.Api/Services/BrokerQueues.cs | 70 ++++++++ SW.Bitween.Api/Services/BusEnvironment.cs | 122 ++++++++++++++ .../Tests/BusEnvironmentTests.cs | 149 ++++++++++++++++++ SW.Bitween.Web/Program.cs | 2 +- docs/configuration.md | 3 +- 5 files changed, 344 insertions(+), 2 deletions(-) create mode 100644 SW.Bitween.Api/Services/BrokerQueues.cs create mode 100644 SW.Bitween.Api/Services/BusEnvironment.cs create mode 100644 SW.Bitween.IntegrationTests/Tests/BusEnvironmentTests.cs diff --git a/SW.Bitween.Api/Services/BrokerQueues.cs b/SW.Bitween.Api/Services/BrokerQueues.cs new file mode 100644 index 00000000..8bb11fa3 --- /dev/null +++ b/SW.Bitween.Api/Services/BrokerQueues.cs @@ -0,0 +1,70 @@ +using System; +using System.Threading.Tasks; +using RabbitMQ.Client; +using RabbitMQ.Client.Exceptions; +using SW.Bus; + +namespace SW.Bitween.Services; + +/// +/// Bitween's own queues on RabbitMQ, looked at before the bus starts: whether one exists, what is in +/// it, and removing one nothing uses any more. The bus itself only ever declares, which would create +/// the very queue being asked about. +/// +public sealed class BrokerQueues : IAsyncDisposable +{ + private readonly IConnection _connection; + + private BrokerQueues(IConnection connection) => _connection = connection; + + public static async Task ConnectAsync(string connectionString) + { + var factory = new ConnectionFactory + { + Uri = new Uri(connectionString), + AutomaticRecoveryEnabled = false, + RequestedConnectionTimeout = TimeSpan.FromSeconds(15) + }; + return new BrokerQueues(await factory.CreateConnectionAsync("bitween-startup")); + } + + /// + /// The name the bus gives a consumer's queue: v3.{environment}.{queue prefix}.{consumer}.{message}. + /// + public static string QueueName(string environment, string queuePrefix, string consumer, string message) => + $"{new BusOptions(environment).ProcessExchange}.{queuePrefix}.{consumer}.{message}".ToLowerInvariant(); + + /// The queue's message and consumer counts, or null when there is no such queue. + public async Task InspectAsync(string queue) + { + // Asking about a missing queue closes the channel it was asked on, so each question gets its own. + await using var channel = await _connection.CreateChannelAsync(); + try + { + return await channel.QueueDeclarePassiveAsync(queue); + } + catch (OperationInterruptedException ex) when (ex.ShutdownReason?.ReplyCode == 404) + { + return null; + } + } + + /// Deletes the queue only while it is empty and nobody consumes it. + /// Whether it was deleted. + public async Task DeleteIfIdleAsync(string queue) + { + await using var channel = await _connection.CreateChannelAsync(); + try + { + await channel.QueueDeleteAsync(queue, ifUnused: true, ifEmpty: true); + return true; + } + catch (OperationInterruptedException) + { + // Not empty, or consumed, by now: it waits for the next start. + return false; + } + } + + public ValueTask DisposeAsync() => _connection.DisposeAsync(); +} diff --git a/SW.Bitween.Api/Services/BusEnvironment.cs b/SW.Bitween.Api/Services/BusEnvironment.cs new file mode 100644 index 00000000..062b70fc --- /dev/null +++ b/SW.Bitween.Api/Services/BusEnvironment.cs @@ -0,0 +1,122 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using SW.Bitween.Domain; + +namespace SW.Bitween.Services; + +/// +/// The bus names its exchange and every queue after ASPNETCORE_ENVIRONMENT. A new name moves Bitween +/// to new queues while the other applications on the same RabbitMQ keep sending to the old ones, and +/// what was waiting in them stays there. So a name Bitween didn't run with before is refused until +/// it is confirmed. +/// +/// +/// Not a hypothetical: the chart's default environment changed from Development to Production, so a +/// deployment that never chose one changed it on upgrade without anyone deciding to. +/// +public static class BusEnvironment +{ + /// + /// The name Bitween last ran with. A Settings row nobody edits and the settings page doesn't list, + /// as . + /// + public const string RecordedKey = "Bitween.BusEnvironment"; + + public const string ConfirmSetting = "Bitween:ConfirmBusEnvironment"; + + private static readonly string[] UsualNames = ["Development", "Staging", "Production"]; + + public static IHost CheckBusEnvironment(this IHost host) + { + using var scope = host.Services.CreateScope(); + var services = scope.ServiceProvider; + var configuration = services.GetRequiredService(); + var current = services.GetRequiredService().EnvironmentName; + var queuePrefix = services.GetRequiredService().QueuePrefix; + + var problem = Check(services.GetRequiredService(), current, configuration[ConfirmSetting], + () => FormerName(configuration.GetConnectionString("RabbitMQ"), queuePrefix, current), + services.GetRequiredService>()) + .GetAwaiter().GetResult(); + if (problem is not null) host.Services.GetRequiredService().Add(problem); + return host; + } + + /// + /// For a Bitween that ran before names were recorded: the name its queues were made under. + /// + /// The startup problem, or null when this name may run. + public static async Task Check(BitweenDbContext db, string current, string confirmed, + Func> formerName, ILogger logger) + { + var row = await db.Set().SingleOrDefaultAsync(s => s.Id == RecordedKey); + var recorded = row?.Value; + + // A new installation has nothing to strand. One already in use tells by its queues. + if (recorded is null && await db.Set().AnyAsync()) + { + try + { + recorded = await formerName(); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Could not reach RabbitMQ to tell which environment this Bitween ran as " + + "before; it is checked on a later start."); + return null; + } + } + + if (recorded is not null && !Same(recorded, current) && !Same(confirmed, current)) + return $"ASPNETCORE_ENVIRONMENT is {current}, but this Bitween's RabbitMQ queues are named after " + + $"{recorded}, which it ran with before. With the new name it would read from new queues, " + + "while other applications on this RabbitMQ keep sending to the old ones. If the change is a " + + $"mistake, set ASPNETCORE_ENVIRONMENT (the chart's global.environment) back to {recorded}. If " + + $"every application on this RabbitMQ moved to {current} on purpose, set " + + $"Bitween__ConfirmBusEnvironment={current}."; + + if (Same(row?.Value, current)) return null; + + if (row is null) db.Add(new Setting { Id = RecordedKey, Value = current }); + else row.Value = current; + try + { + await db.SaveChangesAsync(); + logger.LogInformation("Bitween's RabbitMQ queues are named after the environment {Environment}.", current); + } + catch (DbUpdateException ex) + { + // Another node starting at the same moment recorded it first, with the same name. + logger.LogWarning(ex, "The bus environment was recorded by another node."); + } + + return null; + } + + private static async Task FormerName(string connectionString, string queuePrefix, string current) + { + await using var broker = await BrokerQueues.ConnectAsync(connectionString); + foreach (var name in UsualNames.Prepend(current).Distinct(StringComparer.OrdinalIgnoreCase)) + { + // The lane of exchanges without a work group, which every Bitween since work groups has; + // before them, the queue of exchanges from API calls. + if (await broker.InspectAsync(BrokerQueues.QueueName(name, queuePrefix, nameof(XchangeService), + WorkGroup.None.GetBusMessageName())) is not null || + await broker.InspectAsync(BrokerQueues.QueueName(name, queuePrefix, nameof(XchangeService), + nameof(ApiXchangeCreatedEvent))) is not null) + return name; + } + + // Nothing under any of them, so nothing to leave behind. + return current; + } + + // The bus lowercases the name in every exchange and queue. + private static bool Same(string a, string b) => string.Equals(a, b, StringComparison.OrdinalIgnoreCase); +} diff --git a/SW.Bitween.IntegrationTests/Tests/BusEnvironmentTests.cs b/SW.Bitween.IntegrationTests/Tests/BusEnvironmentTests.cs new file mode 100644 index 00000000..69efd89b --- /dev/null +++ b/SW.Bitween.IntegrationTests/Tests/BusEnvironmentTests.cs @@ -0,0 +1,149 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using RabbitMQ.Client; +using SW.Bitween.Domain; +using SW.Bitween.IntegrationTests.Fixtures; +using SW.Bitween.Model; +using SW.Bitween.Services; +using Xunit; + +namespace SW.Bitween.IntegrationTests.Tests; + +/// +/// A Bitween refuses to start under an environment name its queues weren't made under, until the +/// change is confirmed by naming it. +/// +[Collection("Bitween")] +public class BusEnvironmentTests(BitweenFixture fixture) : IAsyncLifetime +{ + public async Task InitializeAsync() + { + await ForgetRecordedNameAsync(); + await EnsureInUseAsync(); + } + + public Task DisposeAsync() => ForgetRecordedNameAsync(); + + [Fact] + public async Task The_name_it_starts_with_is_recorded() + { + Assert.Null(await CheckAsync("Staging", confirmed: null, formerName: "Staging")); + Assert.Equal("Staging", await RecordedAsync()); + Assert.Null(await CheckAsync("Staging", confirmed: null, formerName: "never asked")); + } + + [Fact] + public async Task A_different_name_is_refused_with_both_names_and_the_way_to_confirm() + { + await CheckAsync("Development", confirmed: null, formerName: "Development"); + + var problem = await CheckAsync("Production", confirmed: null, formerName: "never asked"); + + Assert.Contains("ASPNETCORE_ENVIRONMENT is Production", problem); + Assert.Contains("back to Development", problem); + Assert.Contains("Bitween__ConfirmBusEnvironment=Production", problem); + Assert.Equal("Development", await RecordedAsync()); + } + + [Fact] + public async Task Confirming_the_new_name_records_it_and_a_later_change_is_refused_again() + { + await CheckAsync("Development", confirmed: null, formerName: "Development"); + + Assert.Null(await CheckAsync("Production", confirmed: "production", formerName: "never asked")); + Assert.Equal("Production", await RecordedAsync()); + + // Left in place, the confirmation names Production only. + Assert.NotNull(await CheckAsync("Staging", confirmed: "Production", formerName: "never asked")); + } + + [Fact] + public async Task Before_names_were_recorded_the_queues_tell_what_it_ran_as() + { + var problem = await CheckAsync("Production", confirmed: null, formerName: "Development"); + + Assert.NotNull(problem); + Assert.Null(await RecordedAsync()); + } + + [Fact] + public async Task An_unreachable_broker_leaves_the_check_to_a_later_start() + { + Assert.Null(await CheckAsync("Production", confirmed: null, + formerName: () => throw new InvalidOperationException("broker down"))); + Assert.Null(await RecordedAsync()); + } + + [Fact] + public async Task The_broker_says_which_queues_exist_and_removes_an_idle_one() + { + var name = BrokerQueues.QueueName("Staging", "bitween-test", "XchangeService", $"Probe{Guid.NewGuid():N}"); + Assert.StartsWith("v3.staging.bitween-test.xchangeservice.probe", name); + + var connectionString = fixture.App.Services.GetRequiredService().GetConnectionString("RabbitMQ")!; + await using var broker = await BrokerQueues.ConnectAsync(connectionString); + Assert.Null(await broker.InspectAsync(name)); + + var factory = new ConnectionFactory { Uri = new Uri(connectionString) }; + await using (var connection = await factory.CreateConnectionAsync()) + await using (var channel = await connection.CreateChannelAsync()) + { + await channel.QueueDeclareAsync(name, durable: true, exclusive: false, autoDelete: false); + await channel.BasicPublishAsync("", name, "waiting"u8.ToArray()); + } + + Assert.Equal(1u, (await broker.InspectAsync(name))!.MessageCount); + Assert.False(await broker.DeleteIfIdleAsync(name)); + + await using (var connection = await factory.CreateConnectionAsync()) + await using (var channel = await connection.CreateChannelAsync()) + await channel.QueuePurgeAsync(name); + + Assert.True(await broker.DeleteIfIdleAsync(name)); + Assert.Null(await broker.InspectAsync(name)); + } + + private Task CheckAsync(string current, string? confirmed, string formerName) => + CheckAsync(current, confirmed, () => Task.FromResult(formerName)); + + private async Task CheckAsync(string current, string? confirmed, Func> formerName) + { + await using var scope = fixture.CreateScope(); + return await BusEnvironment.Check(scope.ServiceProvider.GetRequiredService(), + current, confirmed!, formerName, NullLogger.Instance); + } + + private async Task RecordedAsync() + { + await using var scope = fixture.CreateScope(); + return await scope.ServiceProvider.GetRequiredService().Set() + .Where(s => s.Id == BusEnvironment.RecordedKey).Select(s => s.Value).SingleOrDefaultAsync(); + } + + private async Task ForgetRecordedNameAsync() + { + await using var scope = fixture.CreateScope(); + await scope.ServiceProvider.GetRequiredService().Set() + .Where(s => s.Id == BusEnvironment.RecordedKey).ExecuteDeleteAsync(); + } + + /// The queues are only asked about for an installation already in use. + private async Task EnsureInUseAsync() + { + await using var scope = fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + if (await db.Set().AnyAsync()) return; + + var document = new Document(null, $"Bus environment {Guid.NewGuid():N}", DocumentFormat.Json); + db.Add(document); + await db.SaveChangesAsync(); + db.Add(new Subscription($"Bus environment {Guid.NewGuid():N}", document.Id, SubscriptionType.Internal, + Partner.SystemId)); + await db.SaveChangesAsync(); + } +} diff --git a/SW.Bitween.Web/Program.cs b/SW.Bitween.Web/Program.cs index ff7a24b7..088a37d2 100644 --- a/SW.Bitween.Web/Program.cs +++ b/SW.Bitween.Web/Program.cs @@ -55,7 +55,7 @@ public static void Main(string[] args) // These read partners and alerts, whose stored credentials need the encryption key. if (host.CanReadStoredSecrets()) host.SecureSystemPartnerKey().HashStoredPartnerKeys().MoveRetryAlertsToChannels(); - host.ApplyStoredSettings(); + host.ApplyStoredSettings().CheckBusEnvironment(); if (RefusesToStart(host)) return; host.Run(); diff --git a/docs/configuration.md b/docs/configuration.md index b9753ca1..84fdb7e7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -10,7 +10,7 @@ Some values are also **runtime settings** that administrators change in the UI. |---|---| | `ConnectionStrings:BitweenDb` | `Host=db;Port=5432;Database=bitween;Username=bitween;Password=...` | | `ConnectionStrings:RabbitMQ` | `amqp://user:password@rabbitmq:5672/` | -| `ASPNETCORE_ENVIRONMENT` | `Production`, or the value the other applications on the same RabbitMQ use: the bus names its exchanges and queues after it. On an upgrade, keep the running value. Use `Development` only on a developer's machine | +| `ASPNETCORE_ENVIRONMENT` | `Production`, or the value the other applications on the same RabbitMQ use: the bus names its exchanges and queues after it. On an upgrade, keep the running value: Bitween remembers it, and refuses to start under another until `Bitween:ConfirmBusEnvironment` names the new one. Use `Development` only on a developer's machine | | `Token:Key` | A random secret of at least 32 characters, unique to this deployment | | `Token:Issuer`, `Token:Audience` | Values used to sign and validate JWTs | | `Bitween:InitialAdminPassword` | New installations only: the password for `admin@Bitween.systems` | @@ -103,6 +103,7 @@ applications the same `Token` values. Until they match, `Bus:OnInvalidUserToken` | `ConsumeLegacyEventMessages` | `false` | Also drain the queues that versions before work groups published to. | | `CorsOrigins` | empty | Browser origins allowed to call the API with credentials. Set as `Bitween__CorsOrigins__0`, `Bitween__CorsOrigins__1` and so on. | | `SettingsEncryptionKey` | | Passphrase that encrypts secret settings and stored credentials in the database. See [Security](security.md#secrets). | +| `ConfirmBusEnvironment` | | The new `ASPNETCORE_ENVIRONMENT`, after every application on the same RabbitMQ has moved to it. Bitween reads from new queues under a new name, and what waits in the old ones stays there. It only confirms the name it holds, so it can stay set. | | `SettingsEncryptionKeyLost` | `false` | Only when the key that encrypted the stored credentials is gone for good: start anyway, and show what it encrypted as empty until entered again. See [Security](security.md#secrets). | | `ReceiveAttemptRetentionDays` | `30` | Days to keep receive attempts. | | `AuditRetentionDays` | `0` | Days to keep audit entries; the daily clean-up deletes older ones. 0 keeps them indefinitely. | From 726a7a0dde6b2f761cfd77e5054f36223e4340f7 Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Mon, 19 Oct 2026 11:42:09 +0300 Subject: [PATCH 04/10] feat: read queues left by versions before work groups without a setting --- SW.Bitween.Api/Resources/Ops/LaneResolver.cs | 4 +- SW.Bitween.Api/Services/BitweenOptions.cs | 1 - SW.Bitween.Api/Services/LegacyEventQueues.cs | 93 +++++++++++++++++ .../Services/Settings/SettingsCatalog.cs | 4 - SW.Bitween.Api/Services/XchangeService.cs | 16 +-- .../Fixtures/BitweenFixture.cs | 1 + .../Tests/LegacyEventQueuesTests.cs | 99 +++++++++++++++++++ SW.Bitween.Web/Program.cs | 2 +- SW.Bitween.Web/Startup.cs | 1 + docs/architecture.md | 2 +- docs/configuration.md | 4 +- docs/deployment.md | 2 +- 12 files changed, 208 insertions(+), 21 deletions(-) create mode 100644 SW.Bitween.Api/Services/LegacyEventQueues.cs create mode 100644 SW.Bitween.IntegrationTests/Tests/LegacyEventQueuesTests.cs diff --git a/SW.Bitween.Api/Resources/Ops/LaneResolver.cs b/SW.Bitween.Api/Resources/Ops/LaneResolver.cs index 2b0f5a1e..e37b08f0 100644 --- a/SW.Bitween.Api/Resources/Ops/LaneResolver.cs +++ b/SW.Bitween.Api/Resources/Ops/LaneResolver.cs @@ -25,7 +25,7 @@ public enum QueueLane /// A work group's -Result lane. Evaluates notifiers. Notifications, - /// A legacy event type, consumed only while that setting is on. + /// A legacy event type, consumed only while its queue still holds something. Legacy, /// Bookkeeping. No integration traffic. @@ -50,7 +50,7 @@ public class LaneResolver(BitweenDbContext dbContext, BitweenOptions options) ["SubscriptionUnpausedEvent"] = "Resumed integrations", }; - /// Only declared while Bitween.ConsumeLegacyEventMessages is on. + /// Only read while their queues still hold something; see . private static readonly Dictionary Legacy = new(System.StringComparer.OrdinalIgnoreCase) { ["ApiXchangeCreatedEvent"] = "API exchange created", diff --git a/SW.Bitween.Api/Services/BitweenOptions.cs b/SW.Bitween.Api/Services/BitweenOptions.cs index 793fa417..f842ee94 100644 --- a/SW.Bitween.Api/Services/BitweenOptions.cs +++ b/SW.Bitween.Api/Services/BitweenOptions.cs @@ -324,7 +324,6 @@ public BitweenOptions() /// partnerkey keeps working either way. Read per request. /// public string PartnerKeyHeader { get; set; } - public bool ConsumeLegacyEventMessages { get; set; } public string QueuePrefix { get; set; } /// diff --git a/SW.Bitween.Api/Services/LegacyEventQueues.cs b/SW.Bitween.Api/Services/LegacyEventQueues.cs new file mode 100644 index 00000000..90ae0104 --- /dev/null +++ b/SW.Bitween.Api/Services/LegacyEventQueues.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using SW.Bitween.Domain; + +namespace SW.Bitween.Services; + +/// +/// The five queues named after exchange events, which a Bitween before work groups (January 2026) +/// published to. Nothing publishes to them now, but an exchange waiting in one when its Bitween was +/// upgraded runs only if it is read. Each start looks for them: one with something in it is read +/// until it's empty, and an empty one nobody reads is removed. A setting used to switch this on, and +/// an upgrade that didn't know about it left those exchanges behind. +/// +public class LegacyEventQueues +{ + public static readonly string[] Names = + [ + nameof(ApiXchangeCreatedEvent), nameof(InternalXchangeCreatedEvent), nameof(ReceivingXchangeCreatedEvent), + nameof(AggregateXchangeCreatedEvent), nameof(XchangeResultCreatedEvent) + ]; + + private readonly List _toDrain = []; + + /// The ones reads from on this start. + public IReadOnlyList ToDrain => _toDrain; + + /// Looks for them, before the bus starts and would create any it was told to read. + public async Task FindAsync(BrokerQueues broker, string environment, string queuePrefix, ILogger logger) + { + foreach (var name in Names) + { + var queue = BrokerQueues.QueueName(environment, queuePrefix, nameof(XchangeService), name); + var main = await broker.InspectAsync(queue); + if (main is null) continue; + + // A failed message waits out its delay in the retry queue, and then goes back to the main one. + var retry = await broker.InspectAsync($"{queue}.retry"); + if (main.MessageCount > 0 || main.ConsumerCount > 0 || retry?.MessageCount > 0) + { + _toDrain.Add(name); + logger.LogInformation("Reading {Queue}, left by a version of Bitween before work groups, until it is empty.", + queue); + continue; + } + + await broker.DeleteIfIdleAsync(queue); + if (retry is not null) await broker.DeleteIfIdleAsync($"{queue}.retry"); + // What failed for good stays where the queue's dead letters can still be looked at. + if (await broker.InspectAsync($"{queue}.bad") is { MessageCount: 0 }) + await broker.DeleteIfIdleAsync($"{queue}.bad"); + logger.LogInformation("Removed {Queue}, left empty by a version of Bitween before work groups.", queue); + } + } +} + +public static class LegacyEventQueuesHostExtensions +{ + /// + /// Runs just before the bus starts. An unreachable broker is looked at again on the next start; + /// nothing in those queues goes anywhere meanwhile. + /// + public static IHost FindLegacyEventQueues(this IHost host) + { + using var scope = host.Services.CreateScope(); + var services = scope.ServiceProvider; + var logger = services.GetRequiredService>(); + try + { + FindAsync().GetAwaiter().GetResult(); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Could not reach RabbitMQ to look for queues left by a version of Bitween " + + "before work groups; they are looked for again on the next start."); + } + + return host; + + async Task FindAsync() + { + await using var broker = await BrokerQueues.ConnectAsync( + services.GetRequiredService().GetConnectionString("RabbitMQ")); + await services.GetRequiredService().FindAsync(broker, + services.GetRequiredService().EnvironmentName, + services.GetRequiredService().QueuePrefix, logger); + } + } +} diff --git a/SW.Bitween.Api/Services/Settings/SettingsCatalog.cs b/SW.Bitween.Api/Services/Settings/SettingsCatalog.cs index 0eb07ddd..3a6b4fa5 100644 --- a/SW.Bitween.Api/Services/Settings/SettingsCatalog.cs +++ b/SW.Bitween.Api/Services/Settings/SettingsCatalog.cs @@ -400,10 +400,6 @@ public static class SettingsCatalog "How many messages of a lane run at once on each node when nothing more specific says: the ungrouped lane and the bus's other consumers. A work group sets its own. Unset, it is the default prefetch.", SettingKind.Number, t => t.Bitween.ResolvedBusDefaultConcurrency.ToString(CultureInfo.InvariantCulture)), - View("Bitween.ConsumeLegacyEventMessages", "Messaging", "Consume legacy event messages", - "Whether this instance also drains the five queues named after exchange events, which an older Bitween published to before work groups existed. Nothing publishes to them today, so this is only for finishing off messages left behind by an upgrade.", - SettingKind.Boolean, t => Str(t.Bitween.ConsumeLegacyEventMessages)), - Presence("Bitween.RabbitMqManagementUrl", "Messaging", "RabbitMQ management URL", "The management API queue health is read from. All three management values are needed before queue depths can be shown.", t => t.Bitween.RabbitMqManagementUrl), diff --git a/SW.Bitween.Api/Services/XchangeService.cs b/SW.Bitween.Api/Services/XchangeService.cs index 76893637..6dd0638d 100644 --- a/SW.Bitween.Api/Services/XchangeService.cs +++ b/SW.Bitween.Api/Services/XchangeService.cs @@ -12,6 +12,7 @@ using System.Threading.Tasks; using Newtonsoft.Json; using Newtonsoft.Json.Linq; +using SW.Bitween.Services; using SW.Bitween.Services.Adapters; using SW.Bus.RabbitMqExtensions; @@ -22,7 +23,8 @@ public class XchangeService(BitweenOptions BitweenSettings, BitweenDbContext dbC ICloudFilesService cloudFiles, IServiceProvider serviceProvider, IPublishWithValues publish, ILogger logger, IInfolinkCache BitweenCache, IAdapterInvoker adapterInvoker, NativeAdapterDiscoveryService nativeAdapterDiscovery, - FileLinks fileLinks, StorageRetention storageRetention, NotificationChannelSender channelSender) : + FileLinks fileLinks, StorageRetention storageRetention, NotificationChannelSender channelSender, + LegacyEventQueues legacyEventQueues) : // IConsume, // IConsume, // IConsume, @@ -1827,13 +1829,11 @@ public async Task> GetMessageTypeNamesWithO }; } - if (!BitweenSettings.ConsumeLegacyEventMessages) return messageTypeNamesWithOptions; - - messageTypeNamesWithOptions.Add(nameof(ApiXchangeCreatedEvent), new ConsumerOptions() { Priority = 10 }); - messageTypeNamesWithOptions.Add(nameof(InternalXchangeCreatedEvent), new ConsumerOptions()); - messageTypeNamesWithOptions.Add(nameof(ReceivingXchangeCreatedEvent), new ConsumerOptions()); - messageTypeNamesWithOptions.Add(nameof(AggregateXchangeCreatedEvent), new ConsumerOptions()); - messageTypeNamesWithOptions.Add(nameof(XchangeResultCreatedEvent), new ConsumerOptions()); + foreach (var name in legacyEventQueues.ToDrain) + messageTypeNamesWithOptions.Add(name, new ConsumerOptions() + { + Priority = name == nameof(ApiXchangeCreatedEvent) ? 10 : null + }); return messageTypeNamesWithOptions; } } \ No newline at end of file diff --git a/SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs b/SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs index 22bb2819..b3aad5c5 100644 --- a/SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs +++ b/SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs @@ -365,6 +365,7 @@ await Task.WhenAll(_postgres.StartAsync(), _rabbitMq.StartAsync(), _mailHog.Star services.AddScoped(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/SW.Bitween.IntegrationTests/Tests/LegacyEventQueuesTests.cs b/SW.Bitween.IntegrationTests/Tests/LegacyEventQueuesTests.cs new file mode 100644 index 00000000..a426ceab --- /dev/null +++ b/SW.Bitween.IntegrationTests/Tests/LegacyEventQueuesTests.cs @@ -0,0 +1,99 @@ +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using RabbitMQ.Client; +using SW.Bitween.IntegrationTests.Fixtures; +using SW.Bitween.Services; +using Xunit; + +namespace SW.Bitween.IntegrationTests.Tests; + +/// +/// The queues a Bitween before work groups published to are read while they hold something and +/// removed once they don't, with nothing to switch on. +/// +[Collection("Bitween")] +public class LegacyEventQueuesTests(BitweenFixture fixture) +{ + private const string Environment = "Staging"; + private readonly string _queuePrefix = $"legacy-{Guid.NewGuid():N}"; + + [Fact] + public async Task A_queue_with_exchanges_waiting_is_read() + { + var queue = await DeclareAsync("ApiXchangeCreatedEvent", messages: 1); + + var legacy = await FindAsync(); + + Assert.Equal(["ApiXchangeCreatedEvent"], legacy.ToDrain); + Assert.NotNull(await InspectAsync(queue)); + } + + [Fact] + public async Task An_empty_queue_is_removed_but_its_dead_letters_stay() + { + var queue = await DeclareAsync("InternalXchangeCreatedEvent", messages: 0); + await DeclareQueueAsync($"{queue}.retry", messages: 0); + await DeclareQueueAsync($"{queue}.bad", messages: 1); + + var legacy = await FindAsync(); + + Assert.Empty(legacy.ToDrain); + Assert.Null(await InspectAsync(queue)); + Assert.Null(await InspectAsync($"{queue}.retry")); + Assert.NotNull(await InspectAsync($"{queue}.bad")); + } + + [Fact] + public async Task A_queue_whose_messages_wait_to_be_retried_is_read() + { + var queue = await DeclareAsync("XchangeResultCreatedEvent", messages: 0); + await DeclareQueueAsync($"{queue}.retry", messages: 1); + + Assert.Equal(["XchangeResultCreatedEvent"], (await FindAsync()).ToDrain); + } + + [Fact] + public async Task Without_them_nothing_is_read_or_created() + { + Assert.Empty((await FindAsync()).ToDrain); + Assert.Null(await InspectAsync(Name("ApiXchangeCreatedEvent"))); + } + + private async Task FindAsync() + { + var legacy = new LegacyEventQueues(); + await using var broker = await BrokerQueues.ConnectAsync(ConnectionString); + await legacy.FindAsync(broker, Environment, _queuePrefix, NullLogger.Instance); + return legacy; + } + + private async Task InspectAsync(string queue) + { + await using var broker = await BrokerQueues.ConnectAsync(ConnectionString); + return await broker.InspectAsync(queue); + } + + private async Task DeclareAsync(string message, uint messages) + { + var queue = Name(message); + await DeclareQueueAsync(queue, messages); + return queue; + } + + private async Task DeclareQueueAsync(string queue, uint messages) + { + await using var connection = await new ConnectionFactory { Uri = new Uri(ConnectionString) }.CreateConnectionAsync(); + await using var channel = await connection.CreateChannelAsync(); + await channel.QueueDeclareAsync(queue, durable: true, exclusive: false, autoDelete: false); + for (var i = 0; i < messages; i++) + await channel.BasicPublishAsync("", queue, "left behind"u8.ToArray()); + } + + private string Name(string message) => BrokerQueues.QueueName(Environment, _queuePrefix, "XchangeService", message); + + private string ConnectionString => + fixture.App.Services.GetRequiredService().GetConnectionString("RabbitMQ")!; +} diff --git a/SW.Bitween.Web/Program.cs b/SW.Bitween.Web/Program.cs index 088a37d2..7e9d0134 100644 --- a/SW.Bitween.Web/Program.cs +++ b/SW.Bitween.Web/Program.cs @@ -58,7 +58,7 @@ public static void Main(string[] args) host.ApplyStoredSettings().CheckBusEnvironment(); if (RefusesToStart(host)) return; - host.Run(); + host.FindLegacyEventQueues().Run(); } /// diff --git a/SW.Bitween.Web/Startup.cs b/SW.Bitween.Web/Startup.cs index a5652cad..0f4553fd 100644 --- a/SW.Bitween.Web/Startup.cs +++ b/SW.Bitween.Web/Startup.cs @@ -117,6 +117,7 @@ public void ConfigureServices(IServiceCollection services) services.AddSingleton(); // The bell a sync gateway call waits on for its exchange's result. services.AddSingleton(); + services.AddSingleton(); // Metrics: counted in memory as work happens, added to hourly rows once a minute per node. services.AddSingleton(); services.AddSingleton(); diff --git a/docs/architecture.md b/docs/architecture.md index a53f7ca8..bc9acbe0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -108,7 +108,7 @@ RabbitMQ carries all asynchronous work. Queue names are lower-cased and built fr | Front door | `BusService` | Each bus-enabled information type's message type name | Documents from other systems | | Control | `XchangeService` | `SubscriptionUnpausedEvent` | Releasing on-hold exchanges | | Control | `RetryAlertService` | `RetryBudgetExhaustedEvent` | Budget alerts | -| Legacy | `XchangeService` | `ApiXchangeCreatedEvent` and four similar names | Only when `Bitween:ConsumeLegacyEventMessages` is on | +| Legacy | `XchangeService` | `ApiXchangeCreatedEvent` and four similar names | Only while their queues, left by a version before work groups, still hold something. Bitween looks for them at each start and removes them once empty | Creating a work group or a bus-enabled information type asks every instance to refresh its consumers, so new queues are picked up without a restart. Queues left behind by deleted or renamed lanes appear on the Queue health page. diff --git a/docs/configuration.md b/docs/configuration.md index 84fdb7e7..f8666d85 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -23,7 +23,6 @@ Set these too when they apply: - Other applications publish to Bitween's bus: `Bus:OnInvalidUserToken`. See [Bus messages from other applications](#bus-messages-from-other-applications). - Upgrading, and partners must keep getting one call at a time, in order: `Bitween:BusDefaultConcurrency` set to `1`. See [Upgrading to parallel exchanges](deployment.md#upgrading-to-parallel-exchanges). -- Upgrading from a version without work groups: `Bitween:ConsumeLegacyEventMessages`. See [Upgrading](deployment.md#upgrading). - S3-compatible storage other than AWS: `AWS_REQUEST_CHECKSUM_CALCULATION` and `AWS_RESPONSE_CHECKSUM_VALIDATION`. See [Storage](#storage). - A RabbitMQ shared with another Bitween: `Bitween:QueuePrefix`. - MySQL without TLS: `SslMode` in the connection string. See [Databases](deployment.md#databases). @@ -100,7 +99,6 @@ applications the same `Token` values. Until they match, `Bus:OnInvalidUserToken` | `BusDefaultQueuePrefetch` | `12` | Default number of unacknowledged messages per consumer. Work groups can override it. | | `BusDefaultConcurrency` | the prefetch | How many messages of a lane run at once on a node, for the Ungrouped lane and the bus's other consumers (bus-enabled information types, retry and auto-pause alerts). A work group sets its own with **Parallel exchanges**. `1` runs them one at a time, in order, as before. | | `RabbitMqManagementUrl`, `RabbitMqManagementUsername`, `RabbitMqManagementPassword` | | RabbitMQ management API, needed for queue health. | -| `ConsumeLegacyEventMessages` | `false` | Also drain the queues that versions before work groups published to. | | `CorsOrigins` | empty | Browser origins allowed to call the API with credentials. Set as `Bitween__CorsOrigins__0`, `Bitween__CorsOrigins__1` and so on. | | `SettingsEncryptionKey` | | Passphrase that encrypts secret settings and stored credentials in the database. See [Security](security.md#secrets). | | `ConfirmBusEnvironment` | | The new `ASPNETCORE_ENVIRONMENT`, after every application on the same RabbitMQ has moved to it. Bitween reads from new queues under a new name, and what waits in the old ones stays there. It only confirms the name it holds, so it can stay set. | @@ -195,7 +193,7 @@ Settings are stored in the `Settings` table and edited on the Settings page. | `Bitween.MailServer.Password` | Mail server | Yes, secret | empty | | `Bitween.MaxRetryAfterSeconds` | Reliability & jobs | Yes | `21600`, six hours | | `Bitween.RetryJobCron` | Reliability & jobs | Yes | `0 * * * * ?` | -| `Bitween.QueuePrefix`, `BusDefaultQueuePrefetch`, `BusDefaultConcurrency`, `ConsumeLegacyEventMessages` | Messaging | Read-only | | +| `Bitween.QueuePrefix`, `BusDefaultQueuePrefetch`, `BusDefaultConcurrency` | Messaging | Read-only | | | `Bitween.RabbitMqManagementUrl`, `Username`, `Password` | Messaging | Shows whether set | | | `Bitween.UseAzureManagedIdentity` | Database | Read-only | `false` | | `Bitween.AzureManagedIdentityClientId` | Database | Shows whether set | | diff --git a/docs/deployment.md b/docs/deployment.md index 4f6f1728..3440457a 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -131,7 +131,7 @@ Bitween checks its configuration and its database before it serves anything. Whe - Migrations apply at startup, so back up the database before deploying a new version. - Upgrading to this version needs a short downtime: stop every pod of the old version before deploying the new one. - Values already in the `Settings` table are not replaced by new configuration. Change them on the Settings page. -- When upgrading from a version without work groups, set `Bitween__ConsumeLegacyEventMessages=true` until the old event queues are empty. +- Upgrading from a version without work groups needs nothing set: at each start Bitween reads the queues that version left behind until they are empty, then removes them. ### Upgrading to parallel exchanges From a0b2b56f1a9d8dfb75cd5c371d4143629bd5a708 Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Mon, 19 Oct 2026 11:42:09 +0300 Subject: [PATCH 05/10] feat: warn when a stored setting overrides configuration --- .../Settings/SettingsHostExtensions.cs | 1 + .../Services/Settings/SettingsService.cs | 48 +++++++++++++++++++ .../Tests/SettingsTests.cs | 42 ++++++++++++++++ docs/configuration.md | 2 +- docs/deployment.md | 2 +- 5 files changed, 93 insertions(+), 2 deletions(-) diff --git a/SW.Bitween.Api/Services/Settings/SettingsHostExtensions.cs b/SW.Bitween.Api/Services/Settings/SettingsHostExtensions.cs index 46d66169..c7c551b1 100644 --- a/SW.Bitween.Api/Services/Settings/SettingsHostExtensions.cs +++ b/SW.Bitween.Api/Services/Settings/SettingsHostExtensions.cs @@ -25,6 +25,7 @@ public static IHost ApplyStoredSettings(this IHost host) settings.ImportMissing(dbContext).GetAwaiter().GetResult(); settings.Reload(dbContext).GetAwaiter().GetResult(); settings.RememberLegacyDocumentPrefix(dbContext).GetAwaiter().GetResult(); + settings.WarnAboutIgnoredConfiguration(dbContext).GetAwaiter().GetResult(); } catch (Exception ex) { diff --git a/SW.Bitween.Api/Services/Settings/SettingsService.cs b/SW.Bitween.Api/Services/Settings/SettingsService.cs index 30c5da65..36198759 100644 --- a/SW.Bitween.Api/Services/Settings/SettingsService.cs +++ b/SW.Bitween.Api/Services/Settings/SettingsService.cs @@ -155,6 +155,54 @@ public async Task Reload(BitweenDbContext dbContext) } } + /// + /// Names each setting configuration gives a value other than the one stored. The stored one is + /// used, so whoever deploys a changed value would otherwise see nothing happen. Startup only. + /// + public async Task WarnAboutIgnoredConfiguration(BitweenDbContext dbContext) + { + var rows = await dbContext.Set().AsNoTracking() + .ToDictionaryAsync(s => s.Id, s => s.Value, StringComparer.OrdinalIgnoreCase); + + foreach (var definition in SettingsCatalog.All) + { + if (!definition.Stored || !rows.TryGetValue(definition.Key, out var stored)) continue; + + var configured = _configured.GetValueOrDefault(definition.Key) ?? string.Empty; + // Configuration that sets nothing reads as the product default. + if (configured == DefaultOf(definition)) continue; + + string current; + try + { + current = Normalized(definition, definition.Secret ? _protector.Unprotect(stored) : stored); + } + catch + { + // Unreadable or invalid: Reload has already said so. + continue; + } + + if (Normalized(definition, configured) == current) continue; + + if (definition.Secret) + _logger.LogWarning("{Key} is set in configuration, but the Settings page holds a different value, " + + "which is the one used. Change it on the Settings page.", definition.Key); + else + _logger.LogWarning("{Key} is {Configured} in configuration, but the Settings page holds {Stored}, " + + "which is the one used. Change it on the Settings page.", + definition.Key, configured, current); + } + } + + // As the options hold it, so "true" and "True" are the same value. + private static string Normalized(SettingDefinition definition, string value) + { + var target = new SettingsTarget(new BitweenOptions(), new ThemeOptions()); + definition.Write(target, value); + return definition.Read(target) ?? string.Empty; + } + /// /// Writes the first time this version starts - before anyone /// could have changed the prefix, so it's the one every earlier exchange was written under. Runs diff --git a/SW.Bitween.IntegrationTests/Tests/SettingsTests.cs b/SW.Bitween.IntegrationTests/Tests/SettingsTests.cs index 84af6ce3..993ddacf 100644 --- a/SW.Bitween.IntegrationTests/Tests/SettingsTests.cs +++ b/SW.Bitween.IntegrationTests/Tests/SettingsTests.cs @@ -1,8 +1,10 @@ +using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using SW.Bitween.Domain; using SW.Bitween.IntegrationTests.Fixtures; using SW.Bitween.Model; @@ -201,4 +203,44 @@ public async Task A_schedule_that_is_not_a_cron_expression_is_refused() Assert.StartsWith("SETTING_INVALID_VALUE", ex.Message); Assert.Contains("not a valid cron expression", ex.Message); } + + /// + /// The stored value wins, so a deployment that changes only its configuration would otherwise + /// see nothing happen and nothing said. + /// + [Fact] + public async Task A_configured_value_the_stored_one_overrides_is_named_at_startup() + { + await Store(EditableKey, "60"); + + Assert.Contains(await IgnoredConfigurationWarnings(jwtExpiryMinutes: 45), + w => w.Contains(EditableKey) && w.Contains("45") && w.Contains("60")); + Assert.DoesNotContain(await IgnoredConfigurationWarnings(jwtExpiryMinutes: 60), w => w.Contains(EditableKey)); + } + + private async Task> IgnoredConfigurationWarnings(int jwtExpiryMinutes) + { + var logger = new WarningLog(); + var configured = new BitweenOptions { JwtExpiryMinutes = jwtExpiryMinutes }; + var settings = new SettingsService(configured, new ThemeOptions(), new SettingsProtector(configured), logger); + + await using var scope = fixture.CreateScope(); + await settings.WarnAboutIgnoredConfiguration(scope.ServiceProvider.GetRequiredService()); + return logger.Warnings; + } + + private sealed class WarningLog : ILogger + { + public List Warnings { get; } = []; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, + Func formatter) + { + if (logLevel == LogLevel.Warning) Warnings.Add(formatter(state, exception)); + } + + public bool IsEnabled(LogLevel logLevel) => true; + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + } } diff --git a/docs/configuration.md b/docs/configuration.md index f8666d85..75f6a94b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -159,7 +159,7 @@ Logging lives in the `SwLogger` section. Settings are stored in the `Settings` table and edited on the Settings page. -- On first boot, every editable setting is copied from configuration into the table. After that the table wins, and changing configuration has no effect. +- On first boot, every editable setting is copied from configuration into the table. After that the table wins, and changing configuration has no effect: each start logs a warning naming every setting whose configured value differs from the stored one. - A saved change applies immediately on the instance that saved it, and reaches other instances through the cache revoke broadcast. - **Reset to default** writes the product default, not the configured value. - Values captured at startup, such as the bus, storage, CORS and database settings, are shown read-only. Credentials only show whether they are set. diff --git a/docs/deployment.md b/docs/deployment.md index 3440457a..f0bf660c 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -130,7 +130,7 @@ Bitween checks its configuration and its database before it serves anything. Whe - Migrations apply at startup, so back up the database before deploying a new version. - Upgrading to this version needs a short downtime: stop every pod of the old version before deploying the new one. -- Values already in the `Settings` table are not replaced by new configuration. Change them on the Settings page. +- Values already in the `Settings` table are not replaced by new configuration. Change them on the Settings page. A startup warning names each one configuration sets differently. - Upgrading from a version without work groups needs nothing set: at each start Bitween reads the queues that version left behind until they are empty, then removes them. ### Upgrading to parallel exchanges From 8da76ab8f8c868f135b3a3c011322b0680386a5d Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Mon, 19 Oct 2026 11:42:09 +0300 Subject: [PATCH 06/10] feat: check storage can be written and read back at startup --- SW.Bitween.Api/Services/StorageCheck.cs | 56 +++++++++++++++++++ .../Tests/StorageCheckTests.cs | 40 +++++++++++++ SW.Bitween.Web/Program.cs | 2 +- SW.Bitween.Web/Startup.cs | 47 ++++++++++------ docs/deployment.md | 1 + 5 files changed, 127 insertions(+), 19 deletions(-) create mode 100644 SW.Bitween.Api/Services/StorageCheck.cs create mode 100644 SW.Bitween.IntegrationTests/Tests/StorageCheckTests.cs diff --git a/SW.Bitween.Api/Services/StorageCheck.cs b/SW.Bitween.Api/Services/StorageCheck.cs new file mode 100644 index 00000000..ff08cecb --- /dev/null +++ b/SW.Bitween.Api/Services/StorageCheck.cs @@ -0,0 +1,56 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Services; + +/// +/// Whether Bitween can write its files and read them back. Every exchange writes to storage, and a +/// wrong bucket or credential used to show up only as the first exchange failing. +/// +public static class StorageCheck +{ + private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(30); + + /// Runs after the stored settings are applied, which hold the document prefix. + public static IHost CheckStorage(this IHost host) + { + using var scope = host.Services.CreateScope(); + var services = scope.ServiceProvider; + var options = services.GetRequiredService(); + var problem = Check(services.GetRequiredService(), options.DocumentPrefix, + $"{options.StorageProvider}, bucket {services.GetRequiredService()["CloudFiles:BucketName"]}") + .GetAwaiter().GetResult(); + if (problem is not null) host.Services.GetRequiredService().Add(problem); + return host; + } + + /// Which storage, for the message. + /// The startup problem, or null when the file came back as it was written. + public static async Task Check(ICloudFilesService cloudFiles, string documentPrefix, string storage) + { + // The file the settings page writes to tell whether the bucket is open to everyone. + var key = $"{documentPrefix}/{StorageAccess.ProbeName}"; + try + { + await cloudFiles.WriteTextAsync(StorageAccess.ProbeText, + new WriteFileSettings { Key = key, ContentType = "text/plain" }).WaitAsync(Timeout); + await using var stream = await cloudFiles.OpenReadAsync(key).WaitAsync(Timeout); + using var reader = new StreamReader(stream); + if (await reader.ReadToEndAsync().WaitAsync(Timeout) == StorageAccess.ProbeText) return null; + return Problem(storage, "the file read back is not the one written."); + } + catch (Exception ex) + { + return Problem(storage, ex is TimeoutException ? $"no answer within {Timeout.TotalSeconds:0} seconds." : ex.Message); + } + } + + public static string Problem(string storage, string reason) => + $"Bitween can't write its files to storage ({storage}) and read them back: {reason} Check " + + "Bitween:StorageProvider and the CloudFiles settings: the address, the bucket and the credentials."; +} diff --git a/SW.Bitween.IntegrationTests/Tests/StorageCheckTests.cs b/SW.Bitween.IntegrationTests/Tests/StorageCheckTests.cs new file mode 100644 index 00000000..67f167d6 --- /dev/null +++ b/SW.Bitween.IntegrationTests/Tests/StorageCheckTests.cs @@ -0,0 +1,40 @@ +using System.Net.Http; +using System.Reflection; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using SW.Bitween.IntegrationTests.Fixtures; +using SW.Bitween.Services; +using SW.PrimitiveTypes; +using Xunit; + +namespace SW.Bitween.IntegrationTests.Tests; + +/// Storage that can't be written to stops the start, rather than the first exchange. +[Collection("Bitween")] +public class StorageCheckTests(BitweenFixture fixture) +{ + [Fact] + public async Task Storage_that_takes_the_file_and_gives_it_back_passes() + { + await using var scope = fixture.CreateScope(); + Assert.Null(await StorageCheck.Check(scope.ServiceProvider.GetRequiredService(), + "temp30/storage-check", "LocalTests, bucket tests")); + } + + [Fact] + public async Task Storage_that_cannot_be_reached_says_why_and_what_to_check() + { + var problem = await StorageCheck.Check(DispatchProxy.Create(), + "temp30/storage-check", "S3, bucket bitween"); + + Assert.Contains("(S3, bucket bitween)", problem); + Assert.Contains("No such host is known", problem); + Assert.Contains("CloudFiles settings", problem); + } + + public class Unreachable : DispatchProxy + { + protected override object Invoke(MethodInfo? targetMethod, object?[]? args) => + throw new HttpRequestException("No such host is known. (s3.example.invalid:443)"); + } +} diff --git a/SW.Bitween.Web/Program.cs b/SW.Bitween.Web/Program.cs index 7e9d0134..10a5c18c 100644 --- a/SW.Bitween.Web/Program.cs +++ b/SW.Bitween.Web/Program.cs @@ -55,7 +55,7 @@ public static void Main(string[] args) // These read partners and alerts, whose stored credentials need the encryption key. if (host.CanReadStoredSecrets()) host.SecureSystemPartnerKey().HashStoredPartnerKeys().MoveRetryAlertsToChannels(); - host.ApplyStoredSettings().CheckBusEnvironment(); + host.ApplyStoredSettings().CheckBusEnvironment().CheckStorage(); if (RefusesToStart(host)) return; host.FindLegacyEventQueues().Run(); diff --git a/SW.Bitween.Web/Startup.cs b/SW.Bitween.Web/Startup.cs index 0f4553fd..158f4056 100644 --- a/SW.Bitween.Web/Startup.cs +++ b/SW.Bitween.Web/Startup.cs @@ -236,25 +236,36 @@ public void ConfigureServices(IServiceCollection services) ); services.AddApiClient(); - switch (bitweenOptions.StorageProvider.ToUpper()) + // The storage libraries create the bucket and its deletion rules as they register, so + // wrong storage settings stop the start here, before anything else could be registered. + try { - case "AS": - services.AddAsCloudFiles(); - // Azure decides privacy per container, and the storage library creates it public. - services.AddHostedService(); - break; - case "OC": - services.AddOracleCloudFiles(); - break; - case "S3": - services.AddS3CloudFiles(); - break; - case "LOCAL": - services.AddLocalTestsCloudFiles(); - break; - default: - services.AddS3CloudFiles(); - break; + switch (bitweenOptions.StorageProvider.ToUpper()) + { + case "AS": + services.AddAsCloudFiles(); + // Azure decides privacy per container, and the storage library creates it public. + services.AddHostedService(); + break; + case "OC": + services.AddOracleCloudFiles(); + break; + case "S3": + services.AddS3CloudFiles(); + break; + case "LOCAL": + services.AddLocalTestsCloudFiles(); + break; + default: + services.AddS3CloudFiles(); + break; + } + } + catch (Exception ex) + { + problems.Add(StorageCheck.Problem( + $"{bitweenOptions.StorageProvider}, bucket {Configuration["CloudFiles:BucketName"]}", ex.Message)); + throw new StartupProblemException(problems.ToString()); } services.AddServerless(configure => diff --git a/docs/deployment.md b/docs/deployment.md index f0bf660c..6d741c59 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -93,6 +93,7 @@ For a user-assigned identity, set `Bitween__AzureManagedIdentityClientId` or `AZ Set `Bitween:StorageProvider` and the matching `CloudFiles` keys. See [Configuration](configuration.md#storage). +- At each start Bitween writes `.access-check` under the document prefix and reads it back, and [won't start](#when-bitween-wont-start) when it can't. The same file tells the Settings page whether the bucket is open to everyone. - With the default `temp30/` document prefix, the storage libraries expire exchange files after 30 days. Choose `temp1/`, `temp7/` or `temp365/` for other retention, or a prefix outside those to keep files. The prefix can be changed later on the Settings page. - For Azure Blob, set `CloudFiles__SubscriptionId` and `CloudFiles__ResourceGroupName`, and give the identity a role that can manage the account's lifecycle policy. The rules are then created at startup. Without them the rules aren't created, and the Settings page can't show any set up by hand. See [Configuration](configuration.md#storage). - Keep the bucket private: Bitween reads and writes with its own credentials and serves file links itself. On Azure it switches its container to private at startup; with a managed identity, give it a role that can change the container's access level, or set it to Private yourself. From 6ce6f0070f602fd83761e3affc935532809dce97 Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Mon, 19 Oct 2026 11:42:09 +0300 Subject: [PATCH 07/10] docs: upgrading no longer needs downtime --- docs/deployment.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/deployment.md b/docs/deployment.md index 6d741c59..d87f1282 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -130,7 +130,6 @@ Bitween checks its configuration and its database before it serves anything. Whe ## Upgrading - Migrations apply at startup, so back up the database before deploying a new version. -- Upgrading to this version needs a short downtime: stop every pod of the old version before deploying the new one. - Values already in the `Settings` table are not replaced by new configuration. Change them on the Settings page. A startup warning names each one configuration sets differently. - Upgrading from a version without work groups needs nothing set: at each start Bitween reads the queues that version left behind until they are empty, then removes them. From 57997053d7a7f2111bf1530049766d3134a4e6da Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Mon, 19 Oct 2026 12:15:30 +0300 Subject: [PATCH 08/10] fix: refuse to start when RabbitMQ can't tell the old environment name --- SW.Bitween.Api/Services/BusEnvironment.cs | 9 ++++++--- .../Tests/BusEnvironmentTests.cs | 13 ++++++++++--- docs/deployment.md | 2 +- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/SW.Bitween.Api/Services/BusEnvironment.cs b/SW.Bitween.Api/Services/BusEnvironment.cs index 062b70fc..b62f28d5 100644 --- a/SW.Bitween.Api/Services/BusEnvironment.cs +++ b/SW.Bitween.Api/Services/BusEnvironment.cs @@ -67,9 +67,12 @@ public static async Task Check(BitweenDbContext db, string current, stri } catch (Exception ex) { - logger.LogWarning(ex, "Could not reach RabbitMQ to tell which environment this Bitween ran as " + - "before; it is checked on a later start."); - return null; + // Not left to a later start: by then the bus has made queues under the new name, and they + // would be all that start finds. Without the broker the bus can't run anyway. + logger.LogWarning(ex, "Could not reach RabbitMQ to tell which environment this Bitween ran as before."); + return "RabbitMQ could not be reached to tell which ASPNETCORE_ENVIRONMENT this Bitween's queues were " + + $"made under ({ex.GetBaseException().Message}). Check ConnectionStrings:RabbitMQ and that the " + + "broker is up."; } } diff --git a/SW.Bitween.IntegrationTests/Tests/BusEnvironmentTests.cs b/SW.Bitween.IntegrationTests/Tests/BusEnvironmentTests.cs index 69efd89b..aae57a5e 100644 --- a/SW.Bitween.IntegrationTests/Tests/BusEnvironmentTests.cs +++ b/SW.Bitween.IntegrationTests/Tests/BusEnvironmentTests.cs @@ -71,11 +71,18 @@ public async Task Before_names_were_recorded_the_queues_tell_what_it_ran_as() Assert.Null(await RecordedAsync()); } + /// + /// A later start would find queues the bus had made under the new name by then, and take that + /// name for the old one. + /// [Fact] - public async Task An_unreachable_broker_leaves_the_check_to_a_later_start() + public async Task An_unreachable_broker_on_the_first_check_stops_the_start() { - Assert.Null(await CheckAsync("Production", confirmed: null, - formerName: () => throw new InvalidOperationException("broker down"))); + var problem = await CheckAsync("Production", confirmed: null, + formerName: () => throw new InvalidOperationException("broker down")); + + Assert.Contains("broker down", problem); + Assert.Contains("ConnectionStrings:RabbitMQ", problem); Assert.Null(await RecordedAsync()); } diff --git a/docs/deployment.md b/docs/deployment.md index d87f1282..70d764d3 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -55,7 +55,7 @@ Things to know about the chart: - The image tag is the chart version, so a chart must be packaged with the version of an image that exists. - The Service forwards port 80 to container port 8080. - The Ingress exposes `/api`, `/swagger` and `/` by default, so the admin UI and `/blank.html` are served from the same host as the API. A deployment that overrides `ingress.paths` replaces the whole list, so it has to include `/` itself. -- Probes are on by default: a startup probe and a liveness probe on `/health/live`, which only checks that the process answers, and a readiness probe on `/health/ready`, which also checks the database and that the bus is connected to RabbitMQ. A pod whose broker is down starts and stays not ready until the bus connects; it does not restart. The startup probe allows 40 minutes for migrations (`probes.startupFailureThreshold`). +- Probes are on by default: a startup probe and a liveness probe on `/health/live`, which only checks that the process answers, and a readiness probe on `/health/ready`, which also checks the database and that the bus is connected to RabbitMQ. A pod whose broker is down starts and stays not ready until the bus connects; it does not restart. The exception is an installation's first start on a version that records the environment name its queues are made under: it [won't start](#when-bitween-wont-start) until RabbitMQ answers which name that was. The startup probe allows 40 minutes for migrations (`probes.startupFailureThreshold`). - A stopping pod lets the exchanges it is running finish for up to 30 seconds before it closes its queues, so they are acknowledged instead of run again elsewhere. The app waits up to 45 seconds to stop, and the chart gives the pod 60 (`terminationGracePeriodSeconds`); keep that value above 45. - The `rabbitmq` values are not used by any template. - `charts/default/README.md` explains the two routing modes. From 8f26ec0aebaa6601766d6088ffe701be191721e2 Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Mon, 19 Oct 2026 12:15:30 +0300 Subject: [PATCH 09/10] fix: check every stored credential at startup, not the newest 100 --- SW.Bitween.Api/Services/StoredSecretsKey.cs | 17 +++++++++------- .../Tests/SecretColumnEncryptionTests.cs | 20 +++++++++++++++++++ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/SW.Bitween.Api/Services/StoredSecretsKey.cs b/SW.Bitween.Api/Services/StoredSecretsKey.cs index 755c84fb..a683658f 100644 --- a/SW.Bitween.Api/Services/StoredSecretsKey.cs +++ b/SW.Bitween.Api/Services/StoredSecretsKey.cs @@ -19,9 +19,9 @@ public static class StoredSecretsKey { public const string KeyLostSetting = "Bitween:SettingsEncryptionKeyLost"; - // Every value is written with the same key, so a table's newest rows tell, being the likeliest - // written since encryption was turned on; the encryption pass reads all of them minutes later. - private const int Sample = 100; + // Every row, not a sample: a key rotation stopped half way leaves older rows under the old key. + // These are configuration tables, which the encryption pass reads whole on every start too. + private const int BatchSize = 100; /// /// Records a startup problem when they can't be read. Runs after the migrations, before anything @@ -67,10 +67,13 @@ private static async Task Readable(BitweenDbContext db) where T : class var key = db.Model.FindEntityType(typeof(T))!.FindPrimaryKey()!.Properties[0].Name; try { - await db.Set().AsNoTracking() - .OrderByDescending(e => EF.Property(e, key)) - .Take(Sample).ToListAsync(); - return true; + for (var skip = 0; ; skip += BatchSize) + { + var batch = await db.Set().AsNoTracking() + .OrderBy(e => EF.Property(e, key)) + .Skip(skip).Take(BatchSize).ToListAsync(); + if (batch.Count < BatchSize) return true; + } } catch (Exception ex) when (IsUnreadable(ex)) { diff --git a/SW.Bitween.IntegrationTests/Tests/SecretColumnEncryptionTests.cs b/SW.Bitween.IntegrationTests/Tests/SecretColumnEncryptionTests.cs index 30baff35..034fd647 100644 --- a/SW.Bitween.IntegrationTests/Tests/SecretColumnEncryptionTests.cs +++ b/SW.Bitween.IntegrationTests/Tests/SecretColumnEncryptionTests.cs @@ -93,6 +93,26 @@ public async Task A_key_that_did_not_encrypt_them_is_found_before_startup() } } + /// A key rotation stopped half way leaves the older rows under the old key. + [Fact] + public async Task An_older_row_under_another_key_is_found_too() + { + SecretColumnCipher.Configure("the old key"); + var older = await CreatePartnerAsync(); + SecretColumnCipher.Configure("the new key"); + var newer = await CreatePartnerAsync(); + try + { + Assert.False(await AllReadableAsync()); + } + finally + { + await DeletePartnerAsync(older); + await DeletePartnerAsync(newer); + SecretColumnCipher.Configure(null); + } + } + private async Task DeletePartnerAsync(int id) { await using var scope = fixture.CreateScope(); From 3b78c74419681f2b98375fcf2e4f976ca007840b Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Mon, 19 Oct 2026 12:15:30 +0300 Subject: [PATCH 10/10] chore: reword a placeholder address and a docs sentence --- SW.Bitween.Web/Startup.cs | 2 +- docs/security.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/SW.Bitween.Web/Startup.cs b/SW.Bitween.Web/Startup.cs index 158f4056..f03fbee0 100644 --- a/SW.Bitween.Web/Startup.cs +++ b/SW.Bitween.Web/Startup.cs @@ -88,7 +88,7 @@ public void ConfigureServices(IServiceCollection services) foreach (var (name, example) in new[] { (BitweenDbContext.ConnectionString, "the database's connection string"), - ("RabbitMQ", "the broker's address, as amqp://user:password@rabbitmq:5672/") + ("RabbitMQ", "the broker's amqp:// address, with its user and password") }) { if (!string.IsNullOrWhiteSpace(Configuration.GetConnectionString(name))) continue; diff --git a/docs/security.md b/docs/security.md index a1951b3d..a2b24410 100644 --- a/docs/security.md +++ b/docs/security.md @@ -139,7 +139,7 @@ The seeded **SYSTEM** partner has a key named `default` whose value is fixed in - Teams and Slack webhook addresses are secret, because anyone who has one can post to the channel. - A bus gateway's endpoint settings are masked as its data source's settings are: those the data source names as secret, and any with a credential's name (password, token, key and so on). Sending `__private__` back keeps the stored value. - Partner properties, global values, adapter properties, data source settings, notification channel, notifier and retry alert properties, including passwords, are encrypted in the database with AES-256-GCM when `Bitween:SettingsEncryptionKey` is set, and stored in plain text when it isn't. Values stored before the key was set are encrypted a few minutes after the next start. `Bitween:PreviousSettingsEncryptionKey` still decrypts values while the key is rotated. -- Bitween [won't start](deployment.md#when-bitween-wont-start) when it can't read these with the key it has: a key that isn't set, or isn't the one they were encrypted with. A Helm deployment still has the key in an earlier release: `helm history ` lists them and `helm get values --revision ` shows one's values. A database backup doesn't help without it, as it is encrypted with the same key. When the key is lost for good, `Bitween:SettingsEncryptionKeyLost=true` lets Bitween start: every value the lost key encrypted shows as empty until it is entered again, and stays stored until then, so it reads again if the key turns up. Remove the setting once they are entered again. +- Bitween [won't start](deployment.md#when-bitween-wont-start) when it can't read these with the key it has: a key that isn't set, or isn't the one they were encrypted with. A Helm deployment still has the key in an earlier release: `helm history ` lists them and `helm get values --revision ` shows one's values. A database backup doesn't help without it, as it is encrypted with the same key. When the key is lost for good, `Bitween:SettingsEncryptionKeyLost=true` lets Bitween start: every value the lost key encrypted shows as empty until it is entered again, and stays stored until then, so it can still be read if the key turns up. Remove the setting once everything is re-entered. ## Exchange files