hamzahalq · Oct 18, 2026 · Oct 18, 2026 · Oct 18, 2026 · Oct 18, 2026 · Oct 18, 2026
diff --git a/SW.Bitween.Adapters.Tooling/SW.Bitween.Adapters.Tooling.csproj b/SW.Bitween.Adapters.Tooling/SW.Bitween.Adapters.Tooling.csproj
@@ -11,7 +11,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="SimplyWorks.Serverless.Tooling" Version="10.2.2" />
+    <PackageReference Include="SimplyWorks.Serverless.Tooling" Version="10.2.4" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/SW.Bitween.Api/SW.Bitween.Api.csproj b/SW.Bitween.Api/SW.Bitween.Api.csproj
@@ -9,7 +9,7 @@
     <!-- Aliased: another dependency brings the older BouncyCastle.Crypto, with the same type names. -->
     <PackageReference Include="BouncyCastle.Cryptography" Version="2.7.0" Aliases="BouncyCastle2" />
     <PackageReference Include="NJsonSchema" Version="11.6.1" />
-    <PackageReference Include="SimplyWorks.Serverless" Version="10.2.2" />
+    <PackageReference Include="SimplyWorks.Serverless" Version="10.2.4" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj b/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj
@@ -50,10 +50,10 @@
     <PackageReference Include="RabbitMQ.Client" Version="7.2.2" />
 
     <!-- Serverless runtime -->
-    <PackageReference Include="SimplyWorks.Serverless" Version="10.2.2" />
+    <PackageReference Include="SimplyWorks.Serverless" Version="10.2.4" />
     <!-- serverless build and publish as a library: the Python adapters here are built and published
          the way the CLI does it. -->
-    <PackageReference Include="SimplyWorks.Serverless.Tooling" Version="10.2.2" />
+    <PackageReference Include="SimplyWorks.Serverless.Tooling" Version="10.2.4" />
 
     <!-- Bus (for IPublish + AddBus registration) -->
     <PackageReference Include="SimplyWorks.Bus" Version="10.1.0" />

diff --git a/SW.Bitween.IntegrationTests/Tests/HandDroppedIndexMigrationTests.cs b/SW.Bitween.IntegrationTests/Tests/HandDroppedIndexMigrationTests.cs
@@ -0,0 +1,52 @@
+using System.Threading.Tasks;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Npgsql;
+using SW.PrimitiveTypes;
+using Testcontainers.PostgreSql;
+using Xunit;
+
+namespace SW.Bitween.IntegrationTests.Tests;
+
+/// <summary>
+/// A database whose promoted values index was dropped by hand - the way to stop long promoted values
+/// failing to save before DropPromotedValuesIndex dropped it for everyone. That migration assumed the
+/// index was there, so on such a database it failed and the app crash-looped on startup.
+/// </summary>
+public sealed class HandDroppedIndexMigrationTests : IAsyncLifetime
+{
+    private readonly PostgreSqlContainer _postgres = new PostgreSqlBuilder().Build();
+
+    public Task InitializeAsync() => _postgres.StartAsync();
+
+    public async Task DisposeAsync() => await _postgres.DisposeAsync();
+
+    [Fact]
+    public async Task A_database_without_the_promoted_values_index_migrates_to_the_latest()
+    {
+        await using var db = CreateDbContext();
+        await db.GetService<IMigrator>().MigrateAsync("20260930150916_ExchangeRetention");
+        await db.Database.ExecuteSqlRawAsync("DROP INDEX infolink.ix_xchange_promoted_properties_properties_raw;");
+
+        await db.Database.MigrateAsync();
+
+        Assert.Empty(await db.Database.GetPendingMigrationsAsync());
+    }
+
+    private PgSql.BitweenDbContext CreateDbContext()
+    {
+        var dataSource = new NpgsqlDataSourceBuilder(_postgres.GetConnectionString());
+        dataSource.EnableDynamicJson();
+        var options = new DbContextOptionsBuilder<PgSql.BitweenDbContext>()
+            .UseSnakeCaseNamingConvention()
+            .UseNpgsql(dataSource.Build(), b =>
+            {
+                b.MigrationsHistoryTable("_ef_migrations_history", PgSql.BitweenDbContext.Schema);
+                b.MigrationsAssembly(typeof(PgSql.DbType).Assembly.FullName);
+            })
+            .Options;
+
+        return new PgSql.BitweenDbContext(options, new RequestContext(), new NoPublish());
+    }
+}
diff --git a/SW.Bitween.MsSql/Migrations/20261005115325_DropPromotedValuesIndex.cs b/SW.Bitween.MsSql/Migrations/20261005115325_DropPromotedValuesIndex.cs
@@ -10,9 +10,11 @@ public partial class DropPromotedValuesIndex : Migration
         /// <inheritdoc />
         protected override void Up(MigrationBuilder migrationBuilder)
         {
-            migrationBuilder.DropIndex(
-                name: "IX_XchangePromotedProperties_PropertiesRaw",
-                table: "XchangePromotedProperties");
+            // IF EXISTS: before this migration, dropping the index by hand was the way to stop long
+            // values failing to save, and on a database where that was done a plain drop failed and
+            // the app could not start.
+            migrationBuilder.Sql(
+                "DROP INDEX IF EXISTS [IX_XchangePromotedProperties_PropertiesRaw] ON [XchangePromotedProperties];");
 
             migrationBuilder.AlterColumn<string>(
                 name: "PropertiesRaw",

diff --git a/SW.Bitween.MySql/Migrations/20261005115322_DropPromotedValuesIndex.cs b/SW.Bitween.MySql/Migrations/20261005115322_DropPromotedValuesIndex.cs
@@ -10,9 +10,19 @@ public partial class DropPromotedValuesIndex : Migration
         /// <inheritdoc />
         protected override void Up(MigrationBuilder migrationBuilder)
         {
-            migrationBuilder.DropIndex(
-                name: "IX_XchangePromotedProperties_PropertiesRaw",
-                table: "XchangePromotedProperties");
+            // Only if it is there: before this migration, dropping the index by hand was the way to
+            // stop long values failing to save, and on a database where that was done a plain drop
+            // failed and the app could not start. MySQL has no DROP INDEX IF EXISTS.
+            migrationBuilder.Sql("""
+                SET @drop = IF(EXISTS (
+                        SELECT 1 FROM information_schema.statistics
+                        WHERE table_schema = DATABASE() AND table_name = 'XchangePromotedProperties'
+                            AND index_name = 'IX_XchangePromotedProperties_PropertiesRaw'),
+                    'ALTER TABLE `XchangePromotedProperties` DROP INDEX `IX_XchangePromotedProperties_PropertiesRaw`', 'DO 0');
+                PREPARE drop_statement FROM @drop;
+                EXECUTE drop_statement;
+                DEALLOCATE PREPARE drop_statement;
+                """);
 
             migrationBuilder.AlterColumn<string>(
                 name: "PropertiesRaw",

diff --git a/SW.Bitween.PgSql/Migrations/20261005115316_DropPromotedValuesIndex.cs b/SW.Bitween.PgSql/Migrations/20261005115316_DropPromotedValuesIndex.cs
@@ -10,10 +10,10 @@ public partial class DropPromotedValuesIndex : Migration
         /// <inheritdoc />
         protected override void Up(MigrationBuilder migrationBuilder)
         {
-            migrationBuilder.DropIndex(
-                name: "ix_xchange_promoted_properties_properties_raw",
-                schema: "infolink",
-                table: "xchange_promoted_properties");
+            // IF EXISTS: before this migration, dropping the index by hand was the way to stop long
+            // values failing to save, and on a database where that was done a plain drop failed and
+            // the app could not start.
+            migrationBuilder.Sql("DROP INDEX IF EXISTS infolink.ix_xchange_promoted_properties_properties_raw;");
         }
 
         /// <inheritdoc />

diff --git a/SW.Bitween.PgSql/Migrations/20261014121704_ResultSubscriptionAndFailureCount.cs b/SW.Bitween.PgSql/Migrations/20261014121704_ResultSubscriptionAndFailureCount.cs
@@ -15,19 +15,14 @@ public partial class ResultSubscriptionAndFailureCount : Migration
         /// <inheritdoc />
         protected override void Up(MigrationBuilder migrationBuilder)
         {
-            migrationBuilder.AddColumn<int>(
-                name: "consecutive_failures",
-                schema: "infolink",
-                table: "xchange_result",
-                type: "integer",
-                nullable: true);
+            // IF NOT EXISTS: the columns commit before the index build below, which runs outside the
+            // transaction. A build that fails part-way (a timeout, the pod restarted) leaves them
+            // without the migration recorded, and the next start must not fail on them.
+            migrationBuilder.Sql(
+                "ALTER TABLE infolink.xchange_result ADD COLUMN IF NOT EXISTS consecutive_failures integer;");
 
-            migrationBuilder.AddColumn<int>(
-                name: "subscription_id",
-                schema: "infolink",
-                table: "xchange_result",
-                type: "integer",
-                nullable: true);
+            migrationBuilder.Sql(
+                "ALTER TABLE infolink.xchange_result ADD COLUMN IF NOT EXISTS subscription_id integer;");
 
             // Outside the transaction, on what can be the largest table there is: see ConcurrentIndex.
             ConcurrentIndex.Create(migrationBuilder, "xchange_result", "ix_xchange_result_subscription_id_finished_on",

diff --git a/SW.Bitween.Web/Program.cs b/SW.Bitween.Web/Program.cs
@@ -4,13 +4,13 @@
 using System.Linq;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Hosting;
+using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Npgsql;
 using SW.Bitween.Services;
-using SW.EfCoreExtensions;
 using SW.Logger;
 
 namespace SW.Bitween.Web
@@ -27,7 +27,7 @@ public static void Main(string[] args)
             // failure (right code, wrong/stale database) indistinguishable from a code problem.
             try
             {
-                host.MigrateDatabase<BitweenDbContext>();
+                MigrateDatabase(host);
             }
             catch (Exception ex)
             {
@@ -44,6 +44,24 @@ public static void Main(string[] args)
                 .MoveRetryAlertsToChannels().ApplyStoredSettings().Run();
         }
 
+        /// <summary>The longest one migration statement may run, such as an index build on a big table.</summary>
+        private static readonly TimeSpan MigrationCommandTimeout = TimeSpan.FromMinutes(30);
+
+        /// <remarks>
+        /// A migration that builds an index on the exchange tables runs for minutes on a database of
+        /// years of traffic, past the 30 seconds a connection string allows a command by default: on a
+        /// client's production upgrade it timed out and the pod restarted over and over. The longer
+        /// limit is for the migrations only; every other query keeps the connection string's. The
+        /// chart's startup probe allows longer than this, so a migration that does run out logs why.
+        /// </remarks>
+        private static void MigrateDatabase(IHost host)
+        {
+            using var scope = host.Services.CreateScope();
+            var db = scope.ServiceProvider.GetRequiredService<BitweenDbContext>();
+            db.Database.SetCommandTimeout(MigrationCommandTimeout);
+            db.Database.Migrate();
+        }
+
         public static IHostBuilder CreateHostBuilder(string[] args) =>
             Host.CreateDefaultBuilder(args)
                 .ConfigureWebHostDefaults(webBuilder =>

diff --git a/SW.Bitween.Web/SW.Bitween.Web.csproj b/SW.Bitween.Web/SW.Bitween.Web.csproj
@@ -53,7 +53,7 @@
     <PackageReference Include="SimplyWorks.CqApi" Version="8.2.6" />
     <PackageReference Include="SimplyWorks.Logger.Console" Version="8.1.5" />
     <PackageReference Include="SimplyWorks.PrimitiveTypes" Version="10.0.1" />
-    <PackageReference Include="SimplyWorks.Serverless" Version="10.2.2" />
+    <PackageReference Include="SimplyWorks.Serverless" Version="10.2.4" />
     <PackageReference Include="SimplyWorks.SimplyRazor" Version="5.0.5" />
     <PackageReference Include="Swashbuckle.AspNetCore.SwaggerUI" Version="8.1.0" />
 

diff --git a/charts/default/values.yaml b/charts/default/values.yaml
@@ -23,8 +23,9 @@ service:
 probes:
   # Startup /health/live, liveness /health/live, readiness /health/ready (database and RabbitMQ).
   enabled: true
-  # Five-second checks: 60 gives a starting pod five minutes, for migrations, before it is restarted.
-  startupFailureThreshold: 60
+  # Five-second checks: 480 gives a starting pod 40 minutes, for migrations, before it is restarted.
+  # Longer than a migration statement may run (30 minutes), so one that runs out logs why.
+  startupFailureThreshold: 480
 
 # North-south routing. The chart uses ingress-nginx by default. Set
 # gateway.enabled=true to switch to Gateway API (HTTPRoute) instead - this

diff --git a/docs/configuration.md b/docs/configuration.md
@@ -10,19 +10,31 @@ Some values are also **runtime settings** that administrators change in the UI.
 |---|---|
 | `ConnectionStrings:BitweenDb` | `Host=db;Port=5432;Database=bitween;Username=bitween;Password=...` |
 | `ConnectionStrings:RabbitMQ` | `amqp://user:password@rabbitmq:5672/` |
+| `ASPNETCORE_ENVIRONMENT` | `Production`, or the value the other applications on the same RabbitMQ use: the bus names its exchanges and queues after it. On an upgrade, keep the running value. Use `Development` only on a developer's machine |
 | `Token:Key` | A random secret of at least 32 characters, unique to this deployment |
 | `Token:Issuer`, `Token:Audience` | Values used to sign and validate JWTs |
 | `Bitween:InitialAdminPassword` | New installations only: the password for `admin@Bitween.systems` |
+| `Bitween:SettingsEncryptionKey` | Recommended: a random secret of at least 32 characters that encrypts stored passwords. Without it they can't be saved from the Settings page. Never change it once set. See [Security](security.md#secrets) |
 | Storage settings | See [Storage](#storage) |
 
 The service stops at startup with a clear error when the database connection string is missing.
 
+Set these too when they apply:
+
+- Other applications publish to Bitween's bus: `Bus:OnInvalidUserToken`. See [Bus messages from other applications](#bus-messages-from-other-applications).
+- Upgrading, and partners must keep getting one call at a time, in order: `Bitween:BusDefaultConcurrency` set to `1`. See [Upgrading to parallel exchanges](deployment.md#upgrading-to-parallel-exchanges).
+- Upgrading from a version without work groups: `Bitween:ConsumeLegacyEventMessages`. See [Upgrading](deployment.md#upgrading).
+- S3-compatible storage other than AWS: `AWS_REQUEST_CHECKSUM_CALCULATION` and `AWS_RESPONSE_CHECKSUM_VALIDATION`. See [Storage](#storage).
+- A RabbitMQ shared with another Bitween: `Bitween:QueuePrefix`.
+- MySQL without TLS: `SslMode` in the connection string. See [Databases](deployment.md#databases).
+- The Queue health page: `Bitween:RabbitMqManagementUrl`, `Bitween:RabbitMqManagementUsername` and `Bitween:RabbitMqManagementPassword`.
+
 ### `Token:Key` must be your own
 
 Whoever holds this key can mint a token for any identity, including one with no account behind
 it. The service **refuses to start** when it is missing, or when it is still the sample value
-`6547647654764764767657658658758765876532542` - which ships as the chart default and is published
-in this repository, so a deployment that never overrode it was open to anyone who read it.
+`6547647654764764767657658658758765876532542`, which ships as the chart default and is published
+in this repository.
 
 The check runs in every environment, including Development, so **a local checkout needs its own
 key too**: put one in your gitignored `appsettings.Development.json` (or `appsettings.Local.json`).
@@ -127,6 +139,8 @@ Storage credentials live in the `CloudFiles` section and are read by the `Simply
 | `OC` | `TenantId`, `FingerPrint`, `UserId`, `RSAKey`, `Region`, `NamespaceName`, `BucketName` |
 | `Local` | `BucketName`, and optionally `StoragePath` |
 
+For an S3-compatible store other than AWS, set `AWS_REQUEST_CHECKSUM_CALCULATION` and `AWS_RESPONSE_CHECKSUM_VALIDATION` to `when_required`. The AWS SDK otherwise uploads with a trailing checksum, which some of these stores save into the file itself.
+
 Azure keeps deletion rules in the storage account's lifecycle management policy, which is reached through Azure Resource Manager rather than the blob endpoint. With `SubscriptionId` and `ResourceGroupName` set, the storage library creates the temp-prefix rules at startup and the Settings page reads them. The managed identity, or a service principal in `AZURE_TENANT_ID`, `AZURE_CLIENT_ID` and `AZURE_CLIENT_SECRET`, needs `Microsoft.Storage/storageAccounts/managementPolicies/read` and `write`, for example through the Storage Account Contributor role. Without them the rules aren't created on Azure, and any policy set up by hand can't be read.
 
 ## Logging

diff --git a/docs/deployment.md b/docs/deployment.md
@@ -55,7 +55,7 @@ Things to know about the chart:
 - The image tag is the chart version, so a chart must be packaged with the version of an image that exists.
 - The Service forwards port 80 to container port 8080.
 - The Ingress exposes `/api`, `/swagger` and `/` by default, so the admin UI and `/blank.html` are served from the same host as the API. A deployment that overrides `ingress.paths` replaces the whole list, so it has to include `/` itself.
-- Probes are on by default: a startup probe and a liveness probe on `/health/live`, which only checks that the process answers, and a readiness probe on `/health/ready`, which also checks the database and that the bus is connected to RabbitMQ. A pod whose broker is down starts and stays not ready until the bus connects; it does not restart. The startup probe allows five minutes for migrations (`probes.startupFailureThreshold`).
+- Probes are on by default: a startup probe and a liveness probe on `/health/live`, which only checks that the process answers, and a readiness probe on `/health/ready`, which also checks the database and that the bus is connected to RabbitMQ. A pod whose broker is down starts and stays not ready until the bus connects; it does not restart. The startup probe allows 40 minutes for migrations (`probes.startupFailureThreshold`).
 - A stopping pod lets the exchanges it is running finish for up to 30 seconds before it closes its queues, so they are acknowledged instead of run again elsewhere. The app waits up to 45 seconds to stop, and the chart gives the pod 60 (`terminationGracePeriodSeconds`); keep that value above 45.
 - The `rabbitmq` values are not used by any template.
 - `charts/default/README.md` explains the two routing modes.
@@ -122,6 +122,7 @@ Run several replicas behind a load balancer for throughput and availability, and
 ## Upgrading
 
 - Migrations apply at startup, so back up the database before deploying a new version.
+- Upgrading to this version needs a short downtime: stop every pod of the old version before deploying the new one.
 - Values already in the `Settings` table are not replaced by new configuration. Change them on the Settings page.
 - When upgrading from a version without work groups, set `Bitween__ConsumeLegacyEventMessages=true` until the old event queues are empty.