From bebff8dbdeed5d2d3755e50a361cd09accf7b8a6 Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Sun, 18 Oct 2026 13:15:59 +0300 Subject: [PATCH 1/5] Migrations: drop the promoted values index only if it exists --- .../Tests/HandDroppedIndexMigrationTests.cs | 52 +++++++++++++++++++ .../20261005115325_DropPromotedValuesIndex.cs | 8 +-- .../20261005115322_DropPromotedValuesIndex.cs | 16 ++++-- .../20261005115316_DropPromotedValuesIndex.cs | 8 +-- 4 files changed, 74 insertions(+), 10 deletions(-) create mode 100644 SW.Bitween.IntegrationTests/Tests/HandDroppedIndexMigrationTests.cs diff --git a/SW.Bitween.IntegrationTests/Tests/HandDroppedIndexMigrationTests.cs b/SW.Bitween.IntegrationTests/Tests/HandDroppedIndexMigrationTests.cs new file mode 100644 index 00000000..9a0de750 --- /dev/null +++ b/SW.Bitween.IntegrationTests/Tests/HandDroppedIndexMigrationTests.cs @@ -0,0 +1,52 @@ +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Npgsql; +using SW.PrimitiveTypes; +using Testcontainers.PostgreSql; +using Xunit; + +namespace SW.Bitween.IntegrationTests.Tests; + +/// +/// A database whose promoted values index was dropped by hand - the way to stop long promoted values +/// failing to save before DropPromotedValuesIndex dropped it for everyone. That migration assumed the +/// index was there, so on such a database it failed and the app crash-looped on startup. +/// +public sealed class HandDroppedIndexMigrationTests : IAsyncLifetime +{ + private readonly PostgreSqlContainer _postgres = new PostgreSqlBuilder().Build(); + + public Task InitializeAsync() => _postgres.StartAsync(); + + public async Task DisposeAsync() => await _postgres.DisposeAsync(); + + [Fact] + public async Task A_database_without_the_promoted_values_index_migrates_to_the_latest() + { + await using var db = CreateDbContext(); + await db.GetService().MigrateAsync("20260930150916_ExchangeRetention"); + await db.Database.ExecuteSqlRawAsync("DROP INDEX infolink.ix_xchange_promoted_properties_properties_raw;"); + + await db.Database.MigrateAsync(); + + Assert.Empty(await db.Database.GetPendingMigrationsAsync()); + } + + private PgSql.BitweenDbContext CreateDbContext() + { + var dataSource = new NpgsqlDataSourceBuilder(_postgres.GetConnectionString()); + dataSource.EnableDynamicJson(); + var options = new DbContextOptionsBuilder() + .UseSnakeCaseNamingConvention() + .UseNpgsql(dataSource.Build(), b => + { + b.MigrationsHistoryTable("_ef_migrations_history", PgSql.BitweenDbContext.Schema); + b.MigrationsAssembly(typeof(PgSql.DbType).Assembly.FullName); + }) + .Options; + + return new PgSql.BitweenDbContext(options, new RequestContext(), new NoPublish()); + } +} diff --git a/SW.Bitween.MsSql/Migrations/20261005115325_DropPromotedValuesIndex.cs b/SW.Bitween.MsSql/Migrations/20261005115325_DropPromotedValuesIndex.cs index 203fd828..036d53f1 100644 --- a/SW.Bitween.MsSql/Migrations/20261005115325_DropPromotedValuesIndex.cs +++ b/SW.Bitween.MsSql/Migrations/20261005115325_DropPromotedValuesIndex.cs @@ -10,9 +10,11 @@ public partial class DropPromotedValuesIndex : Migration /// protected override void Up(MigrationBuilder migrationBuilder) { - migrationBuilder.DropIndex( - name: "IX_XchangePromotedProperties_PropertiesRaw", - table: "XchangePromotedProperties"); + // IF EXISTS: before this migration, dropping the index by hand was the way to stop long + // values failing to save, and on a database where that was done a plain drop failed and + // the app could not start. + migrationBuilder.Sql( + "DROP INDEX IF EXISTS [IX_XchangePromotedProperties_PropertiesRaw] ON [XchangePromotedProperties];"); migrationBuilder.AlterColumn( name: "PropertiesRaw", diff --git a/SW.Bitween.MySql/Migrations/20261005115322_DropPromotedValuesIndex.cs b/SW.Bitween.MySql/Migrations/20261005115322_DropPromotedValuesIndex.cs index 2906972f..e1ec61a5 100644 --- a/SW.Bitween.MySql/Migrations/20261005115322_DropPromotedValuesIndex.cs +++ b/SW.Bitween.MySql/Migrations/20261005115322_DropPromotedValuesIndex.cs @@ -10,9 +10,19 @@ public partial class DropPromotedValuesIndex : Migration /// protected override void Up(MigrationBuilder migrationBuilder) { - migrationBuilder.DropIndex( - name: "IX_XchangePromotedProperties_PropertiesRaw", - table: "XchangePromotedProperties"); + // Only if it is there: before this migration, dropping the index by hand was the way to + // stop long values failing to save, and on a database where that was done a plain drop + // failed and the app could not start. MySQL has no DROP INDEX IF EXISTS. + migrationBuilder.Sql(""" + SET @drop = IF(EXISTS ( + SELECT 1 FROM information_schema.statistics + WHERE table_schema = DATABASE() AND table_name = 'XchangePromotedProperties' + AND index_name = 'IX_XchangePromotedProperties_PropertiesRaw'), + 'ALTER TABLE `XchangePromotedProperties` DROP INDEX `IX_XchangePromotedProperties_PropertiesRaw`', 'DO 0'); + PREPARE drop_statement FROM @drop; + EXECUTE drop_statement; + DEALLOCATE PREPARE drop_statement; + """); migrationBuilder.AlterColumn( name: "PropertiesRaw", diff --git a/SW.Bitween.PgSql/Migrations/20261005115316_DropPromotedValuesIndex.cs b/SW.Bitween.PgSql/Migrations/20261005115316_DropPromotedValuesIndex.cs index eacc90b0..c2d4106c 100644 --- a/SW.Bitween.PgSql/Migrations/20261005115316_DropPromotedValuesIndex.cs +++ b/SW.Bitween.PgSql/Migrations/20261005115316_DropPromotedValuesIndex.cs @@ -10,10 +10,10 @@ public partial class DropPromotedValuesIndex : Migration /// protected override void Up(MigrationBuilder migrationBuilder) { - migrationBuilder.DropIndex( - name: "ix_xchange_promoted_properties_properties_raw", - schema: "infolink", - table: "xchange_promoted_properties"); + // IF EXISTS: before this migration, dropping the index by hand was the way to stop long + // values failing to save, and on a database where that was done a plain drop failed and + // the app could not start. + migrationBuilder.Sql("DROP INDEX IF EXISTS infolink.ix_xchange_promoted_properties_properties_raw;"); } /// From d381a558401755a80fb3813faeaaef2a5375bdf1 Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Sun, 18 Oct 2026 14:55:38 +0300 Subject: [PATCH 2/5] Bump SW-Serverless to 10.2.4 to log adapter startup failures --- .../SW.Bitween.Adapters.Tooling.csproj | 2 +- SW.Bitween.Api/SW.Bitween.Api.csproj | 2 +- .../SW.Bitween.IntegrationTests.csproj | 4 ++-- SW.Bitween.Web/SW.Bitween.Web.csproj | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/SW.Bitween.Adapters.Tooling/SW.Bitween.Adapters.Tooling.csproj b/SW.Bitween.Adapters.Tooling/SW.Bitween.Adapters.Tooling.csproj index 69ec1658..d16541e1 100644 --- a/SW.Bitween.Adapters.Tooling/SW.Bitween.Adapters.Tooling.csproj +++ b/SW.Bitween.Adapters.Tooling/SW.Bitween.Adapters.Tooling.csproj @@ -11,7 +11,7 @@ - + diff --git a/SW.Bitween.Api/SW.Bitween.Api.csproj b/SW.Bitween.Api/SW.Bitween.Api.csproj index 8df481d8..ae08b10c 100644 --- a/SW.Bitween.Api/SW.Bitween.Api.csproj +++ b/SW.Bitween.Api/SW.Bitween.Api.csproj @@ -9,7 +9,7 @@ - + diff --git a/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj b/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj index 41fec423..d6fc7a61 100644 --- a/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj +++ b/SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj @@ -50,10 +50,10 @@ - + - + diff --git a/SW.Bitween.Web/SW.Bitween.Web.csproj b/SW.Bitween.Web/SW.Bitween.Web.csproj index 4429487e..3fb649e2 100644 --- a/SW.Bitween.Web/SW.Bitween.Web.csproj +++ b/SW.Bitween.Web/SW.Bitween.Web.csproj @@ -53,7 +53,7 @@ - + From f4aed748bd97e18206e57d61619e6dbe3f134162 Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Sun, 18 Oct 2026 15:57:13 +0300 Subject: [PATCH 3/5] fix: startup migrations get 30 minutes and survive a failed index build --- ...21704_ResultSubscriptionAndFailureCount.cs | 19 ++++++---------- SW.Bitween.Web/Program.cs | 22 +++++++++++++++++-- charts/default/values.yaml | 5 +++-- docs/deployment.md | 2 +- 4 files changed, 31 insertions(+), 17 deletions(-) diff --git a/SW.Bitween.PgSql/Migrations/20261014121704_ResultSubscriptionAndFailureCount.cs b/SW.Bitween.PgSql/Migrations/20261014121704_ResultSubscriptionAndFailureCount.cs index 41ebc2eb..f1ed3175 100644 --- a/SW.Bitween.PgSql/Migrations/20261014121704_ResultSubscriptionAndFailureCount.cs +++ b/SW.Bitween.PgSql/Migrations/20261014121704_ResultSubscriptionAndFailureCount.cs @@ -15,19 +15,14 @@ public partial class ResultSubscriptionAndFailureCount : Migration /// protected override void Up(MigrationBuilder migrationBuilder) { - migrationBuilder.AddColumn( - name: "consecutive_failures", - schema: "infolink", - table: "xchange_result", - type: "integer", - nullable: true); + // IF NOT EXISTS: the columns commit before the index build below, which runs outside the + // transaction. A build that fails part-way (a timeout, the pod restarted) leaves them + // without the migration recorded, and the next start must not fail on them. + migrationBuilder.Sql( + "ALTER TABLE infolink.xchange_result ADD COLUMN IF NOT EXISTS consecutive_failures integer;"); - migrationBuilder.AddColumn( - name: "subscription_id", - schema: "infolink", - table: "xchange_result", - type: "integer", - nullable: true); + migrationBuilder.Sql( + "ALTER TABLE infolink.xchange_result ADD COLUMN IF NOT EXISTS subscription_id integer;"); // Outside the transaction, on what can be the largest table there is: see ConcurrentIndex. ConcurrentIndex.Create(migrationBuilder, "xchange_result", "ix_xchange_result_subscription_id_finished_on", diff --git a/SW.Bitween.Web/Program.cs b/SW.Bitween.Web/Program.cs index 57bc9717..acfb4c15 100644 --- a/SW.Bitween.Web/Program.cs +++ b/SW.Bitween.Web/Program.cs @@ -4,13 +4,13 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.AspNetCore.Hosting; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Npgsql; using SW.Bitween.Services; -using SW.EfCoreExtensions; using SW.Logger; namespace SW.Bitween.Web @@ -27,7 +27,7 @@ public static void Main(string[] args) // failure (right code, wrong/stale database) indistinguishable from a code problem. try { - host.MigrateDatabase(); + MigrateDatabase(host); } catch (Exception ex) { @@ -44,6 +44,24 @@ public static void Main(string[] args) .MoveRetryAlertsToChannels().ApplyStoredSettings().Run(); } + /// The longest one migration statement may run, such as an index build on a big table. + private static readonly TimeSpan MigrationCommandTimeout = TimeSpan.FromMinutes(30); + + /// + /// A migration that builds an index on the exchange tables runs for minutes on a database of + /// years of traffic, past the 30 seconds a connection string allows a command by default: on a + /// client's production upgrade it timed out and the pod restarted over and over. The longer + /// limit is for the migrations only; every other query keeps the connection string's. The + /// chart's startup probe allows longer than this, so a migration that does run out logs why. + /// + private static void MigrateDatabase(IHost host) + { + using var scope = host.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + db.Database.SetCommandTimeout(MigrationCommandTimeout); + db.Database.Migrate(); + } + public static IHostBuilder CreateHostBuilder(string[] args) => Host.CreateDefaultBuilder(args) .ConfigureWebHostDefaults(webBuilder => diff --git a/charts/default/values.yaml b/charts/default/values.yaml index 253d95c3..15c1ce89 100644 --- a/charts/default/values.yaml +++ b/charts/default/values.yaml @@ -23,8 +23,9 @@ service: probes: # Startup /health/live, liveness /health/live, readiness /health/ready (database and RabbitMQ). enabled: true - # Five-second checks: 60 gives a starting pod five minutes, for migrations, before it is restarted. - startupFailureThreshold: 60 + # Five-second checks: 480 gives a starting pod 40 minutes, for migrations, before it is restarted. + # Longer than a migration statement may run (30 minutes), so one that runs out logs why. + startupFailureThreshold: 480 # North-south routing. The chart uses ingress-nginx by default. Set # gateway.enabled=true to switch to Gateway API (HTTPRoute) instead - this diff --git a/docs/deployment.md b/docs/deployment.md index c86ab31e..a4be0c79 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -55,7 +55,7 @@ Things to know about the chart: - The image tag is the chart version, so a chart must be packaged with the version of an image that exists. - The Service forwards port 80 to container port 8080. - The Ingress exposes `/api`, `/swagger` and `/` by default, so the admin UI and `/blank.html` are served from the same host as the API. A deployment that overrides `ingress.paths` replaces the whole list, so it has to include `/` itself. -- Probes are on by default: a startup probe and a liveness probe on `/health/live`, which only checks that the process answers, and a readiness probe on `/health/ready`, which also checks the database and that the bus is connected to RabbitMQ. A pod whose broker is down starts and stays not ready until the bus connects; it does not restart. The startup probe allows five minutes for migrations (`probes.startupFailureThreshold`). +- Probes are on by default: a startup probe and a liveness probe on `/health/live`, which only checks that the process answers, and a readiness probe on `/health/ready`, which also checks the database and that the bus is connected to RabbitMQ. A pod whose broker is down starts and stays not ready until the bus connects; it does not restart. The startup probe allows 40 minutes for migrations (`probes.startupFailureThreshold`). - A stopping pod lets the exchanges it is running finish for up to 30 seconds before it closes its queues, so they are acknowledged instead of run again elsewhere. The app waits up to 45 seconds to stop, and the chart gives the pod 60 (`terminationGracePeriodSeconds`); keep that value above 45. - The `rabbitmq` values are not used by any template. - `charts/default/README.md` explains the two routing modes. From ea9b6a1e3ef127733fb1e85b060148306fe7651c Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Sun, 18 Oct 2026 15:57:18 +0300 Subject: [PATCH 4/5] docs: what to set for each install and upgrade --- docs/configuration.md | 18 ++++++++++++++++-- docs/deployment.md | 1 + 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 31bf41b1..caba10a8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -10,19 +10,31 @@ Some values are also **runtime settings** that administrators change in the UI. |---|---| | `ConnectionStrings:BitweenDb` | `Host=db;Port=5432;Database=bitween;Username=bitween;Password=...` | | `ConnectionStrings:RabbitMQ` | `amqp://user:password@rabbitmq:5672/` | +| `ASPNETCORE_ENVIRONMENT` | `Production`, or the value the other applications on the same RabbitMQ use: the bus names its exchanges and queues after it. On an upgrade, keep the running value. Use `Development` only on a developer's machine | | `Token:Key` | A random secret of at least 32 characters, unique to this deployment | | `Token:Issuer`, `Token:Audience` | Values used to sign and validate JWTs | | `Bitween:InitialAdminPassword` | New installations only: the password for `admin@Bitween.systems` | +| `Bitween:SettingsEncryptionKey` | Recommended: a random secret of at least 32 characters that encrypts stored passwords. Without it they can't be saved from the Settings page. Never change it once set. See [Security](security.md#secrets) | | Storage settings | See [Storage](#storage) | The service stops at startup with a clear error when the database connection string is missing. +Set these too when they apply: + +- Other applications publish to Bitween's bus: `Bus:OnInvalidUserToken`. See [Bus messages from other applications](#bus-messages-from-other-applications). +- Upgrading, and partners must keep getting one call at a time, in order: `Bitween:BusDefaultConcurrency` set to `1`. See [Upgrading to parallel exchanges](deployment.md#upgrading-to-parallel-exchanges). +- Upgrading from a version without work groups: `Bitween:ConsumeLegacyEventMessages`. See [Upgrading](deployment.md#upgrading). +- S3-compatible storage other than AWS: `AWS_REQUEST_CHECKSUM_CALCULATION` and `AWS_RESPONSE_CHECKSUM_VALIDATION`. See [Storage](#storage). +- A RabbitMQ shared with another Bitween: `Bitween:QueuePrefix`. +- MySQL without TLS: `SslMode` in the connection string. See [Databases](deployment.md#databases). +- The Queue health page: `Bitween:RabbitMqManagementUrl`, `RabbitMqManagementUsername` and `RabbitMqManagementPassword`. + ### `Token:Key` must be your own Whoever holds this key can mint a token for any identity, including one with no account behind it. The service **refuses to start** when it is missing, or when it is still the sample value -`6547647654764764767657658658758765876532542` - which ships as the chart default and is published -in this repository, so a deployment that never overrode it was open to anyone who read it. +`6547647654764764767657658658758765876532542`, which ships as the chart default and is published +in this repository. The check runs in every environment, including Development, so **a local checkout needs its own key too**: put one in your gitignored `appsettings.Development.json` (or `appsettings.Local.json`). @@ -127,6 +139,8 @@ Storage credentials live in the `CloudFiles` section and are read by the `Simply | `OC` | `TenantId`, `FingerPrint`, `UserId`, `RSAKey`, `Region`, `NamespaceName`, `BucketName` | | `Local` | `BucketName`, and optionally `StoragePath` | +For an S3-compatible store other than AWS, set `AWS_REQUEST_CHECKSUM_CALCULATION` and `AWS_RESPONSE_CHECKSUM_VALIDATION` to `when_required`. The AWS SDK otherwise uploads with a trailing checksum, which some of these stores save into the file itself. + Azure keeps deletion rules in the storage account's lifecycle management policy, which is reached through Azure Resource Manager rather than the blob endpoint. With `SubscriptionId` and `ResourceGroupName` set, the storage library creates the temp-prefix rules at startup and the Settings page reads them. The managed identity, or a service principal in `AZURE_TENANT_ID`, `AZURE_CLIENT_ID` and `AZURE_CLIENT_SECRET`, needs `Microsoft.Storage/storageAccounts/managementPolicies/read` and `write`, for example through the Storage Account Contributor role. Without them the rules aren't created on Azure, and any policy set up by hand can't be read. ## Logging diff --git a/docs/deployment.md b/docs/deployment.md index a4be0c79..fd8483e9 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -122,6 +122,7 @@ Run several replicas behind a load balancer for throughput and availability, and ## Upgrading - Migrations apply at startup, so back up the database before deploying a new version. +- Upgrading to this version needs a short downtime: stop every pod of the old version before deploying the new one. - Values already in the `Settings` table are not replaced by new configuration. Change them on the Settings page. - When upgrading from a version without work groups, set `Bitween__ConsumeLegacyEventMessages=true` until the old event queues are empty. From 5c9b8847d2d67331f26ed7685f6ca4a44c0f82b3 Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Sun, 18 Oct 2026 16:12:47 +0300 Subject: [PATCH 5/5] docs: full names for the Queue health settings --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index caba10a8..afa770b8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -27,7 +27,7 @@ Set these too when they apply: - S3-compatible storage other than AWS: `AWS_REQUEST_CHECKSUM_CALCULATION` and `AWS_RESPONSE_CHECKSUM_VALIDATION`. See [Storage](#storage). - A RabbitMQ shared with another Bitween: `Bitween:QueuePrefix`. - MySQL without TLS: `SslMode` in the connection string. See [Databases](deployment.md#databases). -- The Queue health page: `Bitween:RabbitMqManagementUrl`, `RabbitMqManagementUsername` and `RabbitMqManagementPassword`. +- The Queue health page: `Bitween:RabbitMqManagementUrl`, `Bitween:RabbitMqManagementUsername` and `Bitween:RabbitMqManagementPassword`. ### `Token:Key` must be your own