diff --git a/.dockerignore b/.dockerignore index 3729ff0c..af6e61c9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,7 +13,7 @@ **/*.jfm **/azds.yaml **/bin -**/charts +charts **/docker-compose* **/Dockerfile* **/node_modules diff --git a/.github/workflows/bitween-api-cicd-gateway.yml b/.github/workflows/bitween-api-cicd-gateway.yml index aee651d6..5e101c3b 100644 --- a/.github/workflows/bitween-api-cicd-gateway.yml +++ b/.github/workflows/bitween-api-cicd-gateway.yml @@ -109,10 +109,11 @@ jobs: nameOverride=bitween global.environment=Development replicaCount=2 - resources.requests.cpu=25m - resources.requests.memory=704Mi - resources.limits.memory=1408Mi + resources.requests.cpu=310m + resources.requests.memory=1024Mi + resources.limits.memory=2048Mi environmentVariables.Bitween__BusProvidersEnabled=true + environmentVariables.Bitween__CircuitBreakerPerEndpoint=true environmentVariables.AWS_REQUEST_CHECKSUM_CALCULATION=when_required environmentVariables.AWS_RESPONSE_CHECKSUM_VALIDATION=when_required secrets: @@ -137,7 +138,13 @@ jobs: # holding it can mint a token for any identity, so the app refuses to start on it. # Set the TokenKey secret to a long random string unique to this environment # (openssl rand -base64 48). Changing it signs out everyone holding an older token. - helm-set-secret-values: 'global.token.key=${{ secrets.TokenKey }},db=${{ secrets.dbConnection }},global.bus.rabbitUrl=${{ secrets.rabbitUrl }},global.cloudFiles.secretAccessKey=${{ secrets.SecretAccessKey }},global.cloudFiles.accessKeyId=${{ secrets.AccessKeyId }},global.cloudFiles.serviceUrl=${{ secrets.ServiceUrl }},global.cloudFiles.bucketName=${{ secrets.BucketName }},environmentVariables.Bitween__RabbitMqManagementUrl=${{ secrets.RabbitMqManagementUrl }},environmentVariables.Bitween__RabbitMqManagementPassword=${{ secrets.RabbitMqManagementPassword }},environmentVariables.Bitween__RabbitMqManagementUsername=${{ secrets.RabbitMqManagementUsername }}' + # + # Bitween__SettingsEncryptionKey encrypts stored credentials (partner, adapter and connection + # secrets, the mail server password). Without it they are stored in plain text and secret + # settings can't be saved in the UI. Set the SettingsEncryptionKey secret once + # (openssl rand -base64 32) and never change or lose it: what it encrypted can't be read + # without it. + helm-set-secret-values: 'global.token.key=${{ secrets.TokenKey }},db=${{ secrets.dbConnection }},global.bus.rabbitUrl=${{ secrets.rabbitUrl }},global.cloudFiles.secretAccessKey=${{ secrets.SecretAccessKey }},global.cloudFiles.accessKeyId=${{ secrets.AccessKeyId }},global.cloudFiles.serviceUrl=${{ secrets.ServiceUrl }},global.cloudFiles.bucketName=${{ secrets.BucketName }},environmentVariables.Bitween__RabbitMqManagementUrl=${{ secrets.RabbitMqManagementUrl }},environmentVariables.Bitween__RabbitMqManagementPassword=${{ secrets.RabbitMqManagementPassword }},environmentVariables.Bitween__RabbitMqManagementUsername=${{ secrets.RabbitMqManagementUsername }},secrets.Bitween__SettingsEncryptionKey=${{ secrets.SettingsEncryptionKey }}' Prod: # Release: build and publish for clients, deploy production. releases/r10.0 is the @@ -223,31 +230,33 @@ jobs: # Production's own configuration, the same settings as the develop job's above. PROD_TOKENKEY # is its own random string (openssl rand -base64 48), never the develop key. - helm-set-secret-values: 'global.token.key=${{ secrets.PROD_TOKENKEY }},db=${{ secrets.PROD_DBCONNECTION }},global.bus.rabbitUrl=${{ secrets.PROD_RABBITURL }},global.cloudFiles.secretAccessKey=${{ secrets.PROD_SECRETACCESSKEY }},global.cloudFiles.accessKeyId=${{ secrets.PROD_ACCESSKEYID }},global.cloudFiles.serviceUrl=${{ secrets.PROD_SERVICEURL }},global.cloudFiles.bucketName=${{ secrets.PROD_BUCKETNAME }},environmentVariables.Bitween__RabbitMqManagementUrl=${{ secrets.PROD_RABBITMQMANAGEMENTURL }},environmentVariables.Bitween__RabbitMqManagementPassword=${{ secrets.PROD_RABBITMQMANAGEMENTPASSWORD }},environmentVariables.Bitween__RabbitMqManagementUsername=${{ secrets.PROD_RABBITMQMANAGEMENTUSERNAME }}' + # PROD_SETTINGSENCRYPTIONKEY is production's own too (openssl rand -base64 32), and like develop's + # it must never change or be lost. + helm-set-secret-values: 'global.token.key=${{ secrets.PROD_TOKENKEY }},db=${{ secrets.PROD_DBCONNECTION }},global.bus.rabbitUrl=${{ secrets.PROD_RABBITURL }},global.cloudFiles.secretAccessKey=${{ secrets.PROD_SECRETACCESSKEY }},global.cloudFiles.accessKeyId=${{ secrets.PROD_ACCESSKEYID }},global.cloudFiles.serviceUrl=${{ secrets.PROD_SERVICEURL }},global.cloudFiles.bucketName=${{ secrets.PROD_BUCKETNAME }},environmentVariables.Bitween__RabbitMqManagementUrl=${{ secrets.PROD_RABBITMQMANAGEMENTURL }},environmentVariables.Bitween__RabbitMqManagementPassword=${{ secrets.PROD_RABBITMQMANAGEMENTPASSWORD }},environmentVariables.Bitween__RabbitMqManagementUsername=${{ secrets.PROD_RABBITMQMANAGEMENTUSERNAME }},secrets.Bitween__SettingsEncryptionKey=${{ secrets.PROD_SETTINGSENCRYPTIONKEY }}' # The bitween CLI and the SDKs carry the version the run above computed, so nothing is tagged by # hand. A develop build releases the CLI as a pre-release, cli-v-dev.; the release # from releases/r10.0 releases it as the latest, cli-v, and publishes the SDKs. Each runs # only when the job it needs ran, so the branch is decided above. cli-develop: - needs: build-publish-deploy + needs: Dev uses: ./.github/workflows/cli-release.yml with: - version: ${{ needs.build-publish-deploy.outputs.version }} + version: ${{ needs.Dev.outputs.version }} prerelease: true run-number: ${{ github.run_number }} cli-release: - needs: publish-release + needs: Prod uses: ./.github/workflows/cli-release.yml with: - version: ${{ needs.publish-release.outputs.version }} + version: ${{ needs.Prod.outputs.version }} sdk-release: - needs: publish-release + needs: Prod permissions: contents: read uses: ./.github/workflows/sdk-release.yml with: - version: ${{ needs.publish-release.outputs.version }} + version: ${{ needs.Prod.outputs.version }} secrets: inherit diff --git a/.github/workflows/cli-release.yml b/.github/workflows/cli-release.yml new file mode 100644 index 00000000..24496388 --- /dev/null +++ b/.github/workflows/cli-release.yml @@ -0,0 +1,143 @@ +name: bitween CLI release + +# Publishes the bitween command as self-contained binaries on a GitHub release: nothing needs .NET +# installed to run them. Never run by hand or by a tag: the CI run that publishes this repository +# calls it with the version it computed (determine-semver: the next patch from the git tags), so the +# CLI carries the version of the release it ships with. The computed version is the bare x.y.z on +# develop too, so a develop build says so with prerelease and passes its run number. +# a release, 10.0.61 cli-v10.0.61 the latest release +# a prerelease, 10.0.61 with prerelease and run 121 cli-v10.0.61-dev.121 a pre-release, to try out +# Called by bitween-api-cicd-gateway.yml: after the release from releases/r10.0, and after each develop build. +on: + workflow_call: + inputs: + version: + description: "The version the calling run computed" + type: string + required: true + prerelease: + description: "Release as a pre-release, named after the run" + type: boolean + default: false + run-number: + description: "The calling run's number, for a pre-release" + type: string + default: "" + +permissions: + contents: write + +jobs: + channel: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.channel.outputs.version }} + prerelease: ${{ steps.channel.outputs.prerelease }} + steps: + - name: Channel + id: channel + env: + VERSION: ${{ inputs.version }} + PRERELEASE: ${{ inputs.prerelease }} + RUN_NUMBER: ${{ inputs.run-number }} + run: | + v="$VERSION" + if [ "$PRERELEASE" = "true" ]; then + # A pre-release names its run: 10.0.61 from run 121 is 10.0.61-dev.121. + v="${v%%-*}-dev.${RUN_NUMBER:-${GITHUB_RUN_NUMBER}}" + echo "prerelease=true" >> "$GITHUB_OUTPUT" + elif [[ "$v" == *-* ]]; then + # A version that already carries its branch and run: 10.0.61-releases-r10-0-develop.121 is 10.0.61-dev.121. + v="${v%%-*}-dev.${v##*.}" + echo "prerelease=true" >> "$GITHUB_OUTPUT" + else + echo "prerelease=false" >> "$GITHUB_OUTPUT" + fi + echo "version=$v" >> "$GITHUB_OUTPUT" + + build: + needs: channel + runs-on: ubuntu-latest + strategy: + matrix: + rid: [linux-x64, linux-arm64, linux-musl-x64, linux-musl-arm64, osx-x64, osx-arm64, win-x64] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: "10.0.x" + - name: Publish + run: > + dotnet publish SW.Bitween.Cli -c Release -r ${{ matrix.rid }} --self-contained + -p:PublishSingleFile=true -p:IncludeNativeLibrariesForSelfExtract=true -p:DebugType=none + -p:Version=${{ needs.channel.outputs.version }} -o out + - name: Package + run: | + mkdir -p dist + cp docs/cli.md out/README.md 2>/dev/null || true + if [[ "${{ matrix.rid }}" == win-* ]]; then + (cd out && zip -q -r ../dist/bitween-${{ matrix.rid }}.zip .) + else + tar -czf dist/bitween-${{ matrix.rid }}.tar.gz -C out . + fi + - uses: actions/upload-artifact@v4 + with: + name: bitween-${{ matrix.rid }} + path: dist/* + + # The data source providers (RabbitMQ, SQS and the database engines) as adapter packages, with the + # release's version: installed like any adapter, with bitween adapter publish or apply's adapters + # section. Nothing else publishes them, and a Bitween runs no data source without them. + providers: + needs: channel + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: "10.0.x" + - name: Build + env: + VERSION: ${{ needs.channel.outputs.version }} + run: | + dotnet build SW.Bitween.Cli -c Release -o cli + mkdir -p dist + for project in SW.Bitween.Adapters.Bus.RabbitMq SW.Bitween.Adapters.Bus.Sqs SW.Bitween.Adapters.Db.PostgreSql \ + SW.Bitween.Adapters.Db.MySql SW.Bitween.Adapters.Db.SqlServer SW.Bitween.Adapters.Db.Oracle; do + jq --arg v "$VERSION" '.version = $v' "$project/adapter.json" > adapter.json.tmp && mv adapter.json.tmp "$project/adapter.json" + dotnet cli/bitween.dll adapter build "$project" -o dist + done + ls -la dist + - uses: actions/upload-artifact@v4 + with: + name: bitween-providers + path: dist/*.zip + + release: + needs: [channel, build, providers] + runs-on: ubuntu-latest + steps: + # Only the CLI builds' own artifacts. Without a pattern this fetched every artifact in the run, + # including the build record docker/build-push-action uploads in the deploy job, which can't be + # extracted and failed the release (and would have landed in SHA256SUMS if it could). + - uses: actions/download-artifact@v4 + with: + pattern: bitween-* + path: dist + merge-multiple: true + - name: Checksums + run: cd dist && sha256sum * > SHA256SUMS && cat SHA256SUMS + - name: Release + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: | + v="${{ needs.channel.outputs.version }}" + install="curl -fsSL https://raw.githubusercontent.com/${{ github.repository }}/releases/r10.0/scripts/install-cli.sh" + if [ "${{ needs.channel.outputs.prerelease }}" = "true" ]; then + gh release create "cli-v$v" dist/* --title "bitween CLI $v (pre-release)" --target "${{ github.sha }}" --prerelease \ + --notes "A pre-release of the bitween command, built from ${{ github.ref_name }}, to try out. Install: $install | BITWEEN_CLI_VERSION=$v sh - or download the archive for your platform and check it against SHA256SUMS." + else + gh release create "cli-v$v" dist/* --title "bitween CLI $v" --target "${{ github.sha }}" --latest \ + --notes "The bitween command, self-contained for each platform. Install: $install | sh - or download the archive for your platform and check it against SHA256SUMS." + fi diff --git a/.github/workflows/critical-vuln-check.yml b/.github/workflows/critical-vuln-check.yml index 70f912b7..cab88580 100644 --- a/.github/workflows/critical-vuln-check.yml +++ b/.github/workflows/critical-vuln-check.yml @@ -1,9 +1,9 @@ # ============================================================================= -# Critical Dependabot Vulnerability Check — Caller template +# Critical Dependabot Vulnerability Check - Caller template # ============================================================================= # USAGE: # Copy this file to your repo at .github/workflows/critical-vuln-check.yml. -# No REPLACE values needed — this template requires no customization. +# No REPLACE values needed - this template requires no customization. # # WHAT IT DOES: # Runs on every PR targeting main or develop. Fails if the repository has @@ -11,9 +11,9 @@ # # Enforcement differs by target branch via each repo's OWN branch # protection settings (not by anything in this file): -# - On `main`: mark this check REQUIRED in branch protection — merge +# - On `main`: mark this check REQUIRED in branch protection - merge # is physically blocked while a critical alert is open. -# - On `develop`: leave this check NOT required — it still shows as a +# - On `develop`: leave this check NOT required - it still shows as a # failing/red check (a visible warning) without blocking # the merge. # ============================================================================= diff --git a/.github/workflows/dependabot-auto-merge.yml b/.github/workflows/dependabot-auto-merge.yml index b5fa4546..48ce1b7b 100644 --- a/.github/workflows/dependabot-auto-merge.yml +++ b/.github/workflows/dependabot-auto-merge.yml @@ -1,5 +1,5 @@ # ============================================================================= -# Dependabot Auto-Merge — Caller template +# Dependabot Auto-Merge - Caller template # ============================================================================= # USAGE: # Copy this file to your repo at .github/workflows/dependabot-auto-merge.yml. @@ -10,13 +10,13 @@ # - The PR author is dependabot[bot] # - The update is a semver PATCH bump (never minor/major) # - The ecosystem is npm, nuget, pub, bundler, or github-actions -# (NEVER docker — base image bumps always need manual review) +# (NEVER docker - base image bumps always need manual review) # - This repo currently has no open critical Dependabot alert (re-checked -# here explicitly — see workflow-templates/critical-vuln-check.yml's +# here explicitly - see workflow-templates/critical-vuln-check.yml's # header for why this can't just `needs:` that file's job) # "Auto-merge" here means GitHub's native auto-merge feature: it still # waits for the repo's actual required status checks (build/test) to pass -# before merging — this workflow does not bypass those. +# before merging - this workflow does not bypass those. # ============================================================================= name: Dependabot Auto-Merge @@ -64,5 +64,5 @@ jobs: PR_URL: ${{ github.event.pull_request.html_url }} run: | set -euo pipefail - echo "::notice title=🤖 [AUTO-MERGE] Enabling auto-merge::${PR_URL} — patch-level ${{ steps.metadata.outputs.package-ecosystem }} bump, vuln gate passed" + echo "::notice title=🤖 [AUTO-MERGE] Enabling auto-merge::${PR_URL} - patch-level ${{ steps.metadata.outputs.package-ecosystem }} bump, vuln gate passed" gh pr merge --auto --squash "$PR_URL" diff --git a/.github/workflows/dotnet-pr-checks.yml b/.github/workflows/dotnet-pr-checks.yml new file mode 100644 index 00000000..06025989 --- /dev/null +++ b/.github/workflows/dotnet-pr-checks.yml @@ -0,0 +1,50 @@ +# Builds the solution and runs the .NET unit tests on every pull request into a release branch. +# +# The deploy pipeline runs the .NET tests only after a merge, so a change that broke them was +# found once it had already landed on a release branch. Bitween runs at customer sites that +# upgrade in place, so this check runs before the merge instead. The unit tests need no +# database or broker and take seconds. The integration tests (Testcontainers) still run in the +# deploy pipeline. +name: .NET checks + +on: + pull_request: + branches: [releases/**] + paths-ignore: + # Covered by frontend-tests.yml; the .NET build does not compile the SPA. + - 'SW.Bitween.Web/ClientApp/**' + - 'docs/**' + - '**/*.md' + +permissions: + contents: read + +concurrency: + group: dotnet-pr-checks-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + build-and-unit-test: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + # Pinned to commits, since a tag can be moved. + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4.3.1 + with: + dotnet-version: '10.0.x' + - run: dotnet restore SW.Bitween.sln + # SkipClientBuild keeps MSBuild from building the SPA should any target pull in publish. + - run: dotnet build SW.Bitween.sln --no-restore --configuration Release -p:SkipClientBuild=true + - run: > + dotnet test SW.Bitween.UnitTests/SW.Bitween.UnitTests.csproj + --no-build --configuration Release + --logger "trx;LogFileName=unit-tests.trx" + --results-directory TestResults + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: failure() + with: + name: unit-test-results + path: TestResults diff --git a/.github/workflows/frontend-tests.yml b/.github/workflows/frontend-tests.yml index 0091a1b3..fabe5644 100644 --- a/.github/workflows/frontend-tests.yml +++ b/.github/workflows/frontend-tests.yml @@ -3,7 +3,7 @@ # # The deploy pipeline only runs the .NET test projects, and only after a merge, so until # this existed nothing ever ran the frontend tests outside someone's machine. These need -# no backend or database — the component tests render pages against a mock network — so +# no backend or database - the component tests render pages against a mock network - so # they take seconds. The Playwright e2e suite is not run here: it needs a running backend # with seeded data. name: Frontend tests @@ -40,7 +40,7 @@ jobs: cache: yarn cache-dependency-path: SW.Bitween.Web/ClientApp/yarn.lock - run: yarn install --frozen-lockfile - # The same type-check the Docker build runs, which covers the tests too — so a test file + # The same type-check the Docker build runs, which covers the tests too - so a test file # that breaks it fails here rather than in the deploy after the merge. - run: npx tsc -b - run: yarn test diff --git a/.github/workflows/sdk-release.yml b/.github/workflows/sdk-release.yml new file mode 100644 index 00000000..58af9e68 --- /dev/null +++ b/.github/workflows/sdk-release.yml @@ -0,0 +1,95 @@ +name: SDK release + +# Publishes the adapter contract for Python to PyPI as simplyworks-bitween and for JavaScript and +# TypeScript to npm as @simplyworks/bitween. Never run by hand or by a tag: the release run on +# releases/r10.0 (bitween-api-cicd-gateway.yml) calls it with the version it computed, so they carry +# Bitween's version. Only a release version is published. Each is tested first against +# SW-Serverless's SDK from its default branch, and a version already on its registry is left alone, +# so a failed run can simply be run again. +on: + workflow_call: + inputs: + version: + description: "The version the calling run computed" + type: string + required: true + +permissions: + contents: read + +jobs: + python: + if: ${{ !contains(inputs.version, '-') }} + runs-on: ubuntu-latest + defaults: + run: + working-directory: sdk/python + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: simplify9/SW-Serverless + path: sw-serverless + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Version + id: version + env: + VERSION: ${{ inputs.version }} + run: | + v="$VERSION" + sed -i -E "s/^version = \".*\"/version = \"$v\"/" pyproject.toml + sed -i -E "s/^__version__ = \".*\"/__version__ = \"$v\"/" src/simplyworks_bitween/__init__.py + echo "value=$v" >> "$GITHUB_OUTPUT" + - name: Test + run: PYTHONPATH="src:$GITHUB_WORKSPACE/sw-serverless/sdk/python/src" python -m unittest discover -s tests + - name: Build + run: | + python -m pip install --upgrade build + python -m build + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: sdk/python/dist + password: ${{ secrets.PYPI_API_TOKEN }} + skip-existing: true + + node: + if: ${{ !contains(inputs.version, '-') }} + runs-on: ubuntu-latest + defaults: + run: + working-directory: sdk/node + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: simplify9/SW-Serverless + path: sw-serverless + - uses: actions/setup-node@v4 + with: + node-version: "22" + registry-url: "https://registry.npmjs.org" + - name: Version + id: version + env: + VERSION: ${{ inputs.version }} + run: | + v="$VERSION" + npm version "$v" --no-git-tag-version --allow-same-version + sed -i -E "s/^const VERSION = \".*\";/const VERSION = \"$v\";/" src/index.js + echo "value=$v" >> "$GITHUB_OUTPUT" + - name: Test + env: + SW_SERVERLESS_NODE: ${{ github.workspace }}/sw-serverless/sdk/node/src + run: node --test + - name: Publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: | + name=$(node -p "require('./package.json').name") + if npm view "$name@${{ steps.version.outputs.value }}" version >/dev/null 2>&1; then + echo "$name@${{ steps.version.outputs.value }} is already on npm." + else + npm publish --access public + fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d1b7e245..04e60aac 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -269,7 +269,7 @@ dotnet test --collect:"XPlat Code Coverage" # Frontend, from SW.Bitween.Web/ClientApp yarn test # unit + component tests, no backend needed -yarn test:e2e # end-to-end, needs the backend running on the Local profile +yarn test:e2e # end-to-end, against a running backend; tools/e2e.sh starts one from scratch ``` ### Where a New Test Goes @@ -280,7 +280,7 @@ Four layers, fastest first. Put a test in the lowest layer that can see the bug |---|---|---| | Unit | `SW.Bitween.UnitTests` (MSTest); `ClientApp/src/**/__tests__/*.test.ts` (vitest) | Logic with no database, bus or screen: the mapping engine, transforms, schedules, parsing, the editor's reducer. | | Integration | `SW.Bitween.IntegrationTests` (xUnit, real Postgres and RabbitMQ in Docker) | Anything that crosses into the database, the bus, or a permission guard. | -| Component | `ClientApp/src/**/__tests__/*.test.tsx` (vitest in jsdom, against a mock network — see `src/__tests__/support`) | What one page shows and does: which controls appear, what a click sends, how an error reads. | +| Component | `ClientApp/src/**/__tests__/*.test.tsx` (vitest in jsdom, against a mock network - see `src/__tests__/support`) | What one page shows and does: which controls appear, what a click sends, how an error reads. | | End-to-end | `ClientApp/e2e` (Playwright, against the running backend) | A whole user journey through the real app, and what only a real browser can check: layout widths, drag and drop, two tabs. | An end-to-end test that asserts what the engine returns, or what one page renders, belongs a layer down: it runs far slower, and fails for reasons that have nothing to do with what it tests. @@ -452,7 +452,11 @@ Instances of abusive, harassing, or otherwise unacceptable behavior may be repor ## 📄 License -By contributing to Bitween, you agree that your contributions will be licensed under the MIT License. +By submitting a contribution to Bitween, you agree that: + +- your contribution is licensed under the GNU Affero General Public License v3.0, or under the MIT License if it is in `SW.Bitween.Sdk` or a `SW.Bitween.Sample*` project; +- you grant Simplify9 a perpetual, worldwide, royalty-free right to also license your contribution under other terms, including commercial licenses; and +- you have the right to make the contribution (for example, your employer permits it). --- diff --git a/Dockerfile b/Dockerfile index 676c18a7..dc1d14a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,21 @@ FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS base COPY --from=mcr.microsoft.com/dotnet/aspnet:6.0 /usr/share/dotnet/shared /usr/share/dotnet/shared +# Python, for adapters written in it: SW.Serverless starts them with python3. Ubuntu 24.04's 3.12, +# the oldest the Python SDK supports, patched with every rebuild of the image. Published adapters +# bring their own packages, vendored; pip is here for the adapter editor, which fetches a draft's +# requirements when Bitween:AdapterEditorDependencies is on. +RUN apt-get update \ + && apt-get install -y --no-install-recommends python3 python3-pip \ + && rm -rf /var/lib/apt/lists/* + +# Node, for adapters written in JavaScript or TypeScript, from the official Node 22 LTS image. +# Published adapters bring their dependencies in node_modules; npm is here for the adapter editor, +# as pip is. +COPY --from=node:22-bookworm-slim /usr/local/bin/node /usr/local/bin/node +COPY --from=node:22-bookworm-slim /usr/local/lib/node_modules/npm /usr/local/lib/node_modules/npm +RUN ln -s ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm + WORKDIR /app EXPOSE 8080 EXPOSE 443 @@ -17,6 +32,10 @@ COPY ["SW.Bitween.Web/ClientApp/", "./"] RUN yarn build FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build +# The release and commit, stamped into the assemblies so a container says which it is without being +# told: CI passes both. A build without them is versioned as before. +ARG BITWEEN_VERSION= +ARG BITWEEN_COMMIT= WORKDIR /src COPY ["SW.Bitween.Web/SW.Bitween.Web.csproj", "SW.Bitween.Web/"] COPY ["SW.Bitween.Api/SW.Bitween.Api.csproj", "SW.Bitween.Api/"] @@ -26,14 +45,24 @@ COPY ["SW.Bitween.MsSql/SW.Bitween.MsSql.csproj", "SW.Bitween.MsSql/"] RUN dotnet restore "SW.Bitween.Web/SW.Bitween.Web.csproj" COPY . . WORKDIR "/src/SW.Bitween.Web" -RUN dotnet build "SW.Bitween.Web.csproj" -c Release -o /app/build +RUN dotnet build "SW.Bitween.Web.csproj" -c Release -o /app/build \ + ${BITWEEN_VERSION:+-p:Version=$BITWEEN_VERSION} ${BITWEEN_COMMIT:+-p:SourceRevisionId=$BITWEEN_COMMIT} FROM build AS publish +ARG BITWEEN_VERSION= +ARG BITWEEN_COMMIT= # UI is built in the ui-build stage; the SDK image has no node -RUN dotnet publish "SW.Bitween.Web.csproj" -c Release -o /app/publish -p:SkipClientBuild=true +RUN dotnet publish "SW.Bitween.Web.csproj" -c Release -o /app/publish -p:SkipClientBuild=true \ + ${BITWEEN_VERSION:+-p:Version=$BITWEEN_VERSION} ${BITWEEN_COMMIT:+-p:SourceRevisionId=$BITWEEN_COMMIT} FROM base AS final WORKDIR /app COPY --from=publish /app/publish . COPY --from=ui-build /src/SW.Bitween.Web/wwwroot ./wwwroot +# Run as the image's built-in non-root user. The one place Bitween writes under /app is the cache +# of downloaded adapter packages (SW.Serverless AdapterLocalPath, ./adapters by default), so that +# directory alone is handed over. It is a cache: packages are fetched again from storage if it is +# empty, so a fresh container needs no volume for it. +RUN mkdir -p /app/adapters && chown $APP_UID /app/adapters +USER $APP_UID ENTRYPOINT ["dotnet", "SW.Bitween.Web.dll"] diff --git a/LICENSE b/LICENSE index 50f04ce7..be3f7b28 100644 --- a/LICENSE +++ b/LICENSE @@ -1,21 +1,661 @@ -MIT License - -Copyright (c) 2020 Simplify9 - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/NOTICE b/NOTICE new file mode 100644 index 00000000..d11411dd --- /dev/null +++ b/NOTICE @@ -0,0 +1,67 @@ +Bitween +Copyright (C) 2020-2026 Simplify9 + +This program is free software: you can redistribute it and/or modify it under +the terms of the GNU Affero General Public License as published by the Free +Software Foundation, version 3 (see LICENSE), together with the additional +terms below, which are made under Section 7 of that license. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. + + +ADDITIONAL TERMS (Section 7) +---------------------------- + +1. Attribution (Section 7(b)). Any copy or modified version of this program, + and any work based on it, must preserve the following notice in its source + code, in its documentation, and in the Appropriate Legal Notices it displays + (for example, an "About" screen): + + Based on Bitween by Simplify9 - https://github.com/simplify9/Bitween-api + +2. Modified versions (Section 7(c)). A modified version must be marked in + reasonable ways as different from the original version and must not be + presented as the official Bitween release. + +3. Trademarks (Section 7(e)). This license grants no rights to use the name + "Bitween", the Simplify9 name, or their logos, except as needed to give the + attribution notice above. A modified version distributed or offered to + others must use a different name. + + +ADDITIONAL PERMISSION (Section 7) +--------------------------------- + +Running this program, modified or not, solely for the internal operations of +your organization does not trigger Section 13 merely because your own +employees or contractors acting on your behalf interact with it over a +network. Section 13 still applies when anyone else interacts with it remotely. + + +COMPONENTS UNDER THE MIT LICENSE +-------------------------------- + +So that custom adapters built on them are not affected by the AGPL, the +following directories remain under the MIT License (see the LICENSE file in +each directory): + + SW.Bitween.Sdk (NuGet: SimplyWorks.Bitween.Sdk) + SW.Bitween.Sample* (sample adapters, handlers and mappers) + + +EARLIER VERSIONS +---------------- + +Versions of Bitween released before this notice was added were licensed under +the MIT License, and those versions remain available under it. + + +COMMERCIAL LICENSE +------------------ + +Simplify9 offers commercial licenses for uses the AGPL does not permit without +releasing source code, such as embedding Bitween in a closed-source product or +offering it as a hosted service to third parties. Contact Simplify9 at +info@simplify9.com for details. diff --git a/README.md b/README.md index af83f3a9..753a9b9d 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,9 @@ One .NET 10 service hosts everything: the REST API, the partner-facing gateway e - **Many ways in.** Partners call an API gateway. Other systems publish to Bitween's bus, or to their own RabbitMQ or Amazon SQS broker. Scheduled jobs pull from HTTP APIs, S3, Azure Blob, SFTP/FTP, POP3 mailboxes and databases. Aggregations roll finished exchanges up on a schedule. - **One pipeline.** Every message becomes an *exchange* that runs through the same stages: filter, map, deliver, route the response. -- **Adapters for each stage.** Built-in adapters cover HTTP, S3, Azure Blob, SFTP/FTP, POP3 and SMTP, plus a visual rules-based mapper for JSON and XML. Anything else can be a custom adapter that runs out of process. +- **Adapters for each stage.** Built-in adapters cover HTTP, S3, Azure Blob, SFTP/FTP, POP3 and SMTP, plus a visual rules-based mapper for JSON and XML. Anything else can be a custom adapter, written in .NET, Python, JavaScript or TypeScript, that runs out of process. The `bitween` CLI writes, checks and publishes them, and Python, JavaScript and TypeScript adapters can also be written in the admin UI. - **Long-lived connections.** Data sources hold connections to RabbitMQ and Amazon SQS brokers and to PostgreSQL, MySQL, SQL Server and Oracle databases, so deliveries can publish messages or run SQL. -- **Reliability built in.** Retry policies match failures and retry them with a delay and a shared budget. Every exchange's retries form a chain you can follow. Exhausted budgets raise alerts, and notifiers report results. +- **Reliability built in.** Retry policies match failures and retry them with a delay and a shared budget. Every exchange's retries form a chain you can follow. Exhausted budgets raise alerts, and subscriptions report their results, both through notification channels: Email, Microsoft Teams, Slack or any adapter, each set up once. - **Operable.** Queue health straight from RabbitMQ, data source health, schedule health, run history, receive attempts, a dashboard and an audit trail of every configuration change. - **Administered in the browser.** Roles with fine-grained permissions, Microsoft sign-in, and settings and branding that change at runtime. @@ -41,7 +41,7 @@ flowchart LR X -->|work group queue| P --> R P -->|delivery| OUT[Partner systems] P -->|publish or SQL| EXT - R -->|notifiers and alerts| N[Handler adapters] + R -->|notifications and alerts| N[Notification channels] X -. files .-> S[(Object storage)] R -. rows .-> DB[(Bitween database)] ``` @@ -54,6 +54,7 @@ flowchart LR | Database | PostgreSQL, SQL Server or MySQL 8 | | Message broker | RabbitMQ, with the management plugin for queue health | | Object storage | S3-compatible, Azure Blob Storage, Oracle Cloud Object Storage, or local disk in Development | +| Custom adapters in Python or JavaScript | `python3` 3.12 or later and `node` 22 or later on a Linux or macOS host. The Docker image includes both. | ## Quick start @@ -87,6 +88,7 @@ This runs Bitween locally against PostgreSQL, RabbitMQ and local-disk storage. export Token__Key="replace-with-a-random-string-of-at-least-32-chars" export Token__Issuer=bitween-local export Token__Audience=bitween-local + export Bitween__InitialAdminPassword="replace-with-your-own-password" export Bitween__RabbitMqManagementUrl=http://localhost:15672 export Bitween__RabbitMqManagementUsername=guest export Bitween__RabbitMqManagementPassword=guest @@ -94,14 +96,9 @@ This runs Bitween locally against PostgreSQL, RabbitMQ and local-disk storage. dotnet run --project SW.Bitween.Web ``` -4. Open https://localhost:5000 and sign in as the seeded administrator. +4. Open https://localhost:5000 and sign in as `admin@Bitween.systems` with the password you set in `Bitween__InitialAdminPassword`. - ``` - admin@Bitween.systems - Mtm@dmin!2 - ``` - -The database schema is created on first start. Change the administrator password straight away, and work through the [production checklist](docs/security.md#production-checklist) before exposing an instance. To use brokers or databases as data sources, see [Data sources](docs/data-sources.md#turning-them-on). +The database schema is created on first start. A new database refuses to start without `Bitween__InitialAdminPassword` (see [Configuration](docs/configuration.md#bitweeninitialadminpassword-on-a-new-installation)). Work through the [production checklist](docs/security.md#production-checklist) before exposing an instance. To use brokers or databases as data sources, see [Data sources](docs/data-sources.md#turning-them-on). ## Documentation @@ -111,17 +108,20 @@ The database schema is created on first start. Change the administrator password | [Architecture](docs/architecture.md) | Components, projects, adapter runtimes, data, messaging, storage and caching | | [Exchange pipeline](docs/exchange-pipeline.md) | What happens to a message from arrival to result | | [Entry points](docs/entry-points.md) | API gateways, bus gateways, scheduled jobs, aggregations and legacy entry points | -| [Adapters](docs/adapters.md) | The adapter contract, every built-in adapter and its properties, custom adapters | +| [Adapters](docs/adapters.md) | Every built-in adapter and its properties, custom adapters in .NET, Python and JavaScript, publishing, the adapter editor and the source viewer | +| [Adapter contract](docs/adapter-contract.md) | What Bitween calls on an adapter of each kind and what it passes, in every language | +| [The bitween CLI](docs/cli.md) | Signing in from a terminal, and writing, building, checking and publishing adapters | | [Mapping](docs/mapping.md) | The rules-based mapper and the legacy Scriban JSON mapper | | [Data sources](docs/data-sources.md) | Long-lived connections: enabling them, placement, health and resource limits | | [External brokers](docs/external-brokers.md) | Reading from and publishing to a customer's RabbitMQ or Amazon SQS | | [Databases](docs/databases.md) | PostgreSQL, MySQL, SQL Server and Oracle: statements, polling and writing | -| [Retries and alerts](docs/retries-and-alerts.md) | Manual and bulk retry, retry chains, retry policies, budgets, alerts and notifiers | +| [Retries and alerts](docs/retries-and-alerts.md) | Manual and bulk retry, retry chains, retry policies, budgets, alerts, notification channels and notifications | | [Scheduling](docs/scheduling.md) | Schedules, jobs, run history and schedule health | | [Security](docs/security.md) | Sign-in, tokens, roles and permissions, partner keys, hardening, audit | | [Configuration](docs/configuration.md) | Every configuration key, runtime setting and Helm value | | [Deployment](docs/deployment.md) | Docker image, Helm chart, databases, storage and CI/CD | | [Operations](docs/operations.md) | Monitoring, troubleshooting, logs and data retention | +| [Prod mirror](docs/prod-mirror.md) | Copying a prod's setup onto a staging, every value faked, to try an upgrade there first | | [Admin UI](docs/admin-ui.md) | A tour of the web interface | | [API reference](docs/api-reference.md) | REST conventions and every endpoint | | [Development](docs/development.md) | Local setup, tests, migrations and extending Bitween | @@ -143,13 +143,26 @@ Deeper reference written alongside the features: | `SW.Bitween.Adapters.Bus.RabbitMq`, `SW.Bitween.Adapters.Bus.Sqs` | Resident broker adapters | | `SW.Bitween.Adapters.Db.*` | Resident database adapters and their shared core | | `SW.Bitween.Sdk` | Shared models and the retry policy evaluator, published to NuGet as `SimplyWorks.Bitween.Sdk` | +| `SW.Bitween.Adapters` | The adapter contract and its .NET interfaces, published to NuGet as `SimplyWorks.Bitween.Adapters` | +| `SW.Bitween.Adapters.Tooling` | Bitween's adapters on SW-Serverless's tooling: the contract, templates and Python and Node packages the CLI and the adapter editor use | +| `SW.Bitween.Cli` | The `bitween` command | +| `sdk/python`, `sdk/node` | The adapter contract for Python (`simplyworks-bitween`) and for JavaScript and TypeScript (`@simplyworks/bitween`) | | `SW.Bitween.PgSql`, `SW.Bitween.MySql`, `SW.Bitween.MsSql` | Provider-specific database contexts and migrations | | `SW.Bitween.Sample*` | Sample custom adapters, including a resident handler | -| `SW.Bitween.UnitTests`, `SW.Bitween.IntegrationTests` | Test suites | -| `tools` | Development databases for trying the database adapters | +| `SW.Bitween.UnitTests`, `SW.Bitween.IntegrationTests`, `SW.Bitween.HttpTests`, `SW.Bitween.TransportTests` | Test suites | +| `tools` | Development databases for trying the database adapters, and the end-to-end test script | +| `scripts/install-cli.sh` | Installs the `bitween` CLI from a release | | `charts/default` | Helm chart | | `Dockerfile` | Container image build | ## License -MIT. See [LICENSE](LICENSE). +Bitween is licensed under the [GNU Affero General Public License v3.0](LICENSE), with the additional terms in [NOTICE](NOTICE): + +- You can use, run and modify Bitween for free, including in production inside your organization. +- If you distribute Bitween or offer it as a service to others, you must release your source code under the AGPL and keep the "Based on Bitween by Simplify9" attribution. +- Forks and modified versions must use a different name. "Bitween" is a Simplify9 trademark. +- `SW.Bitween.Sdk` (the `SimplyWorks.Bitween.Sdk` NuGet package) and the `SW.Bitween.Sample*` projects stay under the MIT License, so your custom adapters can use any license. +- Versions released before the license change remain available under the MIT License. + +A commercial license is available from Simplify9 ([info@simplify9.com](mailto:info@simplify9.com)) for embedding Bitween in closed-source products or hosting it for third parties without releasing source code. diff --git a/SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs b/SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs index dfcdfe80..c7cc6096 100644 --- a/SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs +++ b/SW.Bitween.Adapters.Bus.RabbitMq/RabbitBusHandler.cs @@ -15,7 +15,7 @@ namespace SW.Bitween.Adapters.Bus.RabbitMq; /// -/// Bitween's external RabbitMQ bus provider — a broker that is NOT the internal one, owned by +/// Bitween's external RabbitMQ bus provider - a broker that is NOT the internal one, owned by /// someone else, whose queues Bitween consumes and publishes to. /// /// The ack ordering is the contract: @@ -24,7 +24,7 @@ namespace SW.Bitween.Adapters.Bus.RabbitMq; /// -> ONLY THEN BasicAck /// /// A host rejection becomes BasicNack(requeue: true), so a Bitween outage does not lose the -/// customer's messages — it just stops draining their queue, which is the correct failure. +/// customer's messages - it just stops draining their queue, which is the correct failure. /// // Two roles, one package. "bus" is what makes it configurable as a broker connection; "handler" // is what puts it in the delivery picker, because the same resident instance both consumes a @@ -39,27 +39,37 @@ public class RabbitBusHandler(IOptions options, ILogger _endpoints = new(); private readonly Dictionary _consumerTags = new(); - private long _received, _acked, _nacked, _failed, _published; + private long _received, _acked, _nacked, _failed, _published, _poisoned; + + // Failed attempts per message, for brokers that don't count them (only quorum queues send + // x-delivery-count). Keyed by message id, or by a hash of the body when there is none. + private readonly System.Collections.Concurrent.ConcurrentDictionary _attempts = new(); + private DateTimeOffset? _lastMessageOn; private string _lastError; private volatile string _state = "Starting"; + // Consuming can stop with the connection still open: the consume channel closed by the broker + // (an ack it refused, say), or a consumer cancelled by it (its queue deleted). Reported as + // disconnected, so the supervisor restarts the adapter as it does for a dropped connection. + private volatile bool _consumeBroken; + // ---------------------------------------------------------------- lifecycle - public Task StartAsync(IAdapterContext context, CancellationToken cancellationToken) + public async Task StartAsync(IAdapterContext context, CancellationToken cancellationToken) { _context = context; _stopping = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); @@ -76,60 +86,83 @@ public Task StartAsync(IAdapterContext context, CancellationToken cancellationTo VirtualHost = _options.VirtualHost, RequestedConnectionTimeout = TimeSpan.FromSeconds(15), - // The supervisor owns restart policy — backoff, crash-loop quarantine, health + // The supervisor owns restart policy - backoff, crash-loop quarantine, health // write-back. A second, hidden recovery loop in here would fight it. - AutomaticRecoveryEnabled = false + AutomaticRecoveryEnabled = false, + + // Deliveries were handed to the thread pool one task each, so up to the prefetch ran at + // once. This keeps that, and keeps each delivery's body valid until its handler is done. + ConsumerDispatchConcurrency = Math.Max((ushort)1, _options.Prefetch) }; if (_options.UseSsl) factory.Ssl = new SslOption { Enabled = true, ServerName = _options.Host }; - _connection = factory.CreateConnection($"bitween-{context.InstanceKey}"); - _connection.ConnectionShutdown += (_, e) => + _connection = await factory.CreateConnectionAsync($"bitween-{context.InstanceKey}", cancellationToken); + _connection.ConnectionShutdownAsync += (_, e) => { _state = "Disconnected"; _lastError = $"{e.ReplyCode} {e.ReplyText}"; logger.LogWarning("Connection to {Host} closed: {Reason}", _options.Host, e.ReplyText); + return Task.CompletedTask; }; - _publishChannel = _connection.CreateModel(); - _consumeChannel = _connection.CreateModel(); - _consumeChannel.BasicQos(0, _options.Prefetch, global: false); + // With confirmations tracked, a publish completes only once the broker has confirmed it, + // and throws when the broker refuses it or - mandatory - routes it nowhere. + _publishChannel = await _connection.CreateChannelAsync( + new CreateChannelOptions( + publisherConfirmationsEnabled: _options.PublisherConfirms, + publisherConfirmationTrackingEnabled: _options.PublisherConfirms), + cancellationToken); + _consumeChannel = await _connection.CreateChannelAsync(cancellationToken: cancellationToken); + await _consumeChannel.BasicQosAsync(0, _options.Prefetch, global: false, cancellationToken); + _consumeChannel.ChannelShutdownAsync += (_, e) => + { + if (_stopping.IsCancellationRequested || e.Initiator == ShutdownInitiator.Application) return Task.CompletedTask; + _consumeBroken = true; + _lastError = $"consume channel closed: {e.ReplyCode} {e.ReplyText}"; + logger.LogWarning("Consume channel on {Host} closed: {Reason}", _options.Host, e.ReplyText); + return Task.CompletedTask; + }; - DeclareTopology(_consumeChannel); + await DeclareTopologyAsync(_consumeChannel, cancellationToken); // Manage-only: the connection is up and the topology is declared, but nothing is consumed. // TestConnection still checks every endpoint, because it reads _endpoints rather than the - // consumers — so a test proves the queues exist without draining them. + // consumers - so a test proves the queues exist without draining them. if (!_options.Consume) { _state = "Idle"; logger.LogInformation("Connected to {Host} without consuming (Consume=false).", _options.Host); - return Task.CompletedTask; + return; } foreach (var endpoint in _endpoints) { - var consumer = new EventingBasicConsumer(_consumeChannel); - consumer.Received += (_, delivery) => _ = Task.Run(() => HandleAsync(endpoint, delivery)); + var consumer = new BrokerCancelledConsumer(_consumeChannel, () => + { + _consumeBroken = true; + _lastError = $"the broker cancelled the consumer on {endpoint}"; + logger.LogWarning("The broker cancelled the consumer on {Endpoint}.", endpoint); + }); + consumer.ReceivedAsync += (_, delivery) => HandleAsync(endpoint, delivery); // autoAck: false is what makes persist-then-ack possible at all. - _consumerTags[endpoint] = _consumeChannel.BasicConsume(endpoint, autoAck: false, consumer); + _consumerTags[endpoint] = await _consumeChannel.BasicConsumeAsync(endpoint, autoAck: false, consumer, + cancellationToken); } _state = _endpoints.Count == 0 ? "Idle" : "Connected"; logger.LogInformation("Connected to {Host}:{Port}{VHost}, consuming {Count} endpoint(s) with prefetch {Prefetch}.", _options.Host, _options.Port, _options.VirtualHost, _endpoints.Count, _options.Prefetch); - - return Task.CompletedTask; } - private void DeclareTopology(IModel channel) + private async Task DeclareTopologyAsync(IChannel channel, CancellationToken cancellationToken) { var mode = (_options.DeclareMode ?? "assert").ToLowerInvariant(); if (mode == "none") return; if (mode == "create" && !string.IsNullOrWhiteSpace(_options.Exchange)) - channel.ExchangeDeclare(_options.Exchange, _options.ExchangeType ?? "topic", - durable: _options.Durable, autoDelete: false); + await channel.ExchangeDeclareAsync(_options.Exchange, _options.ExchangeType ?? "topic", + durable: _options.Durable, autoDelete: false, cancellationToken: cancellationToken); foreach (var endpoint in _endpoints) { @@ -137,49 +170,54 @@ private void DeclareTopology(IModel channel) { // Throws if it does not exist, which is what we want: better to fail at start than // to silently create a queue on someone else's broker. - channel.QueueDeclarePassive(endpoint); + await channel.QueueDeclarePassiveAsync(endpoint, cancellationToken); continue; } var arguments = new Dictionary(); if (!string.IsNullOrWhiteSpace(_options.QueueType)) arguments["x-queue-type"] = _options.QueueType; - channel.QueueDeclare(endpoint, durable: _options.Durable, exclusive: false, - autoDelete: false, arguments: arguments.Count == 0 ? null : arguments); + await channel.QueueDeclareAsync(endpoint, durable: _options.Durable, exclusive: false, + autoDelete: false, arguments: arguments.Count == 0 ? null : arguments, + cancellationToken: cancellationToken); if (!string.IsNullOrWhiteSpace(_options.Exchange)) - channel.QueueBind(endpoint, _options.Exchange, _options.RoutingKey ?? endpoint); + await channel.QueueBindAsync(endpoint, _options.Exchange, _options.RoutingKey ?? endpoint, + cancellationToken: cancellationToken); } } - public Task StopAsync(CancellationToken cancellationToken) + public async Task StopAsync(CancellationToken cancellationToken) { _state = "Draining"; _stopping?.Cancel(); - lock (_consumeGate) + await _consumeGate.WaitAsync(CancellationToken.None); + try { foreach (var tag in _consumerTags.Values) - try { _consumeChannel?.BasicCancel(tag); } catch { } + try { if (_consumeChannel != null) await _consumeChannel.BasicCancelAsync(tag); } catch { } - try { _consumeChannel?.Close(); } catch { } + try { if (_consumeChannel != null) await _consumeChannel.CloseAsync(); } catch { } } + finally { _consumeGate.Release(); } + + await _publishGate.WaitAsync(CancellationToken.None); + try { if (_publishChannel != null) await _publishChannel.CloseAsync(); } catch { } + finally { _publishGate.Release(); } - lock (_publishGate) - try { _publishChannel?.Close(); } catch { } - try { _connection?.Close(TimeSpan.FromSeconds(3)); } catch { } + try { if (_connection != null) await _connection.CloseAsync(TimeSpan.FromSeconds(3)); } catch { } _connection?.Dispose(); _state = "Stopped"; - return Task.CompletedTask; } - public Task GetStatusAsync() + public async Task GetStatusAsync() { var status = new AdapterStatus { - Connected = _connection?.IsOpen == true, - State = _connection?.IsOpen != true ? "Disconnected" + Connected = _connection?.IsOpen == true && !_consumeBroken, + State = _connection?.IsOpen != true || _consumeBroken ? "Disconnected" : _received == 0 ? "Idle" : _state, LastMessageOn = _lastMessageOn, LastError = _lastError, @@ -195,18 +233,20 @@ public Task GetStatusAsync() status.Details["failed"] = _failed.ToString(); status.Details["published"] = _published.ToString(); + status.Details["poisoned"] = _poisoned.ToString(); + foreach (var endpoint in _endpoints) - status.Details[$"depth:{endpoint}"] = Depth(endpoint)?.ToString() ?? "?"; + status.Details[$"depth:{endpoint}"] = (await DepthAsync(endpoint))?.ToString() ?? "?"; - return Task.FromResult(status); + return status; } - private uint? Depth(string queue) + private async Task DepthAsync(string queue) { try { - using var probe = _connection.CreateModel(); - return probe.QueueDeclarePassive(queue).MessageCount; + await using var probe = await _connection.CreateChannelAsync(); + return (await probe.QueueDeclarePassiveAsync(queue)).MessageCount; } catch { return null; } } @@ -230,8 +270,8 @@ private async Task HandleAsync(string endpoint, BasicDeliverEventArgs delivery) var result = await _context.PublishAsync( delivery.Body, - // The broker's own message id, and nothing else. NOT the delivery tag — tags are - // per channel and restart at 1 on every reconnect — and NOT a content hash: two + // The broker's own message id, and nothing else. NOT the delivery tag - tags are + // per channel and restart at 1 on every reconnect - and NOT a content hash: two // messages that legitimately carry the same body are two messages, and hashing // them would silently drop the second for the whole deduplication window. An // unkeyed delivery is handled once per delivery, which is the honest answer when @@ -246,30 +286,138 @@ private async Task HandleAsync(string endpoint, BasicDeliverEventArgs delivery) if (result.Accepted) { - lock (_consumeGate) _consumeChannel.BasicAck(delivery.DeliveryTag, multiple: false); + await OnConsumeChannelAsync(c => c.BasicAckAsync(delivery.DeliveryTag, multiple: false)); + _attempts.TryRemove(AttemptKey(endpoint, delivery), out _); Interlocked.Increment(ref _acked); _lastMessageOn = DateTimeOffset.UtcNow; _context.Metric("bitween.bus.rabbitmq.acked", 1); } else { - lock (_consumeGate) _consumeChannel.BasicNack(delivery.DeliveryTag, multiple: false, requeue: true); Interlocked.Increment(ref _nacked); _lastError = result.Error; - logger.LogWarning("Bitween rejected a message from {Endpoint}: {Error}. Requeued.", - endpoint, result.Error); + await RejectAsync(endpoint, delivery, result.Error); } } catch (OperationCanceledException) { - try { lock (_consumeGate) _consumeChannel.BasicNack(delivery.DeliveryTag, false, requeue: true); } catch { } + try { await OnConsumeChannelAsync(c => c.BasicNackAsync(delivery.DeliveryTag, false, requeue: true)); } catch { } } catch (Exception ex) { Interlocked.Increment(ref _failed); _lastError = ex.Message; logger.LogError(ex, "Failed to hand a delivery from {Endpoint} to Bitween.", endpoint); - try { lock (_consumeGate) _consumeChannel.BasicNack(delivery.DeliveryTag, false, requeue: true); } catch { } + try { await RejectAsync(endpoint, delivery, ex.Message); } catch { } + } + } + + /// + /// A message Bitween did not take. It used to go straight back on the queue, so one that always + /// fails came straight back too - round and round, as fast as the broker could deliver it, + /// holding a prefetch slot and filling the log. Now each requeue waits a little longer, and + /// after the poison action takes over. + /// + private async Task RejectAsync(string endpoint, BasicDeliverEventArgs delivery, string error) + { + var attempt = AttemptOf(endpoint, delivery); + var action = (_options.PoisonAction ?? "requeue").ToLowerInvariant(); + + if (_options.MaxDeliveryAttempts > 0 && attempt >= _options.MaxDeliveryAttempts && action != "requeue") + { + if (action == "park" && !string.IsNullOrWhiteSpace(_options.ParkingQueue)) + { + await ParkAsync(endpoint, delivery, error, attempt); + await OnConsumeChannelAsync(c => c.BasicAckAsync(delivery.DeliveryTag, multiple: false)); + } + else + { + await OnConsumeChannelAsync(c => c.BasicNackAsync(delivery.DeliveryTag, multiple: false, requeue: false)); + } + + _attempts.TryRemove(AttemptKey(endpoint, delivery), out _); + Interlocked.Increment(ref _poisoned); + logger.LogError("A message from {Endpoint} failed {Attempts} times and was {Action}: {Error}", + endpoint, attempt, action == "park" ? $"parked on {_options.ParkingQueue}" : "rejected", error); + return; + } + + try + { + await Task.Delay(TimeSpan.FromSeconds(Math.Min(attempt, 10)), _stopping.Token); + } + catch (OperationCanceledException) { /* stopping: requeue straight away */ } + + await OnConsumeChannelAsync(c => c.BasicNackAsync(delivery.DeliveryTag, multiple: false, requeue: true)); + logger.LogWarning("Bitween rejected a message from {Endpoint} (attempt {Attempt}): {Error}. Requeued.", + endpoint, attempt, error); + } + + private int AttemptOf(string endpoint, BasicDeliverEventArgs delivery) + { + // Quorum queues count deliveries themselves, across restarts and nodes. + if (delivery.BasicProperties?.Headers?.TryGetValue("x-delivery-count", out var counted) == true && + counted is not null && long.TryParse(counted.ToString(), out var deliveries)) + return (int)deliveries + 1; + + if (_attempts.Count > 100_000) _attempts.Clear(); + return _attempts.AddOrUpdate(AttemptKey(endpoint, delivery), 1, (_, n) => n + 1); + } + + private static string AttemptKey(string endpoint, BasicDeliverEventArgs delivery) => + endpoint + ":" + (delivery.BasicProperties?.MessageId is { Length: > 0 } id + ? id + : Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(delivery.Body.Span))); + + private async Task OnConsumeChannelAsync(Func action) + { + await _consumeGate.WaitAsync(); + try { await action(_consumeChannel); } + finally { _consumeGate.Release(); } + } + + private async Task ParkAsync(string endpoint, BasicDeliverEventArgs delivery, string error, int attempts) + { + var headers = new Dictionary(); + if (delivery.BasicProperties?.Headers is { } original) + foreach (var (key, value) in original) headers[key] = value; + headers["x-bitween-error"] = error ?? ""; + headers["x-bitween-endpoint"] = endpoint; + headers["x-bitween-attempts"] = attempts; + + var properties = new BasicProperties + { + ContentType = delivery.BasicProperties?.ContentType, + MessageId = delivery.BasicProperties?.MessageId, + DeliveryMode = DeliveryModes.Persistent, + Headers = headers + }; + + await PublishConfirmedAsync("", _options.ParkingQueue, mandatory: false, properties, delivery.Body); + } + + /// + /// Publishes and, with confirmations on, returns only once the broker has confirmed it. Without + /// that, a message the broker dropped - a full queue, a failover - was still recorded as + /// delivered. A refusal, or a mandatory message routed nowhere, throws. + /// + private async Task PublishConfirmedAsync(string exchange, string routingKey, bool mandatory, + BasicProperties properties, ReadOnlyMemory body) + { + await _publishGate.WaitAsync(); + try + { + await _publishChannel.BasicPublishAsync(exchange, routingKey, mandatory, properties, body); + } + catch (RabbitMQ.Client.Exceptions.PublishException ex) + { + throw new InvalidOperationException(ex.IsReturn + ? "No queue received the message; the broker returned it as unroutable." + : "The broker refused the message.", ex); + } + finally + { + _publishGate.Release(); } } @@ -291,7 +439,7 @@ private string WarnUnkeyed(string endpoint) /// Egress. Bitween does not have this on the internal gateway yet; an external provider gets /// it for free because the adapter owns the connection either way. /// - public Task Publish(PublishRequest request) + public async Task Publish(PublishRequest request) { if (string.IsNullOrWhiteSpace(request?.Endpoint) && string.IsNullOrWhiteSpace(request?.Exchange)) throw new ArgumentException("Either Endpoint or Exchange is required."); @@ -299,39 +447,36 @@ public Task Publish(PublishRequest request) var body = System.Text.Encoding.UTF8.GetBytes(request.Body ?? ""); var messageId = request.MessageId ?? Guid.NewGuid().ToString("N"); - // CreateBasicProperties is itself a channel operation, so it belongs inside the gate with - // the publish rather than beside it. - lock (_publishGate) + var properties = new BasicProperties { - var properties = _publishChannel.CreateBasicProperties(); - properties.ContentType = request.ContentType ?? "application/json"; - properties.MessageId = messageId; - properties.DeliveryMode = (byte)(_options.Durable ? 2 : 1); - - _publishChannel.BasicPublish( - exchange: request.Exchange ?? "", - routingKey: request.Exchange == null ? request.Endpoint : request.RoutingKey ?? "", - mandatory: false, - basicProperties: properties, - body: body); - } + ContentType = request.ContentType ?? "application/json", + MessageId = messageId, + DeliveryMode = _options.Durable ? DeliveryModes.Persistent : DeliveryModes.Transient + }; + + await PublishConfirmedAsync( + exchange: request.Exchange ?? "", + routingKey: request.Exchange == null ? request.Endpoint : request.RoutingKey ?? "", + mandatory: _options.Mandatory, + properties, + body); Interlocked.Increment(ref _published); - return Task.FromResult(new { messageId, bytes = body.Length }); + return new { messageId, bytes = body.Length }; } /// /// Egress through the PIPELINE: a subscription's delivery stage, publishing the message it was /// given to a queue on this broker. /// - /// has existed since this adapter did, and nothing could reach it — + /// has existed since this adapter did, and nothing could reach it - /// Bitween's pipeline calls Handle on a handler, and this class had none, so an operator /// could consume from a customer's broker and had no way to answer on it. This is the two of /// them joined up; the publishing itself is unchanged. /// /// Where to send is the SUBSCRIPTION's business, not the connection's: one instance serves /// every gateway on this broker, so the endpoint travels with the call rather than with the - /// process. Deliberately NOT restricted to the endpoints the data source consumes — the common + /// process. Deliberately NOT restricted to the endpoints the data source consumes - the common /// case for egress is a queue Bitween does not drain. /// public Task Handle(XchangeFile xchangeFile) @@ -372,7 +517,7 @@ async Task PublishAsFile(XchangeFile xchangeFile, string endpoint, } /// The control the UI needs before a data source is saved. Staged, so a failure names the step. - public Task TestConnection() + public async Task TestConnection() { var steps = new List(); IConnection probe = null; @@ -387,52 +532,60 @@ public Task TestConnection() }; if (_options.UseSsl) factory.Ssl = new SslOption { Enabled = true, ServerName = _options.Host }; - probe = factory.CreateConnection("bitween-probe"); + probe = await factory.CreateConnectionAsync("bitween-probe"); steps.Add(new { step = "connect", ok = true, detail = probe.Endpoint.ToString() }); - using var channel = probe.CreateModel(); + await using var channel = await probe.CreateChannelAsync(); steps.Add(new { step = "authenticate", ok = true, detail = _options.VirtualHost }); foreach (var endpoint in _endpoints) { try { - var declared = channel.QueueDeclarePassive(endpoint); + var declared = await channel.QueueDeclarePassiveAsync(endpoint); steps.Add(new { step = $"queue:{endpoint}", ok = true, detail = $"{declared.MessageCount} message(s)" }); } catch (Exception ex) { steps.Add(new { step = $"queue:{endpoint}", ok = false, detail = ex.Message }); - return Task.FromResult(new { ok = false, steps }); + return new { ok = false, steps }; } } - return Task.FromResult(new { ok = true, steps }); + return new { ok = true, steps }; } catch (Exception ex) { steps.Add(new { step = "failed", ok = false, detail = ex.Message }); - return Task.FromResult(new { ok = false, steps }); + return new { ok = false, steps }; } finally { - try { probe?.Close(); probe?.Dispose(); } catch { } + try { if (probe != null) await probe.CloseAsync(); probe?.Dispose(); } catch { } } } /// What is actually on the broker, for the "pick a queue" step in the UI. - public Task Discover() => Task.FromResult(new + public async Task Discover() { - host = $"{_options.Host}:{_options.Port}", - virtualHost = _options.VirtualHost, - endpoints = _endpoints.Select(e => new { name = e, messages = Depth(e), consuming = _consumerTags.ContainsKey(e) }), - note = "AMQP alone can only report on queues we were told about. " + - "Listing everything on the broker needs the management plugin." - }); + var endpoints = new List(); + foreach (var e in _endpoints) + endpoints.Add(new { name = e, messages = await DepthAsync(e), consuming = _consumerTags.ContainsKey(e) }); + + return new + { + host = $"{_options.Host}:{_options.Port}", + virtualHost = _options.VirtualHost, + endpoints, + note = "AMQP alone can only report on queues we were told about. " + + "Listing everything on the broker needs the management plugin." + }; + } public Task GetStats() => Task.FromResult(new { received = _received, acked = _acked, nacked = _nacked, failed = _failed, published = _published, + poisoned = _poisoned, endpoints = _endpoints, prefetch = _options.Prefetch }); @@ -445,4 +598,17 @@ public class PublishRequest public string ContentType { get; set; } public string Body { get; set; } } + + /// + /// Says when the broker cancels the consumer (its queue deleted, say). The client raises no + /// event for that: UnregisteredAsync is only for a cancel this side asked for. + /// + private sealed class BrokerCancelledConsumer(IChannel channel, Action cancelled) : AsyncEventingBasicConsumer(channel) + { + public override async Task HandleBasicCancelAsync(string consumerTag, CancellationToken cancellationToken = default) + { + cancelled(); + await base.HandleBasicCancelAsync(consumerTag, cancellationToken); + } + } } diff --git a/SW.Bitween.Adapters.Bus.RabbitMq/RabbitOptions.cs b/SW.Bitween.Adapters.Bus.RabbitMq/RabbitOptions.cs index d9cd3d85..7a08a904 100644 --- a/SW.Bitween.Adapters.Bus.RabbitMq/RabbitOptions.cs +++ b/SW.Bitween.Adapters.Bus.RabbitMq/RabbitOptions.cs @@ -3,7 +3,7 @@ namespace SW.Bitween.Adapters.Bus.RabbitMq; /// /// Every one of these arrives as a DataSource property, bound by name. Nothing here is opinionated /// about how the broker should be laid out: the queue may already exist and be owned by someone -/// else, or Bitween may declare it — decides which. +/// else, or Bitween may declare it - decides which. /// /// The attributes are not documentation: Bitween reads them out of this assembly to build the form /// an operator fills in, so a field added here appears there without anyone editing the UI. @@ -15,7 +15,7 @@ namespace SW.Bitween.Adapters.Bus.RabbitMq; public class RabbitOptions { [AdapterSetting(Required = true, - Hint = "Host name only — no amqp:// or amqps:// prefix, and no credentials.")] + Hint = "Host name only - no amqp:// or amqps:// prefix, and no credentials.")] public string Host { get; set; } = "localhost"; [AdapterSetting(Default = "5672", @@ -33,7 +33,7 @@ public class RabbitOptions public string VirtualHost { get; set; } = "/"; [AdapterSetting(Default = "false", AllowedValues = new[] {"true", "false"}, - Hint = "Set for anything that is not localhost — AMQP authenticates in the clear otherwise. " + Hint = "Set for anything that is not localhost - AMQP authenticates in the clear otherwise. " + "Pair it with port 5671.")] public bool UseSsl { get; set; } @@ -42,13 +42,13 @@ public class RabbitOptions public string Endpoints { get; set; } /// - /// none — assume everything exists; never touch the topology. - /// assert — verify it exists and fail loudly if not (passive declare). - /// create — declare queues, and an exchange and binding when Exchange is set. + /// none - assume everything exists; never touch the topology. + /// assert - verify it exists and fail loudly if not (passive declare). + /// create - declare queues, and an exchange and binding when Exchange is set. /// [AdapterSetting(Default = "assert", AllowedValues = new[] {"none", "assert", "create"}, - Hint = "none — assume everything exists. assert — check and fail loudly if not. " - + "create — declare the queues.")] + Hint = "none - assume everything exists. assert - check and fail loudly if not. " + + "create - declare the queues.")] public string DeclareMode { get; set; } = "assert"; /// Optional. When set, each endpoint queue is bound to it. @@ -83,4 +83,45 @@ public class RabbitOptions Hint = "Passed through as x-queue-type when Bitween declares the queue. Leave empty to let " + "the broker choose. It cannot be changed on a queue that already exists.")] public string QueueType { get; set; } + + /// + /// How many times one message may fail to reach Bitween before + /// applies. Zero keeps requeueing it for good, which is what this adapter always did. + /// + [AdapterSetting(Default = "0", + Hint = "How many failed attempts before a message is treated as poison. 0 retries it for good.")] + public int MaxDeliveryAttempts { get; set; } + + /// + /// requeue - put it back and keep trying, however often it fails. + /// reject - reject it without requeue, so the broker dead-letters it if the queue has a + /// dead-letter exchange, and drops it if it does not. + /// park - copy it to with the error in its headers, then ack it. + /// + [AdapterSetting(Default = "requeue", AllowedValues = new[] {"requeue", "reject", "park"}, + Hint = "What to do with a message after MaxDeliveryAttempts failures. reject relies on the " + + "queue's dead-letter exchange; park copies it to ParkingQueue.")] + public string PoisonAction { get; set; } = "requeue"; + + [AdapterSetting(Hint = "Queue that poison messages are copied to when PoisonAction is park. It must exist.")] + public string ParkingQueue { get; set; } + + /// + /// Wait for the broker to confirm each message Bitween publishes, so a delivery that the broker + /// did not take is a failed delivery - retried by the subscription's retry policy - rather + /// than one reported as sent. + /// + [AdapterSetting(Default = "true", AllowedValues = new[] {"true", "false"}, + Hint = "Report a publish as delivered only once the broker confirms it.")] + public bool PublisherConfirms { get; set; } = true; + + /// + /// Fail a publish that no queue receives. Off by default: until now such a message was + /// reported as delivered, and a deployment may be publishing to an exchange with no binding on + /// purpose. + /// + [AdapterSetting(Default = "false", AllowedValues = new[] {"true", "false"}, + Hint = "Fail a publish that is routed to no queue, instead of reporting it as delivered. " + + "Needs PublisherConfirms, which is how the broker's return is waited for.")] + public bool Mandatory { get; set; } } diff --git a/SW.Bitween.Adapters.Bus.RabbitMq/SW.Bitween.Adapters.Bus.RabbitMq.csproj b/SW.Bitween.Adapters.Bus.RabbitMq/SW.Bitween.Adapters.Bus.RabbitMq.csproj index d180c52a..baafa8a2 100644 --- a/SW.Bitween.Adapters.Bus.RabbitMq/SW.Bitween.Adapters.Bus.RabbitMq.csproj +++ b/SW.Bitween.Adapters.Bus.RabbitMq/SW.Bitween.Adapters.Bus.RabbitMq.csproj @@ -1,20 +1,20 @@ Exe - - net8.0 + + net10.0 disable SW.Bitween.Adapters.Bus.RabbitMq - - + + - + diff --git a/SW.Bitween.Adapters.Bus.RabbitMq/adapter.json b/SW.Bitween.Adapters.Bus.RabbitMq/adapter.json new file mode 100644 index 00000000..fe6faffe --- /dev/null +++ b/SW.Bitween.Adapters.Bus.RabbitMq/adapter.json @@ -0,0 +1,20 @@ +{ + "id": "bitween.bus.rabbitmq", + "version": "1.0.0", + "displayName": "RabbitMQ", + "summary": "A customer's RabbitMQ broker as a data source: bus gateways read from its queues, subscriptions publish to it.", + "lifecycle": "resident", + "publisher": { + "name": "Simplify9", + "url": "https://simplify9.com" + }, + "license": "AGPL-3.0-only", + "categories": [ + "Data sources" + ], + "compatibility": { + "applications": { + "bitween": "10.0.0" + } + } +} diff --git a/SW.Bitween.Adapters.Bus.Sqs/SW.Bitween.Adapters.Bus.Sqs.csproj b/SW.Bitween.Adapters.Bus.Sqs/SW.Bitween.Adapters.Bus.Sqs.csproj index 4d35c92c..b6973fa3 100644 --- a/SW.Bitween.Adapters.Bus.Sqs/SW.Bitween.Adapters.Bus.Sqs.csproj +++ b/SW.Bitween.Adapters.Bus.Sqs/SW.Bitween.Adapters.Bus.Sqs.csproj @@ -1,17 +1,17 @@ Exe - net8.0 + net10.0 disable SW.Bitween.Adapters.Bus.Sqs - + - + diff --git a/SW.Bitween.Adapters.Bus.Sqs/SqsBusHandler.cs b/SW.Bitween.Adapters.Bus.Sqs/SqsBusHandler.cs index a16a6e0f..1e2c117c 100644 --- a/SW.Bitween.Adapters.Bus.Sqs/SqsBusHandler.cs +++ b/SW.Bitween.Adapters.Bus.Sqs/SqsBusHandler.cs @@ -22,7 +22,7 @@ namespace SW.Bitween.Adapters.Bus.Sqs; /// /// SQS is a poll-based queue, not a push broker, so the shape differs from RabbitMQ in one way /// that matters: there is no ack, only DELETE. A message stays invisible for the visibility -/// timeout and reappears if it is not deleted — which is the same at-least-once contract by a +/// timeout and reappears if it is not deleted - which is the same at-least-once contract by a /// different mechanism, and maps cleanly onto persist-then-acknowledge: /// /// receive -> PublishAsync to the host -> host persists -> ONLY THEN DeleteMessage @@ -34,8 +34,8 @@ namespace SW.Bitween.Adapters.Bus.Sqs; /// WHY THIS ONE MATTERS BEYOND SQS ITSELF: the Amazon Selling Partner API delivers notifications /// by publishing to an SQS queue that you own. You create the queue, grant SP-API permission to /// send to it, then subscribe notification types to that destination. So this adapter is the -/// transport for SP-API notifications — ORDER_CHANGE, LISTINGS_ITEM_STATUS_CHANGE, REPORT_PROCESSING_FINISHED -/// and the rest — and handles the +/// transport for SP-API notifications - ORDER_CHANGE, LISTINGS_ITEM_STATUS_CHANGE, REPORT_PROCESSING_FINISHED +/// and the rest - and handles the /// envelope they arrive in. The SP-API request/response calls themselves are ordinary HTTPS and /// belong in a mapper or handler, not here. /// @@ -98,7 +98,7 @@ private static IAmazonSQS CreateClient(SqsOptions options) config.AuthenticationRegion = options.Region; } - // No keys means the ambient chain — instance profile, IRSA, environment. That is the + // No keys means the ambient chain - instance profile, IRSA, environment. That is the // correct production answer on AWS, and explicit keys are the exception. return string.IsNullOrWhiteSpace(options.AccessKeyId) ? new AmazonSQSClient(config) @@ -248,7 +248,7 @@ private async Task HandleAsync(string queueUrl, Message message, CancellationTok } else { - // Make it visible again immediately instead of waiting out the timeout, so a + // Back on the queue after a short wait rather than the full visibility timeout, so a // transient Bitween failure retries in seconds rather than minutes. await ReturnToQueueAsync(queueUrl, message, ct); Interlocked.Increment(ref _returned); @@ -267,8 +267,21 @@ private async Task HandleAsync(string queueUrl, Message message, CancellationTok } } - private Task ReturnToQueueAsync(string queueUrl, Message message, CancellationToken ct) => - _sqs.ChangeMessageVisibilityAsync(queueUrl, message.ReceiptHandle, 0, ct); + /// + /// Puts the message back after a wait that grows with each receive: five seconds a time, up to + /// five minutes. It used to be made visible again at once, so a message that always fails was + /// received again straight away, over and over, until the queue's redrive policy moved it - or + /// for good, on a queue without one. + /// + private Task ReturnToQueueAsync(string queueUrl, Message message, CancellationToken ct) + { + var receives = message.Attributes != null && + message.Attributes.TryGetValue("ApproximateReceiveCount", out var count) && + int.TryParse(count, out var parsed) + ? parsed + : 1; + return _sqs.ChangeMessageVisibilityAsync(queueUrl, message.ReceiptHandle, Math.Min(receives * 5, 300), ct); + } /// /// SP-API notifications arrive wrapped: notificationType, notificationVersion, payloadVersion, @@ -332,13 +345,13 @@ public async Task Publish(PublishRequest request) /// Egress through the PIPELINE: a subscription's delivery stage, sending the message it was /// given to a queue these credentials can reach. /// - /// has existed since this adapter did, and nothing could reach it — + /// has existed since this adapter did, and nothing could reach it - /// Bitween's pipeline calls Handle on a handler, and this class had none. This is the /// two joined up; the sending itself is unchanged. /// /// The queue URL is the SUBSCRIPTION's, not the connection's: one instance serves every /// gateway on these credentials, so it travels with the call. Deliberately not restricted to - /// the queues this data source consumes — the common case for egress is one it does not. + /// the queues this data source consumes - the common case for egress is one it does not. /// public async Task Handle(XchangeFile xchangeFile) { @@ -347,7 +360,7 @@ public async Task Handle(XchangeFile xchangeFile) if (string.IsNullOrWhiteSpace(endpoint)) throw new InvalidOperationException( "This delivery has no Endpoint, so there is nowhere to send. Set it to the queue " - + "URL — the full https://sqs..amazonaws.com//, which is " + + "URL - the full https://sqs..amazonaws.com//, which is " + "what the SDK addresses a queue by."); var receipt = await Publish(new PublishRequest diff --git a/SW.Bitween.Adapters.Bus.Sqs/SqsOptions.cs b/SW.Bitween.Adapters.Bus.Sqs/SqsOptions.cs index ccfa419f..abb9f940 100644 --- a/SW.Bitween.Adapters.Bus.Sqs/SqsOptions.cs +++ b/SW.Bitween.Adapters.Bus.Sqs/SqsOptions.cs @@ -2,7 +2,7 @@ namespace SW.Bitween.Adapters.Bus.Sqs; /// /// Bound by name from the DataSource's properties. The attributes are what Bitween's data source -/// form is built from — see AdapterSettingAttribute for why the UI no longer keeps its own copy of +/// form is built from - see AdapterSettingAttribute for why the UI no longer keeps its own copy of /// this list. /// [AdapterSettings( @@ -24,7 +24,7 @@ public class SqsOptions public string Region { get; set; } = "eu-west-1"; /// - /// Leave both blank to use the ambient credential chain — instance profile, IRSA, or the + /// Leave both blank to use the ambient credential chain - instance profile, IRSA, or the /// environment. That is the right answer on AWS; explicit keys are for everything else. /// [AdapterSetting(Secret = true, @@ -42,9 +42,9 @@ public class SqsOptions [AdapterSetting(Hidden = true)] public string Endpoints { get; set; } - /// Long polling. 20 is the maximum and the only sensible value — 0 burns money and API calls. + /// Long polling. 20 is the maximum and the only sensible value - 0 burns money and API calls. [AdapterSetting(Default = "20", - Hint = "Long polling, in seconds. 20 is the maximum and the only sensible value — 0 burns " + Hint = "Long polling, in seconds. 20 is the maximum and the only sensible value - 0 burns " + "money and API calls.")] public int WaitTimeSeconds { get; set; } = 20; diff --git a/SW.Bitween.Adapters.Bus.Sqs/adapter.json b/SW.Bitween.Adapters.Bus.Sqs/adapter.json new file mode 100644 index 00000000..cba280b6 --- /dev/null +++ b/SW.Bitween.Adapters.Bus.Sqs/adapter.json @@ -0,0 +1,20 @@ +{ + "id": "bitween.bus.sqs", + "version": "1.0.0", + "displayName": "Amazon SQS", + "summary": "An Amazon SQS queue, or one compatible with it, as a data source: bus gateways read from it, subscriptions send to it.", + "lifecycle": "resident", + "publisher": { + "name": "Simplify9", + "url": "https://simplify9.com" + }, + "license": "AGPL-3.0-only", + "categories": [ + "Data sources" + ], + "compatibility": { + "applications": { + "bitween": "10.0.0" + } + } +} diff --git a/SW.Bitween.Adapters.Db.Core/DbContracts.cs b/SW.Bitween.Adapters.Db.Core/DbContracts.cs index ff8f02ed..d1dc1203 100644 --- a/SW.Bitween.Adapters.Db.Core/DbContracts.cs +++ b/SW.Bitween.Adapters.Db.Core/DbContracts.cs @@ -8,7 +8,7 @@ namespace SW.Bitween.Adapters.Db; // Every contract below is serialised camelCase, and it is declared per type rather than left to the // host's serializer settings: these objects cross a process boundary as JSON, and the reader on the // far side is Bitween's UI, which reads what the bus adapters already emit. Those are anonymous -// objects, so they are camelCase by construction — a typed contract defaulting to PascalCase would +// objects, so they are camelCase by construction - a typed contract defaulting to PascalCase would // have the data source screen reading `Ok` from one provider and `ok` from the next. // ---------------------------------------------------------------------------- capabilities @@ -16,8 +16,8 @@ namespace SW.Bitween.Adapters.Db; /// /// What this engine, through this login, can actually be asked to do. /// -/// Answered offline-ish — most of it is a constant per adapter, the rest is probed once at connect -/// — and cached by the UI, so a screen can grey out what will not work instead of offering it and +/// Answered offline-ish - most of it is a constant per adapter, the rest is probed once at connect +/// - and cached by the UI, so a screen can grey out what will not work instead of offering it and /// failing at run time. /// /// The privilege list matters more than the feature list. "Oracle supports change notification" is @@ -47,7 +47,7 @@ public class DbCapabilities /// COPY / SqlBulkCopy / LOAD DATA / array binding. public bool BulkCopy { get; set; } - /// MERGE, ON CONFLICT, ON DUPLICATE KEY — whatever the engine calls an upsert. + /// MERGE, ON CONFLICT, ON DUPLICATE KEY - whatever the engine calls an upsert. public bool Merge { get; set; } /// RETURNING / OUTPUT: can a write hand back the rows it wrote. @@ -71,7 +71,7 @@ public class DbCapabilities public string[] ReceiveModes { get; set; } = Array.Empty(); /// - /// How this engine writes a bind placeholder — : or @ — filled in by the base + /// How this engine writes a bind placeholder - : or @ - filled in by the base /// from the adapter's own setting rather than declared per engine, so the two cannot disagree. /// /// Here because a caller that WRITES SQL needs it: the schema browser drafts a statement from @@ -90,7 +90,7 @@ public class DbCapabilities /// public string LimitStyle { get; set; } = "limit"; - /// Probed with the real credentials — what the engine allows AND this login has. + /// Probed with the real credentials - what the engine allows AND this login has. public List Privileges { get; set; } = new(); /// Anything else worth showing: NLS settings, edition, connection pool ceiling. @@ -105,7 +105,7 @@ public class DiscoverRequest /// table | view | procedure | function | sequence | package. Null means tables. public string ObjectType { get; set; } - /// Null means every schema this login can see — which on Oracle is a great many. + /// Null means every schema this login can see - which on Oracle is a great many. public string Schema { get; set; } /// Case-insensitive contains. Not a LIKE pattern: no wildcards to get wrong. @@ -145,7 +145,7 @@ public class DbObject /// Estimated, and null unless asked for. public long? RowCount { get; set; } - /// Empty unless IncludeColumns was set. Empty for a routine — see Parameters. + /// Empty unless IncludeColumns was set. Empty for a routine - see Parameters. public List Columns { get; set; } = new(); /// Routines only. @@ -171,7 +171,7 @@ public class DbColumn public int? Scale { get; set; } public bool PrimaryKey { get; set; } - /// Identity, sequence-defaulted, GENERATED ALWAYS — anything the database fills in. + /// Identity, sequence-defaulted, GENERATED ALWAYS - anything the database fills in. public bool Generated { get; set; } public int Ordinal { get; set; } @@ -205,7 +205,7 @@ public class StatementValidationResult public string Error { get; set; } /// - /// Set when it passed but not everything could be checked — a bare procedure name is accepted + /// Set when it passed but not everything could be checked - a bare procedure name is accepted /// without confirming the routine exists, because it is resolved when called. /// public string Note { get; set; } @@ -218,7 +218,7 @@ public class StatementValidationResult /// /// is the normal path: statements are configured on the data source and the /// message supplies parameter values only. is refused unless the data source -/// explicitly allows ad-hoc SQL, because a mapper is a template evaluated over message content — +/// explicitly allows ad-hoc SQL, because a mapper is a template evaluated over message content - /// if that can emit SQL text, every Xchange is an injection vector into the customer's database. /// [JsonObject(NamingStrategyType = typeof(CamelCaseNamingStrategy))] @@ -300,7 +300,7 @@ public class FetchRequest /// /// Staged on purpose. "It did not work" is not something an operator can act on; the failing stage -/// names what to go and fix — a host name, a password, a grant, a missing table. +/// names what to go and fix - a host name, a password, a grant, a missing table. /// [JsonObject(NamingStrategyType = typeof(CamelCaseNamingStrategy))] public class DbTestResult diff --git a/SW.Bitween.Adapters.Db.Core/DbOptionsBase.cs b/SW.Bitween.Adapters.Db.Core/DbOptionsBase.cs index f2aa810d..4c91cad7 100644 --- a/SW.Bitween.Adapters.Db.Core/DbOptionsBase.cs +++ b/SW.Bitween.Adapters.Db.Core/DbOptionsBase.cs @@ -14,7 +14,7 @@ public abstract class DbOptionsBase /// /// The reason this adapter is resident at all. An ephemeral adapter gets a cold pool per /// process, so every Xchange pays a TCP connect, a TLS handshake and an authentication round - /// trip — 20–150 ms to a remote database, and considerably more to Oracle. A process that + /// trip - 20-150 ms to a remote database, and considerably more to Oracle. A process that /// stays up pays it once. /// [AdapterSetting(Default = "2", Hint = @@ -53,13 +53,13 @@ public abstract class DbOptionsBase /// Named statements as JSON: {"getOrder": "select * from orders where id = :id"}. /// The message supplies parameter values; it never supplies SQL. /// - /// Composed by Bitween from the data source's statement rows, not typed by an operator — which + /// Composed by Bitween from the data source's statement rows, not typed by an operator - which /// is why it is hidden. Statements are their own entity so that writing a query and changing a /// database password are different permissions; the adapter still just receives name-to-SQL and /// knows nothing about where they were kept. /// [AdapterSetting(Hidden = true, Hint = - "Named statements as JSON — {\"name\":\"select …\"}. Composed by Bitween from the data " + "Named statements as JSON - {\"name\":\"select …\"}. Composed by Bitween from the data " + "source's statements; not edited here.")] public string Statements { get; set; } @@ -79,7 +79,7 @@ public abstract class DbOptionsBase // Where these live is the point, and it is not all one place. // // A connection is shared by every subscription pointed at it, so anything that varies per - // reader cannot sit here — one data source could otherwise only ever feed one receiver. What + // reader cannot sit here - one data source could otherwise only ever feed one receiver. What // is left divides cleanly: the SQL and the shape of its rows belong to the STATEMENT, and the // reading policy belongs to the SUBSCRIPTION doing the reading. // @@ -113,7 +113,7 @@ public abstract class DbOptionsBase public string KeyColumn { get; set; } /// - /// Superseded by naming a statement as the subscription's MarkProcessedStatement — Camel's + /// Superseded by naming a statement as the subscription's MarkProcessedStatement - Camel's /// onConsume, run against each row once Bitween has accepted it. /// [AdapterSetting(Hidden = true)] diff --git a/SW.Bitween.Adapters.Db.Core/DbReceiver.cs b/SW.Bitween.Adapters.Db.Core/DbReceiver.cs index 7bb4587d..fcb1228f 100644 --- a/SW.Bitween.Adapters.Db.Core/DbReceiver.cs +++ b/SW.Bitween.Adapters.Db.Core/DbReceiver.cs @@ -13,8 +13,8 @@ namespace SW.Bitween.Adapters.Db; /// /// The receiver half: rows out of the database and into Bitween as Xchanges. /// -/// Modes are Kafka Connect's vocabulary, because it is the one operators already have — -/// bulk, incrementing, timestamp, timestamp+incrementing — plus +/// Modes are Kafka Connect's vocabulary, because it is the one operators already have - +/// bulk, incrementing, timestamp, timestamp+incrementing - plus /// marker, a processed-flag column, which is what enterprise integration tables actually /// tend to use. None of them can see a DELETE. That is a property of polling, not of this adapter, /// and the UI says so rather than letting someone discover it. @@ -22,7 +22,7 @@ namespace SW.Bitween.Adapters.Db; /// Two things make this safe to run at all: /// /// * The cursor is HOST-HELD, through IAdapterContext.SetStateAsync. An adapter cannot keep -/// its own progress — the supervisor restarts it and the next instance may be elsewhere — and a +/// its own progress - the supervisor restarts it and the next instance may be elsewhere - and a /// cursor that resets to the beginning replays every row already processed. /// * The cursor advances per row, in DeleteFile, which the pipeline calls only after Bitween has /// durably accepted that row. Advancing on read instead would lose rows on a crash; advancing @@ -58,13 +58,13 @@ sealed class ReceiveBatch /// /// The host holds state per (adapter, instance, name), and the instance here is the DATA - /// SOURCE — one connection shared by every subscription pointed at it. So the name has to + /// SOURCE - one connection shared by every subscription pointed at it. So the name has to /// carry the reader, or two subscriptions polling one connection share a cursor: whichever /// polls first advances it, and the rows it took are invisible to the other. No error, no /// warning, half the rows each. /// /// The subscription id arrives as a per-invocation value, which is why this is computed per - /// call rather than held in a field — the field would belong to whichever subscription + /// call rather than held in a field - the field would belong to whichever subscription /// happened to call first. /// string CursorStateName() @@ -75,6 +75,41 @@ string CursorStateName() : $"{LegacyCursorStateName}.{subscriptionId}"; } + /// + /// Rows Bitween accepted whose marking failed: the mark-processed statement, or advancing the + /// cursor, threw after the row's exchange was stored. Without this the next poll read them again + /// and made a second exchange of each. Scoped per subscription, as the cursor is. + /// + string UnmarkedStateName() + { + var subscriptionId = Context.ValueOf(SubscriptionIdKey); + return string.IsNullOrWhiteSpace(subscriptionId) ? "receive.unmarked" : $"receive.unmarked.{subscriptionId}"; + } + + /// The most unmarked keys remembered, oldest forgotten first, so the state stays small. + const int MaxUnmarked = 1000; + + async Task> ReadUnmarkedAsync() + { + var saved = await Context.GetStateAsync(UnmarkedStateName(), Stopping); + if (string.IsNullOrWhiteSpace(saved)) return new List(); + try + { + return JsonConvert.DeserializeObject>(saved) ?? new List(); + } + catch (JsonException) + { + return new List(); + } + } + + Task SaveUnmarkedAsync(List keys) => + Context.SetStateAsync(UnmarkedStateName(), + JsonConvert.SerializeObject(keys.Count > MaxUnmarked ? keys.Skip(keys.Count - MaxUnmarked).ToList() : keys), + Stopping); + + static string KeyText(object key) => Convert.ToString(key, CultureInfo.InvariantCulture); + /// /// Set by the host on every invocation. Its Bitween-side name is /// StartupValuesFiller.SubscriptionIdKey; the two are literals on either side of a @@ -94,15 +129,15 @@ string CursorStateName() const string CursorClaimStateName = "receive.cursor.inheritedBy"; /// - /// The saved cursor: this subscription's own, or — once, for whichever subscription asks - /// first — the unscoped one left behind by a version that did not scope them. + /// The saved cursor: this subscription's own, or - once, for whichever subscription asks + /// first - the unscoped one left behind by a version that did not scope them. /// async Task ReadCursorAsync() { var name = CursorStateName(); var saved = await Context.GetStateAsync(name, Stopping); - // Already has its own progress, or there is no subscription id to scope by — either way + // Already has its own progress, or there is no subscription id to scope by - either way // there is nothing to inherit. if (!string.IsNullOrEmpty(saved) || name == LegacyCursorStateName) return saved; @@ -122,7 +157,7 @@ async Task ReadCursorAsync() return legacy; } - // Claimed by this one already, and it has not accepted a row yet — so the inherited value + // Claimed by this one already, and it has not accepted a row yet - so the inherited value // is still where it is up to. if (claimedBy == subscriptionId) return legacy; @@ -142,7 +177,7 @@ async Task ReadCursorAsync() /// /// * The SQL is a named statement, exactly like every other statement. It is a name and /// never text, because a per-invocation property has {{partner.X}} substituted into it - /// before the adapter sees it — SQL there would be steerable by ordinary partner data. + /// before the adapter sees it - SQL there would be steerable by ordinary partner data. /// * The cursor and key columns come from that statement, because they describe what the /// query returns. The same statement returns the same cursor column whoever reads it. /// * The mode and batch size come from the subscription, because they are the reader's @@ -192,7 +227,7 @@ string Given(string name) => mine.TryGetValue(name, out var value) && !string.Is if (int.TryParse(Given("ReceiveBatchSize"), out var parsed) && parsed > 0) plan.BatchSize = parsed; - // A named statement supersedes the data source's own receive settings entirely — SQL and + // A named statement supersedes the data source's own receive settings entirely - SQL and // row shape together, because taking the SQL from one place and the cursor column from // another is how they come to disagree. var name = Given("ReceiveStatement"); @@ -233,7 +268,7 @@ string Given(string name) => mine.TryGetValue(name, out var value) && !string.Is /// /// Nothing to do. The transaction a mark-processed statement might want cannot live here: the /// pipeline calls Initialize, then ListFiles, then a GetFile and DeleteFile per row, then - /// Finalize — and holding one transaction open across all of that would pin a pooled connection + /// Finalize - and holding one transaction open across all of that would pin a pooled connection /// for as long as Bitween takes to persist every row, on a shared instance serving other /// subscriptions at the same time. Each mark-processed runs in its own transaction instead, /// after Bitween has accepted the row, which is the at-least-once boundary anyway. @@ -256,7 +291,7 @@ public virtual async Task> ListFiles() if (string.IsNullOrEmpty(plan.Mode)) throw new InvalidOperationException( "No ReceiveMode, so this cannot be used as a receiver. Set one on the subscription " - + "(or on the data source, as a default) — one of bulk, incrementing, timestamp, " + + "(or on the data source, as a default) - one of bulk, incrementing, timestamp, " + "timestamp+incrementing or marker."); if (string.IsNullOrWhiteSpace(plan.Sql)) @@ -296,7 +331,7 @@ public virtual async Task> ListFiles() MaxRows = plan.BatchSize, // Already resolved from the allow-list by Plan(), so this is configured SQL rather - // than SQL a message supplied — it does not go through the ad-hoc gate again. + // than SQL a message supplied - it does not go through the ad-hoc gate again. Name = null }, adHocAllowed: true); @@ -307,8 +342,10 @@ public virtual async Task> ListFiles() if (page.Rows.Count == 0) return Array.Empty(); var batchId = Guid.NewGuid().ToString("N"); - var batch = new ReceiveBatch { Outstanding = page.Rows.Count }; + var batch = new ReceiveBatch(); var ids = new List(page.Rows.Count); + var unmarked = await ReadUnmarkedAsync(); + var unmarkedBefore = unmarked.Count; foreach (var row in page.Rows) { @@ -317,6 +354,23 @@ public virtual async Task> ListFiles() $"A row came back with no value in the key column '{plan.KeyColumn}'. The " + "receive statement has to select it, spelled as the database returns it."); + // Already accepted, and only its marking failed: finish that, rather than hand it to + // Bitween again as a new exchange. + if (unmarked.Contains(KeyText(key))) + { + try + { + await Acknowledge(row, plan, plan.NeedsCursor && row.TryGetValue(plan.CursorColumn, out var c) ? c : null); + unmarked.Remove(KeyText(key)); + Logger.LogInformation("Row {Key} was already received; marked it processed now instead of receiving it again.", key); + } + catch (Exception ex) + { + Logger.LogWarning(ex, "Row {Key} was already received and still can't be marked processed; it is skipped, not received again.", key); + } + continue; + } + var id = $"{batchId}:{key}"; batch.Rows[id] = row; @@ -326,6 +380,10 @@ public virtual async Task> ListFiles() ids.Add(id); } + if (unmarked.Count != unmarkedBefore) await SaveUnmarkedAsync(unmarked); + if (ids.Count == 0) return Array.Empty(); + + batch.Outstanding = ids.Count; batches[batchId] = batch; Logger.LogInformation("Polled {Count} row(s) in batch {Batch}.", ids.Count, batchId); return ids; @@ -353,24 +411,24 @@ public virtual async Task DeleteFile(string fileId) var key = KeyOf(fileId); var plan = Plan(); - if (!string.IsNullOrWhiteSpace(plan.MarkProcessedSql)) + try { - var parameters = new Dictionary { ["key"] = RawKey(row, plan) }; - - // Every column is offered as a parameter too, so a mark statement can use more than the - // key — a status column, a batch id, the row's own timestamp. - foreach (var kv in row) parameters[kv.Key] = kv.Value; - - await Execute(new StatementRequest + await Acknowledge(row, plan, batch.Cursors.TryGetValue(fileId, out var cursorValue) ? cursorValue : null); + } + catch + { + // The row's exchange is stored. Remembered, so the next poll marks it rather than + // receiving it a second time; the failure still fails this call, so it is reported. + var unmarked = await ReadUnmarkedAsync(); + var text = KeyText(RawKey(row, plan) ?? key); + if (!unmarked.Contains(text)) { - Sql = plan.MarkProcessedSql, - Parameters = parameters - }, adHocAllowed: true); + unmarked.Add(text); + await SaveUnmarkedAsync(unmarked); + } + throw; } - if (batch.Cursors.TryGetValue(fileId, out var cursorValue) && cursorValue != null) - await Context.SetStateAsync(CursorStateName(), FormatCursor(cursorValue), Stopping); - batch.Rows.Remove(fileId); batch.Outstanding--; @@ -379,7 +437,7 @@ await Execute(new StatementRequest /// /// Drops whatever the run did not get through. Rows left here were never accepted by Bitween - /// and the cursor never moved past them, so the next poll reads them again — which is the + /// and the cursor never moved past them, so the next poll reads them again - which is the /// correct at-least-once behaviour, not a leak. /// public virtual Task Finalize() @@ -392,6 +450,31 @@ public virtual Task Finalize() return Task.CompletedTask; } + /// + /// What makes a row's progress real: the mark-processed statement runs, then the cursor advances + /// past it. Both, in that order. Advancing the cursor first would skip a row whose marking failed. + /// + async Task Acknowledge(Dictionary row, ReceivePlan plan, object cursorValue) + { + if (!string.IsNullOrWhiteSpace(plan.MarkProcessedSql)) + { + var parameters = new Dictionary { ["key"] = RawKey(row, plan) }; + + // Every column is offered as a parameter too, so a mark statement can use more than the + // key - a status column, a batch id, the row's own timestamp. + foreach (var kv in row) parameters[kv.Key] = kv.Value; + + await Execute(new StatementRequest + { + Sql = plan.MarkProcessedSql, + Parameters = parameters + }, adHocAllowed: true); + } + + if (cursorValue != null) + await Context.SetStateAsync(CursorStateName(), FormatCursor(cursorValue), Stopping); + } + // ------------------------------------------------------------------ helpers Dictionary Locate(string fileId, out ReceiveBatch batch) @@ -404,7 +487,7 @@ Dictionary Locate(string fileId, out ReceiveBatch batch) if (!batches.TryGetValue(batchId, out batch)) throw new InvalidOperationException( $"Batch {batchId} is no longer held. Either the run already finished, or the adapter " - + "restarted between the listing and this call — in which case the rows were never " + + "restarted between the listing and this call - in which case the rows were never " + "acknowledged and the next poll will read them again."); if (!batch.Rows.TryGetValue(fileId, out var row)) @@ -416,7 +499,7 @@ Dictionary Locate(string fileId, out ReceiveBatch batch) static string KeyOf(string fileId) => fileId.Substring(fileId.IndexOf(':') + 1); /// - /// The row's key as the database returned it — not the string from the file id, which has been + /// The row's key as the database returned it - not the string from the file id, which has been /// through JSON and would bind a number as text. /// static object RawKey(Dictionary row, ReceivePlan plan) => @@ -424,7 +507,7 @@ static object RawKey(Dictionary row, ReceivePlan plan) => /// /// A cursor comes back from the host as text, and has to go into the query as the type the - /// column compares against — a string on one side of a timestamp comparison silently matches + /// column compares against - a string on one side of a timestamp comparison silently matches /// nothing on some engines rather than failing. /// object ParseCursor(string saved, string mode) @@ -456,7 +539,7 @@ object ParseCursor(string saved, string mode) }; /// - /// A batch whose run died mid-way — the job was cancelled, the node went down — is otherwise + /// A batch whose run died mid-way - the job was cancelled, the node went down - is otherwise /// held for the life of the process, and it holds every row of that poll in memory. /// void PruneStaleBatches() diff --git a/SW.Bitween.Adapters.Db.Core/DbResidentAdapterBase.cs b/SW.Bitween.Adapters.Db.Core/DbResidentAdapterBase.cs index 20240051..70072765 100644 --- a/SW.Bitween.Adapters.Db.Core/DbResidentAdapterBase.cs +++ b/SW.Bitween.Adapters.Db.Core/DbResidentAdapterBase.cs @@ -25,7 +25,7 @@ namespace SW.Bitween.Adapters.Db; /// handshake and an authentication round trip on every message. /// /// It is also shared. One instance serves every subscription bound to its data source, concurrently -/// — so nothing here may keep per-message state in a field. Paging cursors and receive batches are +/// - so nothing here may keep per-message state in a field. Paging cursors and receive batches are /// keyed and held in concurrent maps for exactly that reason. /// public abstract partial class DbResidentAdapterBase : IResidentAdapter, IInfolinkHandler, IInfolinkReceiver @@ -83,7 +83,7 @@ protected abstract Task> ProbePrivilegesAsync(DbConnection connecti /// The marker a parameter is written with in SQL: :, @. protected abstract string ParameterPrefix { get; } - /// Anything the driver needs before a command runs — ODP.NET's BindByName, say. + /// Anything the driver needs before a command runs - ODP.NET's BindByName, say. protected virtual void PrepareCommand(DbCommand command) { } /// A one-row, no-side-effect query used to prove the connection answers. @@ -172,7 +172,7 @@ public virtual Task GetStatusAsync() return Task.FromResult(status); } - /// Engine-specific heartbeat detail — Oracle's session count, say. + /// Engine-specific heartbeat detail - Oracle's session count, say. protected virtual IEnumerable> ExtraStatusDetails() => Array.Empty>(); @@ -192,13 +192,13 @@ protected async Task OpenAsync(CancellationToken cancellationToken /// /// The list is written per engine and is therefore about the engine at its newest. Called once /// the server version is known, which is the only point at which the difference can be told. - /// Does nothing unless an engine overrides it — most capabilities have been there for a decade. + /// Does nothing unless an engine overrides it - most capabilities have been there for a decade. /// protected virtual void AdjustForVersion(DbCapabilities described) { } /// /// The major version this server reports, or 0 when it cannot be read. Drivers spell it - /// differently — "16.14", "8.4.11", "16.00.4135" — but all of them lead with the major. + /// differently - "16.14", "8.4.11", "16.00.4135" - but all of them lead with the major. /// protected static int MajorVersionOf(string serverVersion) { @@ -213,7 +213,7 @@ protected static int MajorVersionOf(string serverVersion) /// /// For the settings an engine will not take in a connection string. PostgreSQL puts search_path /// there and Oracle takes CURRENT_SCHEMA the same way; SQL Server has neither, because its - /// default schema belongs to the login rather than to the connection — so it is the one that + /// default schema belongs to the login rather than to the connection - so it is the one that /// needs this. /// /// A pooled connection carries whatever this did into its next use, which is the point: it is @@ -236,7 +236,7 @@ DbCommand CreateCommand(DbConnection connection, string sql, IDictionary /// Values only, always as parameters. A value is never concatenated into the text, whatever it - /// looks like — that rule and the statement allow-list are the two halves of the same defence. + /// looks like - that rule and the statement allow-list are the two halves of the same defence. /// void Bind(DbCommand command, IDictionary parameters) { @@ -318,7 +318,7 @@ public virtual async Task TestConnection() // nothing. // // All of them, not up to the first failure. Stopping early meant fixing one statement - // only to be told about the next, one connection test at a time — and the second + // only to be told about the next, one connection test at a time - and the second // failure is often the same mistake repeated, which is obvious when both are on // screen and invisible when they arrive a fix apart. var bad = 0; @@ -438,7 +438,7 @@ public virtual Task GetStats() => Task.FromResult(new /// /// is for SQL that came from the DATA SOURCE rather than from a - /// message — the receive statement, the mark-processed statement. Those are configuration, so + /// message - the receive statement, the mark-processed statement. Those are configuration, so /// they are already what the allow-list is protecting; running them through it would mean /// forcing AllowAdHocSql on to use the receiver at all. /// @@ -497,7 +497,7 @@ public virtual async Task Fetch(FetchRequest request) if (!cursors.TryGetValue(request.CursorId, out var cursor)) throw new InvalidOperationException( $"Cursor '{request.CursorId}' is not open. It was either already read to the end, " - + $"closed, or left idle longer than {Options.CursorIdleTimeoutSeconds} seconds — " + + $"closed, or left idle longer than {Options.CursorIdleTimeoutSeconds} seconds - " + "an open cursor holds a pooled connection, so idle ones are reclaimed."); var take = request.Take > 0 ? request.Take : Options.MaxRows; @@ -541,7 +541,7 @@ async Task Execute(StatementRequest request, bool adHocAllowed) var result = new QueryResult(); // A write with RETURNING / OUTPUT hands back rows, and losing them would make the - // engine's most useful write form unusable — so read them when they are there. + // engine's most useful write form unusable - so read them when they are there. if (capabilities != null && capabilities.Returning) { using var reader = await command.ExecuteReaderAsync(); @@ -626,7 +626,7 @@ await ReadPageAsync(reader, result.Rows, } /// - /// Whether a statement is a bare routine name — release_order, SALES.PKG.RELEASE — + /// Whether a statement is a bare routine name - release_order, SALES.PKG.RELEASE - /// rather than SQL. Identifier characters only, so anything with a space, a parenthesis or a /// keyword is treated as SQL and prepared. /// @@ -643,19 +643,19 @@ static bool IsBareRoutineName(string sql) /// prepared without being run. Nothing is bound to a value: this is a syntax and schema check. /// /// - /// Asks the database whether one piece of SQL is valid, by PREPARING it — parsed and planned, + /// Asks the database whether one piece of SQL is valid, by PREPARING it - parsed and planned, /// never run. The check behind both the connection test and the one a statement gets when it /// is saved, so the two cannot disagree about what is acceptable. /// async Task<(bool Ok, string Detail)> CheckStatementAsync(DbConnection connection, string sql) { - // A statement meant for Call holds a PROCEDURE NAME, not SQL — that is what + // A statement meant for Call holds a PROCEDURE NAME, not SQL - that is what // CommandType.StoredProcedure takes, and on Oracle it is the only form that works. // Preparing it as text is a syntax error every time, so the check would fail on a // statement that is perfectly correct. Say what was and was not verified instead of // quietly passing it. if (IsBareRoutineName(sql)) - return (true, "procedure name — existence not checked, it is resolved when called"); + return (true, "procedure name - existence not checked, it is resolved when called"); try { @@ -673,7 +673,7 @@ static bool IsBareRoutineName(string sql) /// /// PREPARE is the portable form and is what most drivers want: the server parses the text and /// binds every name in it, which is the whole check. It is overridable because one engine - /// cannot be asked that way — see the SQL Server adapter, whose driver refuses to prepare a + /// cannot be asked that way - see the SQL Server adapter, whose driver refuses to prepare a /// command unless every parameter has been given an explicit type, which is precisely what a /// caller checking someone else's SQL does not know. /// @@ -684,7 +684,7 @@ protected virtual async Task CheckSyntaxAsync(DbConnection connection, string sq command.CommandTimeout = 10; // The placeholders have to be declared before Prepare, because some drivers validate that - // every parameter in the text has been supplied — Npgsql refuses outright — and a check + // every parameter in the text has been supplied - Npgsql refuses outright - and a check // that fell over on every parameterised statement would be worse than none. DeclarePlaceholders(command); PrepareCommand(command); @@ -695,7 +695,7 @@ protected virtual async Task CheckSyntaxAsync(DbConnection connection, string sq /// The one mistake worth naming rather than leaving to a position offset. /// /// Every engine has its own placeholder character, and the driver reports the other one as a - /// bare syntax error at a column number — true, and no help at all to someone who copied a + /// bare syntax error at a column number - true, and no help at all to someone who copied a /// working statement from an Oracle data source into a PostgreSQL one. If the SQL uses the /// other convention, say so. /// @@ -709,7 +709,7 @@ string WrongPrefixHint(string sql) var match = System.Text.RegularExpressions.Regex.Match(sql ?? "", pattern); if (!match.Success) return ""; - return $" — this looks like the wrong placeholder for {DescribeEngine().Engine}: it binds " + return $" - this looks like the wrong placeholder for {DescribeEngine().Engine}: it binds " + $"parameters as {ParameterPrefix}name, so write {ParameterPrefix}{match.Groups[1].Value} " + $"rather than {other}{match.Groups[1].Value}."; } @@ -758,7 +758,7 @@ void DeclarePlaceholders(DbCommand command) /// /// Adds one declared out parameter. Overridden where the driver needs a provider-specific type - /// — Oracle's REF CURSOR being the case that forces this to exist at all. + /// - Oracle's REF CURSOR being the case that forces this to exist at all. /// protected virtual void AddOutParameter(DbCommand command, DbRoutineParameter declared) { @@ -827,7 +827,7 @@ static IsolationLevel ParseIsolation(string level) => /// /// The subscription pipeline's handler and mapper contract. /// - /// The message body is a — or, when the subscription names a + /// The message body is a - or, when the subscription names a /// statement in its adapter properties, just the parameters. That second form is the one worth /// having: the mapper produces a flat object of values and the statement is configuration, so /// nothing about the SQL depends on message content. @@ -893,7 +893,7 @@ static Dictionary ParametersFrom(string body) "With a Statement configured, the message body has to be a JSON object of parameter " + $"names to values. This one is a {token.Type}."); - // A nested object stays as its JSON text — which is what a json column wants, and what a + // A nested object stays as its JSON text - which is what a json column wants, and what a // driver would otherwise refuse outright. return obj.Properties().ToDictionary(p => p.Name, p => (object)p.Value); } @@ -911,7 +911,7 @@ static List ColumnNames(DbDataReader reader) /// Fills one page and reports what is behind it. /// /// Returns the row that PROVES there is more. A reader cannot be rewound, so asking "is there - /// another row" consumes one — and dropping it loses exactly one row per page, which surfaces + /// another row" consumes one - and dropping it loses exactly one row per page, which surfaces /// months later as a single missing order and is close to unfindable. So it is handed back and /// carried into the next page. Null means the result set ended. /// @@ -1014,7 +1014,7 @@ void Failed() new Dictionary { ["engine"] = capabilities?.Engine ?? "" }); } - /// Recorded for the heartbeat. The message only — never the SQL, never the values. + /// Recorded for the heartbeat. The message only - never the SQL, never the values. protected void RecordError(Exception exception) { lastError = exception.Message; diff --git a/SW.Bitween.Adapters.Db.Core/SW.Bitween.Adapters.Db.Core.csproj b/SW.Bitween.Adapters.Db.Core/SW.Bitween.Adapters.Db.Core.csproj index d4e41605..1cec616b 100644 --- a/SW.Bitween.Adapters.Db.Core/SW.Bitween.Adapters.Db.Core.csproj +++ b/SW.Bitween.Adapters.Db.Core/SW.Bitween.Adapters.Db.Core.csproj @@ -1,18 +1,14 @@ - - net8.0 + + net10.0 SW.Bitween.Adapters.Db disable - + diff --git a/SW.Bitween.Adapters.Db.Core/StatementRegistry.cs b/SW.Bitween.Adapters.Db.Core/StatementRegistry.cs index 2ef89d19..0c597050 100644 --- a/SW.Bitween.Adapters.Db.Core/StatementRegistry.cs +++ b/SW.Bitween.Adapters.Db.Core/StatementRegistry.cs @@ -18,7 +18,7 @@ public class StatementRegistry { /// /// One configured statement. Most are just SQL; a statement a receiver polls with also carries - /// the shape of its rows — which column is the cursor, which identifies the row. + /// the shape of its rows - which column is the cursor, which identifies the row. /// /// Those two live with the statement rather than with the subscription reading it because they /// describe what the QUERY returns. A poll statement returns the same cursor column whoever @@ -47,7 +47,7 @@ public StatementRegistry(string json) catch (JsonException ex) { // Named rather than swallowed: an adapter that silently starts with no statements - // fails later, on a message, as "unknown statement" — which sends whoever is debugging + // fails later, on a message, as "unknown statement" - which sends whoever is debugging // it looking in the wrong place entirely. throw new ArgumentException( $"The Statements setting is not valid JSON: {ex.Message}. It should be an object " @@ -84,7 +84,7 @@ public StatementRegistry(string json) throw new ArgumentException( $"Statement '{property.Name}' has to be a string of SQL, or an object with a " - + $"'sql' property — not a {property.Value.Type}."); + + $"'sql' property - not a {property.Value.Type}."); } } @@ -112,7 +112,7 @@ public string Resolve(string name, string sql, bool allowAdHoc) throw new InvalidOperationException( $"'{name}' is not a statement this data source defines. Configured: " + (statements.Count == 0 - ? "none — set the Statements property on the data source." + ? "none - set the Statements property on the data source." : string.Join(", ", statements.Keys.OrderBy(k => k)))); } diff --git a/SW.Bitween.Adapters.Db.MySql/MySqlDbAdapter.cs b/SW.Bitween.Adapters.Db.MySql/MySqlDbAdapter.cs index c53fa7d5..5a795692 100644 --- a/SW.Bitween.Adapters.Db.MySql/MySqlDbAdapter.cs +++ b/SW.Bitween.Adapters.Db.MySql/MySqlDbAdapter.cs @@ -15,13 +15,13 @@ namespace SW.Bitween.Adapters.Db.MySql; /// /// Bitween's MySQL data source provider, which serves MariaDB too. /// -/// Everything generic — the command surface, paging, the statement allow-list, the polling receiver -/// — lives in . What is here is the part that is genuinely +/// Everything generic - the command surface, paging, the statement allow-list, the polling receiver +/// - lives in . What is here is the part that is genuinely /// MySQL: the connection string, information_schema, and what this user is actually granted. /// /// The one structural difference from the other providers is that MySQL has no schema separate /// from the database. A "schema" filter in discovery therefore means a database, and the default -/// is the one this connection opened rather than every database on the server — a service account +/// is the one this connection opened rather than every database on the server - a service account /// on a shared instance can usually see the names of databases it has no business reading. /// // Three roles, one package. "datasource" is what makes it configurable as a connection; @@ -74,7 +74,7 @@ protected override string BuildConnectionString() // A pooled connection MySQL closed at its end is indistinguishable from a healthy one // until it is used. This is what turns that into a fresh connection rather than into a - // failed message — and it matters more here than anywhere, because wait_timeout on a + // failed message - and it matters more here than anywhere, because wait_timeout on a // managed instance is routinely minutes. ConnectionReset = true, @@ -111,7 +111,7 @@ protected override string BuildConnectionString() ProcedureOutParameters = true, // The one place MySQL is more capable than PostgreSQL here: a procedure returns result sets - // simply by SELECTing, with no cursor to declare and nothing to bind — so Call returns rows + // simply by SELECTing, with no cursor to declare and nothing to bind - so Call returns rows // directly, where PostgreSQL needs a set-returning function queried with SELECT and Oracle // needs an explicit REF CURSOR. ProcedureResultSets = true, @@ -120,11 +120,11 @@ protected override string BuildConnectionString() NamedParameters = true, Transactions = true, - // All four, and READ UNCOMMITTED genuinely does something here — unlike PostgreSQL, where + // All four, and READ UNCOMMITTED genuinely does something here - unlike PostgreSQL, where // it is accepted and silently treated as READ COMMITTED. IsolationLevels = ["ReadUncommitted", "ReadCommitted", "RepeatableRead", "Serializable"], - // MySqlBulkCopy exists and is fast, but BulkLoad is not implemented for it yet — false + // MySqlBulkCopy exists and is fast, but BulkLoad is not implemented for it yet - false // rather than advertising a path that would quietly fall back to row-by-row inserts. BulkCopy = false, @@ -151,7 +151,7 @@ protected override string BuildConnectionString() /// /// What this user may do, asked of the server rather than assumed. SHOW GRANTS is the only - /// answer that accounts for roles, wildcards and grants made at every level at once — + /// answer that accounts for roles, wildcards and grants made at every level at once - /// information_schema.user_privileges reports the global ones only. /// protected override async Task> ProbePrivilegesAsync(DbConnection connection, @@ -170,7 +170,7 @@ protected override async Task> ProbePrivilegesAsync(DbConnection co { var grant = reader.GetString(0); - // "GRANT SELECT, INSERT ON `sales`.* TO ..." — the useful half is between GRANT + // "GRANT SELECT, INSERT ON `sales`.* TO ..." - the useful half is between GRANT // and ON, and the rest is the grantee, which is who we already asked about. var on = grant.IndexOf(" ON ", StringComparison.OrdinalIgnoreCase); if (!grant.StartsWith("GRANT ", StringComparison.OrdinalIgnoreCase) || on < 0) @@ -189,7 +189,7 @@ protected override async Task> ProbePrivilegesAsync(DbConnection co } catch (Exception ex) { - // A user that cannot run SHOW GRANTS is unusual but not broken — it just cannot tell + // A user that cannot run SHOW GRANTS is unusual but not broken - it just cannot tell // us what it can do. Saying so beats failing the connection test over it. Logger.LogDebug(ex, "Could not read grants."); privileges.Add($"(could not be read: {ex.Message})"); @@ -209,7 +209,7 @@ protected override IEnumerable> ExtraStatusDetails( /// /// information_schema, which on MySQL is the catalog rather than a slow standard view over one - /// — there is no lower layer to reach for the way pg_class is under PostgreSQL's. + /// - there is no lower layer to reach for the way pg_class is under PostgreSQL's. /// protected override async Task> DiscoverAsync(DbConnection connection, DiscoverRequest request, CancellationToken cancellationToken) @@ -269,7 +269,7 @@ from information_schema.tables t // table_rows is InnoDB's estimate from the index statistics and can be out by // a wide margin on a table that has not been analysed. Reported as an estimate - // everywhere it surfaces — the alternative is COUNT(*) on a stranger's table, + // everywhere it surfaces - the alternative is COUNT(*) on a stranger's table, // which a menu should not do. Always null for a view, which has no rows of its // own to estimate. RowCount = !request.IncludeRowCounts || reader.IsDBNull(4) @@ -293,7 +293,7 @@ async Task FillColumnsAsync(DbConnection connection, List objects, // // MySqlConnector has no array parameter, so the pair of IN lists is built from the page's // own values. They are identifiers this connection just read back from the catalog, not - // anything a caller supplied — but they are quoted anyway, because "it cannot contain a + // anything a caller supplied - but they are quoted anyway, because "it cannot contain a // quote" is the kind of assumption that survives right up until a table is named oddly. var schemas = string.Join(",", objects.Select(o => Quote(o.Schema)).Distinct()); var names = string.Join(",", objects.Select(o => Quote(o.Name)).Distinct()); @@ -327,7 +327,7 @@ where c.table_schema in ({schemas}) and c.table_name in ({names}) Nullable = string.Equals(reader.GetString(4), "YES", StringComparison.OrdinalIgnoreCase), Ordinal = (int)reader.GetInt64(5), - // auto_increment, a generated column, and DEFAULT_GENERATED — which is what a + // auto_increment, a generated column, and DEFAULT_GENERATED - which is what a // CURRENT_TIMESTAMP default reports as. All three are values the database fills in, // which is the only distinction an insert cares about. Generated = extra.IndexOf("auto_increment", StringComparison.OrdinalIgnoreCase) >= 0 @@ -408,7 +408,7 @@ async Task FillParametersAsync(DbConnection connection, List routines, { // information_schema.parameters, one query for the page. The row with ordinal_position 0 // is a function's return value, which RoutinesAsync has already recorded from - // dtd_identifier — so it is skipped here rather than listed twice. + // dtd_identifier - so it is skipped here rather than listed twice. var schemas = string.Join(",", routines.Select(r => Quote(r.Schema)).Distinct()); var names = string.Join(",", routines.Select(r => Quote(r.Name)).Distinct()); @@ -453,7 +453,7 @@ and p.ordinal_position > 0 static string Quote(string value) => "'" + (value ?? "").Replace("'", "''").Replace("\\", "\\\\") + "'"; /// - /// What a value of this column arrives as in a result row. Coarse on purpose — it tells a + /// What a value of this column arrives as in a result row. Coarse on purpose - it tells a /// mapper author whether to expect a string or a number, and is not trying to be a type system. /// static string ClrTypeOf(string dbType) @@ -463,7 +463,7 @@ static string ClrTypeOf(string dbType) return bare switch { - // tinyint(1) is how MySQL stores a boolean — there is no separate type — and the + // tinyint(1) is how MySQL stores a boolean - there is no separate type - and the // driver hands it back as one, so saying "int" here would mislead a mapper author. "tinyint" when dbType.StartsWith("tinyint(1)", StringComparison.OrdinalIgnoreCase) => "bool", "bool" or "boolean" => "bool", diff --git a/SW.Bitween.Adapters.Db.MySql/MySqlOptions.cs b/SW.Bitween.Adapters.Db.MySql/MySqlOptions.cs index 3f8fdefc..44fb6518 100644 --- a/SW.Bitween.Adapters.Db.MySql/MySqlOptions.cs +++ b/SW.Bitween.Adapters.Db.MySql/MySqlOptions.cs @@ -4,7 +4,7 @@ namespace SW.Bitween.Adapters.Db.MySql; /// /// Every setting here arrives as a DataSource property, bound by name, and the form an operator -/// fills in is generated from these attributes — so a field added here appears in Bitween with no +/// fills in is generated from these attributes - so a field added here appears in Bitween with no /// front-end change. /// /// The hints spend their weight on the two things that are genuinely MySQL's own: a database and @@ -25,13 +25,13 @@ public class MySqlOptions : DbOptionsBase public int Port { get; set; } = 3306; /// - /// MySQL has no schema separate from the database — CREATE SCHEMA is a synonym for CREATE - /// DATABASE — so this one value is both, and it is what an unqualified table name resolves + /// MySQL has no schema separate from the database - CREATE SCHEMA is a synonym for CREATE + /// DATABASE - so this one value is both, and it is what an unqualified table name resolves /// against. /// [AdapterSetting(Required = true, Hint = "The database to connect to. In MySQL a database IS a schema, so this is also what an " - + "unqualified table name resolves against — there is no separate schema setting.")] + + "unqualified table name resolves against - there is no separate schema setting.")] public string Database { get; set; } [AdapterSetting(Required = true)] @@ -54,7 +54,7 @@ public class MySqlOptions : DbOptionsBase [AdapterSetting(Default = "180", Hint = "Seconds an idle pooled connection is kept before it is closed. Keep it below the server's " - + "own wait_timeout — the default is 28800, but a managed instance or a proxy in front of " + + "own wait_timeout - the default is 28800, but a managed instance or a proxy in front of " + "one is often far lower, and a connection the server closed first surfaces as a broken " + "pipe on the next message rather than as a timeout.")] public int ConnectionIdleLifetimeSeconds { get; set; } = 180; @@ -68,7 +68,7 @@ public class MySqlOptions : DbOptionsBase [AdapterSetting(Default = "false", AllowedValues = new[] { "true", "false" }, Hint = "Return DATE and DATETIME as strings rather than dates. MySQL permits '0000-00-00', which " - + "is not a date any .NET type can hold — a column holding one throws on read unless this " + + "is not a date any .NET type can hold - a column holding one throws on read unless this " + "is on. Only needed against a schema that has them, which is usually an old one.")] public bool AllowZeroDateTime { get; set; } } diff --git a/SW.Bitween.Adapters.Db.MySql/SW.Bitween.Adapters.Db.MySql.csproj b/SW.Bitween.Adapters.Db.MySql/SW.Bitween.Adapters.Db.MySql.csproj index 10f66a09..9cafff9d 100644 --- a/SW.Bitween.Adapters.Db.MySql/SW.Bitween.Adapters.Db.MySql.csproj +++ b/SW.Bitween.Adapters.Db.MySql/SW.Bitween.Adapters.Db.MySql.csproj @@ -2,16 +2,16 @@ Exe - net8.0 + net10.0 SW.Bitween.Adapters.Db.MySql disable - + diff --git a/SW.Bitween.Adapters.Db.MySql/adapter.json b/SW.Bitween.Adapters.Db.MySql/adapter.json new file mode 100644 index 00000000..a1cd206a --- /dev/null +++ b/SW.Bitween.Adapters.Db.MySql/adapter.json @@ -0,0 +1,20 @@ +{ + "id": "bitween.db.mysql", + "version": "1.0.0", + "displayName": "MySQL", + "summary": "A MySQL database as a data source: its named statements, read from and written to.", + "lifecycle": "resident", + "publisher": { + "name": "Simplify9", + "url": "https://simplify9.com" + }, + "license": "AGPL-3.0-only", + "categories": [ + "Data sources" + ], + "compatibility": { + "applications": { + "bitween": "10.0.0" + } + } +} diff --git a/SW.Bitween.Adapters.Db.Oracle/OracleDbAdapter.cs b/SW.Bitween.Adapters.Db.Oracle/OracleDbAdapter.cs index b50337a1..e30b21a2 100644 --- a/SW.Bitween.Adapters.Db.Oracle/OracleDbAdapter.cs +++ b/SW.Bitween.Adapters.Db.Oracle/OracleDbAdapter.cs @@ -16,8 +16,8 @@ namespace SW.Bitween.Adapters.Db.Oracle; /// /// Bitween's Oracle data source provider. /// -/// Everything generic — the command surface, paging, the statement allow-list, the polling receiver -/// — lives in . What is here is the part that is genuinely +/// Everything generic - the command surface, paging, the statement allow-list, the polling receiver +/// - lives in . What is here is the part that is genuinely /// Oracle: how a connection string is spelled, what the data dictionary is called, which privileges /// are worth probing, and REF CURSOR. /// @@ -98,14 +98,14 @@ string DataSourceOf() if (hasService && hasSid) throw new InvalidOperationException( - "Both ServiceName and Sid are set, and they are alternatives — a service name names " + "Both ServiceName and Sid are set, and they are alternatives - a service name names " + "a database service, a SID names an instance. Set whichever the DBA gave you and " + "clear the other."); if (!hasService && !hasSid) throw new InvalidOperationException( "Neither ServiceName nor Sid is set, so there is nothing to connect to on " - + $"{_options.Host}:{_options.Port}. A modern Oracle wants the SERVICE name — what " + + $"{_options.Host}:{_options.Port}. A modern Oracle wants the SERVICE name - what " + "`lsnrctl services` lists, e.g. FREEPDB1."); // Easy Connect. The slash form is the service name, the colon form is the SID. @@ -116,7 +116,7 @@ string DataSourceOf() protected override void PrepareCommand(DbCommand command) { - // Without this ODP.NET binds by POSITION, in the order parameters were added — so a + // Without this ODP.NET binds by POSITION, in the order parameters were added - so a // statement whose parameters are written in a different order than the caller supplied them // binds the wrong values, silently, with no type error to catch it. if (command is OracleCommand oracle) oracle.BindByName = _options.BindByName; @@ -126,20 +126,60 @@ protected override void PrepareCommand(DbCommand command) // ------------------------------------------------------------------ checking + private static readonly HashSet DdlKeywords = new(StringComparer.OrdinalIgnoreCase) + { + "CREATE", "ALTER", "DROP", "TRUNCATE", "GRANT", "REVOKE", "RENAME", "COMMENT", "AUDIT", + "NOAUDIT", "ANALYZE", "FLASHBACK", "PURGE", "ASSOCIATE", "DISASSOCIATE", + }; + + /// The first word of the statement, past whitespace and comments. + internal static string LeadingKeyword(string sql) + { + var i = 0; + while (i < sql.Length) + { + if (char.IsWhiteSpace(sql[i])) { i++; continue; } + if (sql.AsSpan(i).StartsWith("--")) + { + var end = sql.IndexOf('\n', i); + i = end < 0 ? sql.Length : end + 1; + continue; + } + if (sql.AsSpan(i).StartsWith("/*")) + { + var end = sql.IndexOf("*/", i + 2, StringComparison.Ordinal); + i = end < 0 ? sql.Length : end + 2; + continue; + } + break; + } + + var start = i; + while (i < sql.Length && char.IsLetter(sql[i])) i++; + return i > start ? sql[start..i] : null; + } + /// /// Oracle is checked with DBMS_SQL.PARSE rather than by preparing. /// - /// ODP.NET's Prepare is a client-side no-op — Oracle validates when a statement is - /// executed, not when it is prepared — so the shared check passed everything here, including a + /// ODP.NET's Prepare is a client-side no-op - Oracle validates when a statement is + /// executed, not when it is prepared - so the shared check passed everything here, including a /// select from a table that does not exist. A statement reported as verified without being /// looked at is worse than no check, because it is the one nobody goes back to. /// /// PARSE is the real thing: it compiles the statement and resolves every name in it, raising /// ORA-00942 for a missing table and ORA-00936 for a syntax error, and it runs nothing. Bind - /// placeholders need no values — parsing is exactly the step before binding. + /// placeholders need no values - parsing is exactly the step before binding. /// protected override async Task CheckSyntaxAsync(DbConnection connection, string sql) { + // Except for DDL: Oracle executes a DDL statement the moment it is parsed, so checking + // "drop table orders" would drop it, from a Save or Test button, with no exchange behind it. + if (LeadingKeyword(sql) is { } keyword && DdlKeywords.Contains(keyword)) + throw new InvalidOperationException( + $"{keyword} statements can't be checked on Oracle: parsing one runs it. " + + "Statements are for reading and writing data; run schema changes outside Bitween."); + using var command = connection.CreateCommand(); command.CommandText = @" declare @@ -172,7 +212,7 @@ when others then catch (Exception ex) { // Rethrown without the PL/SQL stack. Running the parse inside a block means the driver - // reports where in OUR block it failed — three ORA-06512 frames through SYS.DBMS_SQL — + // reports where in OUR block it failed - three ORA-06512 frames through SYS.DBMS_SQL - // underneath the one line that is about the operator's SQL. Keeping them would bury // "table does not exist" under the mechanism used to discover it. throw new InvalidOperationException(FirstOracleError(ex.Message), ex); @@ -215,7 +255,7 @@ static string FirstOracleError(string message) IsolationLevels = ["ReadCommitted", "Serializable"], // Array binding exists in ODP.NET, but as a different call shape rather than a flag on this - // path — so it is honestly false until BulkLoad is implemented for it. + // path - so it is honestly false until BulkLoad is implemented for it. BulkCopy = false, Merge = true, Returning = true, @@ -232,13 +272,13 @@ static string FirstOracleError(string message) ReceiveModes = ["bulk", "incrementing", "timestamp", "timestamp+incrementing", "marker"], // `fetch first N rows only`, since 12c. `rownum` is the older idiom and is a trap with an - // ORDER BY — it is applied before the sort, so it takes an arbitrary N and then sorts those. + // ORDER BY - it is applied before the sort, so it takes an arbitrary N and then sorts those. LimitStyle = "fetchFirst" }; /// /// What this login can actually do. Read from the session's own privileges rather than assumed - /// from the engine — an integration account is usually granted a narrow slice, and finding that + /// from the engine - an integration account is usually granted a narrow slice, and finding that /// out at configure time is worth a round trip. /// protected override async Task> ProbePrivilegesAsync(DbConnection connection, @@ -259,7 +299,7 @@ protected override async Task> ProbePrivilegesAsync(DbConnection co } catch (Exception ex) { - // A login without SELECT on session_privs is unusual but not broken — it just cannot + // A login without SELECT on session_privs is unusual but not broken - it just cannot // tell us what it can do. Saying so beats failing the connection test over it. Logger.LogDebug(ex, "Could not read session_privs."); privileges.Add($"(could not be read: {ex.Message})"); @@ -306,7 +346,7 @@ protected override void AddOutParameter(DbCommand command, DbRoutineParameter de /// /// Read from the data dictionary rather than through GetSchema. ODP.NET's collections /// cover tables and columns, but not comments, not routine parameter direction, and not - /// estimated row counts — and the ALL_* views answer all of it in one query per object type. + /// estimated row counts - and the ALL_* views answer all of it in one query per object type. /// /// ALL_, never DBA_: ALL_ is what this login can see, which is the honest answer and the one /// that does not need a privilege an integration account should not have. @@ -380,7 +420,7 @@ from all_objects o Comment = reader.IsDBNull(3) ? null : reader.GetString(3), // From the optimiser's statistics, so it is as fresh as the last gather. Stated - // as an estimate everywhere it surfaces — the alternative is COUNT(*) on a + // as an estimate everywhere it surfaces - the alternative is COUNT(*) on a // stranger's table, which is not a thing a menu should do. RowCount = !request.IncludeRowCounts || reader.IsDBNull(4) ? null @@ -525,7 +565,7 @@ from all_arguments a DbType = dataType, // Surfaced as RefCursor rather than Out, because a caller has to declare it - // differently — this is the field that tells them to. + // differently - this is the field that tells them to. Direction = dataType == "REF CURSOR" ? "RefCursor" : DirectionOf(inOut), Ordinal = reader.IsDBNull(3) ? 0 : Convert.ToInt32(reader.GetValue(3)) }); @@ -578,7 +618,7 @@ from all_sequences s }; /// - /// What a value of this column arrives as in a result row. Coarse on purpose — it is here so a + /// What a value of this column arrives as in a result row. Coarse on purpose - it is here so a /// mapper author knows whether to expect a string or a number, not to be a type system. /// static string ClrTypeOf(string oracleType) => oracleType switch diff --git a/SW.Bitween.Adapters.Db.Oracle/OracleOptions.cs b/SW.Bitween.Adapters.Db.Oracle/OracleOptions.cs index 0bcf48fe..11472643 100644 --- a/SW.Bitween.Adapters.Db.Oracle/OracleOptions.cs +++ b/SW.Bitween.Adapters.Db.Oracle/OracleOptions.cs @@ -4,7 +4,7 @@ namespace SW.Bitween.Adapters.Db.Oracle; /// /// Every setting here arrives as a DataSource property, bound by name, and the form an operator -/// fills in is generated from these attributes — so a field added here appears in Bitween with no +/// fills in is generated from these attributes - so a field added here appears in Bitween with no /// front-end change. /// /// Oracle is the awkward one to configure, and the hints carry that weight deliberately: service @@ -24,13 +24,13 @@ public class OracleOptions : DbOptionsBase public int Port { get; set; } = 1521; [AdapterSetting(Hint = - "The SERVICE name — what `lsnrctl services` lists, e.g. FREEPDB1 or ORCLPDB1. This is what a " + "The SERVICE name - what `lsnrctl services` lists, e.g. FREEPDB1 or ORCLPDB1. This is what a " + "modern Oracle wants. Leave empty only if you are connecting by SID instead.")] public string ServiceName { get; set; } [AdapterSetting(Hint = "The SID, for an older instance that has no service name. Set exactly one of ServiceName " - + "and Sid — a connection naming both is rejected rather than silently preferring one.")] + + "and Sid - a connection naming both is rejected rather than silently preferring one.")] public string Sid { get; set; } [AdapterSetting(Hint = diff --git a/SW.Bitween.Adapters.Db.Oracle/SW.Bitween.Adapters.Db.Oracle.csproj b/SW.Bitween.Adapters.Db.Oracle/SW.Bitween.Adapters.Db.Oracle.csproj index 111cbd23..efb6e8e9 100644 --- a/SW.Bitween.Adapters.Db.Oracle/SW.Bitween.Adapters.Db.Oracle.csproj +++ b/SW.Bitween.Adapters.Db.Oracle/SW.Bitween.Adapters.Db.Oracle.csproj @@ -2,13 +2,13 @@ Exe - net8.0 + net10.0 SW.Bitween.Adapters.Db.Oracle disable - + + + diff --git a/SW.Bitween.Adapters.Db.PostgreSql/adapter.json b/SW.Bitween.Adapters.Db.PostgreSql/adapter.json new file mode 100644 index 00000000..c0d11470 --- /dev/null +++ b/SW.Bitween.Adapters.Db.PostgreSql/adapter.json @@ -0,0 +1,20 @@ +{ + "id": "bitween.db.postgresql", + "version": "1.0.0", + "displayName": "PostgreSQL", + "summary": "A PostgreSQL database as a data source: its named statements, read from and written to.", + "lifecycle": "resident", + "publisher": { + "name": "Simplify9", + "url": "https://simplify9.com" + }, + "license": "AGPL-3.0-only", + "categories": [ + "Data sources" + ], + "compatibility": { + "applications": { + "bitween": "10.0.0" + } + } +} diff --git a/SW.Bitween.Adapters.Db.SqlServer/SW.Bitween.Adapters.Db.SqlServer.csproj b/SW.Bitween.Adapters.Db.SqlServer/SW.Bitween.Adapters.Db.SqlServer.csproj index 5a1650eb..f8b0ed63 100644 --- a/SW.Bitween.Adapters.Db.SqlServer/SW.Bitween.Adapters.Db.SqlServer.csproj +++ b/SW.Bitween.Adapters.Db.SqlServer/SW.Bitween.Adapters.Db.SqlServer.csproj @@ -2,13 +2,13 @@ Exe - net8.0 + net10.0 SW.Bitween.Adapters.Db.SqlServer disable - + + true + + + + + + + + + """); + + yield return ("adapter.json", $$""" + { + "id": "{{id}}", + "version": "0.1.0", + "displayName": "{{Scaffolder.Spaced(name)}}", + "summary": "What this {{kind}} does, in one sentence, for the adapter list.", + "lifecycle": "classic" + } + """); + + yield return ("Program.cs", Program(name, kind)); + + yield return ("settings.example.json", """ + { + "BaseUrl": "https://partner.example.test", + "ApiKey": "put a test key here, and keep this file out of version control once it holds one" + } + """); + + yield return (".gitignore", """ + bin/ + obj/ + settings.json + """); + + yield return ("README.md", $$""" + # {{Scaffolder.Spaced(name)}} + + A Bitween {{kind}} adapter. + + ```sh + bitween adapter build # builds bin/serverless/{{id}}-0.1.0.zip + cp settings.example.json settings.json # then fill in real values + bitween adapter test --settings settings.json # checks it against the Bitween contract + bitween adapter publish bin/serverless/{{id}}-0.1.0.zip # to the Bitween you signed in to + ``` + + Settings are declared in code with `Runner.Expect`; `bitween adapter build` writes them into + the manifest Bitween reads. + """); + } + + static string Program(string name, string kind) => kind switch + { + "receiver" => $$""" + using SW.Bitween.Adapters; + using SW.Serverless.Sdk; + + namespace {{name}}; + + /// Fetches files on a schedule. Bitween calls Initialize, ListFiles, then GetFile and DeleteFile for each file, then Finalize. + [AdapterKind("receiver")] + [AdapterContract("bitween", 1)] + public class Receiver : IBitweenReceiver + { + public Receiver() + { + // Declare settings here; read them in the methods, never in the constructor. + Runner.Expect("BaseUrl", "https://partner.example.test", description: "Where files are fetched from."); + Runner.Expect("ApiKey", isPrivate: true, description: "The partner's key."); + } + + public Task Initialize() => Task.CompletedTask; + + public Task> ListFiles() => + Task.FromResult>(new[] { "example-1" }); + + public Task GetFile(string fileId) => + Task.FromResult(new ExchangeFile("{\"id\":\"" + fileId + "\"}", fileId + ".json")); + + public Task DeleteFile(string fileId) => Task.CompletedTask; + + public Task Finalize() => Task.CompletedTask; + } + + static class Program + { + static Task Main() => Runner.Run(new Receiver()); + } + """, + "validator" => $$""" + using SW.Bitween.Adapters; + using SW.Serverless.Sdk; + + namespace {{name}}; + + /// Checks a message before Bitween accepts it. + [AdapterKind("validator")] + [AdapterContract("bitween", 1)] + public class Validator : IBitweenValidator + { + public Validator() + { + Runner.Expect("MaxBytes", "1000000", description: "The largest message accepted."); + } + + public Task Validate(ExchangeFile file) + { + var result = new ValidationResult(); + if (file.Data.Length > Runner.StartupValueOf("MaxBytes")) + result.AddError("Data", "The message is larger than allowed."); + return Task.FromResult(result); + } + } + + static class Program + { + static Task Main() => Runner.Run(new Validator()); + } + """, + var handlerOrMapper => $$""" + using SW.Bitween.Adapters; + using SW.Serverless.Sdk; + + namespace {{name}}; + + /// {{(handlerOrMapper == "mapper" ? "Maps a message into the shape the next step expects." : "Delivers a message and returns the partner's response.")}} + [AdapterKind("{{handlerOrMapper}}")] + [AdapterContract("bitween", 1)] + public class {{(handlerOrMapper == "mapper" ? "Mapper : IBitweenMapper" : "Handler : IBitweenHandler")}} + { + public {{(handlerOrMapper == "mapper" ? "Mapper" : "Handler")}}() + { + // Declare settings here; read them in the methods, never in the constructor. + Runner.Expect("BaseUrl", "https://partner.example.test", description: "Where messages go."); + Runner.Expect("ApiKey", isPrivate: true, description: "The partner's key."); + } + + public Task Handle(ExchangeFile file) + { + // {{(handlerOrMapper == "mapper" ? "Return the message in its new shape." : "Send file.Data to the partner. A rejection is returned with BadData set, not thrown.")}} + return Task.FromResult(new ExchangeFile(file.Data, file.Filename)); + } + } + + static class Program + { + static Task Main() => Runner.Run(new {{(handlerOrMapper == "mapper" ? "Mapper" : "Handler")}}()); + } + """, + }; + + static IEnumerable<(string File, string Content)> PythonFiles(string name, string id, string kind) + { + yield return ("adapter.json", $$""" + { + "id": "{{id}}", + "version": "0.1.0", + "displayName": "{{Scaffolder.Spaced(name)}}", + "summary": "What this {{kind}} does, in one sentence, for the adapter list.", + "runtime": "python", + "entry": "main.py" + } + """); + + yield return ("main.py", PythonMain(name, kind)); + + yield return ("requirements.txt", $$""" + # What the adapter imports, pinned, one per line; bitween adapter build vendors them into the + # package. The two SDKs are vendored by bitween adapter build itself: they're listed here for + # your editor and for running the tests outside a build. + sw-serverless=={{Scaffolder.PythonSdkVersion}} + simplyworks-bitween>={{BitweenAdaptersPackageVersion}} + """); + + yield return ("settings.example.json", """ + { + "BaseUrl": "https://partner.example.test", + "ApiKey": "put a test key here, and keep this file out of version control once it holds one" + } + """); + + yield return (".gitignore", """ + __pycache__/ + .venv/ + bin/ + settings.json + """); + + yield return ("README.md", $$""" + # {{Scaffolder.Spaced(name)}} + + A Bitween {{kind}} adapter in Python (3.12 or later). + + ```sh + bitween adapter build # builds bin/serverless/{{id}}-0.1.0.zip + cp settings.example.json settings.json # then fill in real values + bitween adapter test --settings settings.json # checks it against the Bitween contract + bitween adapter publish bin/serverless/{{id}}-0.1.0.zip # to the Bitween you signed in to + ``` + + Settings are declared in code with `sw.expect`; `bitween adapter build` writes them into the + manifest Bitween reads. Dependencies go in `requirements.txt`, pinned. + """); + } + + static string PythonMain(string name, string kind) => kind switch + { + "receiver" => $$"""" + import sw_serverless as sw + from simplyworks_bitween import ExchangeFile, Receiver + + + class {{name}}(Receiver): + """Fetches files on a schedule. Bitween calls initialize, list_files, then get_file and + delete_file for each file, then finalize.""" + + def __init__(self): + # Declare settings here; read them in the methods with sw.value_of. + sw.expect("BaseUrl", "https://partner.example.test", description="Where files are fetched from.") + sw.expect("ApiKey", secret=True, description="The partner's key.") + + def list_files(self) -> list[str]: + return ["example-1"] + + def get_file(self, file_id: str) -> ExchangeFile: + return ExchangeFile(data='{"id": "%s"}' % file_id, filename=file_id + ".json") + + def delete_file(self, file_id: str) -> None: + pass + + + if __name__ == "__main__": + sw.run({{name}}) + """", + "validator" => $$"""" + import sw_serverless as sw + from simplyworks_bitween import ExchangeFile, ValidationResult, Validator + + + class {{name}}(Validator): + """Checks a message before Bitween accepts it.""" + + def __init__(self): + sw.expect("MaxBytes", "1000000", type="number", description="The largest message accepted.") + + def validate(self, file: ExchangeFile) -> ValidationResult: + result = ValidationResult() + if len(file.data) > int(sw.value_of("MaxBytes")): + result.add("Data", "The message is larger than allowed.") + return result + + + if __name__ == "__main__": + sw.run({{name}}) + """", + "mapper" => $$"""" + import sw_serverless as sw + from simplyworks_bitween import ExchangeFile, Mapper + + + class {{name}}(Mapper): + """Maps a message into the shape the next step expects.""" + + def map(self, file: ExchangeFile) -> ExchangeFile: + # Return the message in its new shape. + return ExchangeFile(data=file.data, filename=file.filename) + + + if __name__ == "__main__": + sw.run({{name}}) + """", + _ => $$"""" + import sw_serverless as sw + from simplyworks_bitween import ExchangeFile, Handler + + + class {{name}}(Handler): + """Delivers a message and returns the partner's response.""" + + def __init__(self): + # Declare settings here; read them in the methods with sw.value_of. + sw.expect("BaseUrl", "https://partner.example.test", description="Where messages go.") + sw.expect("ApiKey", secret=True, description="The partner's key.") + + def handle(self, file: ExchangeFile) -> ExchangeFile: + # Send file.data to the partner. A rejection is returned with bad_data=True, not raised. + return ExchangeFile(data=file.data, filename=file.filename) + + + if __name__ == "__main__": + sw.run({{name}}) + """", + }; + + static IEnumerable<(string File, string Content)> NodeFiles(string name, string id, string kind, bool typeScript) + { + var entry = typeScript ? "main.ts" : "main.js"; + yield return ("adapter.json", $$""" + { + "id": "{{id}}", + "version": "0.1.0", + "displayName": "{{Scaffolder.Spaced(name)}}", + "summary": "What this {{kind}} does, in one sentence, for the adapter list.", + "runtime": "node", + "entry": "{{entry}}" + } + """); + + yield return (entry, NodeMain(name, kind, typeScript)); + + // The SDKs are named for the editor and for running outside a build; bitween adapter build + // vendors the copies it carries. Other dependencies go here too, and are installed into + // the package. + yield return ("package.json", Scaffolder.NodePackageJson(id, typeScript, + new Dictionary { ["@simplyworks/bitween"] = ">=" + BitweenAdaptersPackageVersion })); + + if (typeScript) + yield return ("tsconfig.json", Scaffolder.TsConfig); + + yield return ("settings.example.json", """ + { + "BaseUrl": "https://partner.example.test", + "ApiKey": "put a test key here, and keep this file out of version control once it holds one" + } + """); + + yield return (".gitignore", """ + node_modules/ + bin/ + settings.json + """); + + yield return ("README.md", $$""" + # {{Scaffolder.Spaced(name)}} + + A Bitween {{kind}} adapter in {{(typeScript ? "TypeScript" : "JavaScript")}}, for Node 22 or later. + + ```sh + bitween adapter build # builds bin/serverless/{{id}}-0.1.0.zip + cp settings.example.json settings.json # then fill in real values + bitween adapter test --settings settings.json # checks it against the Bitween contract + bitween adapter publish bin/serverless/{{id}}-0.1.0.zip # to the Bitween you signed in to + ``` + + Settings are declared in code with `expect`; `bitween adapter build` writes them into the + manifest Bitween reads.{{(typeScript ? " The build strips the types with Node itself, so no compiler is needed; `tsc --noEmit` checks them." : "")}} + """); + } + + static string NodeMain(string name, string kind, bool ts) + { + var header = ts + ? $$""" + import { expect, run, valueOf } from "@simplyworks/sw-serverless"; + import { ExchangeFile, {{Pascal(kind)}}{{(kind == "validator" ? ", ValidationResult" : "")}} } from "@simplyworks/bitween"; + """ + : $$""" + const { expect, run, valueOf } = require("@simplyworks/sw-serverless"); + const { ExchangeFile, {{Pascal(kind)}}{{(kind == "validator" ? ", ValidationResult" : "")}} } = require("@simplyworks/bitween"); + """; + string T(string type) => ts ? type : ""; + var body = kind switch + { + "receiver" => $$""" + /** Fetches files on a schedule. Bitween calls initialize, listFiles, then getFile and deleteFile for each file, then finalize. */ + class {{name}} extends Receiver { + constructor() { + super(); + // Declare settings here; read them in the methods with valueOf. + expect("BaseUrl", { default: "https://partner.example.test", description: "Where files are fetched from." }); + expect("ApiKey", { secret: true, description: "The partner's key." }); + } + + listFiles(){{T(": string[]")}} { + return ["example-1"]; + } + + getFile(fileId{{T(": string")}}){{T(": ExchangeFile")}} { + return new ExchangeFile({ data: JSON.stringify({ id: fileId }), filename: `${fileId}.json` }); + } + + deleteFile(fileId{{T(": string")}}){{T(": void")}} {} + } + """, + "validator" => $$""" + /** Checks a message before Bitween accepts it. */ + class {{name}} extends Validator { + constructor() { + super(); + expect("MaxBytes", { default: "1000000", type: "number", description: "The largest message accepted." }); + } + + validate(file{{T(": ExchangeFile")}}){{T(": ValidationResult")}} { + const result = new ValidationResult(); + if (file.data.length > Number(valueOf("MaxBytes"))) result.add("Data", "The message is larger than allowed."); + return result; + } + } + """, + "mapper" => $$""" + /** Maps a message into the shape the next step expects. */ + class {{name}} extends Mapper { + map(file{{T(": ExchangeFile")}}){{T(": ExchangeFile")}} { + // Return the message in its new shape. + return new ExchangeFile({ data: file.data, filename: file.filename }); + } + } + """, + _ => $$""" + /** Delivers a message and returns the partner's response. */ + class {{name}} extends Handler { + constructor() { + super(); + // Declare settings here; read them in the methods with valueOf. + expect("BaseUrl", { default: "https://partner.example.test", description: "Where messages go." }); + expect("ApiKey", { secret: true, description: "The partner's key." }); + } + + handle(file{{T(": ExchangeFile")}}){{T(": ExchangeFile")}} { + // Send file.data to valueOf("BaseUrl"). A rejection is returned with badData: true, not thrown. + return new ExchangeFile({ data: file.data, filename: file.filename }); + } + } + """, + }; + // valueOf is used by every template but the mapper; keep the import list honest there. + if (kind == "mapper") header = header.Replace("expect, run, valueOf", "run"); + else if (kind == "receiver") header = header.Replace("expect, run, valueOf", "expect, run"); + return header + "\n\n" + body + "\n\nrun(" + name + ");\n"; + } + + static string Pascal(string kind) => char.ToUpperInvariant(kind[0]) + kind[1..]; + +} diff --git a/SW.Bitween.Adapters.Tooling/SW.Bitween.Adapters.Tooling.csproj b/SW.Bitween.Adapters.Tooling/SW.Bitween.Adapters.Tooling.csproj new file mode 100644 index 00000000..d16541e1 --- /dev/null +++ b/SW.Bitween.Adapters.Tooling/SW.Bitween.Adapters.Tooling.csproj @@ -0,0 +1,24 @@ + + + + + net10.0 + latest + disable + false + + + + + + + + + + + + + + diff --git a/SW.Bitween.Adapters/AdapterContract.cs b/SW.Bitween.Adapters/AdapterContract.cs new file mode 100644 index 00000000..c86b5e0d --- /dev/null +++ b/SW.Bitween.Adapters/AdapterContract.cs @@ -0,0 +1,41 @@ +using System.IO; +using System.Reflection; + +namespace SW.Bitween.Adapters; + +/// The Bitween adapter contract, as the JSON it is written in. +public static class AdapterContract +{ + /// The contract version this package describes. + public const int Version = 1; + + /// The kinds of adapter, as adapter manifests name them. + public static class Kinds + { + /// Delivers a message: . + public const string Handler = "handler"; + /// Transforms a message: . + public const string Mapper = "mapper"; + /// Checks a message: . + public const string Validator = "validator"; + /// Fetches files on a schedule: . + public const string Receiver = "receiver"; + } + + /// The contract: kinds, their methods, payload types and encodings. + public static string Json => Read("bitween-adapter-contract.v1.json"); + + /// The JSON Schema of . + public static string ExchangeFileSchema => Read("exchange-file.schema.json"); + + /// The JSON Schema of . + public static string ValidationResultSchema => Read("validation-result.schema.json"); + + private static string Read(string name) + { + using var stream = typeof(AdapterContract).Assembly + .GetManifestResourceStream($"SW.Bitween.Adapters.Contract.{name}")!; + using var reader = new StreamReader(stream); + return reader.ReadToEnd(); + } +} diff --git a/SW.Bitween.Adapters/Contract/bitween-adapter-contract.v1.json b/SW.Bitween.Adapters/Contract/bitween-adapter-contract.v1.json new file mode 100644 index 00000000..a56d1b44 --- /dev/null +++ b/SW.Bitween.Adapters/Contract/bitween-adapter-contract.v1.json @@ -0,0 +1,121 @@ +{ + "contract": "bitween", + "version": 1, + "description": "What Bitween calls on each kind of adapter, and what it passes. Bitween calls adapters by method name over the SW-Serverless protocol; the names below are exact and case-sensitive.", + "encoding": { + "string": "A string argument or result is the raw UTF-8 text, not a JSON string.", + "object": "Any other argument or result is JSON, with property names exactly as its schema gives them.", + "none": "A method with no argument receives an empty payload; one with no result returns an empty payload." + }, + "types": { + "ExchangeFile": { + "schema": "exchange-file.schema.json" + }, + "ValidationResult": { + "schema": "validation-result.schema.json" + }, + "FileId": { + "encoding": "string", + "description": "An id a receiver gave in ListFiles, passed back exactly as given." + }, + "FileIdList": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "errors": "A call fails when the adapter raises an error; Bitween records the error's type and message on the exchange, and retry policies decide what happens next. A handler whose partner rejected the message does not fail: it returns the partner's answer with BadData true.", + "kinds": { + "handler": { + "description": "Delivers a message and returns the partner's response.", + "methods": [ + { + "name": "Handle", + "input": "ExchangeFile", + "output": "ExchangeFile", + "examples": [ + { + "Data": "{\"orderId\":\"SO-1001\",\"lines\":2}", + "Filename": "order-SO-1001.json", + "BadData": false, + "ContentType": "application/json" + } + ] + } + ] + }, + "mapper": { + "description": "Transforms a message into the shape the next step expects.", + "methods": [ + { + "name": "Handle", + "input": "ExchangeFile", + "output": "ExchangeFile", + "examples": [ + { + "Data": "{\"orderId\":\"SO-1001\",\"lines\":2}", + "Filename": "order-SO-1001.json", + "BadData": false, + "ContentType": "application/json" + } + ] + } + ] + }, + "validator": { + "description": "Checks a message before it is accepted.", + "methods": [ + { + "name": "Validate", + "input": "ExchangeFile", + "output": "ValidationResult", + "examples": [ + { + "Data": "{\"orderId\":\"SO-1001\",\"lines\":2}", + "Filename": "order-SO-1001.json", + "BadData": false, + "ContentType": "application/json" + } + ] + } + ] + }, + "receiver": { + "description": "Fetches files from a source on a schedule.", + "session": "One session per run. Calls come in this order: Initialize, ListFiles, then for each file GetFile and, once it is safely taken in, DeleteFile, and finally Finalize, which is also called after a failure.", + "methods": [ + { + "name": "Initialize", + "input": null, + "output": null + }, + { + "name": "ListFiles", + "input": null, + "output": "FileIdList" + }, + { + "name": "GetFile", + "input": "FileId", + "output": "ExchangeFile" + }, + { + "name": "DeleteFile", + "input": "FileId", + "output": null, + "destructive": true + }, + { + "name": "Finalize", + "input": null, + "output": null + } + ] + } + }, + "examples": "Example inputs for each method that takes one. serverless test calls the adapter with them, using the settings it is given, and checks each answer against the method's output type.", + "destructive": "A destructive method changes the source an adapter reads: serverless test calls it only when told it may." +} diff --git a/SW.Bitween.Adapters/Contract/bitween-bookmark-contract.v1.json b/SW.Bitween.Adapters/Contract/bitween-bookmark-contract.v1.json new file mode 100644 index 00000000..f104844d --- /dev/null +++ b/SW.Bitween.Adapters/Contract/bitween-bookmark-contract.v1.json @@ -0,0 +1,29 @@ +{ + "contract": "bitween-bookmark", + "version": 1, + "description": "A receiver that keeps a bookmark through Bitween: where its last good run got to, so the next run takes only what is new. Declared beside the bitween contract. Bitween keeps the bookmark per subscription, hands it to each run as the setting __bookmark__ (absent on a first run), and calls Bookmark once every file of a run was taken in, keeping what it returns. A run in which anything failed doesn't move it.", + "encoding": { + "string": "A string argument or result is the raw UTF-8 text, not a JSON string.", + "none": "A method with no argument receives an empty payload." + }, + "types": { + "BookmarkText": { + "encoding": "string", + "description": "Where the next run starts, in a form of the receiver's own choosing, up to 8000 characters. Empty keeps the bookmark as it was." + } + }, + "errors": "A Bookmark that fails fails the run, and the bookmark stays where it was.", + "kinds": { + "receiver": { + "description": "Keeps a bookmark between runs.", + "session": "Called in the same session as the run, after the last DeleteFile and before Finalize. Called before ListFiles, it returns the bookmark the run was given.", + "methods": [ + { + "name": "Bookmark", + "input": null, + "output": "BookmarkText" + } + ] + } + } +} diff --git a/SW.Bitween.Adapters/Contract/exchange-file.schema.json b/SW.Bitween.Adapters/Contract/exchange-file.schema.json new file mode 100644 index 00000000..52a5fa06 --- /dev/null +++ b/SW.Bitween.Adapters/Contract/exchange-file.schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://bitween.systems/contract/v1/exchange-file.schema.json", + "title": "ExchangeFile", + "description": "A file as Bitween passes it to and from adapters. Property names are exactly as shown; unknown properties are allowed and ignored.", + "type": "object", + "required": ["Data"], + "properties": { + "Filename": { "type": ["string", "null"], "description": "The file's name, when it has one." }, + "Data": { "type": "string", "description": "The content: text, or base64 for binary content." }, + "Hash": { "type": ["string", "null"], "description": "SHA-1 of Data as lower-case hex. Written by .NET adapters; always recomputed from Data when read, never trusted." }, + "BadData": { "type": "boolean", "default": false, "description": "Whether this is a bad response: a delivery the partner rejected." }, + "ContentType": { "type": ["string", "null"], "description": "The content's media type, when known." } + }, + "additionalProperties": true +} diff --git a/SW.Bitween.Adapters/Contract/validation-result.schema.json b/SW.Bitween.Adapters/Contract/validation-result.schema.json new file mode 100644 index 00000000..97d1a839 --- /dev/null +++ b/SW.Bitween.Adapters/Contract/validation-result.schema.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://bitween.systems/contract/v1/validation-result.schema.json", + "title": "ValidationResult", + "description": "What a validator found. Property names are exactly as shown; unknown properties are allowed and ignored.", + "type": "object", + "required": ["Validations"], + "properties": { + "Success": { "type": "boolean", "description": "True when Validations is empty. Bitween derives it from Validations." }, + "Validations": { + "type": "array", + "description": "Each failure: a code, often the field it concerns, and a message.", + "items": { + "type": "object", + "required": ["Key", "Value"], + "properties": { + "Key": { "type": "string" }, + "Value": { "type": "string" } + } + } + } + }, + "additionalProperties": true +} diff --git a/SW.Bitween.Adapters/ExchangeFile.cs b/SW.Bitween.Adapters/ExchangeFile.cs new file mode 100644 index 00000000..8d490643 --- /dev/null +++ b/SW.Bitween.Adapters/ExchangeFile.cs @@ -0,0 +1,57 @@ +using System; +using System.Security.Cryptography; +using System.Text; + +namespace SW.Bitween.Adapters; + +/// +/// A file as Bitween passes it to and from adapters: the content, its name, and whether it is a +/// bad response. On the wire it is the JSON {"Filename","Data","Hash","BadData","ContentType"} +/// that SW.PrimitiveTypes.XchangeFile has always produced, so adapters built on either +/// exchange the same thing. +/// +public class ExchangeFile +{ + private string _data = string.Empty; + + /// For deserializers; is empty until set. + public ExchangeFile() + { + } + + /// The content: text, or base64 for binary content. + /// The file's name, when it has one. + /// Whether this is a bad response - a delivery the partner rejected. + public ExchangeFile(string data, string? filename = null, bool badData = false) + { + Data = data; + Filename = filename; + BadData = badData; + } + + /// The file's name, when it has one. + public string? Filename { get; set; } + + /// The content: text, or base64 for binary content. + public string Data + { + get => _data; + set => _data = value ?? throw new ArgumentNullException(nameof(Data), "A file's data can't be null."); + } + + /// SHA-1 of as lower-case hex. Always derived from the data; a value read from JSON is ignored. + public string Hash + { + get + { + using var sha1 = SHA1.Create(); + return BitConverter.ToString(sha1.ComputeHash(Encoding.UTF8.GetBytes(Data))).Replace("-", "").ToLowerInvariant(); + } + } + + /// Whether this is a bad response - a delivery the partner rejected. + public bool BadData { get; set; } + + /// The content's media type, when known. + public string? ContentType { get; set; } +} diff --git a/SW.Bitween.Adapters/Kinds.cs b/SW.Bitween.Adapters/Kinds.cs new file mode 100644 index 00000000..c935a337 --- /dev/null +++ b/SW.Bitween.Adapters/Kinds.cs @@ -0,0 +1,55 @@ +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace SW.Bitween.Adapters; + +// The .NET form of the Bitween adapter contract (Contract/bitween-adapter-contract.v1.json). Bitween +// calls an adapter by method name, so these names - Handle, Validate, Initialize, ListFiles, +// GetFile, DeleteFile, Finalize - are the contract, and must never change. Adapters written against +// SW.PrimitiveTypes' IInfolinkHandler, IInfolinkValidator and IInfolinkReceiver use the same names +// and are called the same way. + +/// Delivers a message: given the file to send, returns the partner's response. +public interface IBitweenHandler +{ + /// Delivers . A rejection by the partner is returned with set, not thrown. + Task Handle(ExchangeFile file); +} + +/// Transforms a message: given a file, returns it mapped into the shape the next step expects. +public interface IBitweenMapper +{ + /// Returns mapped into the shape the next step expects. + Task Handle(ExchangeFile file); +} + +/// Checks a message before it is accepted, and says what is wrong with it. +public interface IBitweenValidator +{ + /// Checks ; an empty result means it passed. + Task Validate(ExchangeFile file); +} + +/// +/// Fetches files from a source on a schedule. Bitween calls, on one session and in this order: +/// , , then for each file and, +/// once it is safely taken in, , and finally - +/// also after a failure. +/// +public interface IBitweenReceiver +{ + /// Opens the session: connects, signs in. + Task Initialize(); + + /// The ids of the files ready to take in this run. + Task> ListFiles(); + + /// The file with an id gave. + Task GetFile(string fileId); + + /// Removes, or moves aside, a file Bitween has safely taken in. + Task DeleteFile(string fileId); + + /// Closes the session. Called after a failure too. + Task Finalize(); +} diff --git a/SW.Bitween.Adapters/SW.Bitween.Adapters.csproj b/SW.Bitween.Adapters/SW.Bitween.Adapters.csproj new file mode 100644 index 00000000..6897d60c --- /dev/null +++ b/SW.Bitween.Adapters/SW.Bitween.Adapters.csproj @@ -0,0 +1,20 @@ + + + + + netstandard2.0 + latest + enable + SimplyWorks.Bitween.Adapters + The Bitween adapter contract: the kinds of adapter Bitween runs (handler, mapper, validator, receiver), what it calls on each, and the payloads it exchanges with them. + Simplify9 + bitween;adapter;integration + true + + + + + + + + diff --git a/SW.Bitween.Adapters/ValidationResult.cs b/SW.Bitween.Adapters/ValidationResult.cs new file mode 100644 index 00000000..346fe4b4 --- /dev/null +++ b/SW.Bitween.Adapters/ValidationResult.cs @@ -0,0 +1,48 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace SW.Bitween.Adapters; + +/// +/// What a validator found. On the wire it is the JSON +/// {"Success": bool, "Validations": [{"Key": code, "Value": message}]} that +/// SW.PrimitiveTypes.InfolinkValidatorResult has always produced. +/// +public class ValidationResult +{ + private List> _validations = new(); + + /// A result with no failures. + public ValidationResult() + { + } + + /// A result with these failures. + public ValidationResult(IEnumerable> validations) + { + _validations = new List>(validations); + } + + /// True when nothing failed. + public bool Success => _validations.Count == 0; + + /// Each failure: a code (often the field it concerns) and a message. + public List> Validations + { + get => _validations; + set => _validations = value ?? new List>(); + } + + /// Records one failure. + public ValidationResult AddError(string code, string message) + { + _validations.Add(new KeyValuePair( + code ?? throw new ArgumentNullException(nameof(code)), + message ?? throw new ArgumentNullException(nameof(message)))); + return this; + } + + /// Whether a failure with this code was recorded. + public bool Has(string code) => _validations.Any(v => v.Key == code); +} diff --git a/SW.Bitween.Api/Controllers/AdapterPackagesController.cs b/SW.Bitween.Api/Controllers/AdapterPackagesController.cs new file mode 100644 index 00000000..b3567227 --- /dev/null +++ b/SW.Bitween.Api/Controllers/AdapterPackagesController.cs @@ -0,0 +1,67 @@ +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain; +using SW.Bitween.Services.Adapters; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Controllers; + +/// +/// Publishing a package through Bitween: what bitween adapter publish does by default, so the people +/// who write adapters need a Bitween account rather than the storage's keys, and every version +/// published is in the audit trail with who published it. +/// +/// +/// A controller rather than a CqApi handler because the package is the request body itself - the zip +/// bytes, as built - not JSON. Refusals answer as CqApi's do: 403 without the permission, 400 with +/// { field: [message] }. +/// +[ApiController] +[Route("api/adapters/packages")] +public class AdapterPackagesController(BitweenDbContext dbContext, RequestContext requestContext, AdapterWorkshop workshop) + : ControllerBase +{ + /// major, minor, patch or an exact version; the package's own when empty. + /// Make it the version that runs where nothing is pinned. + /// Replaces the package's own, when given. + [HttpPost] + [RequestSizeLimit(200 * 1024 * 1024)] + public async Task Post([FromQuery] string version, [FromQuery] bool current, [FromQuery] string releaseNotes) + { + try + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.AdapterSource.Operate); + } + catch (SWUnauthorizedException) + { + return StatusCode(403); + } + + var who = requestContext.GetNameIdentifier(); + var account = int.TryParse(who, out var accountId) + ? await dbContext.Set().AsNoTracking().FirstOrDefaultAsync(a => a.Id == accountId) + : null; + + try + { + var published = await workshop.UploadAsync(Request.Body, version, current, releaseNotes, account?.DisplayName ?? who); + dbContext.Add(new AdapterRelease(published.Manifest.Id, published.Version, AdapterRelease.PublishedAction, null, who)); + if (current) + dbContext.Add(new AdapterRelease(published.Manifest.Id, published.Version, AdapterRelease.PromotedAction, null, who)); + await dbContext.SaveChangesAsync(); + return Ok(new { AdapterId = published.Manifest.Id, published.Version, Current = current, published.Sha256 }); + } + catch (SWValidationException ex) + { + foreach (var v in ex.Validations) ModelState.AddModelError(v.Key, v.Value); + return BadRequest(ModelState); + } + catch (SWException ex) + { + ModelState.AddModelError("Package", ex.Message); + return BadRequest(ModelState); + } + } +} diff --git a/SW.Bitween.Api/Controllers/FileLinksController.cs b/SW.Bitween.Api/Controllers/FileLinksController.cs index 760dc732..3b739436 100644 --- a/SW.Bitween.Api/Controllers/FileLinksController.cs +++ b/SW.Bitween.Api/Controllers/FileLinksController.cs @@ -10,7 +10,7 @@ namespace SW.Bitween.Controllers; /// the public storage URL it stands in for worked, so a reader built for those links needs no change. /// /// -/// The literal "files" keeps this clear of CqApi, which owns the rest of /api/ — see +/// The literal "files" keeps this clear of CqApi, which owns the rest of /api/ - see /// . /// [ApiController] diff --git a/SW.Bitween.Api/Controllers/GatewayController.cs b/SW.Bitween.Api/Controllers/GatewayController.cs index da321abb..56512b94 100644 --- a/SW.Bitween.Api/Controllers/GatewayController.cs +++ b/SW.Bitween.Api/Controllers/GatewayController.cs @@ -21,11 +21,21 @@ public class GatewayController( RequestContext requestContext, IInfolinkCache cache, XchangeService xchangeService, - GatewayCallers callers) : ControllerBase + GatewayCallers callers, + GatewayActivity activity, + BitweenOptions options, + Services.Metrics.MetricsRecorder metrics, + ResultSignals signals) : ControllerBase { + /// A sync call's wait for its own exchange's result, from before it is published to the answer. + private ResultSignals.Wait signal; + + /// Answers a call whose payload the type dropped as a duplicate, naming the exchange it repeats. + public const string DuplicateOfHeader = "Bitween-Duplicate-Of"; + /// /// {gateway's url name}/sync or /async. The url name can be several segments, - /// so it is everything before the last one — a route can't put a literal after a catch-all. + /// so it is everything before the last one - a route can't put a literal after a catch-all. /// /// /// The literal "gateway" is what keeps this clear of CqApi, which owns the rest of /api/ with @@ -49,46 +59,96 @@ public Task Post([FromRoute] string path) private async Task ProcessAsync(string gatewayApiName, bool resultSync) { + // Creation and the sync wait, counted together; see DbCommandTally. + var commands = Services.Metrics.DbCommandTally.Open("gateway"); + try + { + return await ProcessCallAsync(gatewayApiName, resultSync); + } + finally + { + signal?.Dispose(); + commands.Record(metrics); + } + } + + private async Task ProcessCallAsync(string gatewayApiName, bool resultSync) + { + // The creation in a method of its own, so that once it returns nothing reaches what it read: + // the request body and its parsed trees, which a sync call would otherwise hold through the + // whole wait (up to Bitween:MaxResponseWaitSeconds), are garbage before the wait starts. + var call = await CreateAsync(gatewayApiName, resultSync); + if (call.Answer != null) return call.Answer; + if (!resultSync) return Accepted(call.XchangeId); + return await WaitAsync(call); + } + + /// + /// What a call's creation leaves for the rest of it: an answer when it is done already, else the + /// exchange it waits for, how long, and how the gateway answers a failure. Nothing of the payload. + /// + private sealed record CreatedCall(IActionResult Answer, string XchangeId = null, int WaitSeconds = 0, + GatewayFailureAnswer FailureAnswer = default); + + private async Task CreateAsync(string gatewayApiName, bool resultSync) + { + using var created = BitweenTelemetry.Stage("gateway.create"); var globalAdapterValuesSet = await cache.ListGlobalAdapterValuesSetsAsync(); - // Lowered on the column too: rows saved before names had to be lowercase may not be. - var apiGateway = await dbContext.Set() - .Include(ag => ag.Partners) - .ThenInclude(agp => agp.Partner) - .FirstOrDefaultAsync(ag => ag.UrlName.ToLower() == gatewayApiName); + // From the cache, with its attachments and partners, as the subscriptions are: these were two + // queries on every call. A save to a gateway, an attachment, a partner or a key revokes it. + var apiGateway = await cache.ApiGatewayByUrlNameAsync(gatewayApiName); + var address = HttpContext.Connection.RemoteIpAddress?.ToString(); if (apiGateway == null) - return NotFound(); + { + activity.Refused(gatewayApiName, "unknown-gateway", 404, address); + return new CreatedCall(NotFound()); + } // Resolve the partner the way this gateway asks callers to prove who they are - var (authorized, partner, callerReference) = await callers.Identify(apiGateway, requestContext); + var (authorized, partner, callerReference, keyName) = await callers.Identify(apiGateway, requestContext); if (!authorized) - return Unauthorized(); + { + activity.Refused(gatewayApiName, "not-authenticated", 401, address); + return new CreatedCall(Unauthorized()); + } // Verify partner is part of the API Gateway var apiGatewayPartner = apiGateway.Partners.FirstOrDefault(agp => agp.PartnerId == partner.Id); if (apiGatewayPartner == null) - return Unauthorized(); + { + activity.Refused(gatewayApiName, "not-attached", 401, address); + return new CreatedCall(Unauthorized()); + } + await activity.KeyUsedAsync(partner.Id, keyName); + Services.Metrics.DbCommandTally.Current?.AttributeTo(DateTime.UtcNow, apiGatewayPartner.SubscriptionId, partner.Id); // After authorisation on purpose: whether a gateway exists and is switched off is - // something only an attached partner should learn — checking it earlier would + // something only an attached partner should learn - checking it earlier would // answer that for anyone who guessed the url. // // 503 rather than 404 because the url is right and the partner should keep it: a // 404 reads as "wrong address" and sends someone hunting for a new one, where this // is a gateway somebody switched off and will switch back on. if (apiGateway.Inactive) - return StatusCode(StatusCodes.Status503ServiceUnavailable, - $"The '{apiGateway.Name}' gateway is currently deactivated."); + { + activity.Refused(gatewayApiName, "gateway-off", 503, address); + return new CreatedCall(StatusCode(StatusCodes.Status503ServiceUnavailable, + $"The '{apiGateway.Name}' gateway is currently deactivated.")); + } var subscription = await cache.SubscriptionByIdAsync(apiGatewayPartner.SubscriptionId); if (subscription == null) - return NotFound(); + { + activity.Refused(gatewayApiName, "no-subscription", 404, address); + return new CreatedCall(NotFound()); + } var json = await new StreamReader(HttpContext.Request.Body).ReadToEndAsync(); - // The file name travels with the exchange into handlers, some of which write it to disk — + // The file name travels with the exchange into handlers, some of which write it to disk - // a slash there is a directory nobody asked for. JSON and XML types keep the ".json" // they have always had, since whatever picks those files up may expect it. var document = await cache.DocumentByIdAsync(subscription.DocumentId); @@ -98,63 +158,164 @@ private async Task ProcessAsync(string gatewayApiName, bool resul DocumentFormat.Other => "", _ => ".json", }; - var xchangeFile = new XchangeFile(json, $"{gatewayApiName.Replace('/', '-')}{extension}"); + // Carrying its parsed trees, so the schema check below and the promoted-property read at + // creation share one parse, unless Bitween:SharedPayloadParse is off (finding 6c). + var fileName = $"{gatewayApiName.Replace('/', '-')}{extension}"; + var xchangeFile = options.SharedPayloadParse + ? new NativeAdapters.ParsedXchangeFile(json, fileName) + : new XchangeFile(json, fileName); + + // Told now, while the partner is still on the line, rather than as a failed exchange later. + var schemaErrors = document is null + ? [] + : await DocumentSchema.Check(document.DocumentFormat, document.ValidationSchema, xchangeFile); + if (schemaErrors.Count > 0) + return new CreatedCall(BadRequest(new { error = "SCHEMA_MISMATCH", errors = schemaErrors })); var validatorProperties = subscription.ValidatorProperties.ToDictionary() .Fill(partner, globalAdapterValuesSet); - await xchangeService.RunValidator(subscription.ValidatorId, validatorProperties, + await xchangeService.RunValidator(subscription.ValidatorRef, validatorProperties, xchangeFile); var xchangeReferences = new List { callerReference }; - var globalAdapterValuesSets = await dbContext.Set().ToArrayAsync(); - var xchangeId = await xchangeService.SubmitSubscriptionXchange(subscription.Id, xchangeFile, - xchangeReferences.ToArray(), partner, globalAdapterValuesSets); - if (!resultSync) + + // The caller may ask for its own wait in Wait-Period; it is capped, see ResultWait. + var waitResponse = ResultWait.Clamp( + int.TryParse(Request.Headers["Wait-Period"].FirstOrDefault(), out var asked) ? asked : null, + options.MaxResponseWaitSeconds); + + // A repeat of a call the partner couldn't be sure arrived - same Idempotency-Key - gets the + // exchange the first one made, not a second. Scoped to this gateway and this partner. + var idempotencyKey = Request.Headers[IdempotencyGuard.Header].FirstOrDefault()?.Trim(); + if (idempotencyKey?.Length > IdempotencyGuard.MaxKeyLength) + return new CreatedCall(BadRequest($"{IdempotencyGuard.Header} is longer than {IdempotencyGuard.MaxKeyLength} characters.")); + var idempotencyScope = $"gateway:{apiGateway.Id}:partner:{partner.Id}"; + + var xchangeId = string.IsNullOrEmpty(idempotencyKey) + ? null + : await IdempotencyGuard.FindAsync(dbContext, idempotencyScope, idempotencyKey); + // A mirror's rehearsal calls as prod's partner did, under a correlation of its own, and sends + // what the rehearsal before it sent. + var rehearsal = Services.Mirror.MirrorTraffic.CorrelationFor(options, Request.Headers); + // Not for a replayed call: its payload is a repeat by definition, and it is answered already. + var duplicate = xchangeId is null && rehearsal is null + ? await new DuplicateArrivals(dbContext).CheckAsync(document, partner.Id, xchangeFile) + : null; + + if (xchangeId is not null) { - return Accepted(xchangeId); + Response.Headers["Idempotent-Replay"] = "true"; } - - var waitResponse = 120; - // check headers for wait response value - var waitResponseHeader = Request.Headers["Wait-Period"].FirstOrDefault(); - if (int.TryParse(waitResponseHeader, out var waitResponseValue)) + else if (duplicate?.Drop == true) { - waitResponse = waitResponseValue <= 0 ? 120 : waitResponseValue; + // The same payload from this partner within the type's duplicate window, which the type + // drops: answered with the exchange it repeats, as a replayed call would be. + await dbContext.SaveChangesAsync(); + xchangeId = duplicate.OriginalXchangeId; + Response.Headers[DuplicateOfHeader] = xchangeId; } - - var currentFibTerm = 1; - var previousTerm = 1; - while (currentFibTerm <= waitResponse) + else { - await Task.Delay(TimeSpan.FromSeconds(currentFibTerm)); - var nextTerm = Math.Min(currentFibTerm + previousTerm, 8); - previousTerm = currentFibTerm; - currentFibTerm = nextTerm; - if (!await dbContext.Set() - .AsNoTracking() - .AnyAsync(i => i.Id == xchangeId)) continue; + if (duplicate != null && duplicate.FitsWith(xchangeReferences)) xchangeReferences.Add(duplicate.Reference); + + // The sets already read at the top of the request, from the cache. + var xchange = await xchangeService.CreateXchange(subscription, xchangeFile, xchangeReferences.ToArray(), + rehearsal ?? Guid.NewGuid().ToString("N"), partner, globalAdapterValuesSet); + if (resultSync) + { + // Before the save publishes the exchange, so its result can't arrive unannounced. The + // created message names this node, and the worker rings the bell here once the result + // is saved. + signal = signals.Register(xchange.Id, TimeSpan.FromSeconds(waitResponse + 5)); + dbContext.Handoff.AwaitedOn(xchange.Id, signals.Node); + } + + if (string.IsNullOrEmpty(idempotencyKey)) + { + await dbContext.SaveChangesAsync(); + xchangeId = xchange.Id; + } + else + { + await IdempotencyGuard.StageAsync(dbContext, idempotencyScope, idempotencyKey, xchange.Id); + (xchangeId, var replayed) = + await IdempotencyGuard.SaveAsync(dbContext, idempotencyScope, idempotencyKey, xchange.Id); + if (replayed) Response.Headers["Idempotent-Replay"] = "true"; + } - var xchangeResult = await dbContext.FindAsync(xchangeId); + // Answered with an earlier call's exchange instead: this one was never published. + if (xchangeId != xchange.Id) + { + signal?.Dispose(); + signal = null; + } + } + + return new CreatedCall(null, xchangeId, waitResponse, apiGateway.FailureAnswer); + } + + private async Task WaitAsync(CreatedCall call) + { + var xchangeId = call.XchangeId; + var waitResponse = call.WaitSeconds; + using var waited = BitweenTelemetry.Stage("gateway.wait"); + // Each check reads the result itself rather than asking whether there is one and then reading + // it: one query once it is there, where it was two. + async Task ReadResultAsync() => XchangeResultReady.From( + await dbContext.Set().AsNoTracking().FirstOrDefaultAsync(i => i.Id == xchangeId)); - switch (xchangeResult!.Success) + XchangeResultReady ready; + if (signal != null) + { + // The worker's signal, with the database asked only as a backstop: a signal lost to a + // restart, or a worker on a version that sends none. + ready = await ResultWait.UntilAsync(signal.Result, ReadResultAsync, waitResponse, HttpContext.RequestAborted); + } + else + { + // A replayed or dropped call, answered with an exchange this call didn't create: usually + // finished already, so asked at once, then polled. + ready = await ReadResultAsync(); + if (ready == null) + await ResultWait.UntilAsync(async () => (ready = await ReadResultAsync()) != null, + waitResponse, HttpContext.RequestAborted); + } + + if (ready != null) + { + switch (ready.Success) { - case true when xchangeResult.ResponseSize == 0: + // A delivered response whose file couldn't be stored has no size, but the signal still + // brings it when it is small enough to travel; without it there is nothing to read. + case true when ready.ResponseSize == 0 && ready.Response == null: { return Ok(xchangeId); } - case true when xchangeResult.ResponseSize != 0: + case true: { - var response = await xchangeService.GetFile(xchangeId, XchangeFileType.Response); + // From the signal when it brought the response; from storage when it was too + // large to travel, or the backstop found the result. + var response = ready.Response ?? await xchangeService.GetFile(xchangeId, XchangeFileType.Response); return new ContentResult { - StatusCode = xchangeResult.ResponseBad ? 400 : 200, + StatusCode = ready.ResponseBad ? 400 : 200, Content = response, - ContentType = xchangeResult.ResponseContentType ?? MediaTypeNames.Application.Json, + ContentType = ready.ResponseContentType ?? MediaTypeNames.Application.Json, }; } case false: - return BadRequest(); + { + if (call.FailureAnswer == GatewayFailureAnswer.Legacy) return BadRequest(); + // The retry, if any, is scheduled in the same save as the result, so it is there now. + var retryOn = await dbContext.Set().AsNoTracking() + .Where(r => r.Id == xchangeId).Select(r => (DateTime?)r.On).FirstOrDefaultAsync(); + var hasMapper = await dbContext.Set().AsNoTracking() + .Where(x => x.Id == xchangeId).Select(x => x.MapperId != null).FirstOrDefaultAsync(); + var answer = GatewayFailureAnswers.For(call.FailureAnswer, xchangeId, hasMapper, + mapped: ready.OutputSize > 0 || ready.OutputName != null, retryOn); + return new ObjectResult(answer.Body) { StatusCode = answer.Status }; + } } } diff --git a/SW.Bitween.Api/Controllers/GatewayFailureAnswers.cs b/SW.Bitween.Api/Controllers/GatewayFailureAnswers.cs new file mode 100644 index 00000000..480a467d --- /dev/null +++ b/SW.Bitween.Api/Controllers/GatewayFailureAnswers.cs @@ -0,0 +1,49 @@ +using System; +using SW.Bitween.Domain; +using SW.Bitween.Model; + +namespace SW.Bitween.Controllers; + +/// +/// What a sync gateway call is told when the delivery behind it failed. Legacy says 400 whatever +/// happened, which tells the caller its request was wrong: in the lab, orders refused by a warehouse +/// that was down were answered 400 and then delivered by retry minutes later, so the marketplace was +/// told "rejected" about orders that went through. Accurate says what Bitween will do next. +/// +public static class GatewayFailureAnswers +{ + public record Answer(int Status, object Body); + + /// Whether the subscription maps the message before delivering it. + /// Whether mapping produced an output, so the failure came after it. + /// When the retry Bitween scheduled is due; null when there is none. + public static Answer For(GatewayFailureAnswer mode, string exchangeId, bool hasMapper, bool mapped, DateTime? retryOn) + { + if (mode == GatewayFailureAnswer.Legacy) return new Answer(400, null); + + // Kept until the destination's retry budget is lifted: Bitween has it, and will try it again. + if (retryOn >= DelayedRetry.HeldOn) + return new Answer(202, new + { + exchangeId, status = "held", + error = "The destination has been failing past its retry budget. Bitween keeps the message and tries it again once the destination recovers.", + }); + + // Bitween has the message and will deliver it again: sending it again would make a second one. + if (retryOn is { } next) + return new Answer(202, new { exchangeId, status = "retrying", nextAttemptOn = next }); + + if (hasMapper && !mapped) + return new Answer(422, new + { + exchangeId, status = "failed", + error = "The message couldn't be mapped to what the destination takes. It won't be retried.", + }); + + return new Answer(502, new + { + exchangeId, status = "failed", + error = "The delivery to the destination failed. It won't be retried.", + }); + } +} diff --git a/SW.Bitween.Api/Controllers/KitPlanController.cs b/SW.Bitween.Api/Controllers/KitPlanController.cs new file mode 100644 index 00000000..d2874998 --- /dev/null +++ b/SW.Bitween.Api/Controllers/KitPlanController.cs @@ -0,0 +1,67 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using SW.Bitween.Services.Marketplace; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Controllers; + +/// +/// Plans a kit sent as its zip, changing nothing: what installing it here would do, or why it can't be. +/// The registry calls it on a reference Bitween for every kit published, so a kit whose bundle doesn't +/// plan never reaches review; a publisher can call it on their own Bitween too. +/// +/// +/// A controller, as is: the zip is the request body. It plans as +/// the caller, through this Bitween's own API, and reads nothing but the kit's own files and parameters. +/// +[ApiController] +[Route("api/kits/plan")] +public class KitPlanController(BitweenDbContext dbContext, RequestContext requestContext, KitInstaller kits) : ControllerBase +{ + /// The largest kit planned: a kit is YAML and a few schemas, never a package's binaries. + const int MaxKitBytes = 20 * 1024 * 1024; + + /// What the installation would be called; the kit's id when empty. + /// A required parameter not given takes the kit's example, or a placeholder. + /// Parameters as NAME=value, repeated. + [HttpPost] + [RequestSizeLimit(MaxKitBytes)] + public async Task Post([FromQuery] string name, [FromQuery] bool examples, [FromQuery] string[] set) + { + try + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.AdapterSource.Operate); + } + catch (SWUnauthorizedException) + { + return StatusCode(403); + } + + var parameters = new Dictionary(); + foreach (var pair in set ?? []) + { + var at = pair.IndexOf('='); + if (at <= 0) + { + ModelState.AddModelError("set", $"{pair}: give a parameter as NAME=value."); + return BadRequest(ModelState); + } + parameters[pair[..at].Trim()] = pair[(at + 1)..]; + } + + using var body = new MemoryStream(); + await Request.Body.CopyToAsync(body); + try + { + return Ok(await kits.PlanPackageAsync(body.ToArray(), name, parameters, examples)); + } + catch (SWValidationException ex) + { + foreach (var v in ex.Validations) ModelState.AddModelError(v.Key, v.Value); + return BadRequest(ModelState); + } + } +} diff --git a/SW.Bitween.Api/Controllers/MarketplaceBundleController.cs b/SW.Bitween.Api/Controllers/MarketplaceBundleController.cs new file mode 100644 index 00000000..7c1af908 --- /dev/null +++ b/SW.Bitween.Api/Controllers/MarketplaceBundleController.cs @@ -0,0 +1,57 @@ +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using SW.Bitween.Domain; +using SW.Bitween.Services.Marketplace; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Controllers; + +/// +/// Install from bundle: the zip bitween registry bundle downloaded, for a Bitween that can't reach +/// the registry. A controller because the bundle is the request body itself. Needs the trusted keys +/// in Settings, not the marketplace to be on: an offline site turns the marketplace off. +/// +[ApiController] +[Route("api/marketplace/bundle")] +public class MarketplaceBundleController(BitweenDbContext dbContext, RequestContext requestContext, MarketplaceInstaller installer) + : ControllerBase +{ + /// Make each installed version the one that runs where nothing is pinned. + [HttpPost] + [RequestSizeLimit(600 * 1024 * 1024)] + public async Task Post([FromQuery] bool current) + { + try + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.AdapterSource.Operate); + } + catch (SWUnauthorizedException) + { + return StatusCode(403); + } + + var who = requestContext.GetNameIdentifier(); + try + { + var result = await installer.InstallBundleAsync(Request.Body, current, who); + foreach (var item in result.Items.Where(i => i.Outcome == "Installed")) + { + dbContext.Add(new AdapterRelease(item.Id, item.Version, AdapterRelease.InstalledAction, null, who)); + if (current) dbContext.Add(new AdapterRelease(item.Id, item.Version, AdapterRelease.PromotedAction, null, who)); + } + await dbContext.SaveChangesAsync(); + return Ok(result); + } + catch (SWValidationException ex) + { + foreach (var v in ex.Validations) ModelState.AddModelError(v.Key, v.Value); + return BadRequest(ModelState); + } + catch (SWException ex) + { + ModelState.AddModelError("Bundle", ex.Message); + return BadRequest(ModelState); + } + } +} diff --git a/SW.Bitween.Api/Controllers/XchangeExportController.cs b/SW.Bitween.Api/Controllers/XchangeExportController.cs index 394c1708..22d32355 100644 --- a/SW.Bitween.Api/Controllers/XchangeExportController.cs +++ b/SW.Bitween.Api/Controllers/XchangeExportController.cs @@ -22,7 +22,7 @@ namespace SW.Bitween.Controllers; /// /// The files of several exchanges as one zip, for the exchange list's "Export files": one folder per /// exchange, named like its archive ({main value}_{id}), holding input, mapped -/// and handled — whichever it has. says beforehand how many exchanges are +/// and handled - whichever it has. says beforehand how many exchanges are /// older than storage keeps their files, so the person hears it before choosing to export. Files missing /// anyway, or for another reason, are listed in missing.txt, so one of them never costs the rest. /// @@ -33,8 +33,8 @@ namespace SW.Bitween.Controllers; /// {prefix}/{resource}/{token}, the way 's does. /// /// -/// Refusals answer the way CqApi does — 401 without the permission, 400 with { code: [message] } -/// — so the client reads them like any other call. They're all decided before the first byte is sent. +/// Refusals answer the way CqApi does - 401 without the permission, 400 with { code: [message] } +/// - so the client reads them like any other call. They're all decided before the first byte is sent. /// /// [ApiController] @@ -44,7 +44,7 @@ public class XchangeExportController(BitweenDbContext dbContext, RequestContext : ControllerBase { /// - /// The most exchanges one zip takes — the bulk retry's ceiling, for the same reason: every file is + /// The most exchanges one zip takes - the bulk retry's ceiling, for the same reason: every file is /// read from storage in sequence, inside the one request. /// public const int Limit = 500; @@ -102,6 +102,11 @@ public async Task Post([FromBody] XchangeFilesExport request) var missing = ids.Except(xchanges.Select(x => x.Id)) .Select(id => $"{id}: this exchange no longer exists.").ToList(); + // Exchange files leave Bitween here, so who took which is recorded before they do. + await dbContext.RecordNowAsync(requestContext, OperatorAction.Export, + xchanges.Count == 1 ? xchanges[0].Id : $"{xchanges.Count} exchanges", + string.Join(", ", xchanges.Take(20).Select(x => x.Id)) + (xchanges.Count > 20 ? ", …" : "")); + Response.ContentType = "application/zip"; Response.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment") { @@ -124,9 +129,11 @@ await Add(zip, xchange, XchangeFileType.Input, $"{folder}/input", if (!results.TryGetValue(xchange.Id, out var result)) continue; await Add(zip, xchange, XchangeFileType.Output, $"{folder}/mapped", - result.OutputName, result.OutputSize, result.OutputContentType, missing, aborted); + result.OutputName, result.OutputSize, result.OutputContentType, missing, aborted, + result.NotStoredReason(XchangeFileType.Output)); await Add(zip, xchange, XchangeFileType.Response, $"{folder}/handled", - result.ResponseName, result.ResponseSize, result.ResponseContentType, missing, aborted); + result.ResponseName, result.ResponseSize, result.ResponseContentType, missing, aborted, + result.NotStoredReason(XchangeFileType.Response)); } if (missing.Count > 0) @@ -143,11 +150,18 @@ await list.WriteAsync(Encoding.UTF8.GetBytes( /// /// One file into the zip, named plus the extension its own name or content type - /// gives. A file the exchange never had is left out; one storage no longer has goes on the missing list. + /// gives. A file the exchange never had is left out; one storage no longer has, or never got because it + /// couldn't be stored after the delivery (), goes on the missing list. /// private async Task Add(ZipArchive zip, Xchange xchange, XchangeFileType type, string path, string name, int size, - string contentType, List missing, CancellationToken aborted) + string contentType, List missing, CancellationToken aborted, string notStored = null) { + if (notStored != null) + { + missing.Add($"{path + ExtensionOf(name, contentType)}: it was delivered, but could not be stored: {notStored}"); + return; + } + if (size == 0) return; path += ExtensionOf(name, contentType); diff --git a/SW.Bitween.Api/Data/AuditPolicy.cs b/SW.Bitween.Api/Data/AuditPolicy.cs index 27fe0a9e..0e445377 100644 --- a/SW.Bitween.Api/Data/AuditPolicy.cs +++ b/SW.Bitween.Api/Data/AuditPolicy.cs @@ -4,6 +4,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using SW.Bitween.Domain; using SW.Bitween.Domain.Accounts; +using SW.Bitween.Domain.Connections; using SW.Bitween.Domain.Gateway; using SW.Bitween.Services; using SW.EfCoreExtensions; @@ -34,8 +35,8 @@ public static class AuditPolicy { /// /// The configuration entities, their owned children, and the account/role records. Runtime - /// traffic — every Xchange, result, delivery, receive attempt, retry usage counter and - /// refresh token — is deliberately absent. + /// traffic - every Xchange, result, delivery, receive attempt, retry usage counter and + /// refresh token - is deliberately absent. /// private static readonly HashSet Audited = [ @@ -53,17 +54,31 @@ public static class AuditPolicy typeof(RetryPolicy), typeof(RetryAlertOverride), typeof(Notifier), + typeof(NotificationChannel), typeof(GlobalAdapterValuesSet), + // Who added a connection, changed its client, or authorized it. Token renewals bypass the + // change tracker, so the hourly refresh of a token isn't an entry. + typeof(Connection), typeof(Setting), typeof(Account), typeof(Role), - typeof(AccountRoleLink) + typeof(AccountRoleLink), + // Issued and revoked: who made a credential, and who ended it. + typeof(ServiceToken), + // Not configuration, but reading an adapter's source is what the trail is for. + typeof(AdapterSourceAccess), + // Code written in Bitween, and what was published and made current from it. + typeof(AdapterDraft), + typeof(AdapterRelease), + // Retries, exports, requeues and queue deletions: not configuration, but who did them is + // what the trail is asked. + typeof(OperatorAction) ]; /// /// Properties whose values never enter the trail. The adapter property bags are here because /// each is stored as a single JSON column, so the change tracker sees one property rather than - /// the individual keys — there is no way to keep Host and drop Password without + /// the individual keys - there is no way to keep Host and drop Password without /// cracking the JSON open and asking the adapter which of its keys are [Secure], which /// would mean starting a serverless adapter in the middle of a save. Recording that a /// subscription was edited without saying which adapter key changed is the deliberate trade. @@ -79,6 +94,7 @@ public static class AuditPolicy ], [typeof(Partner)] = [nameof(Partner.AdapterProperties)], [typeof(Notifier)] = [nameof(Notifier.HandlerProperties)], + [typeof(NotificationChannel)] = [nameof(NotificationChannel.HandlerProperties)], [typeof(RetryPolicy)] = [nameof(RetryPolicy.AlertHandlerProperties)], [typeof(RetryAlertOverride)] = [nameof(RetryAlertOverride.AlertHandlerProperties)], @@ -89,7 +105,22 @@ public static class AuditPolicy // The key itself is the credential. Name survives, so adding or revoking one is visible. [typeof(ApiCredential)] = [nameof(ApiCredential.Key)], - [typeof(Account)] = [nameof(Account.Password)] + [typeof(Account)] = [nameof(Account.Password)], + + [typeof(Connection)] = + [ + nameof(Connection.ClientSecret), + nameof(Connection.Password), + nameof(Connection.RefreshToken), + nameof(Connection.AccessToken) + ], + + // A hash of a random token, so no use to anyone; kept out anyway, like a partner's key. + [typeof(ServiceToken)] = [nameof(ServiceToken.Hash)], + + // Not a secret, but the code would be in the trail twice on every save. FilesHash is + // recorded instead: each save shows, and the code itself is in the published package. + [typeof(AdapterDraft)] = [nameof(AdapterDraft.FilesJson)] }; public static readonly AuditOptions Options = new() @@ -105,7 +136,7 @@ private static bool ShouldAuditProperty(EntityEntry entry, IProperty property) return false; // A setting's value is only sometimes a secret, and unlike an adapter's properties the - // catalog says which — a static lookup, no adapter to start — so the theme colour stays + // catalog says which - a static lookup, no adapter to start - so the theme colour stays // readable in the trail while the licence key never enters it. if (entry.Entity is Setting setting && property.Name == nameof(Setting.Value)) return SettingsCatalog.Find(setting.Id)?.Secret != true; diff --git a/SW.Bitween.Api/Data/BitweenDbContext.cs b/SW.Bitween.Api/Data/BitweenDbContext.cs index f50b5cf7..1b569444 100644 --- a/SW.Bitween.Api/Data/BitweenDbContext.cs +++ b/SW.Bitween.Api/Data/BitweenDbContext.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System; using Microsoft.EntityFrameworkCore; using SW.EfCoreExtensions; @@ -15,13 +16,15 @@ namespace SW.Bitween { public class BitweenDbContext(DbContextOptions options, RequestContext requestContext, - IPublish publish) : DbContext(options) + IPublish publish, IInfolinkCache cache = null, OutboxAcknowledger outboxAcknowledger = null) : DbContext(options) { private readonly RequestContext requestContext = requestContext; private readonly IPublish publish = publish; + private readonly IInfolinkCache cache = cache; + private readonly OutboxAcknowledger outboxAcknowledger = outboxAcknowledger; // Parsed as Unspecified kind, so the .ToUniversalTime() calls at every use site below used - // to convert using whatever timezone the current machine happened to be in — deterministic + // to convert using whatever timezone the current machine happened to be in - deterministic // on any one developer's machine, but different on every other machine/CI runner, which // permanently disagreed with whichever machine last regenerated the migration snapshot. protected readonly DateTime defaultCreatedOn = DateTime.SpecifyKind(DateTime.Parse("1/1/2022"), DateTimeKind.Utc); @@ -32,6 +35,19 @@ public class BitweenDbContext(DbContextOptions options, RequestContext requestCo public const string ConnectionString = "BitweenDb"; + /// What this unit of work's exchanges hand to its save; see . + internal XchangeHandoff Handoff { get; } = new(); + + /// + /// Every provider's context counts its commands against the exchange stage running, so the + /// round trips an exchange costs are measured rather than estimated (see DbCommandTally). + /// + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + { + base.OnConfiguring(optionsBuilder); + optionsBuilder.AddInterceptors(Services.Metrics.DbCommandCounter.Instance); + } + protected override void OnModelCreating(ModelBuilder modelBuilder) { base.OnModelCreating(modelBuilder); @@ -130,7 +146,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) ds.Property(p => p.Name).IsRequired().HasMaxLength(200); ds.Property(p => p.AdapterId).IsRequired().HasMaxLength(200).IsUnicode(false); ds.Property(p => p.Kind).HasConversion(); - ds.Property(p => p.Properties).StoreAsJson(); + ds.Property(p => p.Properties).StoreAsJson().Encrypted(); ds.Property(p => p.SecretProperties).StoreAsJson(); ds.Property(p => p.LastKnownState).HasMaxLength(100).IsUnicode(false); ds.Property(p => p.OwnedByNode).HasMaxLength(200).IsUnicode(false); @@ -146,7 +162,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) st.Property(p => p.Sql).IsRequired(); st.Property(p => p.Description).HasMaxLength(1000); - // Column names, so the database's own identifier limit is the ceiling — 128 is + // Column names, so the database's own identifier limit is the ceiling - 128 is // above every engine's (Oracle allows 128, PostgreSQL 63). st.Property(p => p.CursorColumn).HasMaxLength(128).IsUnicode(false); st.Property(p => p.KeyColumn).HasMaxLength(128).IsUnicode(false); @@ -180,7 +196,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) // Declared here rather than only in the PgSql context: leader election needs this table // on every provider Bitween supports, and a node whose database has no cluster_lease - // cannot fence anything — which means two nodes can consume one queue, silently. + // cannot fence anything - which means two nodes can consume one queue, silently. modelBuilder.Entity(cl => { cl.ToTable("ClusterLeases"); @@ -194,7 +210,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) im.ToTable("InboundMessages"); // The dedupe key IS the key. A unique constraint the database enforces is the - // whole mechanism — see the type's remarks. + // whole mechanism - see the type's remarks. im.HasKey(i => i.Id); im.Property(i => i.Id).HasMaxLength(400).IsUnicode(false); im.Property(i => i.XchangeId).HasMaxLength(50).IsUnicode(false); @@ -225,7 +241,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) gav.ToTable("GlobalAdapterValuesSets"); gav.HasKey(i => i.Id); gav.Property(p => p.Id).IsUnicode(false).HasMaxLength(200); - gav.Property(p => p.Values).StoreAsJson(); + gav.Property(p => p.Values).StoreAsJson().Encrypted(); gav.Property(p => p.SecretProperties).StoreAsJson(); }); modelBuilder.Entity(b => @@ -233,7 +249,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) b.ToTable("Partners"); b.Metadata.SetNavigationAccessMode(PropertyAccessMode.Field); b.Property(p => p.Name).IsRequired().IsUnicode(false).HasMaxLength(200); - b.Property(p => p.AdapterProperties).StoreAsJson(); + b.Property(p => p.AdapterProperties).StoreAsJson().Encrypted(); b.Property(p => p.SecretProperties).StoreAsJson(); b.Property(p => p.LoginIdentity).HasMaxLength(500); b.HasIndex(p => p.LoginIdentity).IsUnique(); @@ -244,6 +260,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) apicred.ToTable("PartnerApiCredentials"); apicred.Property(p => p.Name).IsRequired().HasMaxLength(500); apicred.Property(p => p.Key).IsRequired().IsUnicode(false).HasMaxLength(500); + apicred.Property(p => p.KeyPrefix).HasMaxLength(10); apicred.HasIndex(p => p.Key).IsUnique(); apicred.WithOwner().HasForeignKey("PartnerId"); @@ -276,16 +293,21 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) schedules.Property(p => p.Recurrence).HasConversion(); }); - b.Property(p => p.HandlerProperties).StoreAsJson(); - b.Property(p => p.MapperProperties).StoreAsJson(); - b.Property(p => p.ReceiverProperties).StoreAsJson(); - b.Property(p => p.ValidatorProperties).StoreAsJson(); + b.Property(p => p.HandlerProperties).StoreAsJson().Encrypted(); + b.Property(p => p.MapperProperties).StoreAsJson().Encrypted(); + b.Property(p => p.ReceiverProperties).StoreAsJson().Encrypted(); + b.Property(p => p.ValidatorProperties).StoreAsJson().Encrypted(); b.Property(p => p.DocumentFilter).StoreAsJson(); b.Property(p => p.ResponseMessageTypeName).IsUnicode(false).HasMaxLength(500); b.Property(p => p.MapperId).HasMaxLength(200).IsUnicode(false); b.Property(p => p.HandlerId).HasMaxLength(200).IsUnicode(false); + // Pinned versions, null to follow whatever is current. See SubscriptionConfiguration.HandlerVersion. + b.Property(p => p.ReceiverVersion).HasMaxLength(50).IsUnicode(false); + b.Property(p => p.ValidatorVersion).HasMaxLength(50).IsUnicode(false); + b.Property(p => p.MapperVersion).HasMaxLength(50).IsUnicode(false); + b.Property(p => p.HandlerVersion).HasMaxLength(50).IsUnicode(false); b.Property(p => p.ReceiverId).HasMaxLength(200).IsUnicode(false); b.Property(p => p.ValidatorId).HasMaxLength(200).IsUnicode(false); @@ -306,6 +328,8 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) b.HasOne(i => i.RetryPolicy).WithMany().HasForeignKey(i => i.RetryPolicyId).IsRequired(false) .OnDelete(DeleteBehavior.SetNull); b.Property(p => p.CustomRetryPolicy).StoreAsJson(); + b.Property(p => p.ExpectTraffic).StoreAsJson(); + b.Property(p => p.Notifications).StoreAsJson(); b.Property(p => p.MatchExpression).HasMatchExpressionConversion(); }); @@ -317,7 +341,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) b.Property(p => p.Name).IsRequired().HasMaxLength(200); b.Property(p => p.Groups).StoreAsJson(); b.Property(p => p.AlertHandlerId).HasMaxLength(200).IsUnicode(false); - b.Property(p => p.AlertHandlerProperties).StoreAsJson(); + b.Property(p => p.AlertHandlerProperties).StoreAsJson().Encrypted(); }); modelBuilder.Entity(b => @@ -329,6 +353,152 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) b.HasIndex(p => p.On); }); + modelBuilder.Entity(b => + { + b.ToTable("IdempotencyKeys"); + b.Property(p => p.Id).ValueGeneratedOnAdd(); + b.Property(p => p.Scope).IsRequired().HasMaxLength(200); + b.Property(p => p.Key).IsRequired().HasMaxLength(200); + b.Property(p => p.XchangeId).IsRequired().IsUnicode(false).HasMaxLength(50); + b.HasIndex(p => new { p.Scope, p.Key }).IsUnique(); + b.HasIndex(p => p.CreatedOn); + }); + + modelBuilder.Entity(b => + { + b.ToTable("OutboxMessages"); + b.Property(p => p.Id).ValueGeneratedOnAdd(); + b.Property(p => p.MessageType).IsRequired().HasMaxLength(500); + b.Property(p => p.Body).IsRequired(); + b.Property(p => p.Headers).StoreAsJson(); + b.Property(p => p.LastError).HasMaxLength(2000); + // The dispatcher's index on unpublished rows differs by provider: see the SQL Server + // and MySQL contexts. + }); + + modelBuilder.Entity(b => + { + b.ToTable("OperatorActions"); + b.HasKey(a => a.Id); + b.Property(a => a.Id).ValueGeneratedOnAdd(); + b.Property(a => a.Action).HasMaxLength(50).IsRequired(); + b.Property(a => a.Target).HasMaxLength(500); + b.Property(a => a.Detail).HasMaxLength(2000); + b.Property(a => a.AccountId).HasMaxLength(50); + b.HasIndex(a => a.OccurredOn); + }); + + modelBuilder.Entity(b => + { + b.ToTable("ApiKeyUses"); + b.HasKey(u => new { u.PartnerId, u.KeyName }); + b.Property(u => u.KeyName).HasMaxLength(500); + }); + + modelBuilder.Entity(b => + { + b.ToTable("ClusterNodes"); + b.Property(p => p.Id).HasMaxLength(200).IsUnicode(false); + b.Property(p => p.Host).HasMaxLength(200).IsUnicode(false); + b.Property(p => p.Version).HasMaxLength(50).IsUnicode(false); + b.Property(p => p.Runtimes).HasMaxLength(100).IsUnicode(false); + b.Property(p => p.ResidentHealth).HasMaxLength(4000); + b.HasIndex(p => p.LastSeenOn); + }); + + // Metrics: hourly rollups each node adds to once a minute, and broker snapshots. Keys lead + // with the hour, so reading a time range and pruning old hours both use the key itself. + modelBuilder.Entity(b => + { + b.ToTable("ExchangeStats"); + b.HasKey(p => new { p.Hour, p.Node, p.SubscriptionId, p.PartnerId }); + b.Property(p => p.Node).HasMaxLength(100).IsUnicode(false); + }); + + modelBuilder.Entity(b => + { + b.ToTable("AdapterStats"); + b.HasKey(p => new { p.Hour, p.Node, p.AdapterId, p.Version, p.Role, p.Runtime }); + b.Property(p => p.Node).HasMaxLength(100).IsUnicode(false); + b.Property(p => p.AdapterId).HasMaxLength(200).IsUnicode(false); + b.Property(p => p.Version).HasMaxLength(50).IsUnicode(false); + b.Property(p => p.Role).HasMaxLength(20).IsUnicode(false); + b.Property(p => p.Runtime).HasMaxLength(20).IsUnicode(false); + }); + + modelBuilder.Entity(b => + { + b.ToTable("ErrorStats"); + b.HasKey(p => new { p.Hour, p.Node, p.Scope, p.ScopeKey, p.Fingerprint }); + b.Property(p => p.Node).HasMaxLength(100).IsUnicode(false); + b.Property(p => p.Scope).HasMaxLength(20).IsUnicode(false); + b.Property(p => p.ScopeKey).HasMaxLength(200).IsUnicode(false); + b.Property(p => p.Fingerprint).HasMaxLength(16).IsUnicode(false); + b.Property(p => p.Sample).HasMaxLength(500); + }); + + modelBuilder.Entity(b => + { + b.ToTable("QueueSnapshots"); + b.HasKey(p => p.At); + }); + + modelBuilder.Entity(b => + { + b.ToTable("AdapterDrafts"); + b.Property(p => p.Id).ValueGeneratedOnAdd(); + b.Property(p => p.AdapterId).HasMaxLength(200).IsUnicode(false).IsRequired(); + b.Property(p => p.Language).HasMaxLength(20).IsUnicode(false).IsRequired(); + b.Property(p => p.Kind).HasMaxLength(20).IsUnicode(false); + b.Property(p => p.BaseVersion).HasMaxLength(50).IsUnicode(false); + b.Property(p => p.FilesJson).IsRequired(); + b.Property(p => p.FilesHash).HasMaxLength(64).IsUnicode(false).IsRequired(); + b.Property(p => p.CreatedBy).HasMaxLength(200); + b.Property(p => p.ModifiedBy).HasMaxLength(200); + b.HasIndex(p => p.AdapterId); + }); + + modelBuilder.Entity(b => + { + b.ToTable("AdapterReleases"); + b.Property(p => p.Id).ValueGeneratedOnAdd(); + b.Property(p => p.AdapterId).HasMaxLength(200).IsUnicode(false).IsRequired(); + b.Property(p => p.Version).HasMaxLength(50).IsUnicode(false).IsRequired(); + b.Property(p => p.Action).HasMaxLength(20).IsUnicode(false).IsRequired(); + b.Property(p => p.AccountId).HasMaxLength(100).IsUnicode(false); + b.HasIndex(p => new { p.AdapterId, p.OccurredOn }); + }); + + modelBuilder.Entity(b => + { + b.ToTable("AdapterSourceAccesses"); + b.Property(p => p.Id).ValueGeneratedOnAdd(); + b.Property(p => p.AdapterId).HasMaxLength(200).IsUnicode(false).IsRequired(); + b.Property(p => p.Version).HasMaxLength(50).IsUnicode(false); + b.Property(p => p.Path).HasMaxLength(500); + b.Property(p => p.AccountId).HasMaxLength(100).IsUnicode(false); + b.HasIndex(p => new { p.AdapterId, p.OccurredOn }); + }); + + modelBuilder.Entity(b => + { + b.ToTable("SubscriptionRevisions"); + b.Property(p => p.Id).ValueGeneratedOnAdd(); + b.Property(p => p.Snapshot).IsRequired(); + b.Property(p => p.SavedBy).HasMaxLength(200); + b.HasIndex(p => new { p.SubscriptionId, p.Number }).IsUnique(); + b.HasOne().WithMany().HasForeignKey(p => p.SubscriptionId) + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity(b => + { + b.ToTable("DuplicateArrivals"); + b.Property(p => p.Id).ValueGeneratedOnAdd(); + b.Property(p => p.OriginalXchangeId).IsUnicode(false).HasMaxLength(50).IsRequired(); + b.HasIndex(p => new { p.DocumentId, p.OccurredOn }); + }); + modelBuilder.Entity(b => { b.ToTable("ReceiveAttempts"); @@ -353,7 +523,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) b.HasKey(p => new { p.SubscriptionId, p.GroupId }); b.Property(p => p.AlertMode).HasConversion(); b.Property(p => p.AlertHandlerId).HasMaxLength(200).IsUnicode(false); - b.Property(p => p.AlertHandlerProperties).StoreAsJson(); + b.Property(p => p.AlertHandlerProperties).StoreAsJson().Encrypted(); }); modelBuilder.Entity(b => @@ -366,8 +536,8 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) b.Property(p => p.InputName).HasMaxLength(200); b.Property(p => p.MapperId).HasMaxLength(200).IsUnicode(false); b.Property(p => p.HandlerId).HasMaxLength(200).IsUnicode(false); - b.Property(p => p.HandlerProperties).StoreAsJson(); - b.Property(p => p.MapperProperties).StoreAsJson(); + b.Property(p => p.HandlerProperties).StoreAsJson().Encrypted(); + b.Property(p => p.MapperProperties).StoreAsJson().Encrypted(); b.Property(p => p.SourceValues).StoreAsJson(); b.Property(p => p.SourceXchangeId).IsUnicode(false).HasMaxLength(50); b.Property(p => p.InputContentType).IsUnicode(false).HasMaxLength(200); @@ -404,10 +574,15 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) b.Property(p => p.ResponseContentType).IsUnicode(false).HasMaxLength(200); b.Property(p => p.OutputContentType).IsUnicode(false).HasMaxLength(200); b.Property(p => p.RetryBlockedReason).HasMaxLength(500); + b.Property(p => p.Warning).HasMaxLength(XchangeResult.WarningMaxLength); b.Property(p => p.RetryGroupId); b.Property(p => p.AttemptNumber); b.HasIndex(p => p.RetryGroupId); - + b.Property(p => p.NodeName).HasMaxLength(100).IsUnicode(false); + b.Property(p => p.MapperVersion).HasMaxLength(50).IsUnicode(false); + b.Property(p => p.HandlerVersion).HasMaxLength(50).IsUnicode(false); + // A subscription's latest result: what auto-pause counts from. + b.HasIndex(p => new { p.SubscriptionId, p.FinishedOn }); b.HasOne().WithOne().HasForeignKey(p => p.Id).OnDelete(DeleteBehavior.Cascade); }); @@ -419,14 +594,18 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) b.Property(p => p.XchangeId).IsUnicode(false).HasMaxLength(50); // Retention deletes an exchange's notifications by exchange id. b.HasIndex(p => p.XchangeId); + // The quiet window looks up a notifier's recent sends. + b.HasIndex(p => new { p.NotifierId, p.FinishedOn }); + // And a channel's. + b.HasIndex(p => new { p.ChannelId, p.FinishedOn }); }); - modelBuilder.Entity(b => + modelBuilder.Entity(b => { - b.ToTable("XchangeDeliveries"); + b.ToTable("XchangeRetryClaims"); b.Property(p => p.Id).IsUnicode(false).HasMaxLength(50); - b.HasIndex(i => i.DeliveredOn); - b.HasOne().WithOne().HasForeignKey(p => p.Id) + b.Property(p => p.RetryXchangeId).IsUnicode(false).HasMaxLength(50).IsRequired(); + b.HasOne().WithOne().HasForeignKey(p => p.Id) .OnDelete(DeleteBehavior.Cascade); }); @@ -454,23 +633,56 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) // No b-tree on PropertiesRaw. The only query on it is the Exchanges page's substring // search, a leading-wildcard LIKE no b-tree can serve (Postgres has a trigram index // for it, PromotedPropertiesTrigramIndex). Having one made long values fail the - // save — after the delivery, so the retry delivered again — and it is what had + // save - after the delivery, so the retry delivered again - and it is what had // capped the column at 450 characters on SQL Server and 255 on MySQL. b.HasOne().WithOne().HasForeignKey(p => p.Id) .OnDelete(DeleteBehavior.Cascade); }); + modelBuilder.Entity(c => + { + c.ToTable("Connections"); + c.Property(p => p.Id).ValueGeneratedOnAdd(); + c.Property(p => p.Name).IsRequired().HasMaxLength(100).IsUnicode(false); + c.HasIndex(p => p.Name).IsUnique(); + c.Property(p => p.Grant).HasConversion(); + c.Property(p => p.Status).HasConversion(); + c.Property(p => p.AuthorizeUrl).HasMaxLength(1000); + c.Property(p => p.TokenUrl).HasMaxLength(1000); + c.Property(p => p.ClientId).HasMaxLength(500); + c.Property(p => p.ClientSecret).EncryptedText(); + c.Property(p => p.Scopes).HasMaxLength(2000); + c.Property(p => p.AuthorizeParameters).HasMaxLength(2000); + c.Property(p => p.TokenAuth).HasMaxLength(20).IsUnicode(false); + c.Property(p => p.Username).HasMaxLength(500); + c.Property(p => p.Password).EncryptedText(); + c.Property(p => p.RefreshToken).EncryptedText(); + c.Property(p => p.AccessToken).EncryptedText(); + c.Property(p => p.LastError).HasMaxLength(2000); + c.Property(p => p.AuthorizedBy).HasMaxLength(200); + }); + modelBuilder.Entity(b => { b.ToTable("Notifiers"); b.Property(p => p.Id).ValueGeneratedOnAdd(); b.Property(p => p.Name).HasMaxLength(100).IsRequired(); - b.Property(p => p.HandlerProperties).StoreAsJson(); + b.Property(p => p.HandlerProperties).StoreAsJson().Encrypted(); b.Property(p => p.HandlerId).HasMaxLength(200).IsUnicode(false); b.Property(p => p.RunOnSubscriptions).IsSeparatorDelimited(); }); + modelBuilder.Entity(b => + { + b.ToTable("NotificationChannels"); + b.Property(p => p.Id).ValueGeneratedOnAdd(); + b.Property(p => p.Name).HasMaxLength(100).IsRequired(); + b.HasIndex(p => p.Name).IsUnique(); + b.Property(p => p.HandlerId).HasMaxLength(200).IsUnicode(false).IsRequired(); + b.Property(p => p.HandlerProperties).StoreAsJson().Encrypted(); + }); + modelBuilder.Entity(b => { b.ToTable("Accounts"); @@ -484,6 +696,8 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) b.Property(p => p.EmailProvider).HasConversion(); b.Property(p => p.LoginMethods).HasConversion(); + // Added with service accounts; every account before them is a person (0). + b.Property(p => p.Kind).HasConversion(); b.HasData( new @@ -498,6 +712,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) Password = defaultPasswordHash, Deleted = false, Role = AccountRole.Admin, + Kind = AccountKind.Person, FailedLoginCount = 0, // True for a fresh installation, whose password is the published default. // Installations that already exist are handled by the migration, which @@ -517,6 +732,36 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) b.Property(p => p.LoginMethod).HasConversion(); }); + modelBuilder.Entity(b => + { + b.ToTable("ServiceTokens"); + b.HasKey(p => p.Id); + b.Property(p => p.Id).ValueGeneratedOnAdd(); + b.HasOne().WithMany().HasForeignKey(p => p.AccountId).OnDelete(DeleteBehavior.Cascade); + + b.Property(p => p.Name).IsRequired().HasMaxLength(100); + b.Property(p => p.Hash).IsRequired().IsUnicode(false).HasMaxLength(64); + b.Property(p => p.Shown).IsRequired().IsUnicode(false).HasMaxLength(20); + b.Property(p => p.RevokedBy).HasMaxLength(200); + b.Property(p => p.LastUsedFrom).IsUnicode(false).HasMaxLength(64); + // Every sign-in looks a token up by its hash. + b.HasIndex(p => p.Hash).IsUnique(); + }); + + modelBuilder.Entity(b => + { + b.ToTable("AccountLinks"); + b.HasKey(p => p.Id); + b.Property(p => p.Id).ValueGeneratedOnAdd(); + b.HasOne().WithMany().HasForeignKey(p => p.AccountId).OnDelete(DeleteBehavior.Cascade); + + b.Property(p => p.Purpose).HasConversion(); + b.Property(p => p.Hash).IsRequired().IsUnicode(false).HasMaxLength(64); + b.Property(p => p.EmailError).HasMaxLength(500); + // A link is looked up by its hash when it is opened. + b.HasIndex(p => p.Hash).IsUnique(); + }); + modelBuilder.Entity(b => { b.ToTable("Roles"); @@ -548,7 +793,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) b.Property(p => p.Id).IsUnicode(false).HasMaxLength(200); }); - // ——— Audit trail ——— + // --- Audit trail --- // A table that only ever grows and is only ever appended to. The indexes answer the two // questions asked of it: the history of one row, and everything one save changed. modelBuilder.Entity(b => @@ -571,8 +816,8 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) } /// - /// The built-in roles. Their grants aren't stored — - /// derives them from the catalog — so adding a permission never needs a data fix-up here. + /// The built-in roles. Their grants aren't stored - + /// derives them from the catalog - so adding a permission never needs a data fix-up here. /// protected Role[] SystemRoleSeed() => [ @@ -589,31 +834,58 @@ protected Role[] SystemRoleSeed() => /// /// Refused, because it would save without auditing. Only - /// captures the trail, and an audit table with a silent hole in it is worse than none — + /// captures the trail, and an audit table with a silent hole in it is worse than none - /// nobody would know which changes it had missed. Nothing in Bitween calls this today; /// this makes sure a future caller finds out immediately rather than quietly. /// public override int SaveChanges() => throw new NotSupportedException( - "Use SaveChangesAsync — the synchronous path would skip the audit trail."); + "Use SaveChangesAsync - the synchronous path would skip the audit trail."); /// /// Saves, and records what was saved. The audit rows are written inside the same transaction - /// as the change they describe, so the trail can never disagree with the data — a save that + /// as the change they describe, so the trail can never disagree with the data - a save that /// rolls back takes its audit rows with it. /// /// - /// The transaction is opened only when there is something to audit. Runtime traffic — every - /// xchange, result and receive attempt — is excluded by and so + /// The transaction is opened only when there is something to audit. Runtime traffic - every + /// xchange, result and receive attempt - is excluded by and so /// still saves in exactly one round trip, unchanged. When a caller has already opened a /// transaction of its own, that one is used rather than nested. /// async public override Task SaveChangesAsync(CancellationToken cancellationToken = default) { + // Exchange files written beside the work rather than before it: none of the rows that + // name them is saved until storage has them all. A write that failed fails the save. + await Handoff.WrittenAsync(); + var userId = requestContext.GetNameIdentifier(); ChangeTracker.ApplyAuditValues(userId); var pendingAudit = ChangeTracker.CapturePendingAuditDiffs(userId, AuditPolicy.Options); + // Domain events go into the outbox in the same save as the change that raised them, so + // one can no longer be committed without the other. They are taken off the entities + // only once the save has succeeded; a save that fails leaves them where they were. + var raised = ChangeTracker.Entries() + .Select(e => e.Entity) + .Where(e => e.Events.Any()) + .ToArray(); + var fromEvents = raised + .SelectMany(entity => entity.Events) + .Select(domainEvent => Handoff.Announcing(domainEvent, ToOutboxMessage(domainEvent))) + .ToList(); + foreach (var message in fromEvents) Add(message); + + var revokesCache = cache != null && ChangeTracker.Entries().Any(e => + e.State is EntityState.Added or EntityState.Modified or EntityState.Deleted && + e.Entity is Partner or ApiCredential or ApiGateway or ApiGatewayPartner or RetryPolicy); + + // Plus any a caller queued itself, such as a delivery's response. + var outgoing = ChangeTracker.Entries() + .Where(e => e.State == EntityState.Added) + .Select(e => e.Entity) + .ToList(); + var transaction = pendingAudit.Count > 0 && Database.CurrentTransaction is null ? await Database.BeginTransactionAsync(cancellationToken) : null; @@ -631,40 +903,121 @@ async public override Task SaveChangesAsync(CancellationToken cancellationT Add(new AuditEntry(diff)); // base, deliberately: re-entering this override would audit the audit rows and - // publish every domain event a second time. + // write every domain event to the outbox a second time. await base.SaveChangesAsync(cancellationToken); } if (transaction is not null) await transaction.CommitAsync(cancellationToken); } + catch + { + // Not saved, so not to be published - and not left tracked, or the next save + // would insert them alongside a second copy made from the same events. + foreach (var message in outgoing) Entry(message).State = EntityState.Detached; + throw; + } finally { if (transaction is not null) await transaction.DisposeAsync(); } - // Published after the commit. An event announcing a change that then rolled back would - // send every consumer after a row that never existed. - //await ChangeTracker.PublishDomainEvents(publish); - var entitiesWithEvents = ChangeTracker.Entries() - .Select(e => e.Entity) - .Where(e => e.Events.Any()) - .ToArray(); + foreach (var entity in raised) entity.Events.Clear(); - foreach (var entity in entitiesWithEvents) + if (revokesCache) { - var events = entity.Events.ToArray(); - entity.Events.Clear(); - foreach (var domainEvent in events) - if (domainEvent is IHasWorkGroup hasWorkGroup) - await publish.Publish(hasWorkGroup.GetBusMessageName(), - JsonConvert.SerializeObject(new XchangeMessage { Id = hasWorkGroup.Id })); - else - await publish.Publish(domainEvent.GetType().Name, JsonConvert.SerializeObject(domainEvent)); + // Gateway calls and exchanges read these from the cache. Revoked here rather than by + // each handler, so no writer can forget: a key removed must stop opening the gateway. + // This instance at once, the others through the broadcast. + cache.Revoke(); + await cache.BroadcastRevoke(); } + // Published straight after the commit, as before; the outbox is for when that fails. + // Inside a caller's transaction nothing is committed yet - announcing a row that may + // still roll back would send every consumer after a row that never existed - so those + // wait for the dispatcher. + if (outgoing.Count > 0 && Database.CurrentTransaction is null) + await PublishOutboxAsync(outgoing, cancellationToken); return affectedRecords; } + + /// + /// Saves without an audit entry or an outbox message - for rewriting stored values in a new + /// form, which changes nothing a person did. Only + /// should need it. + /// + internal Task SaveRewrittenValuesAsync(CancellationToken cancellationToken) => + base.SaveChangesAsync(cancellationToken); + + private static OutboxMessage ToOutboxMessage(object domainEvent) => + domainEvent is IHasWorkGroup hasWorkGroup + ? new OutboxMessage(hasWorkGroup.GetBusMessageName(), + JsonConvert.SerializeObject(new XchangeMessage { Id = hasWorkGroup.Id })) + : new OutboxMessage(domainEvent.GetType().Name, JsonConvert.SerializeObject(domainEvent)); + + /// + /// Publishes saved outbox rows and deletes those the broker took, through + /// , off this caller's path. One that fails stays unpublished, + /// with the error on it, for . Never throws: the change is + /// committed, and failing the caller now would only make it try the change a second time. + /// + /// How many the broker took. + internal async Task PublishOutboxAsync(IReadOnlyCollection messages, + CancellationToken cancellationToken = default) + { + using var stage = BitweenTelemetry.Stage("publish"); + var published = new List(); + foreach (var message in messages) + { + try + { + // Bitween's bus publisher takes values as well; anything that only publishes + // (a test double) gets the message without them. + // A created message goes with what its exchange handed over, such as the input; + // the stored row, and so a later publish of it, is the id alone. + var body = Handoff.BodyToPublish(message) ?? message.Body; + if (message.Headers is { Count: > 0 } && publish is SW.Bus.RabbitMqExtensions.IPublishWithValues withValues) + await withValues.Publish(message.MessageType, body, message.Headers); + else + await publish.Publish(message.MessageType, body); + published.Add(message.Id); + } + catch (Exception ex) + { + var error = ex.Message.Length > 2000 ? ex.Message[..2000] : ex.Message; + try + { + await Set().Where(m => m.Id == message.Id) + .ExecuteUpdateAsync(u => u + .SetProperty(m => m.Attempts, m => m.Attempts + 1) + .SetProperty(m => m.LastError, error), CancellationToken.None); + } + catch + { + // The dispatcher finds it either way; the error text is only a courtesy. + } + } + } + + if (published.Count == 0) return 0; + + // Deleted rather than marked, and in a batch a second later rather than here: marking + // was a round trip per save on every exchange, and a day of dead rows. + if (outboxAcknowledger?.Acknowledge(published) == true) return published.Count; + + try + { + await OutboxAcknowledger.DeleteAsync(this, published, CancellationToken.None); + } + catch + { + // Published but not deleted: the dispatcher will publish them again. Consumers take a + // repeat - Process checks for an existing result before doing anything. + } + + return published.Count; + } } } diff --git a/SW.Bitween.Api/Data/MySqlTls.cs b/SW.Bitween.Api/Data/MySqlTls.cs new file mode 100644 index 00000000..fd7bf2f5 --- /dev/null +++ b/SW.Bitween.Api/Data/MySqlTls.cs @@ -0,0 +1,22 @@ +using System.Text.RegularExpressions; + +namespace SW.Bitween; + +/// +/// TLS for Bitween's own MySQL database. The driver's default, Preferred, encrypts only when the server +/// offers it and falls back to plain text otherwise, so a server without TLS, or someone in between who +/// strips it, gets credentials and payloads in the clear. Required is used instead, unless the +/// connection string chooses a mode itself, which is how a deployment whose server has no TLS keeps +/// connecting (SslMode=Preferred, or None). +/// +public static class MySqlTls +{ + // SslMode, Ssl Mode and Ssl-Mode are all one key to the drivers. + static readonly Regex SslModeKey = new(@"(^|;)\s*ssl[\s-]*mode\s*=", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + + public static string RequiredUnlessChosen(string connectionString) + { + if (string.IsNullOrWhiteSpace(connectionString) || SslModeKey.IsMatch(connectionString)) return connectionString; + return connectionString.TrimEnd().TrimEnd(';') + ";SslMode=Required"; + } +} diff --git a/SW.Bitween.Api/Data/SecretColumnCipher.cs b/SW.Bitween.Api/Data/SecretColumnCipher.cs new file mode 100644 index 00000000..afe9bd09 --- /dev/null +++ b/SW.Bitween.Api/Data/SecretColumnCipher.cs @@ -0,0 +1,205 @@ +using System; +using System.Security.Cryptography; +using System.Text; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.ChangeTracking; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Newtonsoft.Json; + +namespace SW.Bitween; + +/// +/// Encrypts the columns that hold credentials - partner and global values, data source settings, +/// the adapter properties of subscriptions, notifiers and alerts, and the copies of them each +/// exchange keeps - so a copy of the database, or a backup, is not a copy of every password. +/// +/// +/// +/// The whole column is encrypted, not the values marked secret. Adapters published as packages +/// rarely declare which of their properties are secret, so encrypting only declared ones would +/// leave exactly those exposed. +/// +/// +/// Keyed by Bitween:SettingsEncryptionKey, the passphrase the settings secrets already use. +/// Without it nothing is encrypted and nothing changes. A stored value carries the enc:v1: +/// prefix, so plaintext written before this - or by a deployment with no key - still reads. To +/// change the key, move the old one to Bitween:PreviousSettingsEncryptionKey: values are read +/// with either and written with the new one. +/// +/// +/// A key that is gone for good leaves nothing to read them with. Bitween:SettingsEncryptionKeyLost +/// then reads such a value as empty, so it can be entered again, and leaves what is stored in place +/// until it is: should the key turn up after all, whatever wasn't entered again reads once more. +/// +/// +public static class SecretColumnCipher +{ + public const string Prefix = "enc:v1:"; + + private const int NonceBytes = 12; + private const int TagBytes = 16; + + // Derived once, not per value: these columns are read on every exchange, and the per-value + // PBKDF2 the settings use would cost a hundred thousand rounds each time. + private static readonly byte[] Salt = Encoding.UTF8.GetBytes("bitween/secret-columns/v1"); + + private static byte[] _key; + private static byte[] _previousKey; + private static bool _keyLost; + + /// Whether values are being encrypted as they are written. + public static bool Enabled => _key is not null; + + /// Whether a value no key here can read is taken as empty rather than refused. + public static bool KeyLost => _keyLost; + + public static void Configure(string passphrase, string previousPassphrase = null, bool keyLost = false) + { + _key = string.IsNullOrWhiteSpace(passphrase) ? null : Derive(passphrase); + _previousKey = string.IsNullOrWhiteSpace(previousPassphrase) ? null : Derive(previousPassphrase); + _keyLost = keyLost; + } + + public static string Protect(string plaintext) + { + if (_key is null || plaintext is null || plaintext.StartsWith(Prefix, StringComparison.Ordinal)) + return plaintext; + + var data = Encoding.UTF8.GetBytes(plaintext); + var output = new byte[NonceBytes + data.Length + TagBytes]; + var nonce = output.AsSpan(0, NonceBytes); + RandomNumberGenerator.Fill(nonce); + + using var aes = new AesGcm(_key, TagBytes); + aes.Encrypt(nonce, data, output.AsSpan(NonceBytes, data.Length), output.AsSpan(NonceBytes + data.Length)); + return Prefix + Convert.ToBase64String(output); + } + + public static string Unprotect(string stored) => Unprotect(stored, unreadable: null); + + /// The same for JSON, where a value that can't be read is JSON's own empty value. + private static string UnprotectJson(string stored) => Unprotect(stored, unreadable: "null"); + + private static string Unprotect(string stored, string unreadable) + { + if (stored is null || !stored.StartsWith(Prefix, StringComparison.Ordinal)) + return stored; + + var input = Convert.FromBase64String(stored[Prefix.Length..]); + foreach (var key in new[] { _key, _previousKey }) + { + if (key is null) continue; + try + { + return Decrypt(key, input); + } + catch (AuthenticationTagMismatchException) + { + // Not this key; try the previous one. + } + } + + if (_keyLost) return unreadable; + throw new UnreadableSecretException(); + } + + private static string Decrypt(byte[] key, byte[] input) + { + var length = input.Length - NonceBytes - TagBytes; + var plaintext = new byte[length]; + using var aes = new AesGcm(key, TagBytes); + aes.Decrypt(input.AsSpan(0, NonceBytes), input.AsSpan(NonceBytes, length), + input.AsSpan(NonceBytes + length), plaintext); + return Encoding.UTF8.GetString(plaintext); + } + + private static byte[] Derive(string passphrase) => + Rfc2898DeriveBytes.Pbkdf2(passphrase, Salt, 210_000, HashAlgorithmName.SHA256, 32); + + /// + /// Wraps the converter already on the property - StoreAsJson, typically - so the JSON it + /// produces is encrypted on the way to the database and decrypted on the way back. + /// + public static PropertyBuilder Encrypted(this PropertyBuilder builder) + { + var inner = builder.Metadata.GetValueConverter() + ?? throw new InvalidOperationException( + $"{builder.Metadata.Name} has no converter to encrypt the output of; call StoreAsJson first."); + var comparer = builder.Metadata.GetValueComparer(); + + var cipher = new ValueConverter(v => Protect(v), v => UnprotectJson(v)); + builder.Metadata.SetValueConverter(inner.ComposeWith(cipher)); + if (comparer is not null) builder.Metadata.SetValueComparer(comparer); + return builder; + } + + /// A text column holding a credential itself, encrypted as the JSON columns are. + public static PropertyBuilder EncryptedText(this PropertyBuilder builder) => + builder.HasConversion(new ValueConverter(v => Protect(v), v => Unprotect(v))); + + /// + /// The same for a PostgreSQL jsonb column, which must always hold valid JSON. The + /// encrypted payload is stored as a JSON string, so the column stays valid; a column written + /// before encryption - a plain JSON object - still reads. + /// + public static PropertyBuilder EncryptedJsonb(this PropertyBuilder builder) where T : class + { + builder.HasConversion( + new ValueConverter(v => ToJsonb(v), s => FromJsonb(s)), + new ValueComparer( + (a, b) => JsonConvert.SerializeObject(a) == JsonConvert.SerializeObject(b), + v => v == null ? 0 : JsonConvert.SerializeObject(v).GetHashCode(), + v => v == null ? null : JsonConvert.DeserializeObject(JsonConvert.SerializeObject(v)))); + builder.HasColumnType("jsonb"); + return builder; + } + + private static string ToJsonb(T value) + { + if (value is null) return null; + var json = JsonConvert.SerializeObject(value); + return Enabled ? JsonConvert.SerializeObject(Protect(json)) : json; + } + + private static T FromJsonb(string stored) where T : class + { + if (stored is null) return null; + var json = stored.TrimStart().StartsWith("\"" + Prefix, StringComparison.Ordinal) + ? UnprotectJson(JsonConvert.DeserializeObject(stored)) + : stored; + return JsonConvert.DeserializeObject(json); + } + + /// + /// For a key-value column (PostgreSQL hstore) rather than JSON: each value is encrypted + /// on its own, and the keys stay readable. + /// + public static PropertyBuilder> EncryptedValues( + this PropertyBuilder> builder) + { + builder.HasConversion( + new ValueConverter, System.Collections.Generic.Dictionary>( + v => MapValues(v, true), v => MapValues(v, false)), + new ValueComparer>( + (a, b) => JsonConvert.SerializeObject(a) == JsonConvert.SerializeObject(b), + v => v == null ? 0 : JsonConvert.SerializeObject(v).GetHashCode(), + v => v == null ? null : new System.Collections.Generic.Dictionary(v))); + return builder; + } + + private static System.Collections.Generic.Dictionary MapValues( + System.Collections.Generic.Dictionary values, bool protect) + { + if (values is null) return null; + var result = new System.Collections.Generic.Dictionary(values.Count); + foreach (var (key, value) in values) result[key] = protect ? Protect(value) : Unprotect(value); + return result; + } +} + +/// A stored credential that neither of this instance's keys can decrypt. +public class UnreadableSecretException() : InvalidOperationException( + "A stored value is encrypted with a key this instance does not have. Set " + + "Bitween:SettingsEncryptionKey to the key it was written with, or put that key in " + + "Bitween:PreviousSettingsEncryptionKey while moving to a new one."); diff --git a/SW.Bitween.Api/Domain/Accounts/Account.cs b/SW.Bitween.Api/Domain/Accounts/Account.cs index eadb08fe..d6e87bc8 100644 --- a/SW.Bitween.Api/Domain/Accounts/Account.cs +++ b/SW.Bitween.Api/Domain/Accounts/Account.cs @@ -17,6 +17,23 @@ public Account(string displayName, string email, string password, AccountRole ro Role = role; } + /// + /// An account for a program rather than a person: no email, no password, nothing to sign + /// in to the admin UI with. It holds roles like anyone, and signs in with its tokens. + /// + public static Account ForService(string name, AccountRole role) => new() + { + DisplayName = name, + Role = role, + Kind = AccountKind.Service, + LoginMethods = LoginMethod.ApiKey, + }; + + /// A person unless created as a service account; every account before these existed is a person. + public AccountKind Kind { get; private set; } + + public bool IsService => Kind == AccountKind.Service; + public string Email { get; private set; } public string DisplayName { get; set; } @@ -44,6 +61,23 @@ public Account(string displayName, string email, string password, AccountRole ro /// public bool MustChangePassword { get; private set; } + /// + /// The Microsoft identity - object id and tenant id - this account signs in as, bound on its + /// first Microsoft sign-in. Matching by address alone let anyone who could get Microsoft to + /// put that address in a token sign in as the account. + /// + public string MicrosoftIdentity { get; private set; } + + public void BindMicrosoftIdentity(string identity) => MicrosoftIdentity = identity; + + /// + /// The OpenID Connect identity - subject and issuer - this account signs in as, bound on its + /// first single sign-on, as is. + /// + public string OidcIdentity { get; private set; } + + public void BindOidcIdentity(string identity) => OidcIdentity = identity; + public bool IsLockedOut(DateTime nowUtc) => LockoutEnd.HasValue && LockoutEnd.Value > nowUtc; public void RegisterSuccessfulLogin() @@ -52,6 +86,14 @@ public void RegisterSuccessfulLogin() LockoutEnd = null; } + /// + /// When the member last signed in with a password, Microsoft or the CLI; not when a session + /// renewed itself. Being on the audited Account, each sign-in is in the audit trail too. + /// + public DateTime? LastSignInOn { get; private set; } + + public void SignedIn() => LastSignInOn = DateTime.UtcNow; + // Admin action: clear a lockout before it expires. public void Unlock() { @@ -81,6 +123,15 @@ private bool AddLoginMethod(LoginMethod loginMethod) return true; } + /// + /// The same password, hashed again as passwords are hashed now: on a sign-in that just proved it, + /// so an account stored with an older, weaker hash is upgraded without anyone choosing a new one. + /// + public void UpgradePasswordHash(string password) + { + if (SecurePasswordHasher.NeedsRehash(Password)) Password = SecurePasswordHasher.Hash(password); + } + public void SetPassword(string password) { Password = SecurePasswordHasher.Hash(password); diff --git a/SW.Bitween.Api/Domain/Accounts/AccountKind.cs b/SW.Bitween.Api/Domain/Accounts/AccountKind.cs new file mode 100644 index 00000000..14e76e14 --- /dev/null +++ b/SW.Bitween.Api/Domain/Accounts/AccountKind.cs @@ -0,0 +1,13 @@ +namespace SW.Bitween.Domain.Accounts; + +/// Who an account is: a person who signs in, or a program that holds service tokens. +public enum AccountKind +{ + Person = 0, + + /// + /// Signs in only by trading one of its s; never with a password, + /// Microsoft or the CLI's browser sign-in, and never into the admin UI. + /// + Service = 1 +} diff --git a/SW.Bitween.Api/Domain/Accounts/AccountLink.cs b/SW.Bitween.Api/Domain/Accounts/AccountLink.cs new file mode 100644 index 00000000..c30bbd19 --- /dev/null +++ b/SW.Bitween.Api/Domain/Accounts/AccountLink.cs @@ -0,0 +1,77 @@ +using System; +using System.Security.Cryptography; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Domain.Accounts; + +/// +/// A link emailed to a person to choose their password: an invite to a new member, or a reset +/// asked for from the sign-in page. It works once, and only until it expires. +/// +/// +/// Only a hash is kept, as for service tokens: the link itself is in the email, and whoever reads +/// this table can't use it. A new invite replaces the old one; reset links each last their hour. +/// Using any link ends every link the member has. +/// +public class AccountLink : BaseEntity, IHasCreationTime +{ + public static readonly TimeSpan InviteLifetime = TimeSpan.FromHours(48); + public static readonly TimeSpan ResetLifetime = TimeSpan.FromHours(1); + + private AccountLink() + { + } + + AccountLink(int accountId, AccountLinkPurpose purpose, string hash, DateTime expiresOn) + { + AccountId = accountId; + Purpose = purpose; + Hash = hash; + ExpiresOn = expiresOn; + } + + /// A new link for the account, and the only time its token is known. + public static (AccountLink Link, string Token) Issue(int accountId, AccountLinkPurpose purpose, DateTime nowUtc) + { + // 32 random bytes, URL-safe: it goes in a link's query string. + var token = Convert.ToBase64String(RandomNumberGenerator.GetBytes(32)) + .TrimEnd('=').Replace('+', '-').Replace('/', '_'); + var lifetime = purpose == AccountLinkPurpose.Invite ? InviteLifetime : ResetLifetime; + return (new AccountLink(accountId, purpose, HashOf(token), nowUtc + lifetime) { CreatedOn = nowUtc }, token); + } + + public static string HashOf(string token) => ServiceToken.HashOf(token); + + public int AccountId { get; private set; } + public AccountLinkPurpose Purpose { get; private set; } + + /// SHA-256 of the token, hex. + public string Hash { get; private set; } + + public DateTime ExpiresOn { get; private set; } + + /// When the email carrying it was handed to the mail server; null when it wasn't sent. + public DateTime? EmailedOn { get; private set; } + + /// Why the email couldn't be sent, for the member's page; null once one was. + public string EmailError { get; private set; } + + public DateTime CreatedOn { get; set; } + + public bool IsUsable(DateTime nowUtc) => ExpiresOn > nowUtc; + + public void Emailed(DateTime nowUtc) + { + EmailedOn = nowUtc; + EmailError = null; + } + + public void NotEmailed(string error) => + EmailError = error is { Length: > 500 } ? error[..500] : error; +} + +public enum AccountLinkPurpose : byte +{ + Invite = 1, + PasswordReset = 2, +} diff --git a/SW.Bitween.Api/Domain/Accounts/AccountRoleLink.cs b/SW.Bitween.Api/Domain/Accounts/AccountRoleLink.cs index acc1047a..489773fe 100644 --- a/SW.Bitween.Api/Domain/Accounts/AccountRoleLink.cs +++ b/SW.Bitween.Api/Domain/Accounts/AccountRoleLink.cs @@ -1,6 +1,6 @@ namespace SW.Bitween.Domain.Accounts; -/// Which roles an account holds. Composite key — one row per account/role pair. +/// Which roles an account holds. Composite key - one row per account/role pair. public class AccountRoleLink { private AccountRoleLink() diff --git a/SW.Bitween.Api/Domain/Accounts/RefreshToken.cs b/SW.Bitween.Api/Domain/Accounts/RefreshToken.cs index 7cf11a37..e2317492 100644 --- a/SW.Bitween.Api/Domain/Accounts/RefreshToken.cs +++ b/SW.Bitween.Api/Domain/Accounts/RefreshToken.cs @@ -15,5 +15,12 @@ public RefreshToken(int accountId, LoginMethod loginMethod) public int AccountId { get; private set; } public LoginMethod LoginMethod { get; set; } public DateTime CreatedOn { get ; set ; } + + /// + /// When a refresh replaced this token. It stays usable for a short grace after that: the + /// browser only learns the new token from the response, and a response lost to a navigation + /// would otherwise leave it holding a token that no longer worked - signed out mid-task. + /// + public DateTime? SupersededOn { get; set; } } } \ No newline at end of file diff --git a/SW.Bitween.Api/Domain/Accounts/Role.cs b/SW.Bitween.Api/Domain/Accounts/Role.cs index 08c6193e..10301102 100644 --- a/SW.Bitween.Api/Domain/Accounts/Role.cs +++ b/SW.Bitween.Api/Domain/Accounts/Role.cs @@ -23,7 +23,7 @@ private Role() { } - /// Seeding path for the built-in roles — fixed Id, grants computed at runtime. + /// Seeding path for the built-in roles - fixed Id, grants computed at runtime. public Role(int id, string name, string description) { Id = id; @@ -43,7 +43,7 @@ public Role(string name, string description, IEnumerable permissions) public string Description { get; private set; } /// - /// Catalog keys, stored as one JSON column. Empty for built-in roles — see + /// Catalog keys, stored as one JSON column. Empty for built-in roles - see /// . /// public List Permissions { get; private set; } = []; @@ -61,7 +61,7 @@ public void Update(string name, string description, IEnumerable permissi /// /// What this role actually grants. Built-in roles derive their grants from the catalog at /// runtime instead of storing them, so adding a permission to the catalog reaches - /// Administrators — and the right groups — without a migration or a data fix-up. + /// Administrators - and the right groups - without a migration or a data fix-up. /// public List GetEffectivePermissions() => IsSystem ? SystemPermissions(Id) : Permissions; diff --git a/SW.Bitween.Api/Domain/Accounts/ServiceToken.cs b/SW.Bitween.Api/Domain/Accounts/ServiceToken.cs new file mode 100644 index 00000000..8949499e --- /dev/null +++ b/SW.Bitween.Api/Domain/Accounts/ServiceToken.cs @@ -0,0 +1,89 @@ +using System; +using System.Security.Cryptography; +using System.Text; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Domain.Accounts; + +/// +/// A credential a service account signs in with, unattended: a job applying configuration, a +/// pipeline publishing adapters. Traded for an ordinary short session at +/// POST /api/accounts/servicetoken, so everything a session can do is checked the same way. +/// +/// +/// Only a hash is kept; the token itself is shown once, when it is made. Every token expires, and +/// can be revoked on its own or with its account, by disabling it. +/// +public class ServiceToken : BaseEntity, IAudited +{ + public const string Prefix = "bwt_"; + public const int RandomLength = 40; + public const int MaxLifetimeDays = 365; + public const int DefaultLifetimeDays = 90; + + /// Unrevoked, unexpired tokens one account may hold: two, so one can replace the other. + public const int MaxActivePerAccount = 2; + + /// How much of the token is kept to tell tokens apart on screen: the prefix and 8 more. + public const int ShownLength = 12; + + const string Alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + + private ServiceToken() + { + } + + ServiceToken(int accountId, string name, string hash, string shown, DateTime expiresOn) + { + AccountId = accountId; + Name = name; + Hash = hash; + Shown = shown; + ExpiresOn = expiresOn; + } + + /// A new token for the account, and the only time its value is known. + public static (ServiceToken Token, string Value) Issue(int accountId, string name, DateTime expiresOn) + { + var value = Prefix + RandomNumberGenerator.GetString(Alphabet, RandomLength); + return (new ServiceToken(accountId, name, HashOf(value), value[..ShownLength], expiresOn), value); + } + + public static string HashOf(string value) => + Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value))).ToLowerInvariant(); + + /// Whether a string is shaped like a token at all, before looking it up. + public static bool LooksLikeOne(string value) => + value is { Length: 44 } && value.StartsWith(Prefix, StringComparison.Ordinal); + + public int AccountId { get; private set; } + public string Name { get; private set; } + + /// SHA-256 of the token, hex. The tokens are random, so a plain hash is enough. + public string Hash { get; private set; } + + /// The token's first characters, for telling it apart on screen and in logs. + public string Shown { get; private set; } + + public DateTime ExpiresOn { get; private set; } + public DateTime? RevokedOn { get; private set; } + public string RevokedBy { get; private set; } + + /// When it was last traded for a session, and from which address. Not audited: it moves on every use. + public DateTime? LastUsedOn { get; private set; } + public string LastUsedFrom { get; private set; } + + public bool IsActive(DateTime nowUtc) => RevokedOn is null && ExpiresOn > nowUtc; + + public void Revoke(string by) + { + if (RevokedOn is not null) return; + RevokedOn = DateTime.UtcNow; + RevokedBy = by; + } + + public DateTime CreatedOn { get; set; } + public string CreatedBy { get; set; } + public DateTime? ModifiedOn { get; set; } + public string ModifiedBy { get; set; } +} diff --git a/SW.Bitween.Api/Domain/AdapterDraft.cs b/SW.Bitween.Api/Domain/AdapterDraft.cs new file mode 100644 index 00000000..b4cb20fa --- /dev/null +++ b/SW.Bitween.Api/Domain/AdapterDraft.cs @@ -0,0 +1,117 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using Newtonsoft.Json; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Domain; + +/// +/// An adapter being written in Bitween's code editor: its files, until they are published as a +/// version. Python and JavaScript/TypeScript only - they build from their source with nothing but +/// their runtime, which the image has; .NET and Go adapters are built with the CLI. +/// +public class AdapterDraft : BaseEntity, IAudited +{ + public static readonly string[] Languages = ["python", "node", "typescript"]; + + /// What a draft may hold, so a draft can't become a way to store anything in the database. + public const int MaxFiles = 200; + public const int MaxBytes = 2 * 1024 * 1024; + + private AdapterDraft() + { + } + + public AdapterDraft(string adapterId, string language, string kind, string baseVersion, IDictionary files) + { + if (string.IsNullOrWhiteSpace(adapterId)) throw new ArgumentException("A draft needs an adapter id.", nameof(adapterId)); + if (!Languages.Contains(language)) throw new SWValidationException("Language", $"'{language}' isn't a language the editor builds: {string.Join(", ", Languages)}."); + AdapterId = adapterId; + Language = language; + Kind = kind; + BaseVersion = baseVersion; + SetFiles(files); + } + + public string AdapterId { get; private set; } + public string Language { get; private set; } + public string Kind { get; private set; } + + /// The version it was started from or last published as; null for a new adapter not yet published. + public string BaseVersion { get; private set; } + + /// The files as JSON, path to content. Kept out of the audit trail; is in it. + public string FilesJson { get; private set; } + + /// SHA-256 of the files, so the trail shows every save without holding the code twice. + public string FilesHash { get; private set; } + + public DateTime CreatedOn { get; set; } + public string CreatedBy { get; set; } + public DateTime? ModifiedOn { get; set; } + public string ModifiedBy { get; set; } + + public Dictionary Files => + JsonConvert.DeserializeObject>(FilesJson ?? "{}") ?? new Dictionary(); + + public void SetFiles(IDictionary files) + { + if (files == null || files.Count == 0) throw new SWValidationException("Files", "A draft needs its files."); + if (files.Count > MaxFiles) throw new SWValidationException("Files", $"A draft holds at most {MaxFiles} files."); + foreach (var path in files.Keys) + if (!IsSafePath(path)) throw new SWValidationException("Files", $"'{path}' isn't a path inside the adapter."); + if (files.Values.Sum(v => Encoding.UTF8.GetByteCount(v ?? "")) > MaxBytes) + throw new SWValidationException("Files", $"A draft holds at most {MaxBytes / (1024 * 1024)} MB of code."); + + var sorted = new SortedDictionary(files.ToDictionary(f => f.Key, f => f.Value ?? ""), StringComparer.Ordinal); + FilesJson = JsonConvert.SerializeObject(sorted); + FilesHash = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(FilesJson))).ToLowerInvariant(); + } + + public void Published(string version) => BaseVersion = version; + + /// Relative, forward slashes, nothing that climbs out or hides: what a project folder holds. + public static bool IsSafePath(string path) + { + if (string.IsNullOrWhiteSpace(path) || path.Length > 300) return false; + if (path.StartsWith('/') || path.Contains('\\') || path.Contains(':') || path.Contains('\0')) return false; + var segments = path.Split('/'); + return segments.All(s => s.Length > 0 && s != "." && s != ".."); + } +} + +/// A version published or made current from Bitween: in the audit trail, with who did it. +public class AdapterRelease +{ + public const string PublishedAction = "published"; + public const string PromotedAction = "promoted"; + public const string WithdrawnAction = "withdrawn"; + + /// Installed from a Bitween Registry, through the Marketplace tab. + public const string InstalledAction = "installed"; + + private AdapterRelease() + { + } + + public AdapterRelease(string adapterId, string version, string action, int? draftId, string accountId) + { + AdapterId = adapterId; + Version = version; + Action = action; + DraftId = draftId; + AccountId = accountId; + OccurredOn = DateTime.UtcNow; + } + + public long Id { get; private set; } + public string AdapterId { get; private set; } + public string Version { get; private set; } + public string Action { get; private set; } + public int? DraftId { get; private set; } + public string AccountId { get; private set; } + public DateTime OccurredOn { get; private set; } +} diff --git a/SW.Bitween.Api/Domain/AdapterSourceAccess.cs b/SW.Bitween.Api/Domain/AdapterSourceAccess.cs new file mode 100644 index 00000000..d809e8e4 --- /dev/null +++ b/SW.Bitween.Api/Domain/AdapterSourceAccess.cs @@ -0,0 +1,36 @@ +using System; + +namespace SW.Bitween.Domain; + +/// +/// Someone read an adapter's source in Bitween. Kept as a row, rather than only logged, so the +/// audit trail records it as it records every other change: who, which adapter and version, +/// which file, and when. Source is a partner's or our own code, and reading it is worth knowing. +/// +public class AdapterSourceAccess +{ + private AdapterSourceAccess() + { + } + + public AdapterSourceAccess(string adapterId, string version, string path, string accountId) + { + AdapterId = adapterId; + Version = version; + Path = path; + AccountId = accountId; + OccurredOn = DateTime.UtcNow; + } + + public long Id { get; private set; } + public string AdapterId { get; private set; } + + /// The version read; empty for a package published without one. + public string Version { get; private set; } + + /// The file read, relative to the package's source folder; null when only the list was. + public string Path { get; private set; } + + public string AccountId { get; private set; } + public DateTime OccurredOn { get; private set; } +} diff --git a/SW.Bitween.Api/Domain/Audit/AuditEntry.cs b/SW.Bitween.Api/Domain/Audit/AuditEntry.cs index 05bb1c7f..2d848114 100644 --- a/SW.Bitween.Api/Domain/Audit/AuditEntry.cs +++ b/SW.Bitween.Api/Domain/Audit/AuditEntry.cs @@ -14,7 +14,7 @@ namespace SW.Bitween.Domain; /// /// /// Rows are written by for the entities -/// admits, in the same transaction as the change itself — so an entry +/// admits, in the same transaction as the change itself - so an entry /// exists for every audited change that committed, and for no change that didn't. /// /// @@ -53,7 +53,7 @@ public AuditEntry(GenericAuditDiffJson diff) public DateTime OccurredOn { get; private set; } /// - /// The account id behind the change, or null for a save with no signed-in user — a bus consumer + /// The account id behind the change, or null for a save with no signed-in user - a bus consumer /// or a scheduled job. Kept as the id rather than a name so that renaming an account doesn't /// rewrite history. /// diff --git a/SW.Bitween.Api/Domain/Cluster/ClusterLease.cs b/SW.Bitween.Api/Domain/Cluster/ClusterLease.cs index bf1a9dae..b93f4908 100644 --- a/SW.Bitween.Api/Domain/Cluster/ClusterLease.cs +++ b/SW.Bitween.Api/Domain/Cluster/ClusterLease.cs @@ -6,7 +6,7 @@ namespace SW.Bitween.Domain.Cluster; /// /// The fencing token for one exclusively-owned resource. /// -/// This is not the lock — the lock lives on the bus. This is the monotonic counter the bus cannot +/// This is not the lock - the lock lives on the bus. This is the monotonic counter the bus cannot /// provide, so that a node which was paused while ownership moved can discover it lost. /// public class ClusterLease : BaseEntity diff --git a/SW.Bitween.Api/Domain/Cluster/ClusterNode.cs b/SW.Bitween.Api/Domain/Cluster/ClusterNode.cs new file mode 100644 index 00000000..f0c564d8 --- /dev/null +++ b/SW.Bitween.Api/Domain/Cluster/ClusterNode.cs @@ -0,0 +1,47 @@ +using System; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Domain.Cluster; + +/// +/// A Bitween node, as it last said it was: written by every node on a heartbeat, so the cluster +/// can be seen from any of them. Not audited: it is the nodes describing themselves, not +/// configuration anyone changes. +/// +public class ClusterNode : BaseEntity +{ + private ClusterNode() + { + } + + /// The node's name as leases record it: host and process id. + public ClusterNode(string name) + { + Id = name ?? throw new ArgumentNullException(nameof(name)); + } + + public string Host { get; set; } + public DateTime StartedOn { get; set; } + public DateTime LastSeenOn { get; set; } + public string Version { get; set; } + + /// Bitween:BusProvidersEnabled on this node: whether it runs data sources. + public bool DataSources { get; set; } + + /// The adapter runtimes this node has, comma-separated: dotnet,python,node. + public string Runtimes { get; set; } + + /// + /// The resident adapter instances running on this node at its last beat - state, uptime, + /// restarts, quarantine - as JSON, so any node can show every node's. Null on a node without + /// resident adapters, and on beats from before this was recorded. + /// + public string ResidentHealth { get; set; } + + /// + /// Outbox messages this node published in the hour before its last beat. Published rows are + /// deleted now rather than kept, so this is where the queue health page counts them from. Null + /// on beats from nodes from before this was recorded. + /// + public int? OutboxPublishedLastHour { get; set; } +} diff --git a/SW.Bitween.Api/Domain/Connections/Connection.cs b/SW.Bitween.Api/Domain/Connections/Connection.cs new file mode 100644 index 00000000..4b822ac1 --- /dev/null +++ b/SW.Bitween.Api/Domain/Connections/Connection.cs @@ -0,0 +1,111 @@ +using System; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Domain.Connections; + +/// How a connection gets its access tokens. +public enum ConnectionGrant +{ + /// OAuth2 client credentials: the client id and secret alone. Nobody has to approve anything. + ClientCredentials = 1, + + /// + /// OAuth2 authorization code: someone approves access once at the provider, and the refresh token + /// that brings keeps the connection going, as Salesforce, Zoho, Shopify and Dynamics apps need. + /// + AuthorizationCode = 2, + + /// OAuth2 password grant: a service user's name and password, for providers that still offer it. + Password = 3, +} + +/// Where a connection stands, for the Connections page and the error an exchange reports. +public enum ConnectionStatus +{ + /// Saved, never yet used or authorized. + New = 0, + + Connected = 1, + + /// An authorization code connection with no refresh token, or one the provider stopped honouring. + NeedsAuthorizing = 2, + + /// The last attempt to get a token failed; says why. + Error = 3, +} + +/// +/// Credentials Bitween keeps fresh for adapters: an OAuth2 client whose access tokens Bitween asks +/// for, renews and rotates, so that any adapter - built in or published, in any language - can write +/// {{connections.NAME.accessToken}} in a setting and get a valid token when it runs. +/// +/// +/// The token is resolved when the adapter starts, never when an exchange is made, so no exchange +/// stores one and a retry hours later still gets a token that works. +/// +public class Connection : BaseEntity, IAudited +{ + /// What settings write after connections.: letters, digits, '_' and '-'. + public string Name { get; set; } + + public ConnectionGrant Grant { get; set; } + + /// For : the provider's page that asks for approval. + public string AuthorizeUrl { get; set; } + + public string TokenUrl { get; set; } + public string ClientId { get; set; } + public string ClientSecret { get; set; } + + /// Separated by spaces. + public string Scopes { get; set; } + + /// + /// Extra query parameters for the approval page, as access_type=offline&prompt=consent, + /// which Google needs before it hands out a refresh token. + /// + public string AuthorizeParameters { get; set; } + + /// post sends the client id and secret in the token request's form; basic in an Authorization header. + public string TokenAuth { get; set; } = "post"; + + /// For . + public string Username { get; set; } + + public string Password { get; set; } + + public string RefreshToken { get; set; } + public string AccessToken { get; set; } + public DateTime? AccessTokenExpiresOn { get; set; } + + /// + /// Set while one node renews the token, so another waits for its result rather than renewing too: + /// a provider that rotates refresh tokens honours only the first renewal, and the second would + /// leave the connection needing authorizing again. + /// + public DateTime? RenewingUntil { get; set; } + + public ConnectionStatus Status { get; set; } + public string LastError { get; set; } + public string AuthorizedBy { get; set; } + public DateTime? AuthorizedOn { get; set; } + + public DateTime CreatedOn { get; set; } + public string CreatedBy { get; set; } + public DateTime? ModifiedOn { get; set; } + public string ModifiedBy { get; set; } + + /// Whether a token is in hand that is good for at least more. + public bool HasFreshToken(DateTime now, TimeSpan margin) => + !string.IsNullOrEmpty(AccessToken) && AccessTokenExpiresOn is { } expires && expires - margin > now; + + /// Forgets the tokens, as when the client or the provider changes: they belonged to the old one. + public void ForgetTokens() + { + AccessToken = null; + AccessTokenExpiresOn = null; + RefreshToken = null; + Status = Grant == ConnectionGrant.AuthorizationCode ? ConnectionStatus.NeedsAuthorizing : ConnectionStatus.New; + LastError = null; + } +} diff --git a/SW.Bitween.Api/Domain/DataSources/AdapterState.cs b/SW.Bitween.Api/Domain/DataSources/AdapterState.cs index 1ff5fb6d..74793f9d 100644 --- a/SW.Bitween.Api/Domain/DataSources/AdapterState.cs +++ b/SW.Bitween.Api/Domain/DataSources/AdapterState.cs @@ -5,7 +5,7 @@ namespace SW.Bitween.Domain.DataSources; /// /// A bookmark a resident adapter asked Bitween to hold for it. /// -/// The motivating case is a polling database receiver's cursor — the last incrementing id or +/// The motivating case is a polling database receiver's cursor - the last incrementing id or /// timestamp it consumed. The adapter cannot keep it: the supervisor restarts it, the next instance /// may come up on another node, and a pooled one is not the same process twice. Any of those resets /// a cursor held in a field back to the beginning, which means replaying every row already @@ -26,7 +26,7 @@ public class AdapterState public string AdapterId { get; set; } /// - /// Which instance owns it — the data source id for an exclusive resident. Part of the key + /// Which instance owns it - the data source id for an exclusive resident. Part of the key /// because two instances of one adapter are two different connections, and one reading the /// other's cursor would skip rows that were never processed. /// diff --git a/SW.Bitween.Api/Domain/DataSources/DataSource.cs b/SW.Bitween.Api/Domain/DataSources/DataSource.cs index d1c5634b..2793ecff 100644 --- a/SW.Bitween.Api/Domain/DataSources/DataSource.cs +++ b/SW.Bitween.Api/Domain/DataSources/DataSource.cs @@ -11,8 +11,8 @@ namespace SW.Bitween.Domain.DataSources; /// every gateway that exists today keeps working untouched. Setting one moves that gateway onto an /// external broker, served by a resident serverless adapter. /// -/// The split matters: this holds the connection — endpoint, credentials, health — and the gateway -/// holds the meaning — which Document, which routes, which filters. One data source can serve many +/// The split matters: this holds the connection - endpoint, credentials, health - and the gateway +/// holds the meaning - which Document, which routes, which filters. One data source can serve many /// gateways, exactly as one RabbitMQ connection serves many queues. /// public class DataSource : BaseEntity, IAudited @@ -31,7 +31,7 @@ public class DataSource : BaseEntity, IAudited /// /// Connection settings handed to the adapter as startup values. Deliberately untyped: a /// provider must not be constrained to the subset of a broker's model that Bitween happens to - /// have modelled. Secrets are protected at rest — see . + /// have modelled. Secrets are protected at rest - see . /// public Dictionary Properties { get; set; } = new(); @@ -43,7 +43,7 @@ public class DataSource : BaseEntity, IAudited /// /// How many nodes may run this source's adapter at once. Left - /// it follows from , which is right almost always — set it only to override. + /// it follows from , which is right almost always - set it only to override. /// public DataSourcePlacement Placement { get; set; } = DataSourcePlacement.Auto; @@ -52,8 +52,8 @@ public class DataSource : BaseEntity, IAudited /// /// How long a message's dedupe key is remembered. It has to exceed the widest redelivery - /// window this broker can actually produce — its message TTL, a dead-letter replay, someone - /// re-driving a queue by hand — because a key forgotten too early lets a redelivery through + /// window this broker can actually produce - its message TTL, a dead-letter replay, someone + /// re-driving a queue by hand - because a key forgotten too early lets a redelivery through /// as a fresh message. That number is a property of the customer's broker, not of Bitween, /// which is why it lives here rather than in configuration. /// @@ -70,7 +70,7 @@ public class DataSource : BaseEntity, IAudited /// /// A hard ceiling in megabytes, enforced by the runtime rather than by the supervisor's - /// goodwill — it becomes the adapter process's GC heap hard limit, so an allocation past it + /// goodwill - it becomes the adapter process's GC heap hard limit, so an allocation past it /// fails inside the adapter instead of taking the node down with it. Zero leaves the host's /// own default in place. /// @@ -81,12 +81,12 @@ public class DataSource : BaseEntity, IAudited public int HardMemoryLimitMb { get; set; } /// - /// Sustained CPU ceiling for the adapter process, as a percentage of the WHOLE node — the same + /// Sustained CPU ceiling for the adapter process, as a percentage of the WHOLE node - the same /// figure the heartbeat reports. Worth being exact about, because the intuitive reading is /// wrong in an expensive direction: one core pegged flat out on a sixteen-core node reads about /// 6%, so a ceiling set at "50%, surely that's half a core" would in fact allow eight. /// - /// Deliberately sustained rather than instantaneous — an adapter draining a backlog is supposed + /// Deliberately sustained rather than instantaneous - an adapter draining a backlog is supposed /// to work hard, and recycling it for that would be a bug wearing a limit's clothes. Zero /// leaves the host default in place. /// @@ -120,7 +120,7 @@ public class DataSource : BaseEntity, IAudited } /// -/// Broker today. The others are declared now because the adapter contract is identical for them — +/// Broker today. The others are declared now because the adapter contract is identical for them - /// a relational or object-store source differs in whether it pushes or is polled, not in how it is /// configured, supervised or observed. /// @@ -137,7 +137,7 @@ public enum DataSourceKind /// How many nodes may hold this connection. /// /// The distinction is not cosmetic and the default is not safe in both directions. A broker queue -/// consumed by two nodes is duplicate processing — the failure the whole leased design exists to +/// consumed by two nodes is duplicate processing - the failure the whole leased design exists to /// prevent. A database connection pool held by only one node is the opposite failure: every other /// node's Xchanges have nowhere to run, and the health page reports "not running here" as though /// that were normal. diff --git a/SW.Bitween.Api/Domain/DataSources/DataSourceStatement.cs b/SW.Bitween.Api/Domain/DataSources/DataSourceStatement.cs index 28908c50..49eac644 100644 --- a/SW.Bitween.Api/Domain/DataSources/DataSourceStatement.cs +++ b/SW.Bitween.Api/Domain/DataSources/DataSourceStatement.cs @@ -7,7 +7,7 @@ namespace SW.Bitween.Domain.DataSources; /// One named piece of SQL a data source is allowed to run. /// /// This is deliberately an entity rather than a JSON blob on , and the -/// reason is permissions before anything else. Statements have to live on the connection — the +/// reason is permissions before anything else. Statements have to live on the connection - the /// alternative, SQL in a subscription's adapter properties, is a live injection surface, because /// those property values have {{partner.X}} substituted into them before the adapter ever /// sees them, and partner records are ordinary data. But putting the SQL in a field on the data @@ -20,8 +20,8 @@ namespace SW.Bitween.Domain.DataSources; /// one field, where a bad edit fails the connection test for everyone. /// * Namespacing. A unique index on (data source, name) makes a collision an error at the /// point of saving rather than a silent overwrite. -/// * Dead SQL. A statement nothing references is now a countable fact — see the usage -/// endpoint — instead of a line nobody dares delete. +/// * Dead SQL. A statement nothing references is now a countable fact - see the usage +/// endpoint - instead of a line nobody dares delete. /// /// What does NOT change is the adapter contract. The supervisor composes these rows into the same /// Statements JSON the adapter has always received, so an adapter still resolves a name and @@ -33,7 +33,7 @@ public class DataSourceStatement : BaseEntity, IAudited public DataSource DataSource { get; set; } /// - /// What a subscription names to run it. Unique within the data source, case-insensitively — + /// What a subscription names to run it. Unique within the data source, case-insensitively - /// the adapter resolves names that way, so allowing getOrder and GetOrder to /// coexist would make which one runs a matter of dictionary ordering. /// @@ -56,7 +56,7 @@ public class DataSourceStatement : BaseEntity, IAudited public int? WorkGroupId { get; set; } /// - /// Kept out of the composed statement set without being deleted — the same idea as + /// Kept out of the composed statement set without being deleted - the same idea as /// . Useful for retiring a statement while the subscriptions /// that used it are still being migrated: they fail loudly on a missing name rather than /// quietly running SQL nobody meant to keep. @@ -64,7 +64,7 @@ public class DataSourceStatement : BaseEntity, IAudited public bool Inactive { get; set; } /// - /// For a statement a receiver polls with: the column carrying the cursor — the incrementing + /// For a statement a receiver polls with: the column carrying the cursor - the incrementing /// id, or the modified-at timestamp. Its value in the last row read is what gets saved. /// /// It lives here rather than on the subscription because it describes the SHAPE of what this @@ -78,7 +78,7 @@ public class DataSourceStatement : BaseEntity, IAudited /// /// For a polled statement: the column identifying a row, used for mark-processed and for - /// deduplication. Here for the same reason as — it is a fact about + /// deduplication. Here for the same reason as - it is a fact about /// the query's result, not about who reads it. /// public string KeyColumn { get; set; } diff --git a/SW.Bitween.Api/Domain/DataSources/InboundMessage.cs b/SW.Bitween.Api/Domain/DataSources/InboundMessage.cs index 39dc153d..af5fb506 100644 --- a/SW.Bitween.Api/Domain/DataSources/InboundMessage.cs +++ b/SW.Bitween.Api/Domain/DataSources/InboundMessage.cs @@ -6,7 +6,7 @@ namespace SW.Bitween.Domain.DataSources; /// /// One inbound message we have already persisted, remembered by its dedupe key. /// -/// At-least-once delivery is not an edge case here — it is what persist-then-acknowledge buys. +/// At-least-once delivery is not an edge case here - it is what persist-then-acknowledge buys. /// A crash between committing the Xchange and acknowledging the broker redelivers by design, so /// duplicates are normal and something has to recognise them. /// diff --git a/SW.Bitween.Api/Domain/DelayedRetry.cs b/SW.Bitween.Api/Domain/DelayedRetry.cs index c8bd1ffc..0c82ae06 100644 --- a/SW.Bitween.Api/Domain/DelayedRetry.cs +++ b/SW.Bitween.Api/Domain/DelayedRetry.cs @@ -1,4 +1,8 @@ using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; using SW.PrimitiveTypes; namespace SW.Bitween.Domain; @@ -6,4 +10,51 @@ namespace SW.Bitween.Domain; public class DelayedRetry : BaseEntity { public DateTime On { get; set; } + + /// + /// for a held retry: one kept because its group's total budget is spent, never + /// due until makes it due. A date rather than a column, so holding + /// needed no schema change; the retry job never reaches it. + /// + public static readonly DateTime HeldOn = new(9999, 12, 31, 0, 0, 0, DateTimeKind.Utc); + + public bool Held => On >= HeldOn; + + /// + /// Makes the held retries of these subscriptions due now: their budget has been lifted, by a + /// success or by hand. + /// + /// How many were released. + public static Task ReleaseHeldAsync(BitweenDbContext dbContext, IReadOnlyCollection subscriptionIds) + { + var now = DateTime.UtcNow; + return dbContext.Set() + .Where(r => r.On >= HeldOn && dbContext.Set() + .Any(x => x.Id == r.Id && x.SubscriptionId != null && subscriptionIds.Contains(x.SubscriptionId.Value))) + .ExecuteUpdateAsync(set => set.SetProperty(r => r.On, now)); + } + + /// + /// Makes due every held retry whose subscription has no spent budget left: whatever lifted it. + /// The release that rides on a success can miss: a success that started before the last charge + /// leaves the budget spent, and a failure a moment later can hold more after the budget is lifted. + /// The retry job calls this every pass, so nothing stays held past a minute once the budget is free. + /// A spent budget is one whose exhaustion was claimed (). + /// + /// How many were released. + public static Task ReleaseHeldWithBudgetAsync(BitweenDbContext dbContext) + { + var now = DateTime.UtcNow; + return dbContext.Set() + .Where(r => r.On >= HeldOn && dbContext.Set() + .Any(x => x.Id == r.Id && x.SubscriptionId != null && !dbContext.Set() + .Any(u => u.SubscriptionId == x.SubscriptionId && u.ExhaustedNotifiedOn != null))) + .ExecuteUpdateAsync(set => set.SetProperty(r => r.On, now)); + } + + /// + /// How many times running this retry has failed. A failure is rescheduled, not dropped - a + /// database blip used to delete the retry for good - up to . + /// + public int RunFailures { get; set; } } diff --git a/SW.Bitween.Api/Domain/Document/Document.cs b/SW.Bitween.Api/Domain/Document/Document.cs index ce7e595d..6e6e4b1e 100644 --- a/SW.Bitween.Api/Domain/Document/Document.cs +++ b/SW.Bitween.Api/Domain/Document/Document.cs @@ -24,7 +24,7 @@ public Document(int id, string name, DocumentFormat format) DocumentFormat = format; } - /// Production creation path — Id is database-generated. Code is optional. + /// Production creation path - Id is database-generated. Code is optional. public Document(string code, string name, DocumentFormat format) { Code = code; @@ -37,7 +37,18 @@ public Document(string code, string name, DocumentFormat format) public string Code { get; private set; } public bool BusEnabled { get; set; } public string BusMessageTypeName { get; set; } + /// + /// Minutes within which the same payload from the same partner counts as a duplicate. 0 for + /// no check. A duplicate is reported, and dropped only when says so. + /// public int DuplicateInterval { get; set; } + + /// + /// Whether a duplicate is dropped rather than only reported. Off by default, and on every type + /// that existed before duplicates were checked: the window was shown for years without doing + /// anything, so turning it on for them would start dropping messages nobody expected to lose. + /// + public bool DropDuplicates { get; set; } public bool? DisregardsUnfilteredMessages { get; set; } public DocumentFormat DocumentFormat { get; set; } @@ -69,6 +80,18 @@ public void SetCode(string code) Code = code; } + /// + /// JSON Schema for a JSON type, XSD for an XML one; null checks nothing. What arrives at a + /// gateway that doesn't match is refused, and an exchange whose input doesn't match fails. + /// A private setter so a save that doesn't mention it leaves it alone. + /// + public string ValidationSchema { get; private set; } + + public void SetValidationSchema(string schema) + { + ValidationSchema = string.IsNullOrWhiteSpace(schema) ? null : schema; + } + public void Retire() { RetiredOn = DateTime.UtcNow; diff --git a/SW.Bitween.Api/Domain/Document/DuplicateArrival.cs b/SW.Bitween.Api/Domain/Document/DuplicateArrival.cs new file mode 100644 index 00000000..3e010584 --- /dev/null +++ b/SW.Bitween.Api/Domain/Document/DuplicateArrival.cs @@ -0,0 +1,35 @@ +using System; + +namespace SW.Bitween.Domain; + +/// +/// A payload that arrived again within its information type's duplicate window: what the type's +/// page counts, and how an administrator decides whether to have duplicates dropped. +/// +public class DuplicateArrival +{ + private DuplicateArrival() + { + } + + public DuplicateArrival(int documentId, int? partnerId, string originalXchangeId, bool dropped) + { + DocumentId = documentId; + PartnerId = partnerId; + OriginalXchangeId = originalXchangeId; + Dropped = dropped; + OccurredOn = DateTime.UtcNow; + } + + public long Id { get; private set; } + public int DocumentId { get; private set; } + public int? PartnerId { get; private set; } + + /// The earlier exchange this payload repeats. + public string OriginalXchangeId { get; private set; } + + /// Dropped, or let through and reported. + public bool Dropped { get; private set; } + + public DateTime OccurredOn { get; private set; } +} diff --git a/SW.Bitween.Api/Domain/Gateway/ApiGateway.cs b/SW.Bitween.Api/Domain/Gateway/ApiGateway.cs index da311309..8973be05 100644 --- a/SW.Bitween.Api/Domain/Gateway/ApiGateway.cs +++ b/SW.Bitween.Api/Domain/Gateway/ApiGateway.cs @@ -12,7 +12,7 @@ public class ApiGateway : BaseEntity,IAudited /// /// Turns the gateway off without deleting it. Deleting is the only alternative today, - /// and it takes the partner attachments with it — so a gateway that needs stopping for + /// and it takes the partner attachments with it - so a gateway that needs stopping for /// an afternoon gets rebuilt by hand afterwards, or left running. /// public bool Inactive { get; set; } @@ -34,6 +34,12 @@ public class ApiGateway : BaseEntity,IAudited public string JwtAudience { get; set; } public string JwtPartnerClaim { get; set; } + /// + /// What a sync call is told when the delivery behind it fails. Legacy (400, no body) for gateways + /// made before this existed: a caller may be written against it. + /// + public GatewayFailureAnswer FailureAnswer { get; set; } + public ICollection Partners { get; set; } public DateTime CreatedOn { get; set; } public string CreatedBy { get; set; } diff --git a/SW.Bitween.Api/Domain/Gateway/BusGateway.cs b/SW.Bitween.Api/Domain/Gateway/BusGateway.cs index 31909816..2691d333 100644 --- a/SW.Bitween.Api/Domain/Gateway/BusGateway.cs +++ b/SW.Bitween.Api/Domain/Gateway/BusGateway.cs @@ -10,13 +10,13 @@ public class BusGateway : BaseEntity, IAudited public int DocumentId { get; set; } /// - /// Turns the gateway off without deleting it — its routes stop being offered the + /// Turns the gateway off without deleting it - its routes stop being offered the /// message. See . /// public bool Inactive { get; set; } /// - /// Null means the INTERNAL bus — the only behaviour that existed before, and still the + /// Null means the INTERNAL bus - the only behaviour that existed before, and still the /// default, so every gateway already in a database keeps working with no migration of data. /// Set it and this gateway is fed by an external broker through a resident adapter instead. /// @@ -30,14 +30,14 @@ public class BusGateway : BaseEntity, IAudited public string Endpoint { get; set; } /// - /// Per-gateway overrides handed to the adapter alongside the data source's own properties — + /// Per-gateway overrides handed to the adapter alongside the data source's own properties - /// prefetch, consumer group, visibility timeout. Connection settings belong on the DataSource; /// these are about this one subscription to it. /// /// NOT YET CONSUMED. The supervisor namespaces these and passes them through as /// Endpoint:<endpoint>:<key> startup values, but neither bundled adapter reads /// them, so setting one currently has no effect. The transport is in place; binding them is - /// per-adapter work — RabbitMQ can apply queue type and durability per endpoint at declare + /// per-adapter work - RabbitMQ can apply queue type and durability per endpoint at declare /// time, SQS a per-queue visibility timeout, and prefetch would need a consume channel per /// endpoint rather than the single one it has now. /// diff --git a/SW.Bitween.Api/Domain/GlobalAdapterValue/GlobalAdapterValuesSet.cs b/SW.Bitween.Api/Domain/GlobalAdapterValue/GlobalAdapterValuesSet.cs index cb686ff2..326c7618 100644 --- a/SW.Bitween.Api/Domain/GlobalAdapterValue/GlobalAdapterValuesSet.cs +++ b/SW.Bitween.Api/Domain/GlobalAdapterValue/GlobalAdapterValuesSet.cs @@ -12,7 +12,7 @@ public class GlobalAdapterValuesSet:BaseEntity /// /// Names within whose values are never returned by the API in clear. /// A set is shared across every adapter that references it, so one secret in a set must not - /// force the whole set out of sight — the flag is per value, not per set. + /// force the whole set out of sight - the flag is per value, not per set. /// public List SecretProperties { get; set; } = new(); } \ No newline at end of file diff --git a/SW.Bitween.Api/Domain/IdempotencyKey.cs b/SW.Bitween.Api/Domain/IdempotencyKey.cs new file mode 100644 index 00000000..b240e088 --- /dev/null +++ b/SW.Bitween.Api/Domain/IdempotencyKey.cs @@ -0,0 +1,32 @@ +using System; + +namespace SW.Bitween.Domain; + +/// +/// A partner's Idempotency-Key, and the exchange the first call carrying it created. A partner +/// whose call timed out can't tell whether it arrived; sending it again with the same key returns +/// that exchange instead of making a second one. +/// +public class IdempotencyKey +{ + private IdempotencyKey() + { + } + + public IdempotencyKey(string scope, string key, string xchangeId) + { + Scope = scope; + Key = key; + XchangeId = xchangeId; + CreatedOn = DateTime.UtcNow; + } + + public long Id { get; private set; } + + /// Who the key belongs to - gateway and partner - so two callers can't collide. + public string Scope { get; private set; } + + public string Key { get; private set; } + public string XchangeId { get; private set; } + public DateTime CreatedOn { get; private set; } +} diff --git a/SW.Bitween.Api/Domain/Metrics/AdapterStats.cs b/SW.Bitween.Api/Domain/Metrics/AdapterStats.cs new file mode 100644 index 00000000..656c220e --- /dev/null +++ b/SW.Bitween.Api/Domain/Metrics/AdapterStats.cs @@ -0,0 +1,40 @@ +using System; + +namespace SW.Bitween.Domain.Metrics; + +/// +/// One hour of calls to one version of one adapter, in one role, on one runtime, as one node made +/// them. Counted where every adapter call goes through (AdapterInvoker), so receivers, validators, +/// mappers and handlers, native or not, are all here. +/// +public class AdapterStats : TimedStats +{ + public DateTime Hour { get; set; } + public string Node { get; set; } + public string AdapterId { get; set; } + /// The version that ran; empty for a built-in adapter or one published before catalogs. + public string Version { get; set; } + /// receiver, validator, mapper, handler. + public string Role { get; set; } + /// native (in-process), resident (pooled process), process (one process per call). + public string Runtime { get; set; } + + public int Calls { get; set; } + /// Every call that failed, whatever the cause; the three below are some of them. + public int Failures { get; set; } + public int Timeouts { get; set; } + /// Stopped by the runtime for going over a memory or CPU limit. + public int LimitStops { get; set; } + /// The adapter's process died or stopped answering. + public int Crashes { get; set; } + + public void Add(AdapterStats other) + { + Calls += other.Calls; + Failures += other.Failures; + Timeouts += other.Timeouts; + LimitStops += other.LimitStops; + Crashes += other.Crashes; + AddTimes(other); + } +} diff --git a/SW.Bitween.Api/Domain/Metrics/DurationBuckets.cs b/SW.Bitween.Api/Domain/Metrics/DurationBuckets.cs new file mode 100644 index 00000000..4dc23296 --- /dev/null +++ b/SW.Bitween.Api/Domain/Metrics/DurationBuckets.cs @@ -0,0 +1,89 @@ +using System; + +namespace SW.Bitween.Domain.Metrics; + +/// +/// Durations counted into ten fixed buckets rather than kept one by one, so an hour of a million +/// exchanges is still ten numbers, buckets from many hours and nodes add up, and a percentile is +/// read off the counts. The answer is the bucket's upper edge: "p95 under 2.5 s", which is as exact +/// as a dashboard needs. +/// +public static class DurationBuckets +{ + /// Upper edges in milliseconds; the last bucket has no edge. + public static readonly int[] UpperMs = [50, 100, 250, 500, 1_000, 2_500, 5_000, 10_000, 30_000]; + + public const int Count = 10; + + public static int IndexOf(long ms) + { + for (var i = 0; i < UpperMs.Length; i++) + if (ms < UpperMs[i]) return i; + return Count - 1; + } + + /// + /// The duration under which of the counted ones fell (0.95 for p95), + /// or null when nothing was counted. Past the last edge, the longest one seen. + /// + public static long? Percentile(long[] buckets, double fraction, long maxMs) + { + long total = 0; + foreach (var b in buckets) total += b; + if (total == 0) return null; + var target = (long)Math.Ceiling(total * fraction); + long seen = 0; + for (var i = 0; i < buckets.Length; i++) + { + seen += buckets[i]; + if (seen >= target) return i < UpperMs.Length ? Math.Min(UpperMs[i], Math.Max(maxMs, 1)) : maxMs; + } + return maxMs; + } +} + +/// The ten bucket counts, as columns, on every rollup that times something. +public abstract class TimedStats +{ + public long DurationSumMs { get; set; } + public long DurationMaxMs { get; set; } + public int D0 { get; set; } + public int D1 { get; set; } + public int D2 { get; set; } + public int D3 { get; set; } + public int D4 { get; set; } + public int D5 { get; set; } + public int D6 { get; set; } + public int D7 { get; set; } + public int D8 { get; set; } + public int D9 { get; set; } + + public long[] Buckets() => [D0, D1, D2, D3, D4, D5, D6, D7, D8, D9]; + + public void AddDuration(long ms, int times = 1) + { + DurationSumMs += ms * times; + if (ms > DurationMaxMs) DurationMaxMs = ms; + switch (DurationBuckets.IndexOf(ms)) + { + case 0: D0 += times; break; + case 1: D1 += times; break; + case 2: D2 += times; break; + case 3: D3 += times; break; + case 4: D4 += times; break; + case 5: D5 += times; break; + case 6: D6 += times; break; + case 7: D7 += times; break; + case 8: D8 += times; break; + default: D9 += times; break; + } + } + + public void AddTimes(TimedStats other) + { + DurationSumMs += other.DurationSumMs; + if (other.DurationMaxMs > DurationMaxMs) DurationMaxMs = other.DurationMaxMs; + D0 += other.D0; D1 += other.D1; D2 += other.D2; D3 += other.D3; D4 += other.D4; + D5 += other.D5; D6 += other.D6; D7 += other.D7; D8 += other.D8; D9 += other.D9; + } +} diff --git a/SW.Bitween.Api/Domain/Metrics/ErrorStats.cs b/SW.Bitween.Api/Domain/Metrics/ErrorStats.cs new file mode 100644 index 00000000..be6e4698 --- /dev/null +++ b/SW.Bitween.Api/Domain/Metrics/ErrorStats.cs @@ -0,0 +1,26 @@ +using System; + +namespace SW.Bitween.Domain.Metrics; + +/// +/// How often one kind of error happened in an hour, for an adapter, a subscription or a gateway. +/// Errors are told apart by a fingerprint of their message with the parts that change every time +/// (ids, numbers, dates) taken out, so "Order 1043 not found" and "Order 2210 not found" count as +/// one, with the latest whole message kept as an example. +/// +public class ErrorStats +{ + public const string AdapterScope = "adapter"; + public const string SubscriptionScope = "subscription"; + public const string GatewayScope = "gateway"; + + public DateTime Hour { get; set; } + public string Node { get; set; } + public string Scope { get; set; } + /// The adapter id, subscription id or gateway url name. + public string ScopeKey { get; set; } + public string Fingerprint { get; set; } + public string Sample { get; set; } + public int Count { get; set; } + public DateTime LastSeenOn { get; set; } +} diff --git a/SW.Bitween.Api/Domain/Metrics/ExchangeStats.cs b/SW.Bitween.Api/Domain/Metrics/ExchangeStats.cs new file mode 100644 index 00000000..ad384d79 --- /dev/null +++ b/SW.Bitween.Api/Domain/Metrics/ExchangeStats.cs @@ -0,0 +1,70 @@ +using System; + +namespace SW.Bitween.Domain.Metrics; + +/// +/// One hour of exchanges for one subscription and partner, as one node processed them. Counted in +/// memory as each exchange finishes and added in here once a minute (see MetricsFlusher), so no +/// page ever counts exchanges themselves. Hours are those the exchanges started in. +/// +/// +/// A node's rows carry its name, so nodes never write the same row. History from before metrics +/// were recorded is folded in once under . +/// +public class ExchangeStats : TimedStats +{ + /// The node name history rows are written under. + public const string HistoryNode = "(history)"; + + /// 0 when there is none: an exchange routed by information type alone. + public const int None = 0; + + /// The subscription and partner of the row that marks an hour of history as done. + public const int HourDone = -1; + + public DateTime Hour { get; set; } + public string Node { get; set; } + public int SubscriptionId { get; set; } + public int PartnerId { get; set; } + + public int Total { get; set; } + public int Succeeded { get; set; } + public int BadResponse { get; set; } + public int Failed { get; set; } + public int FailedAtMapper { get; set; } + public int FailedAtHandler { get; set; } + /// Exchanges that were themselves retries of another. + public int Retries { get; set; } + public int ManualRetries { get; set; } + public long BytesIn { get; set; } + public long BytesOut { get; set; } + + /// + /// Database commands the exchanges sent, summed over their stages: creation, processing, the + /// result lane and a gateway's sync wait. Over it is commands per exchange. + /// + public long DbCommands { get; set; } + + /// + /// Full parses of the exchanges' payloads into a tree, summed over the same stages. Over + /// it is parses per exchange. Null on rows written before it was counted. + /// + public long? PayloadParses { get; set; } + + public void Add(ExchangeStats other) + { + Total += other.Total; + Succeeded += other.Succeeded; + BadResponse += other.BadResponse; + Failed += other.Failed; + FailedAtMapper += other.FailedAtMapper; + FailedAtHandler += other.FailedAtHandler; + Retries += other.Retries; + ManualRetries += other.ManualRetries; + BytesIn += other.BytesIn; + BytesOut += other.BytesOut; + DbCommands += other.DbCommands; + if (other.PayloadParses is { } parses) PayloadParses = (PayloadParses ?? 0) + parses; + AddTimes(other); + } +} diff --git a/SW.Bitween.Api/Domain/Metrics/QueueSnapshot.cs b/SW.Bitween.Api/Domain/Metrics/QueueSnapshot.cs new file mode 100644 index 00000000..10806d2f --- /dev/null +++ b/SW.Bitween.Api/Domain/Metrics/QueueSnapshot.cs @@ -0,0 +1,20 @@ +using System; + +namespace SW.Bitween.Domain.Metrics; + +/// +/// The broker's totals at one moment, taken every few minutes, so the backlog can be shown as a +/// trend and not only as it is right now. Kept for a month. +/// +public class QueueSnapshot +{ + /// The time, cut to the sampling interval: two nodes sampling together keep one. + public DateTime At { get; set; } + public long QueueDepth { get; set; } + public long RetryBacklog { get; set; } + public long DeadLetterBacklog { get; set; } + public double IncomingRate { get; set; } + public double AckRate { get; set; } + public int Consumers { get; set; } + public int UnhealthyConsumers { get; set; } +} diff --git a/SW.Bitween.Api/Domain/NotificationChannel.cs b/SW.Bitween.Api/Domain/NotificationChannel.cs new file mode 100644 index 00000000..ec40b48e --- /dev/null +++ b/SW.Bitween.Api/Domain/NotificationChannel.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Generic; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Domain; + +/// +/// Somewhere Bitween can send a notification: a handler adapter and everything it needs to deliver, +/// such as the recipients or a webhook url. Set up once in Settings, then picked by name wherever +/// something is worth telling someone about. +/// +/// +/// A channel is complete. The places that use one choose only when to send, never how, so +/// what the Settings page shows for a channel is exactly what every use of it sends. Different +/// recipients are a second channel. +/// +public class NotificationChannel : BaseEntity, IAudited +{ + public string Name { get; set; } + + /// Any handler adapter, built in or custom. + public string HandlerId { get; set; } + + /// The adapter's settings. Encrypted at rest, since they can hold passwords and keys. + public IReadOnlyDictionary HandlerProperties { get; set; } = + new Dictionary(); + + public DateTime CreatedOn { get; set; } + public string CreatedBy { get; set; } + public DateTime? ModifiedOn { get; set; } + public string ModifiedBy { get; set; } +} diff --git a/SW.Bitween.Api/Domain/OperatorAction.cs b/SW.Bitween.Api/Domain/OperatorAction.cs new file mode 100644 index 00000000..41c8e1ec --- /dev/null +++ b/SW.Bitween.Api/Domain/OperatorAction.cs @@ -0,0 +1,52 @@ +using System; + +namespace SW.Bitween.Domain; + +/// +/// Something a member did that changes what Bitween does without changing its configuration: +/// retrying exchanges, exporting them, requeueing dead letters, deleting queues, running a scheduled +/// retry now. Recorded so the audit trail answers "who did that", as it does for configuration. +/// +public class OperatorAction +{ + public const string Retry = "retry"; + public const string BulkRetry = "bulk-retry"; + public const string Export = "export"; + public const string RunRetryNow = "run-retry-now"; + public const string RequeueDeadLetters = "requeue-dead-letters"; + public const string DeleteQueues = "delete-queues"; + public const string ForgetBookmark = "forget-bookmark"; + public const string InstallKit = "install-kit"; + public const string AskExchanges = "ask-exchanges"; + + /// A failed exchange sent, masked, to the AI provider under Settings, AI for an explanation. + public const string ExplainFailure = "explain-failure"; + public const string SuggestRetryPolicy = "suggest-retry-policy"; + + private OperatorAction() + { + } + + public OperatorAction(string action, string target, string detail, string accountId) + { + Action = action; + Target = Clip(target, 500); + Detail = Clip(detail, 2000); + AccountId = accountId; + OccurredOn = DateTime.UtcNow; + } + + public long Id { get; private set; } + public string Action { get; private set; } + + /// What it was done to: an exchange id, a queue name, a selection. + public string Target { get; private set; } + + /// How much it touched, in a sentence: "12 exchanges", "3 queues deleted, 1 skipped". + public string Detail { get; private set; } + + public string AccountId { get; private set; } + public DateTime OccurredOn { get; private set; } + + static string Clip(string value, int max) => value == null || value.Length <= max ? value : value[..max]; +} diff --git a/SW.Bitween.Api/Domain/OutboxMessage.cs b/SW.Bitween.Api/Domain/OutboxMessage.cs new file mode 100644 index 00000000..52490f13 --- /dev/null +++ b/SW.Bitween.Api/Domain/OutboxMessage.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.Generic; + +namespace SW.Bitween.Domain; + +/// +/// A bus message waiting to be published, written in the same transaction as the change that +/// raised it. +/// +/// +/// Messages used to be published straight after the commit. When the broker was unreachable at +/// that moment, the change was saved and its message was lost: an exchange that was stored and +/// never processed, with nothing to notice. Now the message is stored with the change, published +/// straight after the commit as before, and - when that fails - published later by +/// OutboxDispatcher. Delivery is at least once, so consumers must tolerate a repeat. +/// +public class OutboxMessage +{ + private OutboxMessage() + { + } + + public OutboxMessage(string messageType, string body, Dictionary headers = null) + { + MessageType = messageType; + Body = body; + Headers = headers; + CreatedOn = DateTime.UtcNow; + } + + public long Id { get; private set; } + public string MessageType { get; private set; } + public string Body { get; private set; } + public Dictionary Headers { get; private set; } + public DateTime CreatedOn { get; private set; } + + /// Null until the broker has taken it. + public DateTime? PublishedOn { get; private set; } + + /// A dispatcher's claim on it, so two replicas never publish the same row at once. + public DateTime? ClaimedUntil { get; private set; } + + public int Attempts { get; private set; } + public string LastError { get; private set; } +} diff --git a/SW.Bitween.Api/Domain/Partner/ApiCredential.cs b/SW.Bitween.Api/Domain/Partner/ApiCredential.cs index da21c7e7..57de7900 100644 --- a/SW.Bitween.Api/Domain/Partner/ApiCredential.cs +++ b/SW.Bitween.Api/Domain/Partner/ApiCredential.cs @@ -8,15 +8,23 @@ private ApiCredential() { } + /// The key itself; only its hash is kept. See . public ApiCredential(string name, string key) { Name = name ?? throw new ArgumentNullException(nameof(name)); - Key = key ?? throw new ArgumentNullException(nameof(key)); + if (key == null) throw new ArgumentNullException(nameof(key)); + Key = PartnerKeyHash.Stored(key); + if (!PartnerKeyHash.IsHashed(key)) KeyPrefix = key.Length > 5 ? key[..5] : key; } public string Name { get; private set; } + + /// The key's hash - sha256:... - or, on a row not yet converted, the key. public string Key { get; private set; } + /// The first characters of the key, so a screen can tell one key from another. + public string KeyPrefix { get; private set; } + public override bool Equals(object obj) { return obj is ApiCredential credential && diff --git a/SW.Bitween.Api/Domain/Partner/ApiKeyUse.cs b/SW.Bitween.Api/Domain/Partner/ApiKeyUse.cs new file mode 100644 index 00000000..85c1ee22 --- /dev/null +++ b/SW.Bitween.Api/Domain/Partner/ApiKeyUse.cs @@ -0,0 +1,26 @@ +using System; + +namespace SW.Bitween.Domain; + +/// +/// When a partner's API key last got a call through a gateway. Kept beside the key rather than on +/// it, so a gateway call writes one small row instead of the partner; written at most every few +/// minutes per key and node (see ), so it is approximate by that much. +/// +public class ApiKeyUse +{ + private ApiKeyUse() + { + } + + public ApiKeyUse(int partnerId, string keyName, DateTime usedOn) + { + PartnerId = partnerId; + KeyName = keyName; + LastUsedOn = usedOn; + } + + public int PartnerId { get; private set; } + public string KeyName { get; private set; } + public DateTime LastUsedOn { get; set; } +} diff --git a/SW.Bitween.Api/Domain/Partner/Partner.cs b/SW.Bitween.Api/Domain/Partner/Partner.cs index 33332e54..c0d0975d 100644 --- a/SW.Bitween.Api/Domain/Partner/Partner.cs +++ b/SW.Bitween.Api/Domain/Partner/Partner.cs @@ -36,7 +36,7 @@ public Partner(string name) /// Names within whose values are never returned by the API /// in clear. Opt-in and freely reversible: a partner property is ordinary text until /// someone marks it, which is why the list lives here rather than being inferred from the - /// key's name — inferring would have hidden values that were readable yesterday. + /// key's name - inferring would have hidden values that were readable yesterday. /// public List SecretProperties { get; set; } = new(); diff --git a/SW.Bitween.Api/Domain/Partner/PartnerKeyHash.cs b/SW.Bitween.Api/Domain/Partner/PartnerKeyHash.cs new file mode 100644 index 00000000..f6fdb033 --- /dev/null +++ b/SW.Bitween.Api/Domain/Partner/PartnerKeyHash.cs @@ -0,0 +1,28 @@ +using System; +using System.Security.Cryptography; +using System.Text; + +namespace SW.Bitween.Domain +{ + /// + /// Partner API keys are stored as a SHA-256 of the key, never the key itself, so a copy of the + /// database no longer lets anyone call Bitween as every partner. + /// + /// + /// Unsalted on purpose. Keys are random - 128 bits from a GUID - so there is nothing to look up + /// in a table, and an unsalted hash keeps the lookup a single indexed equality. + /// + public static class PartnerKeyHash + { + public const string Prefix = "sha256:"; + + public static string Of(string key) => + Prefix + Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(key))).ToLowerInvariant(); + + public static bool IsHashed(string stored) => + stored != null && stored.StartsWith(Prefix, StringComparison.Ordinal); + + /// The stored form of a key, whichever form it arrives in. + public static string Stored(string key) => IsHashed(key) ? key : Of(key); + } +} diff --git a/SW.Bitween.Api/Domain/ReceiveAttempt.cs b/SW.Bitween.Api/Domain/ReceiveAttempt.cs index a7ceaa54..e887a211 100644 --- a/SW.Bitween.Api/Domain/ReceiveAttempt.cs +++ b/SW.Bitween.Api/Domain/ReceiveAttempt.cs @@ -6,7 +6,7 @@ namespace SW.Bitween.Domain; /// /// One execution of a Receiving subscription's receive step, written by ReceivingJob -/// itself right where it already catches the receiver's own failures — kept independent of +/// itself right where it already catches the receiver's own failures - kept independent of /// Quartz's own run history and unaffected by how Quartz treats a thrown exception. /// public class ReceiveAttempt : BaseEntity diff --git a/SW.Bitween.Api/Domain/RetryAlertOverride.cs b/SW.Bitween.Api/Domain/RetryAlertOverride.cs index 61d1b6b9..c7d1c2c0 100644 --- a/SW.Bitween.Api/Domain/RetryAlertOverride.cs +++ b/SW.Bitween.Api/Domain/RetryAlertOverride.cs @@ -27,9 +27,15 @@ public class RetryAlertOverride /// public RetryAlertMode AlertMode { get; set; } - /// Adapter that delivers the alert. Required when is Send. + /// Channel that delivers the alert. Required when is Send. + public int? AlertChannelId { get; set; } + + /// + /// Retired: alerts go through . Only the one-time move onto a + /// channel reads it, and clears it. + /// public string AlertHandlerId { get; set; } - /// That adapter's own settings — api key, recipients, subject. + /// Retired along with . public IReadOnlyDictionary AlertHandlerProperties { get; set; } } diff --git a/SW.Bitween.Api/Domain/RetryBudgetExhaustedEvent.cs b/SW.Bitween.Api/Domain/RetryBudgetExhaustedEvent.cs index b5f7e29f..f9c72524 100644 --- a/SW.Bitween.Api/Domain/RetryBudgetExhaustedEvent.cs +++ b/SW.Bitween.Api/Domain/RetryBudgetExhaustedEvent.cs @@ -33,7 +33,7 @@ public class RetryBudgetExhaustedEvent : BaseDomainEvent /// /// The ceiling that was reached. Not paired with an "used" count, because at exhaustion the two - /// are the same number — except when the ceiling was lowered below what had already been spent, + /// are the same number - except when the ceiling was lowered below what had already been spent, /// where the ceiling is still the meaningful figure. /// public int MaxAttemptsTotal { get; set; } diff --git a/SW.Bitween.Api/Domain/RetryGroupUsage.cs b/SW.Bitween.Api/Domain/RetryGroupUsage.cs index b19f7a9b..fddb123c 100644 --- a/SW.Bitween.Api/Domain/RetryGroupUsage.cs +++ b/SW.Bitween.Api/Domain/RetryGroupUsage.cs @@ -10,8 +10,8 @@ namespace SW.Bitween.Domain; /// /// Once reaches the group's MaxAttemptsTotal the group stops /// retrying for that integration until this row is cleared. A row that has reached the cap is cleared -/// by the integration's next success — the only signal that the downstream it was failing against has -/// recovered — or by one of the reset endpoints. A row still below the cap is left alone by a success: +/// by the integration's next success - the only signal that the downstream it was failing against has +/// recovered - or by one of the reset endpoints. A row still below the cap is left alone by a success: /// the cap is there for a downstream that fails some messages and succeeds others, which is exactly /// when crediting it back would stop it ever being reached. /// @@ -25,7 +25,7 @@ public class RetryGroupUsage public int AttemptsUsed { get; set; } - /// When the last attempt was claimed — the only clue left once a group is exhausted. + /// When the last attempt was claimed - the only clue left once a group is exhausted. public DateTime LastAttemptOn { get; set; } /// diff --git a/SW.Bitween.Api/Domain/RetryPolicy.cs b/SW.Bitween.Api/Domain/RetryPolicy.cs index c452e6f3..71e68e34 100644 --- a/SW.Bitween.Api/Domain/RetryPolicy.cs +++ b/SW.Bitween.Api/Domain/RetryPolicy.cs @@ -11,12 +11,18 @@ public class RetryPolicy : BaseEntity, IAudited, IRetryPolicy public List Groups { get; set; } = []; /// - /// Default destination for "retry budget exhausted" alerts, used by every group that does not + /// Default channel for "retry budget exhausted" alerts, used by every group that does not /// override it. Null means no alert unless a group or a subscription+group override defines one. /// + public int? AlertChannelId { get; set; } + + /// + /// Retired: alerts go through . Only the one-time move onto a + /// channel reads it, and clears it. + /// public string AlertHandlerId { get; set; } - /// That adapter's own settings — api key, recipients, subject. + /// Retired along with . public IReadOnlyDictionary AlertHandlerProperties { get; set; } public DateTime CreatedOn { get; set; } public string CreatedBy { get; set; } diff --git a/SW.Bitween.Api/Domain/Setting.cs b/SW.Bitween.Api/Domain/Setting.cs index 61d566e8..d86e5d91 100644 --- a/SW.Bitween.Api/Domain/Setting.cs +++ b/SW.Bitween.Api/Domain/Setting.cs @@ -5,14 +5,14 @@ namespace SW.Bitween.Domain; /// /// One instance-wide setting, keyed by the setting's catalog key (e.g. Theme.PrimaryColor). -/// This table is the single source of truth: configuration seeds a key once — on the first boot -/// after that key exists — and is ignored for it from then on. Every catalog key normally has a +/// This table is the single source of truth: configuration seeds a key once - on the first boot +/// after that key exists - and is ignored for it from then on. Every catalog key normally has a /// row; "reset to default" rewrites the row with the product default rather than removing it. /// /// Deliberately a plain key/value store: the definition of a key (label, section, type, /// whether it's a secret) lives in , so adding a /// setting never needs a migration or a data fix-up. A secret's value is encrypted before it -/// gets here — see . +/// gets here - see . /// /// public class Setting : BaseEntity, IAudited diff --git a/SW.Bitween.Api/Domain/Subscription/Schedule.cs b/SW.Bitween.Api/Domain/Subscription/Schedule.cs index eaed7744..3a4b06c4 100644 --- a/SW.Bitween.Api/Domain/Subscription/Schedule.cs +++ b/SW.Bitween.Api/Domain/Subscription/Schedule.cs @@ -57,7 +57,7 @@ public override int GetHashCode() public DateTime Next(DateTime? currentDate = null) { - // On.Hours/On.Minutes are UTC wall-clock time — the Quartz cron trigger that + // On.Hours/On.Minutes are UTC wall-clock time - the Quartz cron trigger that // actually fires this schedule is explicitly pinned to TimeZoneInfo.Utc // (see SW.Scheduler's WithCronSchedule(...).InTimeZone(TimeZoneInfo.Utc) call // sites), so this must agree rather than each side defaulting independently. diff --git a/SW.Bitween.Api/Domain/Subscription/Subscription.cs b/SW.Bitween.Api/Domain/Subscription/Subscription.cs index 7647247d..e8522e01 100644 --- a/SW.Bitween.Api/Domain/Subscription/Subscription.cs +++ b/SW.Bitween.Api/Domain/Subscription/Subscription.cs @@ -3,6 +3,7 @@ using SW.PrimitiveTypes; using System; using System.Collections.Generic; +using System.Linq; namespace SW.Bitween.Domain; @@ -75,8 +76,42 @@ private Subscription(string name, int documentId, SubscriptionType type, int? pa public WorkGroup WorkGroup { get; set; } public bool Temporary { get; private set; } public DateTime? PausedOn { get; private set; } + + /// + /// Failed deliveries in a row that pause the subscription; null never does. Counted in + /// , which receivers and aggregations use for their runs. + /// + public int? AutoPauseAfterFailures { get; set; } + + /// Paused by rather than by a person. + public bool PausedAutomatically { get; private set; } + + /// How often an exchange should arrive; null expects nothing. Validated by . + public ExpectTraffic ExpectTraffic { get; private set; } + + /// + /// When QuietSubscriptionsJob found no exchange where wanted one; + /// null while traffic arrives. Written by the job alone, with a conditional update, so two nodes + /// can't both raise it. + /// + public DateTime? QuietSince { get; private set; } + + /// Which finished exchanges are reported, and through which notification channel. + public List Notifications { get; private set; } = []; + + public void SetNotifications(IEnumerable notifications) + { + Notifications = notifications?.ToList() ?? []; + } + + public void SetExpectTraffic(ExpectTraffic expectTraffic) + { + ExpectTraffic = expectTraffic; + // Nothing is expected any more, so nothing can be missing; left set it would show for good. + if (expectTraffic == null) QuietSince = null; + } /// - /// Which external system this subscription's adapters connect through — a database, typically. + /// Which external system this subscription's adapters connect through - a database, typically. /// /// Null keeps every existing subscription exactly as it was: an adapter carries its own /// connection settings in its properties. Setting one moves that job to the data source, which @@ -94,6 +129,21 @@ private Subscription(string name, int documentId, SubscriptionType type, int? pa public string ReceiverId { get; set; } public string MapperId { get; set; } + + /// + /// Published versions this subscription is pinned to, one per adapter slot; null follows + /// whatever version is current - which is how every subscription behaved before versions. + /// + public string ReceiverVersion { get; set; } + public string ValidatorVersion { get; set; } + public string MapperVersion { get; set; } + public string HandlerVersion { get; set; } + + /// What actually runs for a slot: the adapter id, or {id}/{version} when pinned. + public string ReceiverRef => SW.Serverless.Contract.Catalog.AdapterCatalogPaths.Ref(ReceiverId, ReceiverVersion); + public string ValidatorRef => SW.Serverless.Contract.Catalog.AdapterCatalogPaths.Ref(ValidatorId, ValidatorVersion); + public string MapperRef => SW.Serverless.Contract.Catalog.AdapterCatalogPaths.Ref(MapperId, MapperVersion); + public string HandlerRef => SW.Serverless.Contract.Catalog.AdapterCatalogPaths.Ref(HandlerId, HandlerVersion); public IReadOnlyDictionary ValidatorProperties { get; private set; } public IReadOnlyDictionary HandlerProperties { get; private set; } public IReadOnlyDictionary MapperProperties { get; private set; } @@ -102,6 +152,19 @@ private Subscription(string name, int documentId, SubscriptionType type, int? pa public IPropertyMatchSpecification MatchExpression { get; private set; } public bool IsRunning { get; set; } + + /// + /// When the run that set started. A process killed mid-run never clears + /// the flag, so without a time the subscription stayed "running" - and never ran - for good. + /// + public DateTime? RunningSince { get; private set; } + + /// Lets the next scheduled run start: for a flag left behind by a run that died. + public void ClearRunning() + { + IsRunning = false; + RunningSince = null; + } public bool Inactive { get; set; } public int? ResponseSubscriptionId { get; set; } /// @@ -183,7 +246,7 @@ public void SetMatchExpression(IPropertyMatchSpecification matchExpression) /// /// Assigns the subscription's retry policy. An inline - /// always takes precedence over a referenced — the two are + /// always takes precedence over a referenced - the two are /// mutually exclusive, matching the resolution order used at evaluation time /// (CustomRetryPolicy ?? RetryPolicy). Setting a custom policy clears any referenced /// one and vice versa, so the two fields can never disagree. @@ -205,11 +268,21 @@ public void SetRetryPolicy(int? retryPolicyId, CustomRetryPolicy customRetryPoli public void Pause() { PausedOn = DateTime.UtcNow; + PausedAutomatically = false; } public void UnPause() { + // Resuming after an automatic pause starts the count again; otherwise the next failure + // would pause it straight back. + if (PausedAutomatically) + { + ConsecutiveFailures = 0; + LastException = null; + } + PausedOn = null; + PausedAutomatically = false; Events.Add(new SubscriptionUnpausedEvent { Id = Id diff --git a/SW.Bitween.Api/Domain/Subscription/SubscriptionAutoPausedEvent.cs b/SW.Bitween.Api/Domain/Subscription/SubscriptionAutoPausedEvent.cs new file mode 100644 index 00000000..22aed1cc --- /dev/null +++ b/SW.Bitween.Api/Domain/Subscription/SubscriptionAutoPausedEvent.cs @@ -0,0 +1,20 @@ +using System; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Domain; + +/// +/// A subscription paused itself after failing deliveries in a row. Picked up +/// by its own consumer, as is, so a broken alert handler +/// holds up nothing else. +/// +public class SubscriptionAutoPausedEvent : BaseDomainEvent +{ + public int SubscriptionId { get; set; } + + /// The failure that paused it. + public string XchangeId { get; set; } + + public int Failures { get; set; } + public DateTime OccurredOn { get; set; } +} diff --git a/SW.Bitween.Api/Domain/Subscription/SubscriptionRevision.cs b/SW.Bitween.Api/Domain/Subscription/SubscriptionRevision.cs new file mode 100644 index 00000000..0b44030e --- /dev/null +++ b/SW.Bitween.Api/Domain/Subscription/SubscriptionRevision.cs @@ -0,0 +1,41 @@ +using System; + +namespace SW.Bitween.Domain; + +/// +/// A subscription's configuration as one save left it: what its history lists, compares and restores. +/// +/// +/// The configuration as the API reads it, so a secret is the __private__ sentinel, never its +/// value: restoring a revision keeps the secrets the subscription has now. Running state (failures, +/// pauses, last runs) is left out, since it isn't something a person chose. +/// +public class SubscriptionRevision +{ + private SubscriptionRevision() + { + } + + public SubscriptionRevision(int subscriptionId, int number, string snapshot, string savedBy) + { + SubscriptionId = subscriptionId; + Number = number; + Snapshot = snapshot; + SavedBy = savedBy; + SavedOn = DateTime.UtcNow; + } + + public long Id { get; private set; } + public int SubscriptionId { get; private set; } + + /// 1 for the first save recorded, counting up. + public int Number { get; private set; } + + /// The configuration, as camel-cased JSON in the API's own shape. + public string Snapshot { get; private set; } + + /// The account that saved it, as the audit trail names accounts. + public string SavedBy { get; private set; } + + public DateTime SavedOn { get; private set; } +} diff --git a/SW.Bitween.Api/Domain/WorkGroup/WorkGroup.cs b/SW.Bitween.Api/Domain/WorkGroup/WorkGroup.cs index 23cef165..fa5ebf7a 100644 --- a/SW.Bitween.Api/Domain/WorkGroup/WorkGroup.cs +++ b/SW.Bitween.Api/Domain/WorkGroup/WorkGroup.cs @@ -21,4 +21,12 @@ public class WorkGroup : BaseEntity,IWorkGroup //public string public static WorkGroup None => new() { BusMessageName = "Ungrouped"}; public WorkGroupOptions Options { get; set; } + + /// + /// How many exchanges of this group's work lane run at once on a node: the group's parallel + /// exchanges, else its prefetch, else the default prefetch. Its -Result lane always runs one + /// at a time. + /// + public ushort WorkLaneConcurrency(ushort defaultPrefetch) => + Options?.RabbitMqOptions?.Concurrency ?? Options?.RabbitMqOptions?.Prefetch ?? defaultPrefetch; } \ No newline at end of file diff --git a/SW.Bitween.Api/Domain/Xchange/OnHoldXchange.cs b/SW.Bitween.Api/Domain/Xchange/OnHoldXchange.cs index f62ee190..505fb8f3 100644 --- a/SW.Bitween.Api/Domain/Xchange/OnHoldXchange.cs +++ b/SW.Bitween.Api/Domain/Xchange/OnHoldXchange.cs @@ -29,7 +29,7 @@ public OnHoldXchange(Subscription subscription, string data, string fileName = n public string[] References { get; private set; } /// - /// The partner to run as once released, when it is not the subscription's own — a response + /// The partner to run as once released, when it is not the subscription's own - a response /// runs as the partner of the subscription that fed it. Null means the subscription's own. /// public int? PartnerId { get; private set; } diff --git a/SW.Bitween.Api/Domain/Xchange/Xchange.cs b/SW.Bitween.Api/Domain/Xchange/Xchange.cs index 658a24a7..ff8a100e 100644 --- a/SW.Bitween.Api/Domain/Xchange/Xchange.cs +++ b/SW.Bitween.Api/Domain/Xchange/Xchange.cs @@ -54,8 +54,9 @@ public Xchange(Subscription subscription, XchangeFile file, string[] references this(subscription.DocumentId, subscription.WorkGroup, file, references, subscription.Type) { SubscriptionId = subscription.Id; - MapperId = subscription.MapperId; - HandlerId = subscription.HandlerId; + // The pinned version travels with the exchange, so a retry runs the same package. + MapperId = subscription.MapperRef; + HandlerId = subscription.HandlerRef; ResponseSubscriptionId = subscription.ResponseSubscriptionId; ResponseMessageTypeName = subscription.ResponseMessageTypeName; PartnerId = gatewayPartner?.Id ?? subscription.PartnerId; @@ -96,8 +97,9 @@ public Xchange(Subscription subscription, Xchange xchange, XchangeFile file, Par ManualRetry = manualRetry; SubscriptionId = xchange.SubscriptionId; PartnerId = xchange.PartnerId ?? subscription.PartnerId; - MapperId = subscription.MapperId; - HandlerId = subscription.HandlerId; + // The pinned version travels with the exchange, so a retry runs the same package. + MapperId = subscription.MapperRef; + HandlerId = subscription.HandlerRef; MapperProperties = (subscription.MapperProperties ?? new Dictionary()).ToDictionary() .Fill(gatewayPartner, globalAdapterValuesSets).WithDataSource(subscription.DataSourceId); HandlerProperties = (subscription.HandlerProperties ?? new Dictionary()).ToDictionary() @@ -134,8 +136,8 @@ public Xchange(Subscription subscription, Xchange xchange, XchangeFile file, Par public string SourceXchangeId { get; private set; } /// - /// What the subscription reads from the input of — the original - /// document — by path, as it was when this exchange was created. A path the document had no + /// What the subscription reads from the input of - the original + /// document - by path, as it was when this exchange was created. A path the document had no /// value at is left out. /// /// diff --git a/SW.Bitween.Api/Domain/Xchange/XchangeCreatedEvent.cs b/SW.Bitween.Api/Domain/Xchange/XchangeCreatedEvent.cs index 734d4b71..836f45a3 100644 --- a/SW.Bitween.Api/Domain/Xchange/XchangeCreatedEvent.cs +++ b/SW.Bitween.Api/Domain/Xchange/XchangeCreatedEvent.cs @@ -1,4 +1,5 @@ -using SW.Bitween.Model; +using Newtonsoft.Json; +using SW.Bitween.Model; using SW.PrimitiveTypes; namespace SW.Bitween.Domain @@ -6,6 +7,26 @@ namespace SW.Bitween.Domain internal class XchangeMessage { public string Id { get; set; } + + /// + /// The exchange's input, as written to storage, when it is no larger than + /// Bitween:MessagePayloadMaxBytes: the worker uses it instead of reading storage. Only on + /// the publish straight after the commit; the outbox row stays the id alone, so a message the + /// dispatcher publishes again has none, and the worker reads storage, as it always did. + /// Storage stays the record: this is a copy for the trip. Left out when there is none, so a + /// stored body is exactly what it always was. + /// + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public string Input { get; set; } + + /// + /// The node holding the synchronous gateway call that waits for this exchange's result + /// (), which the worker tells once the result is saved. Set by + /// the gateway's sync calls only, on the publish straight after the commit; a message without + /// it is answered by the call's backstop poll. + /// + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public string AwaitedOn { get; set; } } internal abstract class XchangeCreatedEvent : BaseDomainEvent,IHasWorkGroup { diff --git a/SW.Bitween.Api/Domain/Xchange/XchangeRetryClaim.cs b/SW.Bitween.Api/Domain/Xchange/XchangeRetryClaim.cs new file mode 100644 index 00000000..7118e6a2 --- /dev/null +++ b/SW.Bitween.Api/Domain/Xchange/XchangeRetryClaim.cs @@ -0,0 +1,32 @@ +using System; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Domain; + +/// +/// That an exchange has been retried, keyed by the exchange retried, so the database itself refuses +/// a second retry of it: two committed at the very same moment can't both be kept. +/// +/// +/// A table of its own rather than a unique index on Xchange.RetryFor: exchanges retried +/// before the one-retry rule existed can already have forked, and an index can't be created over +/// them. Claims start with the first retry made after this table, so nothing old stands in the way. +/// +public class XchangeRetryClaim : BaseEntity +{ + private XchangeRetryClaim() + { + } + + public XchangeRetryClaim(string retriedXchangeId, string retryXchangeId) + { + Id = retriedXchangeId; + RetryXchangeId = retryXchangeId; + ClaimedOn = DateTime.UtcNow; + } + + /// The exchange made by the retry. + public string RetryXchangeId { get; private set; } + + public DateTime ClaimedOn { get; private set; } +} diff --git a/SW.Bitween.Api/Domain/Xchange/XchangeSource.cs b/SW.Bitween.Api/Domain/Xchange/XchangeSource.cs index 644f52e8..e8045884 100644 --- a/SW.Bitween.Api/Domain/Xchange/XchangeSource.cs +++ b/SW.Bitween.Api/Domain/Xchange/XchangeSource.cs @@ -8,7 +8,7 @@ namespace SW.Bitween.Domain /// /// The exchange whose delivery's response this one runs on. /// - /// Values read from the original document, by path. Null until they are read — a message off the + /// Values read from the original document, by path. Null until they are read - a message off the /// bus carries only the id, and each route reads what it uses. /// public record XchangeSource(string XchangeId, IReadOnlyDictionary Values = null); diff --git a/SW.Bitween.Api/Domain/XchangeDelivery.cs b/SW.Bitween.Api/Domain/XchangeDelivery.cs deleted file mode 100644 index 7232d3b8..00000000 --- a/SW.Bitween.Api/Domain/XchangeDelivery.cs +++ /dev/null @@ -1,22 +0,0 @@ -using SW.PrimitiveTypes; -using System; -using System.Collections.Generic; -using System.Text; - -namespace SW.Bitween.Domain -{ - public class XchangeDelivery : BaseEntity - { - private XchangeDelivery() - { - } - - public XchangeDelivery(string xchangeId) - { - Id = xchangeId; - DeliveredOn = DateTime.UtcNow; - } - - public DateTime DeliveredOn { get; private set; } - } -} diff --git a/SW.Bitween.Api/Domain/XchangeNotification.cs b/SW.Bitween.Api/Domain/XchangeNotification.cs index 97e8094b..24428415 100644 --- a/SW.Bitween.Api/Domain/XchangeNotification.cs +++ b/SW.Bitween.Api/Domain/XchangeNotification.cs @@ -23,21 +23,46 @@ public XchangeNotification(string xchangeId, int? notifierId, string notifierNam /// /// Logs an attempt to deliver a "retry budget exhausted" alert. These rows have no /// because the alert is configured on the retry policy rather than - /// on a notifier — which is also how the send is recognised as already done on a redelivery. + /// on a notifier - which is also how the send is recognised as already done on a redelivery. /// - public static XchangeNotification ForRetryBudgetAlert(string xchangeId, string exception = null) => - new(xchangeId, null, RetryBudgetAlertName, exception); + public static XchangeNotification ForRetryBudgetAlert(string xchangeId, int channelId, + string exception = null) => + new(xchangeId, null, RetryBudgetAlertName, exception) { ChannelId = channelId }; + + public const string AutoPauseAlertName = "Auto-pause alert"; + + /// + /// The alert that the subscription this exchange belongs to was paused by its failure. One row + /// per channel told, so a redelivery skips the channels that already heard. + /// + public static XchangeNotification ForAutoPauseAlert(string xchangeId, int channelId, + string exception = null) => + new(xchangeId, null, AutoPauseAlertName, exception) { ChannelId = channelId }; + + /// Logs a send a subscription's own notifications made through a channel. + public static XchangeNotification ForChannel(string xchangeId, int channelId, string channelName, + string exception = null) => + new(xchangeId, null, channelName, exception) { ChannelId = channelId }; public string XchangeId { get; private set; } public bool Success { get; set; } - /// The notifier that produced this row, or null for a retry-budget alert. + /// The notifier that produced this row, or null for an alert or a channel send. public int? NotifierId { get; set; } + + /// The channel the send went through. Null on rows written by a legacy notifier. + public int? ChannelId { get; private set; } public string NotifierName { get; set; } public string Exception { get; private set; } public DateTime FinishedOn { get; private set; } + + /// + /// Not sent: the notifier or channel had already sent for this subscription and outcome within its + /// quiet window. Counted into the next notification that is sent. + /// + public bool Suppressed { get; set; } } } \ No newline at end of file diff --git a/SW.Bitween.Api/Domain/XchangePromotedProperties.cs b/SW.Bitween.Api/Domain/XchangePromotedProperties.cs index f3638f9b..b7660abf 100644 --- a/SW.Bitween.Api/Domain/XchangePromotedProperties.cs +++ b/SW.Bitween.Api/Domain/XchangePromotedProperties.cs @@ -11,7 +11,7 @@ public class XchangePromotedProperties : BaseEntity /// The most a promoted value is stored as. A longer one is cut, ending in "…" to show it. /// /// - /// Promoted values find and name an exchange — an order number, a customer — and they are + /// Promoted values find and name an exchange - an order number, a customer - and they are /// shown on the Exchanges page and name its archived files. A path that picks up a notes /// field or a whole object would put thousands of characters in all of those. Routing is /// not affected: match expressions read the payload, not these. diff --git a/SW.Bitween.Api/Domain/XchangeResult/XchangeResult.cs b/SW.Bitween.Api/Domain/XchangeResult/XchangeResult.cs index ed666dee..7e80e490 100644 --- a/SW.Bitween.Api/Domain/XchangeResult/XchangeResult.cs +++ b/SW.Bitween.Api/Domain/XchangeResult/XchangeResult.cs @@ -1,5 +1,7 @@ -using SW.PrimitiveTypes; +using SW.Bitween.Model; +using SW.PrimitiveTypes; using System; +using System.Linq; namespace SW.Bitween.Domain { @@ -45,6 +47,27 @@ public XchangeResult(string xchangeId,WorkGroup workGroup, XchangeFile outputFil }); } + /// + /// Which node processed it, which versions of the mapper and handler ran (the pinned one, or + /// the current one at the time), and how long it took from arriving to finishing. Null on + /// results from before these were recorded. + /// + public string NodeName { get; private set; } + public string MapperVersion { get; private set; } + public string HandlerVersion { get; private set; } + public int? DurationMs { get; private set; } + + public void Stamp(string nodeName, string mapperVersion, string handlerVersion, long durationMs) + { + NodeName = Cut(nodeName, 100); + MapperVersion = Cut(mapperVersion, 50); + HandlerVersion = Cut(handlerVersion, 50); + DurationMs = (int)System.Math.Clamp(durationMs, 0, int.MaxValue); + } + + static string Cut(string value, int length) => + string.IsNullOrEmpty(value) ? null : value.Length <= length ? value : value[..length]; + public bool Success { get; private set; } public string Exception { get; private set; } public DateTime FinishedOn { get; private set; } @@ -62,6 +85,60 @@ public XchangeResult(string xchangeId,WorkGroup workGroup, XchangeFile outputFil public bool ResponseBad { get; private set; } public string ResponseContentType { get; private set; } + /// The longest kept. + public const int WarningMaxLength = 2000; + + /// + /// Something about a successful result its reader should know, or null for nothing: + /// a delivery that went through while a file of it could not be stored, one line per file. + /// Null on results from before it was recorded. + /// + public string Warning { get; private set; } + + /// + /// Records that the delivery went through but the output or response file could not be + /// stored: the file is recorded as missing (no size, name or hash, so nothing links to it) + /// and the warning says why. Its content type and bad flag stay, since they describe what + /// the handler did rather than the file. + /// + public void FileNotStored(XchangeFileType type, string reason) + { + if (type == XchangeFileType.Output) + { + OutputSize = 0; + OutputName = null; + OutputHash = null; + } + else if (type == XchangeFileType.Response) + { + ResponseSize = 0; + ResponseName = null; + ResponseHash = null; + } + else throw new ArgumentOutOfRangeException(nameof(type), type, "Only the output and response files are stored after delivery."); + + // Each line gets its share of the column, so a long first reason never cuts the second off. + var line = NotStoredPrefix(type) + Cut(reason ?? "unknown error", WarningMaxLength / 2 - 60); + Warning = Warning == null ? line : Warning + "\n" + line; + } + + /// + /// Why 's file could not be stored after the delivery, read back from + /// ; null when it was stored, or never written. + /// + public string NotStoredReason(XchangeFileType type) => NotStoredReason(Warning, type); + + /// + public static string NotStoredReason(string warning, XchangeFileType type) + { + if (warning == null) return null; + var prefix = NotStoredPrefix(type); + return warning.Split('\n').FirstOrDefault(l => l.StartsWith(prefix, StringComparison.Ordinal))?[prefix.Length..]; + } + + static string NotStoredPrefix(XchangeFileType type) => + $"Delivered; the {type.ToString().ToLowerInvariant()} file could not be stored: "; + /// /// Why the retry policy declined to schedule another attempt for this failure, or /// null when a retry was scheduled or no policy applied. Without it a group that @@ -93,6 +170,37 @@ public void SetRetryEvaluation(Guid groupId, int attemptNumber) AttemptNumber = attemptNumber; } + /// + /// The subscription the exchange ran for, copied here so a subscription's latest result is + /// one indexed read. Null on results from before it was recorded, and for routing exchanges. + /// + public int? SubscriptionId { get; private set; } + + /// + /// How many failed deliveries in a row this result makes for its subscription: 0 for a + /// delivery that went through (a bad response included), the previous result's count plus + /// one for a failure. Counted for subscriptions set to pause after failures; null otherwise, + /// and on results from before it was recorded. What auto-pause counts, instead of a counter + /// on the subscription row that every exchange had to update. + /// + public int? ConsecutiveFailures { get; private set; } + + public void SetSubscription(int? subscriptionId) => SubscriptionId = subscriptionId; + + public void SetConsecutiveFailures(int? count) => ConsecutiveFailures = count; + + /// + /// Drops the result event when nothing is set to hear about it: no notifier and no + /// notification of the subscription's covers this outcome. Saves the result lane a message, + /// and the two reads it made only to find that out. Any other event raised on the result, + /// such as a spent retry budget, still goes. + /// + public void NothingToNotify() + { + foreach (var created in Events.OfType().ToList()) + Events.Remove(created); + } + /// /// Announces that this failure was the one that emptied the group's shared budget. Only ever /// called by the caller that won the claim, so the event is raised once per exhaustion. diff --git a/SW.Bitween.Api/Extensions/AccountExtensions.cs b/SW.Bitween.Api/Extensions/AccountExtensions.cs index e160fd12..d20c02ed 100644 --- a/SW.Bitween.Api/Extensions/AccountExtensions.cs +++ b/SW.Bitween.Api/Extensions/AccountExtensions.cs @@ -16,52 +16,87 @@ namespace SW.Bitween { public static class AccountExtensions { - public static Task GetEmailFromAzureJwtDefault(this UserLogin model) - => model.GetEmailFromAzureJwtDefault(null); + /// Who a Microsoft ID token says it is: the address to match, and the identity to bind. + public record MicrosoftIdentity(string Email, string ObjectAndTenant); - public static async Task GetEmailFromAzureJwtDefault(this UserLogin model, ILogger logger) + // Cached and refreshed by the manager itself. It used to be built on every sign-in, which + // fetched Microsoft's metadata and keys each time. + private static readonly ConfigurationManager MicrosoftMetadata = new( + "https://login.microsoftonline.com/common/v2.0/.well-known/openid-configuration", + new OpenIdConnectConfigurationRetriever(), + new HttpDocumentRetriever()); + + /// + /// Where Microsoft's signing keys come from: its published metadata. Replaced only by tests, + /// which sign their own tokens to take sign-in through the whole login handler. + /// + internal static Func>> MicrosoftSigningKeys { get; set; } = + async () => (await MicrosoftMetadata.GetConfigurationAsync()).SigningKeys; + + /// + /// Validates a Microsoft ID token for this Bitween's own app registration, or returns null. + /// + /// + /// The audience and issuer used to go unchecked, so a token Microsoft issued to any app, in + /// any tenant, signed in here - one a user handed to some unrelated site would do - and the + /// account was chosen by preferred_username, which Microsoft says not to authorize on. + /// + public static async Task ValidateMicrosoftTokenAsync(string jwt, string clientId, + string tenantId, ILogger logger) { + if (string.IsNullOrEmpty(jwt) || string.IsNullOrWhiteSpace(clientId)) return null; try { - var jwt = model.MsToken; - if (string.IsNullOrEmpty(jwt)) - { - logger?.LogWarning("GetEmailFromAzureJwtDefault: MsToken is null or empty."); - return null; - } - - var configurationManager = new ConfigurationManager( - "https://login.microsoftonline.com/common/v2.0/.well-known/openid-configuration", - new OpenIdConnectConfigurationRetriever(), - new HttpDocumentRetriever() - ); - var openIdConfig = await configurationManager.GetConfigurationAsync(); - var handler = new JwtSecurityTokenHandler(); - var tokenValidationParameters = new TokenValidationParameters - { - ValidateIssuer = false, - ValidateAudience = false, - ValidateLifetime = true, - ValidateIssuerSigningKey = true, - IssuerSigningKeys = openIdConfig.SigningKeys - }; - handler.ValidateToken(jwt, tokenValidationParameters, out _); - var jwtSecurityToken = handler.ReadJwtToken(jwt); - - var allClaims = string.Join(", ", jwtSecurityToken.Claims.Select(c => $"{c.Type}={c.Value}")); - logger?.LogInformation("MS token claims: {Claims}", allClaims); - - var email = jwtSecurityToken!.Claims.FirstOrDefault(i => i.Type.Contains("preferred_username"))?.Value; - logger?.LogInformation("MS token preferred_username claim value: '{Email}'", email ?? "(not found)"); - return string.IsNullOrWhiteSpace(email) ? null : email; + return ValidateMicrosoftToken(jwt, await MicrosoftSigningKeys(), clientId, tenantId); } catch (Exception ex) { - logger?.LogError(ex, "GetEmailFromAzureJwtDefault: failed to extract email from MS token."); + logger?.LogWarning("Microsoft sign-in refused: {Reason}", ex.Message); return null; } } + /// The checks themselves, against the given keys, so they can be tested. + public static MicrosoftIdentity ValidateMicrosoftToken(string jwt, IEnumerable signingKeys, + string clientId, string tenantId) + { + var handler = new JwtSecurityTokenHandler { MapInboundClaims = false }; + var parameters = new TokenValidationParameters + { + ValidateIssuerSigningKey = true, + IssuerSigningKeys = signingKeys, + ValidateLifetime = true, + ValidateAudience = true, + ValidAudience = clientId, + ValidateIssuer = true, + IssuerValidator = (issuer, token, _) => + { + var tid = ((JwtSecurityToken)token).Claims.FirstOrDefault(c => c.Type == "tid")?.Value; + var fromTenant = tid is not null && + (issuer == $"https://login.microsoftonline.com/{tid}/v2.0" || + issuer == $"https://sts.windows.net/{tid}/"); + // A tenant given as an id pins sign-in to it; "common", "organizations" or a + // domain name can't be compared with the token's tid and leave it open. + var rightTenant = !Guid.TryParse(tenantId, out var pinned) || + string.Equals(tid, pinned.ToString(), StringComparison.OrdinalIgnoreCase); + if (!fromTenant || !rightTenant) + throw new SecurityTokenInvalidIssuerException("The token was not issued by the expected tenant."); + return issuer; + } + }; + + handler.ValidateToken(jwt, parameters, out var validated); + var claims = ((JwtSecurityToken)validated).Claims.ToList(); + + var email = claims.FirstOrDefault(c => c.Type == "preferred_username")?.Value + ?? claims.FirstOrDefault(c => c.Type == "email")?.Value; + var oid = claims.FirstOrDefault(c => c.Type == "oid")?.Value; + var tid = claims.FirstOrDefault(c => c.Type == "tid")?.Value; + if (string.IsNullOrWhiteSpace(email) || oid is null || tid is null) return null; + + return new MicrosoftIdentity(email.ToLowerInvariant(), $"{oid}@{tid}"); + } + private static ClaimsIdentity CreateClaimsIdentity(this Account account, LoginMethod loginMethod) { var claims = new List diff --git a/SW.Bitween.Api/Extensions/IReadOnlyDictionaryExtensions.cs b/SW.Bitween.Api/Extensions/IReadOnlyDictionaryExtensions.cs index b88c9171..21646fca 100644 --- a/SW.Bitween.Api/Extensions/IReadOnlyDictionaryExtensions.cs +++ b/SW.Bitween.Api/Extensions/IReadOnlyDictionaryExtensions.cs @@ -29,7 +29,7 @@ public static ICollection ToKeyAndValueCollection( } /// - /// An exchange's promoted values in the order its information type lists the properties — the + /// An exchange's promoted values in the order its information type lists the properties - the /// order they're shown in, and the first one names an archived exchange. The stored values can't /// be trusted to keep it (Postgres re-sorts a jsonb object's keys), and the type's order can /// change after the exchange ran. Values for properties the type no longer has go last. diff --git a/SW.Bitween.Api/Extensions/InfolinkDbContextExtensions.cs b/SW.Bitween.Api/Extensions/InfolinkDbContextExtensions.cs index a1c424a3..26696a39 100644 --- a/SW.Bitween.Api/Extensions/InfolinkDbContextExtensions.cs +++ b/SW.Bitween.Api/Extensions/InfolinkDbContextExtensions.cs @@ -29,8 +29,14 @@ static class BitweenDbContextExtensions if (string.IsNullOrEmpty(partnerKey)) return (false, null, null); + // Stored hashed. The plain form still matches a row the startup pass has not converted + // yet, so an upgrade does not lock anyone out for the moment between the two - but never + // for a value in the stored form itself, or the hash out of a copied database would be + // a working key. Real keys never look like one. + var hashedKey = PartnerKeyHash.Of(partnerKey); + var plainKey = PartnerKeyHash.IsHashed(partnerKey) ? hashedKey : partnerKey; var partnerQuery = from partner in dbContext.Set() - where partner.ApiCredentials.Any(cred => cred.Key == partnerKey) + where partner.ApiCredentials.Any(cred => cred.Key == hashedKey || cred.Key == plainKey) select partner; var par = await partnerQuery.AsNoTracking().SingleOrDefaultAsync(); @@ -39,8 +45,7 @@ where partner.ApiCredentials.Any(cred => cred.Key == partnerKey) // The database may compare without regard to case (SQL Server and MySQL often do), so // the row found can hold a key that only matches ignoring case. Keys are exact. - var credential = par.ApiCredentials - .SingleOrDefault(c => string.Equals(c.Key, partnerKey, StringComparison.Ordinal)); + var credential = ExactKey(par, hashedKey, plainKey); if (credential == null) return (false, null, null); if (keyName != null && !string.Equals(keyName, credential.Name, StringComparison.OrdinalIgnoreCase)) @@ -49,6 +54,38 @@ where partner.ApiCredentials.Any(cred => cred.Key == partnerKey) return (true, par, credential.Name); } + /// + /// against the + /// partners in the cache, for the gateway, which checks a key on every call. The same rules; + /// a key removed or changed stops working once the save's revoke reaches the cache. + /// + public static async Task<(bool Authorized, Partner Partner, string KeyName)> CheckPartnerAuthorized( + this IInfolinkCache cache, + RequestContext requestContext, + string keyHeader) + { + var (partnerKey, keyName) = ReadPartnerKey(requestContext, keyHeader); + if (string.IsNullOrEmpty(partnerKey)) + return (false, null, null); + + var hashedKey = PartnerKeyHash.Of(partnerKey); + var plainKey = PartnerKeyHash.IsHashed(partnerKey) ? hashedKey : partnerKey; + foreach (var partner in await cache.ListPartnersAsync()) + { + var credential = ExactKey(partner, hashedKey, plainKey); + if (credential == null) continue; + if (keyName != null && !string.Equals(keyName, credential.Name, StringComparison.OrdinalIgnoreCase)) + return (false, null, null); + return (true, partner, credential.Name); + } + + return (false, null, null); + } + + static ApiCredential ExactKey(Partner partner, string hashedKey, string plainKey) => + partner.ApiCredentials?.SingleOrDefault(c => string.Equals(c.Key, hashedKey, StringComparison.Ordinal) || + string.Equals(c.Key, plainKey, StringComparison.Ordinal)); + /// /// The same key, from any of these places: the header named for keys, partnerkey /// (always, so renaming the header cuts no one off), a bearer token, or Basic auth. Basic @@ -88,7 +125,7 @@ where partner.ApiCredentials.Any(cred => cred.Key == partnerKey) return (null, null); } - // The username is everything before the first colon — Basic has no way to escape one. + // The username is everything before the first colon - Basic has no way to escape one. var colon = decoded.IndexOf(':'); return colon < 0 ? (null, null) : (decoded[(colon + 1)..], decoded[..colon]); } diff --git a/SW.Bitween.Api/Extensions/OperatorActions.cs b/SW.Bitween.Api/Extensions/OperatorActions.cs new file mode 100644 index 00000000..c4684943 --- /dev/null +++ b/SW.Bitween.Api/Extensions/OperatorActions.cs @@ -0,0 +1,21 @@ +using System.Threading.Tasks; +using SW.Bitween.Domain; +using SW.PrimitiveTypes; + +namespace SW.Bitween; + +public static class OperatorActions +{ + /// Adds an operator action to be saved with the caller's next save. + public static void Record(this BitweenDbContext dbContext, RequestContext requestContext, + string action, string target, string detail = null) => + dbContext.Add(new OperatorAction(action, target, detail, requestContext.GetNameIdentifier())); + + /// Records and saves at once, for an action that saves nothing else of its own. + public static async Task RecordNowAsync(this BitweenDbContext dbContext, RequestContext requestContext, + string action, string target, string detail = null) + { + dbContext.Record(requestContext, action, target, detail); + await dbContext.SaveChangesAsync(); + } +} diff --git a/SW.Bitween.Api/Extensions/RequestContextExtensions.cs b/SW.Bitween.Api/Extensions/RequestContextExtensions.cs index cb3ccc06..897c29ef 100644 --- a/SW.Bitween.Api/Extensions/RequestContextExtensions.cs +++ b/SW.Bitween.Api/Extensions/RequestContextExtensions.cs @@ -28,8 +28,8 @@ public static class RequestContextExtensions /// /// Present on a token issued to an account whose password nobody has chosen. Such a token - /// authenticates but grants nothing, so the account can reach self-service — changing the - /// password — and nothing else. + /// authenticates but grants nothing, so the account can reach self-service - changing the + /// password - and nothing else. /// /// /// A sign-in has to succeed for the password to be changeable at all: the change requires @@ -40,12 +40,10 @@ public static class RequestContextExtensions public const string MustChangePasswordClaim = "bitween_must_change_password"; /// - /// Throws unless the caller holds at least one of . This is really a - /// "forbidden" — the caller is signed in and simply isn't allowed — but CqApi renders - /// SWForbiddenException as a 401 that's byte-identical to sending no token, so there is no - /// distinction to be had on the wire. The client answers a 401 by refreshing the token and - /// retrying once, which means every denial costs a wasted round trip. Worth revisiting if - /// CqApi ever maps forbidden to 403. + /// Throws unless the caller holds at least one of . CqApi renders + /// the exception as a 401; for a signed-in caller the web host rewrites that to 403 on the + /// way out (see Startup), so a client can tell "not allowed" from "not signed in" and does + /// not spend a token refresh on a refusal. /// public static async Task EnsurePermission(this RequestContext requestContext, BitweenDbContext dbContext, params string[] anyOf) @@ -82,6 +80,11 @@ public static async Task> GetPermissions(this RequestContext req if (!int.TryParse(requestContext.GetNameIdentifier(), out var accountId)) throw new SWUnauthorizedException("INSUFFICIENT_PERMISSIONS"); + // A token outlives the account's state: disabling someone has to take effect now, not + // when their last token expires. Same for an account that has since been removed. + if (!await dbContext.Set().AnyAsync(a => a.Id == accountId && !a.Disabled && !a.Deleted)) + return []; + return await GetPermissionsOf(dbContext, accountId); } diff --git a/SW.Bitween.Api/Helpers/StartupValuesFiller.cs b/SW.Bitween.Api/Helpers/StartupValuesFiller.cs index 7c7c13ca..acff1b53 100644 --- a/SW.Bitween.Api/Helpers/StartupValuesFiller.cs +++ b/SW.Bitween.Api/Helpers/StartupValuesFiller.cs @@ -12,7 +12,7 @@ public static class StartupValuesFiller /// How the pipeline tells a runtime that this subscription runs through a data source, without /// every call site along the way having to grow a parameter for it. /// - /// Reserved, and stripped before the adapter ever sees it — the adapter's own settings come + /// Reserved, and stripped before the adapter ever sees it - the adapter's own settings come /// from the data source itself. The double-underscore convention matches the /// __partner__ and __globals__ injections the mapper already relies on. /// @@ -23,7 +23,7 @@ public static class StartupValuesFiller /// /// Unlike this is NOT stripped: the adapter reads it. A resident /// data source is one instance shared by every subscription pointed at it, so anything the - /// adapter remembers between calls — a receive cursor above all — has to be namespaced by the + /// adapter remembers between calls - a receive cursor above all - has to be namespaced by the /// reader, or two subscriptions polling one connection share one cursor and each sees half the /// rows. The adapter side of this contract is DbReceiver's CursorStateName. /// @@ -66,8 +66,8 @@ public static Dictionary WithSubscription(this Dictionary /// Fills {{source.PATH}} from the values read out of the original document: the input of - /// the exchange whose delivery got the response this one runs on. Null values leave the tokens as they are — every exchange but - /// a response or bus gateway subscription's — and so does a path with no value, for + /// the exchange whose delivery got the response this one runs on. Null values leave the tokens as they are - every exchange but + /// a response or bus gateway subscription's - and so does a path with no value, for /// to find. /// public static Dictionary FillSource(this Dictionary properties, @@ -83,6 +83,42 @@ public static IEnumerable SourcePathsIn(IEnumerable values) => .SelectMany(v => SourceToken.Matches(v).Select(m => m.Groups[1].Value)) .Distinct(); + // As permissive as the resolver below: it splits a globals token on the first dot only and + // restricts neither half. + private static readonly Regex GlobalToken = new(@"\{\{globals\.([^.}]+)\.([^}]+)\}\}", RegexOptions.IgnoreCase); + private static readonly Regex PartnerToken = new(@"\{\{partner\.([^}]+)\}\}", RegexOptions.IgnoreCase); + + /// + /// The global values and partner properties name, by + /// {{globals.SET.KEY}} and {{partner.KEY}}. Names only, never what they resolve + /// to, so this can go where the values themselves can't. + /// + /// + /// Set ids and keys are told apart ignoring case, the way the resolver looks them up, and kept + /// as first written. + /// + public static Model.SubscriptionReferences ReferencesIn(IEnumerable values) + { + var present = (values ?? []).Where(v => v != null).ToList(); + + var globals = present + .SelectMany(v => GlobalToken.Matches(v)) + .GroupBy(m => m.Groups[1].Value, StringComparer.OrdinalIgnoreCase) + .Select(set => new Model.GlobalValuesReference + { + SetId = set.Key, + Keys = set.Select(m => m.Groups[2].Value).Distinct(StringComparer.OrdinalIgnoreCase).ToList() + }) + .ToList(); + + var partnerProperties = present + .SelectMany(v => PartnerToken.Matches(v).Select(m => m.Groups[1].Value)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + + return new Model.SubscriptionReferences { Globals = globals, PartnerProperties = partnerProperties }; + } + public static Dictionary Fill(this IDictionary inputTemplated, Partner partner, GlobalAdapterValuesSet[] globals) { diff --git a/SW.Bitween.Api/Interfaces/IInfolinkCache.cs b/SW.Bitween.Api/Interfaces/IInfolinkCache.cs index c9bda388..1d9a4798 100644 --- a/SW.Bitween.Api/Interfaces/IInfolinkCache.cs +++ b/SW.Bitween.Api/Interfaces/IInfolinkCache.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Threading.Tasks; using SW.Bitween.Domain; using SW.Bitween.Domain.Gateway; @@ -8,7 +9,16 @@ public interface IInfolinkCache { public Task ListSubscriptionsByDocumentAsync(int documentId); public Task ListBusGatewayRoutesByDocumentAsync(int documentId); + + /// + /// The routes, and those of the switched-off gateways in too: a + /// mirror's copy switches off gateways prod has on, so nothing runs on its own, and its rehearsal's + /// traffic still goes where prod's would. + /// + public Task ListBusGatewayRoutesByDocumentAsync(int documentId, IReadOnlySet alsoSwitchedOff) => + ListBusGatewayRoutesByDocumentAsync(documentId); public Task ListNotifiersAsync(); + public Task NotificationChannelByIdAsync(int channelId); public Task SubscriptionByIdAsync(int subscriptionId); public Task DocumentByIdAsync(int documentId); @@ -24,4 +34,17 @@ public interface IInfolinkCache Task GlobalAdapterValuesSetById (string globalAdapterValuesSetId); Task ListGlobalAdapterValuesSetsAsync(); + + Task ListDocumentsAsync(); + + /// + /// A partner with its API keys. Shared by every caller: read it, never change it or attach it to a + /// context. + /// + Task PartnerByIdAsync(int partnerId); + + Task ListPartnersAsync(); + + /// An API gateway with its attachments and their partners, shared like a partner. + Task ApiGatewayByUrlNameAsync(string urlName); } \ No newline at end of file diff --git a/SW.Bitween.Api/Resources/Accounts/AccountRoles.cs b/SW.Bitween.Api/Resources/Accounts/AccountRoles.cs index 6f6117e5..18849e61 100644 --- a/SW.Bitween.Api/Resources/Accounts/AccountRoles.cs +++ b/SW.Bitween.Api/Resources/Accounts/AccountRoles.cs @@ -10,7 +10,7 @@ namespace SW.Bitween.Resources.Accounts; internal static class AccountRoles { - /// Role summaries for a set of accounts, in one query — keyed by account id. + /// Role summaries for a set of accounts, in one query - keyed by account id. public static async Task>> For(BitweenDbContext dbContext, List accountIds) { @@ -57,7 +57,7 @@ public static async Task Set(BitweenDbContext dbContext, int accountId, List /// Keeps the legacy column meaningful for older API consumers. It no - /// longer drives authorization — it's derived from whichever built-in role the member holds, + /// longer drives authorization - it's derived from whichever built-in role the member holds, /// falling back to Member for someone who only holds custom roles. /// public static AccountRole LegacyRoleFor(List roleIds) diff --git a/SW.Bitween.Api/Resources/Accounts/Administrators.cs b/SW.Bitween.Api/Resources/Accounts/Administrators.cs index fbd18109..928faac5 100644 --- a/SW.Bitween.Api/Resources/Accounts/Administrators.cs +++ b/SW.Bitween.Api/Resources/Accounts/Administrators.cs @@ -9,13 +9,15 @@ namespace SW.Bitween.Resources.Accounts; internal static class Administrators { /// - /// How many other enabled accounts still hold the Administrator role. Guards every operation - /// that could otherwise leave an instance with nobody able to manage members and roles. + /// How many other enabled people still hold the Administrator role. Guards every operation + /// that could otherwise leave an instance with nobody able to manage members and roles. A + /// service account holding it doesn't count: no person could sign in as it to put things right. /// public static Task OtherThan(BitweenDbContext dbContext, int accountId) => (from link in dbContext.Set() join account in dbContext.Set() on link.AccountId equals account.Id - where link.RoleId == Role.AdministratorId && link.AccountId != accountId && !account.Disabled + where link.RoleId == Role.AdministratorId && link.AccountId != accountId && !account.Disabled && + account.Kind == AccountKind.Person select link.AccountId).CountAsync(); public static Task Holds(BitweenDbContext dbContext, int accountId) => @@ -24,7 +26,7 @@ public static Task Holds(BitweenDbContext dbContext, int accountId) => /// /// Blocks an operation that would remove the last administrator. Only applies when the account - /// actually holds the role — otherwise nothing is being taken away. + /// actually holds the role - otherwise nothing is being taken away. /// public static async Task EnsureNotTheLast(BitweenDbContext dbContext, int accountId) { diff --git a/SW.Bitween.Api/Resources/Accounts/ChangePassword.cs b/SW.Bitween.Api/Resources/Accounts/ChangePassword.cs index 3c13599c..ba9a120e 100644 --- a/SW.Bitween.Api/Resources/Accounts/ChangePassword.cs +++ b/SW.Bitween.Api/Resources/Accounts/ChangePassword.cs @@ -10,7 +10,8 @@ namespace SW.Bitween.Resources.Accounts; [HandlerName("changePassword")] -public class ChangePassword(BitweenDbContext dbContext, RequestContext requestContext) +public class ChangePassword(BitweenDbContext dbContext, RequestContext requestContext, SignInThrottle throttle, + Microsoft.AspNetCore.Http.IHttpContextAccessor httpContextAccessor, Services.AccountMail mail) : ICommandHandler { public async Task Handle(ChangePasswordModel request) @@ -18,18 +19,34 @@ public async Task Handle(ChangePasswordModel request) // Self-service: this only ever changes the caller's own password, and the old one has to // be supplied. The guard it replaces listed every role, so it granted nothing. var accountId = Convert.ToInt32(requestContext.GetNameIdentifier()); - var account = await dbContext.Set().FindAsync(accountId); + var account = await Members.FindAsync(dbContext, accountId); - if (!SecurePasswordHasher.Verify(request.OldPassword, account!.Password)) + // Checking the old password is guessing it, so it is limited as sign-in is: the same count of + // failures from one address locks it out for the same time, and a stolen session can't be + // used to try passwords until one fits. + var nowUtc = DateTime.UtcNow; + var address = httpContextAccessor.HttpContext?.Connection.RemoteIpAddress?.ToString() ?? "unknown"; + if (throttle.LockedUntil(accountId, address, nowUtc) is { } lockedUntil) { + var minutes = (int)Math.Ceiling((lockedUntil - nowUtc).TotalMinutes); + throw new SWValidationException("TOO_MANY_ATTEMPTS", + $"Too many wrong passwords. Try again in {minutes} minute{(minutes == 1 ? "" : "s")}."); + } + + // A service account has no password to change, and isn't found here. + if (account is null || string.IsNullOrEmpty(account.Password) || + !SecurePasswordHasher.Verify(request.OldPassword, account.Password)) + { + throttle.RegisterFailure(accountId, address, nowUtc); throw new SWValidationException("AUTHENTICATION_ERROR", "The Password entered does not match the user's password"); } + throttle.Clear(accountId, address); account.SetPassword(request.NewPassword); await dbContext.SaveChangesAsync(); - // Signing in with a refresh token skips password verification entirely — it looks the + // Signing in with a refresh token skips password verification entirely - it looks the // account up by id and issues a fresh token from whatever state it is now in. So a session // opened with the old password outlives the change unless the tokens go with it, and // "change the password" would not actually remove whoever you changed it because of. @@ -37,6 +54,10 @@ await dbContext.Set() .Where(t => t.AccountId == account.Id) .ExecuteDeleteAsync(); + // Whoever holds a stolen session can change the password too; the owner hears of it. + mail.SendInBackground(account.Email, Services.AccountEmails.PasswordChanged( + Services.AccountEmails.ChangedBy.Themselves, account.Email, mail.SignedInBase, mail.ResetAvailable)); + return null; } diff --git a/SW.Bitween.Api/Resources/Accounts/CliGrant.cs b/SW.Bitween.Api/Resources/Accounts/CliGrant.cs new file mode 100644 index 00000000..3ab6b5ec --- /dev/null +++ b/SW.Bitween.Api/Resources/Accounts/CliGrant.cs @@ -0,0 +1,30 @@ +using System; +using System.Threading.Tasks; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Accounts; + +public class CliGrantRequest +{ + /// The CLI's S256 code challenge. + public string CodeChallenge { get; set; } +} + +/// +/// A signed-in member signing the bitween CLI in as themselves, from the admin UI's confirmation +/// page: the code it returns goes back to the CLI, which redeems it at . It +/// works whichever way the member signed in to the browser, Microsoft included. An account that +/// must change its password never gets here; the pipeline refuses its token everywhere else. +/// +[HandlerName("cligrant")] +public class CliGrant(RequestContext requestContext, CliSignInCodes codes) : ICommandHandler +{ + public Task Handle(CliGrantRequest request) + { + if (!CliSignInCodes.IsChallenge(request?.CodeChallenge)) + throw new SWValidationException("CodeChallenge", "This isn't a sign-in request from the bitween CLI."); + + var accountId = Convert.ToInt32(requestContext.GetNameIdentifier()); + return Task.FromResult(new { Code = codes.Issue(accountId, request.CodeChallenge, DateTimeOffset.UtcNow) }); + } +} diff --git a/SW.Bitween.Api/Resources/Accounts/CliToken.cs b/SW.Bitween.Api/Resources/Accounts/CliToken.cs new file mode 100644 index 00000000..86b34ada --- /dev/null +++ b/SW.Bitween.Api/Resources/Accounts/CliToken.cs @@ -0,0 +1,51 @@ +using System; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using SW.Bitween.Domain.Accounts; +using SW.HttpExtensions; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Accounts; + +public class CliTokenRequest +{ + public string Code { get; set; } + public string CodeVerifier { get; set; } +} + +/// +/// Where the bitween CLI trades the code from , and the verifier only it +/// holds, for a session: an access token and a refresh token, both in the body since the CLI keeps +/// no cookies. The refresh token renews and ends like the browser's (see ). +/// +[HandlerName("clitoken")] +[Unprotect] +public class CliToken(BitweenDbContext dbContext, JwtTokenParameters jwtTokenParameters, BitweenOptions options, + CliSignInCodes codes, ILogger logger) : ICommandHandler +{ + public async Task Handle(CliTokenRequest request) + { + var accountId = codes.Redeem(request?.Code, request?.CodeVerifier, DateTimeOffset.UtcNow); + var account = accountId is null ? null : await Members.FindAsync(dbContext, accountId.Value); + + // Disabled or told to change its password since the code was issued: no session either way. + if (account is null || account.Disabled || account.MustChangePassword) + { + logger.LogWarning("CLI sign-in refused: the code was invalid, expired, or its account can't sign in."); + throw new SWException("This sign-in has expired or isn't valid. Run bitween login again."); + } + + account.SignedIn(); + var refreshToken = new RefreshToken(account.Id, LoginMethod.EmailAndPassword); + dbContext.Add(refreshToken); + await dbContext.SaveChangesAsync(); + + return new + { + Jwt = account.CreateJwt(LoginMethod.EmailAndPassword, jwtTokenParameters, TimeSpan.FromMinutes(options.JwtExpiryMinutes)), + RefreshToken = refreshToken.Id, + account.Email, + }; + } +} diff --git a/SW.Bitween.Api/Resources/Accounts/Create.cs b/SW.Bitween.Api/Resources/Accounts/Create.cs index 284c8f7e..7173fbc2 100644 --- a/SW.Bitween.Api/Resources/Accounts/Create.cs +++ b/SW.Bitween.Api/Resources/Accounts/Create.cs @@ -20,14 +20,20 @@ public async Task Handle(CreateAccountModel request) await requestContext.EnsurePermission(dbContext, Model.Permissions.Users.Create); if (string.IsNullOrEmpty(request.Name) || string.IsNullOrEmpty(request.Email) || - (!_bitweenOptions.DisableEmailPasswordLogin && string.IsNullOrEmpty(request.Password))) + (!_bitweenOptions.DisableEmailPasswordLogin && !request.Invite && string.IsNullOrEmpty(request.Password))) throw new SWValidationException("INVALID_PAYLOAD", "The payload is invalid"); + // An invited member chooses their own password; one set here too would be a second + // way in that nobody asked them about. + if (request.Invite && !string.IsNullOrEmpty(request.Password)) + throw new SWValidationException("INVITE_OR_PASSWORD", + "Send either a password or an invite, not both."); + if (await dbContext.Set().AnyAsync(a => a.Email == request.Email)) throw new SWValidationException("ACCOUNT_EXISTS", $"Account with email {request.Email} exists"); // Prefer explicit role ids, and fall back to the legacy coarse role only when one was - // actually sent. It used to be a plain int, so omitting it meant 0 — which is Admin — + // actually sent. It used to be a plain int, so omitting it meant 0 - which is Admin - // and adding a member with no roles ticked handed them the run of the instance. var roleIds = request.RoleIds is { Count: > 0 } ? request.RoleIds.Distinct().ToList() @@ -35,9 +41,11 @@ public async Task Handle(CreateAccountModel request) ? [] : [BuiltInRoleFor((AccountRole)request.Role)]; - // No password at all when this instance signs in through Microsoft only — the account + await GrantLimits.EnsureCallerHoldsRoles(dbContext, requestContext, roleIds); + + // No password at all when this instance signs in through Microsoft only - the account // exists purely to be matched by email. - var password = _bitweenOptions.DisableEmailPasswordLogin + var password = _bitweenOptions.DisableEmailPasswordLogin || request.Invite ? null : SecurePasswordHasher.Hash(request.Password); @@ -68,10 +76,11 @@ public Validate(BitweenOptions bitweenOptions) { RuleFor(i => i.Name).NotEmpty(); RuleFor(i => i.Email).NotEmpty(); - RuleFor(i => i.Password).Password().When(_ => !bitweenOptions.DisableEmailPasswordLogin); + // An invited member chooses their own password later, against the same rules. + RuleFor(i => i.Password).Password().When(i => !bitweenOptions.DisableEmailPasswordLogin && !i.Invite); // The handler prefers explicit role ids and only falls back to the legacy coarse // Role, so demanding Role unconditionally rejected the payload the UI actually - // sends — adding a member failed outright. Still required when no ids were sent, + // sends - adding a member failed outright. Still required when no ids were sent, // which is what keeps an account from being created with no roles at all. RuleFor(i => i.Role).NotNull().When(i => i.RoleIds is not { Count: > 0 }); } diff --git a/SW.Bitween.Api/Resources/Accounts/ForgotPassword.cs b/SW.Bitween.Api/Resources/Accounts/ForgotPassword.cs new file mode 100644 index 00000000..12d9e043 --- /dev/null +++ b/SW.Bitween.Api/Resources/Accounts/ForgotPassword.cs @@ -0,0 +1,82 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using SW.Bitween.Domain.Accounts; +using SW.Bitween.Model; +using SW.Bitween.Services; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Accounts; + +/// +/// "Forgot password?": emails a reset link to the address, when it is a member's. +/// +/// +/// The answer is the same whether the address has an account or not, and so is the work done +/// before it: one lookup. Making the link and sending it happen after the answer, so neither the +/// reply nor its timing says who has an account here. A member gets at most one email a minute, +/// however often anyone sends the form. +/// +[HandlerName("forgotPassword")] +[Unprotect] +public class ForgotPassword(BitweenDbContext dbContext, AccountMail mail, IServiceScopeFactory scopes, + ILogger logger) : ICommandHandler +{ + static readonly TimeSpan OneEmailPer = TimeSpan.FromMinutes(1); + + public async Task Handle(ForgotPasswordModel request) + { + if (!mail.ResetAvailable) + throw new SWValidationException("PASSWORD_RESET_OFF", + "This Bitween can't email password resets. Ask your administrator to set a new password for you."); + + var email = request.Email?.Trim(); + if (string.IsNullOrEmpty(email)) return null; + + var account = await dbContext.Set().AsNoTracking() + .Where(a => a.Email.ToLower() == email.ToLower() && a.Kind == AccountKind.Person && !a.Disabled) + .Select(a => new { a.Id, a.Email }) + .FirstOrDefaultAsync(); + + if (account is not null) + _ = Task.Run(() => IssueAsync(account.Id, account.Email)); + + return null; + } + + /// Makes the link and emails it, on a scope of its own: the request's is gone by then. + async Task IssueAsync(int accountId, string email) + { + try + { + await using var scope = scopes.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var now = DateTime.UtcNow; + + if (await db.Set().AnyAsync(l => l.AccountId == accountId && + l.Purpose == AccountLinkPurpose.PasswordReset && + l.CreatedOn > now - OneEmailPer)) + return; + + // Earlier reset links keep working until they expire, and using any of them ends them all. + // Replacing them here instead let two requests at once each delete the other's link, so + // one of the two emails carried a link that no longer worked. Only expired ones go. + await db.Set() + .Where(l => l.AccountId == accountId && l.ExpiresOn <= now) + .ExecuteDeleteAsync(); + + var (link, token) = AccountLink.Issue(accountId, AccountLinkPurpose.PasswordReset, now); + db.Add(link); + await db.SaveChangesAsync(); + + await mail.SendAsync(email, AccountEmails.Reset(email, mail.PublicBase, token)); + } + catch (Exception ex) + { + logger.LogWarning(ex, "A password reset for account {AccountId} could not be made.", accountId); + } + } +} diff --git a/SW.Bitween.Api/Resources/Accounts/GrantLimits.cs b/SW.Bitween.Api/Resources/Accounts/GrantLimits.cs new file mode 100644 index 00000000..520d58ff --- /dev/null +++ b/SW.Bitween.Api/Resources/Accounts/GrantLimits.cs @@ -0,0 +1,59 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain.Accounts; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Accounts; + +/// +/// Nobody hands out more than they hold. Managing members and roles is delegable - a custom role +/// can carry users.edit or roles.edit without being Administrator - and without this a holder of +/// either could grant themselves everything: put themselves in Administrator, add every permission +/// to a role they hold, or set an administrator's password and sign in as them. +/// +/// +/// The rule is a subset check against the caller's own grants, resolved the same way +/// resolves them for every guard. An +/// administrator holds the whole catalog, so nothing an administrator could do before is refused. +/// +internal static class GrantLimits +{ + /// Refuses unless the caller holds every permission grant. + public static async Task EnsureCallerHoldsRoles(BitweenDbContext dbContext, RequestContext requestContext, + IEnumerable roleIds) + { + var ids = (roleIds ?? []).Distinct().ToList(); + if (ids.Count == 0) return; + + var roles = await dbContext.Set().AsNoTracking().Where(r => ids.Contains(r.Id)).ToListAsync(); + await EnsureCallerHoldsPermissions(dbContext, requestContext, + roles.SelectMany(r => r.GetEffectivePermissions() ?? [])); + } + + /// Refuses unless the caller holds every one of . + public static async Task EnsureCallerHoldsPermissions(BitweenDbContext dbContext, RequestContext requestContext, + IEnumerable permissions) + { + var granted = await requestContext.GetPermissions(dbContext); + var beyond = (permissions ?? []).Distinct().Where(p => !granted.Contains(p)).OrderBy(p => p).ToList(); + if (beyond.Count > 0) + throw new SWValidationException("BEYOND_YOUR_PERMISSIONS", + "You can only grant permissions you hold yourself. Not held: " + string.Join(", ", beyond) + "."); + } + + /// + /// Refuses unless the caller holds everything the account holds - so a member can't reset, + /// re-role, disable or remove someone with more access than they have. + /// + public static async Task EnsureCallerOutranks(BitweenDbContext dbContext, RequestContext requestContext, + int accountId) + { + var target = await RequestContextExtensions.GetPermissionsOf(dbContext, accountId); + var granted = await requestContext.GetPermissions(dbContext); + if (!target.IsSubsetOf(granted)) + throw new SWValidationException("ACCOUNT_OUTRANKS_YOU", + "This member holds permissions you don't, so you can't change their account."); + } +} diff --git a/SW.Bitween.Api/Resources/Accounts/Invite.cs b/SW.Bitween.Api/Resources/Accounts/Invite.cs new file mode 100644 index 00000000..c2259e6d --- /dev/null +++ b/SW.Bitween.Api/Resources/Accounts/Invite.cs @@ -0,0 +1,95 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain.Accounts; +using SW.Bitween.Model; +using SW.Bitween.Services; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Accounts; + +/// +/// Invites a member who has no password yet to choose one: emailed when there is a mail server, +/// otherwise handed back for whoever invited to pass on. Sending it again replaces the last one. +/// +/// +/// On an instance that signs in only with Microsoft or single sign-on there is no password to +/// choose, so the invite is an email saying where to sign in, and needs a mail server. +/// +[HandlerName("invite")] +public class Invite(BitweenDbContext dbContext, RequestContext requestContext, BitweenOptions options, + AccountMail mail) : ICommandHandler +{ + public async Task Handle(int key, InviteAccountModel request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Users.Create); + + var account = await Members.FindAsync(dbContext, key); + if (account is null) + throw new SWValidationException("ACCOUNT_NOT_FOUND", $"No account exists with the id {key}"); + + // The link chooses their password, which is signing in as them: as with Set password, only + // someone who holds everything they hold may have it. + await GrantLimits.EnsureCallerOutranks(dbContext, requestContext, key); + + if (account.Disabled) + throw new SWValidationException("ACCOUNT_DISABLED", + $"{account.DisplayName} is disabled. Re-enable them before inviting them."); + + var callerId = Convert.ToInt32(requestContext.GetNameIdentifier()); + var inviter = await dbContext.Set().Where(a => a.Id == callerId) + .Select(a => a.DisplayName).FirstOrDefaultAsync() ?? "An administrator"; + var linkBase = mail.SignedInBase; + + if (options.DisableEmailPasswordLogin) + { + if (!mail.CanSend) + throw new SWValidationException("NO_MAIL_SERVER", + "Bitween can't email an invite until a mail server is set up in Settings, under Mail server."); + + var sent = await mail.SendAsync(account.Email, AccountEmails.InviteToSignIn(inviter, account.Email, linkBase)); + return new InviteResultModel { Emailed = sent is null, EmailError = sent }; + } + + if (!string.IsNullOrEmpty(account.Password)) + throw new SWValidationException("ALREADY_HAS_PASSWORD", + $"{account.DisplayName} already has a password. Set a new one for them, " + + "or have them use Forgot password on the sign-in page."); + + var now = DateTime.UtcNow; + var (link, token) = AccountLink.Issue(key, AccountLinkPurpose.Invite, now); + + // Only the newest invite works, so a resend makes any earlier email useless. The old link goes + // and the new one is stored as one change, committed before anything is sent: an email is + // never sent for a link that wasn't saved, and a failed save leaves the old link working. + await using (var transaction = await dbContext.Database.BeginTransactionAsync()) + { + await dbContext.Set() + .Where(l => l.AccountId == key && l.Purpose == AccountLinkPurpose.Invite) + .ExecuteDeleteAsync(); + dbContext.Add(link); + await dbContext.SaveChangesAsync(); + await transaction.CommitAsync(); + } + + string error = null; + if (mail.CanSend) + { + error = await mail.SendAsync(account.Email, AccountEmails.Invite(inviter, account.Email, linkBase, token)); + if (error is null) link.Emailed(now); + else link.NotEmailed(error); + await dbContext.SaveChangesAsync(); + } + + var emailed = mail.CanSend && error is null; + return new InviteResultModel + { + Emailed = emailed, + EmailError = error, + ExpiresOn = link.ExpiresOn, + // Handed back only when nobody else will get it to them. + Link = emailed ? null : $"{linkBase}/set-password?token={token}" + }; + } +} diff --git a/SW.Bitween.Api/Resources/Accounts/Login.cs b/SW.Bitween.Api/Resources/Accounts/Login.cs index 1e8d1152..553cc3f7 100644 --- a/SW.Bitween.Api/Resources/Accounts/Login.cs +++ b/SW.Bitween.Api/Resources/Accounts/Login.cs @@ -14,10 +14,20 @@ namespace SW.Bitween.Resources.Accounts [HandlerName("login")] [Unprotect] public class Login(JwtTokenParameters jwtTokenParameters, BitweenDbContext dbContext, - BitweenOptions BitweenSettings, IHttpContextAccessor httpContextAccessor, ILogger logger) : ICommandHandler + BitweenOptions BitweenSettings, IHttpContextAccessor httpContextAccessor, ILogger logger, + SignInThrottle throttle, Services.OidcSignIn oidc, Services.AccountMail mail) : ICommandHandler { - private const int MaxFailedLoginAttempts = 5; + // Account-wide, across every address: the backstop for guessing spread over many of them. + // Five from any one address already locks that address out - see SignInThrottle. + private const int MaxFailedLoginAttempts = 20; private static readonly TimeSpan LockoutDuration = TimeSpan.FromMinutes(15); + private static readonly TimeSpan RefreshTokenLifetime = TimeSpan.FromDays(30); + private static readonly TimeSpan RefreshGrace = TimeSpan.FromSeconds(30); + + // Verified against when no account matches, so an unknown email costs the same PBKDF2 work + // as a known one. Without it the response time alone said which addresses have accounts. + private static readonly Lazy UnknownAccountHash = + new(() => SecurePasswordHasher.Hash(Guid.NewGuid().ToString("N"))); public async Task Handle(UserLogin request) { @@ -26,17 +36,67 @@ public async Task Handle(UserLogin request) var accountQ = dbContext .Set() .AsQueryable(); + AccountExtensions.MicrosoftIdentity microsoftIdentity = null; + Services.OidcSignIn.OidcIdentity oidcIdentity = null; + + // Single sign-on, with Microsoft or an OpenID Connect provider: the provider vouched for + // who this is, so there is no password to check. + var ssoLogin = !string.IsNullOrEmpty(request.MsToken) || !string.IsNullOrEmpty(request.OidcCode); + + // Credentials sent with the request are what the caller is asking to sign in as. A refresh + // cookie can still be in the browser from the previous session - a sign-out whose request + // never completed leaves it there - and preferring it signed the next person in as the + // previous one. So the cookie is only read when nothing else was sent. + var signingInWithCredentials = ssoLogin || + !string.IsNullOrEmpty(request.Username) || + !string.IsNullOrEmpty(request.Password); - // Prefer refresh token from HttpOnly cookie (secure), fall back to body (legacy) - var refreshTokenValue = httpContextAccessor.HttpContext?.Request.Cookies["refresh_token"]; - if (string.IsNullOrEmpty(refreshTokenValue)) + // Refresh token from the HttpOnly cookie (secure), falling back to the body (legacy). + var refreshTokenValue = signingInWithCredentials + ? null + : httpContextAccessor.HttpContext?.Request.Cookies["refresh_token"]; + if (string.IsNullOrEmpty(refreshTokenValue) && !signingInWithCredentials) refreshTokenValue = request.RefreshToken; if (!string.IsNullOrEmpty(refreshTokenValue)) { - var refreshToken = await dbContext.Set() + var refreshToken = await dbContext.Set().AsNoTracking() .SingleOrDefaultAsync(x => x.Id == refreshTokenValue); - if (refreshToken is null) + var now = DateTime.UtcNow; + + // Marked superseded rather than deleted, with one conditional UPDATE so two requests + // with the same token can't both be the first. A token superseded moments ago still + // works, for the browser whose copy of the new one went missing; one superseded longer + // ago is refused. Deleting it outright used to make the loser of a race throw a + // concurrency error - a 500 from the sign-in endpoint - and a lost response sign out. + var spent = false; + if (refreshToken is not null) + { + spent = await dbContext.Set() + .Where(t => t.Id == refreshTokenValue && t.SupersededOn == null) + .ExecuteUpdateAsync(u => u.SetProperty(t => t.SupersededOn, now)) == 1; + + if (!spent) + spent = await dbContext.Set().AsNoTracking() + .AnyAsync(t => t.Id == refreshTokenValue && t.SupersededOn > now - RefreshGrace); + + // Superseded ones past their grace are of no further use to anyone. + var graceEnded = now - RefreshGrace; + await dbContext.Set() + .Where(t => t.AccountId == refreshToken.AccountId && t.SupersededOn < graceEnded) + .ExecuteDeleteAsync(); + } + + // The cookie expires after 30 days, but the row did not, and a refresh token sent in + // the body has no cookie to expire. Each refresh issues a new token, so this is an + // idle limit: a session in use keeps renewing, one left alone for 30 days ends. + if (spent && refreshToken.CreatedOn < now - RefreshTokenLifetime) + { + logger.LogInformation("Refresh token older than {Days} days refused.", RefreshTokenLifetime.TotalDays); + spent = false; + } + + if (!spent) { logger.LogWarning("Refresh token not found in DB, clearing cookie and falling back to credentials."); httpContextAccessor.HttpContext?.Response.Cookies.Delete("refresh_token"); @@ -44,22 +104,30 @@ public async Task Handle(UserLogin request) } else { - dbContext.Remove(refreshToken); accountQ = accountQ.Where(u => u.Id == refreshToken.AccountId); } } + else if (signingInWithCredentials && + httpContextAccessor.HttpContext?.Request.Cookies["refresh_token"] is { Length: > 0 } leftover) + { + // The previous session's token, left behind: it is replaced below, so it goes now + // rather than lingering as a second way in. + await dbContext.Set().Where(t => t.Id == leftover).ExecuteDeleteAsync(); + } - if (string.IsNullOrEmpty(refreshTokenValue) && string.IsNullOrEmpty(request.MsToken) && + if (string.IsNullOrEmpty(refreshTokenValue) && !ssoLogin && BitweenSettings.DisableEmailPasswordLogin) { logger.LogWarning("Email/password login attempt rejected: DisableEmailPasswordLogin is enabled."); - throw new SWException("Email and password login is disabled. Please sign in with Microsoft."); + throw new SWException(oidc.IsConfigured && string.IsNullOrWhiteSpace(BitweenSettings.MsalClientId) + ? "Email and password login is disabled. Please sign in with single sign-on." + : "Email and password login is disabled. Please sign in with Microsoft."); } // A credential login must carry both a username and a password. Without this guard a // request with a valid username but empty/missing password would skip verification // below and still be issued a token. - if (string.IsNullOrEmpty(refreshTokenValue) && string.IsNullOrEmpty(request.MsToken) && + if (string.IsNullOrEmpty(refreshTokenValue) && !ssoLogin && (string.IsNullOrEmpty(request.Username) || string.IsNullOrEmpty(request.Password))) { logger.LogWarning("Login rejected: missing username or password on a credential login."); @@ -72,14 +140,56 @@ public async Task Handle(UserLogin request) } else if (!string.IsNullOrEmpty(request.MsToken)) { - var email = (await request.GetEmailFromAzureJwtDefault(logger))?.ToLower(); - if (string.IsNullOrEmpty(email)) + if (string.IsNullOrWhiteSpace(BitweenSettings.MsalClientId)) + throw new SWException("Microsoft sign-in is not configured."); + + microsoftIdentity = await AccountExtensions.ValidateMicrosoftTokenAsync( + request.MsToken, BitweenSettings.MsalClientId, BitweenSettings.MsalTenantId, logger); + if (microsoftIdentity is null) { - logger.LogWarning("MS login failed: could not extract email from token."); - throw new SWException("Could not retrieve your email from Microsoft. Please ensure your Microsoft account has a valid email address and try again."); + logger.LogWarning("MS login failed: the token was not valid for this application."); + throw new SWException("Could not sign you in with Microsoft. Please try again, or contact your administrator."); } - logger.LogInformation("MS login attempt. Extracted email from token: '{Email}'", email); - accountQ = accountQ.Where(u => u.Email.ToLower() == email); + + // The bound identity first; the address only for an account that has none yet. + var identity = microsoftIdentity.ObjectAndTenant; + var email = microsoftIdentity.Email; + var accountId = await dbContext.Set().AsNoTracking() + .Where(u => u.MicrosoftIdentity == identity) + .Select(u => (int?)u.Id).FirstOrDefaultAsync() + ?? await dbContext.Set().AsNoTracking() + .Where(u => u.Email.ToLower() == email) + .Select(u => (int?)u.Id).FirstOrDefaultAsync(); + accountQ = accountQ.Where(u => u.Id == accountId); + } + else if (!string.IsNullOrEmpty(request.OidcCode)) + { + try + { + oidcIdentity = await oidc.CompleteAsync(request.OidcCode, request.OidcState); + } + catch (Services.OidcSignIn.Refused ex) + { + logger.LogWarning("Single sign-on refused: {Reason}", ex.Message); + throw new SWException(ex.Message); + } + catch (Exception ex) + { + logger.LogWarning("Single sign-on refused: {Reason}", ex.Message); + throw new SWException("Could not sign you in with single sign-on. Please try again, or contact your administrator."); + } + + // The bound identity first; the address, when the provider vouches for it, only for + // an account that has none yet. + var identity = oidcIdentity.SubjectAndIssuer; + var email = oidcIdentity.Email; + var accountId = await dbContext.Set().AsNoTracking() + .Where(u => u.OidcIdentity == identity) + .Select(u => (int?)u.Id).FirstOrDefaultAsync() + ?? (email == null ? null : await dbContext.Set().AsNoTracking() + .Where(u => u.Email.ToLower() == email) + .Select(u => (int?)u.Id).FirstOrDefaultAsync()); + accountQ = accountQ.Where(u => u.Id == accountId); } else { @@ -89,18 +199,35 @@ public async Task Handle(UserLogin request) var account = await accountQ .SingleOrDefaultAsync(); + // Service accounts sign in only with their tokens. They have no email or password to + // match, so this is the second lock on that door, not the first. + if (account is { IsService: true }) account = null; + + var credentialLogin = string.IsNullOrEmpty(refreshTokenValue) && !ssoLogin; + if (account is null) { + if (credentialLogin) + SecurePasswordHasher.Verify(request.Password, UnknownAccountHash.Value); + if (!string.IsNullOrEmpty(request.MsToken)) { logger.LogWarning("MS login failed: no account found matching the token email."); throw new SWException("Your Microsoft account is not registered in the system. Please contact your administrator to be added."); } + if (oidcIdentity is not null) + { + logger.LogWarning("Single sign-on failed: no account is bound to the identity or has its verified address."); + throw new SWException("Your account is not registered in the system. Please contact your administrator to be added."); + } + throw new SWException("Invalid username or password."); } - if (account.Disabled) + // A password sign-in learns the account is disabled only once the password is right - + // see below - so the message can't be used to find out which addresses exist. + if (account.Disabled && !credentialLogin) { if (!string.IsNullOrEmpty(request.MsToken)) { @@ -111,20 +238,46 @@ public async Task Handle(UserLogin request) throw new SWException("Your account has been disabled. Please contact your administrator."); } + if (microsoftIdentity is not null) + { + if (account.MicrosoftIdentity is null) + account.BindMicrosoftIdentity(microsoftIdentity.ObjectAndTenant); + else if (account.MicrosoftIdentity != microsoftIdentity.ObjectAndTenant) + { + logger.LogWarning("MS login refused: account {AccountId} is bound to a different Microsoft identity.", account.Id); + throw new SWException("This account is linked to a different Microsoft account. Please contact your administrator."); + } + } + + if (oidcIdentity is not null) + { + if (account.OidcIdentity is null) + account.BindOidcIdentity(oidcIdentity.SubjectAndIssuer); + else if (account.OidcIdentity != oidcIdentity.SubjectAndIssuer) + { + logger.LogWarning("Single sign-on refused: account {AccountId} is bound to a different identity.", account.Id); + throw new SWException("This account is linked to a different single sign-on identity. Please contact your administrator."); + } + } + if (string.IsNullOrEmpty(refreshTokenValue) && !string.IsNullOrEmpty(request.Username) && - !string.IsNullOrEmpty(request.Password) && string.IsNullOrEmpty(request.MsToken)) + !string.IsNullOrEmpty(request.Password) && !ssoLogin) { var nowUtc = DateTime.UtcNow; - if (account.IsLockedOut(nowUtc)) + var address = httpContextAccessor.HttpContext?.Connection.RemoteIpAddress?.ToString() ?? "unknown"; + var lockedUntil = new[] { account.IsLockedOut(nowUtc) ? account.LockoutEnd : null, + throttle.LockedUntil(account.Id, address, nowUtc) } + .Max(); + if (lockedUntil is not null) { - var minutes = (int)Math.Ceiling((account.LockoutEnd!.Value - nowUtc).TotalMinutes); + var minutes = (int)Math.Ceiling((lockedUntil.Value - nowUtc).TotalMinutes); logger.LogWarning("Login rejected: account '{Email}' is temporarily locked.", account.Email); throw new SWException( $"Your account is temporarily locked due to multiple failed login attempts. " + $"Please try again in {minutes} minute{(minutes == 1 ? "" : "s")}."); } - // An account can have no stored password at all — that is how one created while + // An account can have no stored password at all - that is how one created while // Microsoft-only sign-in was on looks, and that setting can be turned back off. // Verify dereferences the stored hash, so without this the attempt is a 500 from // an unauthenticated endpoint instead of an ordinary failed sign-in. @@ -132,6 +285,8 @@ public async Task Handle(UserLogin request) string.IsNullOrEmpty(account.Password) || !SecurePasswordHasher.Verify(request.Password, account.Password)) { + throttle.RegisterFailure(account.Id, address, nowUtc); + // Atomic DB-side update so concurrent wrong-password attempts can't read the // same count and lose increments, which would let them slip past the lockout. var lockoutEnd = nowUtc.Add(LockoutDuration); @@ -142,16 +297,30 @@ await dbContext.Set() a => a.FailedLoginCount + 1 >= MaxFailedLoginAttempts ? lockoutEnd : a.LockoutEnd) .SetProperty(a => a.FailedLoginCount, a => a.FailedLoginCount + 1 >= MaxFailedLoginAttempts ? 0 : a.FailedLoginCount + 1)); + + // It wasn't locked a moment ago - that was checked above - so a lock now is this + // attempt's, and the owner should know someone is guessing. + if (await dbContext.Set().AnyAsync(a => a.Id == account.Id && a.LockoutEnd > nowUtc)) + mail.SendInBackground(account.Email, Services.AccountEmails.Locked( + account.Email, mail.PublicBase, MaxFailedLoginAttempts, lockoutEnd)); throw new SWException("Invalid username or password."); } + if (account.Disabled) + throw new SWException("Your account has been disabled. Please contact your administrator."); + + throttle.Clear(account.Id, address); account.RegisterSuccessfulLogin(); + account.UpgradePasswordHash(request.Password); } + // A sign-in, not a session renewing itself: the member's last sign-in, and the trail's. + if (string.IsNullOrEmpty(refreshTokenValue)) account.SignedIn(); + var newRefreshToken = CreateRefreshToken(account, LoginMethod.EmailAndPassword); await dbContext.SaveChangesAsync(); - // Set refresh token as a secure, HttpOnly cookie — not accessible to JavaScript. + // Set refresh token as a secure, HttpOnly cookie - not accessible to JavaScript. // Secure is always on: the app is served over HTTPS, and TLS is terminated at the // reverse proxy, so Request.IsHttps would otherwise be false and drop the attribute. httpContextAccessor.HttpContext?.Response.Cookies.Append("refresh_token", newRefreshToken, new CookieOptions @@ -159,10 +328,10 @@ await dbContext.Set() HttpOnly = true, Secure = true, SameSite = SameSiteMode.Lax, - Expires = DateTimeOffset.UtcNow.AddDays(30) + Expires = DateTimeOffset.UtcNow.Add(RefreshTokenLifetime) }); - // Return only the JWT — refresh token stays in the cookie, not in the response body. + // Return only the JWT - refresh token stays in the cookie, not in the response body. // MustChangePassword rides along so the client can send them straight to the change // form; the token grants nothing until they do, so this is a courtesy, not the control. return new diff --git a/SW.Bitween.Api/Resources/Accounts/Logout.cs b/SW.Bitween.Api/Resources/Accounts/Logout.cs index bdb0089d..c0472d21 100644 --- a/SW.Bitween.Api/Resources/Accounts/Logout.cs +++ b/SW.Bitween.Api/Resources/Accounts/Logout.cs @@ -6,17 +6,25 @@ namespace SW.Bitween.Resources.Accounts { - public class UserLogout { } + public class UserLogout + { + /// + /// The refresh token to end, for a client without cookies such as the bitween CLI. Ending + /// one needs only the token itself, which is what renewing a session with it needs too. + /// + public string RefreshToken { get; set; } + } [HandlerName("logout")] [Unprotect] -public class Logout(BitweenDbContext dbContext, IHttpContextAccessor httpContextAccessor) + public class Logout(BitweenDbContext dbContext, IHttpContextAccessor httpContextAccessor) : ICommandHandler { public async Task Handle(UserLogout request) { var httpContext = httpContextAccessor.HttpContext; - var refreshTokenValue = httpContext?.Request.Cookies["refresh_token"]; + var cookie = httpContext?.Request.Cookies["refresh_token"]; + var refreshTokenValue = string.IsNullOrEmpty(cookie) ? request?.RefreshToken : cookie; if (!string.IsNullOrEmpty(refreshTokenValue)) { @@ -29,7 +37,7 @@ public async Task Handle(UserLogout request) await dbContext.SaveChangesAsync(); } - httpContext.Response.Cookies.Delete("refresh_token"); + if (!string.IsNullOrEmpty(cookie)) httpContext.Response.Cookies.Delete("refresh_token"); } // Tell the browser to wipe cookies, cache and storage for this origin on logout. diff --git a/SW.Bitween.Api/Resources/Accounts/Members.cs b/SW.Bitween.Api/Resources/Accounts/Members.cs new file mode 100644 index 00000000..88af1369 --- /dev/null +++ b/SW.Bitween.Api/Resources/Accounts/Members.cs @@ -0,0 +1,17 @@ +using System.Threading.Tasks; +using SW.Bitween.Domain.Accounts; + +namespace SW.Bitween.Resources.Accounts; + +internal static class Members +{ + /// + /// A person's account by id, or null. Service accounts aren't members: what can be done to them + /// is under service accounts, behind its own permissions, so the member endpoints don't find them. + /// + public static async Task FindAsync(BitweenDbContext dbContext, int id) + { + var account = await dbContext.Set().FindAsync(id); + return account is { IsService: true } ? null : account; + } +} diff --git a/SW.Bitween.Api/Resources/Accounts/OidcStart.cs b/SW.Bitween.Api/Resources/Accounts/OidcStart.cs new file mode 100644 index 00000000..950b225e --- /dev/null +++ b/SW.Bitween.Api/Resources/Accounts/OidcStart.cs @@ -0,0 +1,30 @@ +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using SW.Bitween.Model; +using SW.Bitween.Services; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Accounts; + +/// Starts a single sign-on: the provider's address to send the browser to. +[HandlerName("oidcstart")] +[Unprotect] +public class OidcStart(OidcSignIn oidc, ILogger logger) : ICommandHandler +{ + public async Task Handle(OidcStartRequest request) + { + try + { + return new OidcStartResult { Url = await oidc.StartAsync() }; + } + catch (OidcSignIn.Refused ex) + { + throw new SWException(ex.Message); + } + catch (System.Exception ex) + { + logger.LogWarning(ex, "Single sign-on couldn't start: the provider's metadata couldn't be read."); + throw new SWException("Single sign-on couldn't reach the provider. Please try again, or contact your administrator."); + } + } +} diff --git a/SW.Bitween.Api/Resources/Accounts/PasswordLink.cs b/SW.Bitween.Api/Resources/Accounts/PasswordLink.cs new file mode 100644 index 00000000..33278691 --- /dev/null +++ b/SW.Bitween.Api/Resources/Accounts/PasswordLink.cs @@ -0,0 +1,101 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using FluentValidation; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain.Accounts; +using SW.Bitween.Model; +using SW.Bitween.Services; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Accounts; + +/// The links invite and reset emails carry, and the account each one is for. +internal static class PasswordLinks +{ + public const string Invalid = "This link has expired or has already been used. Ask for a new one."; + + /// The usable link the token belongs to and its member, or a refusal. + public static async Task<(AccountLink Link, Account Account)> FindAsync(BitweenDbContext dbContext, string token) + { + if (string.IsNullOrWhiteSpace(token) || token.Length > 100) + throw new SWValidationException("LINK_INVALID", Invalid); + + var hash = AccountLink.HashOf(token.Trim()); + var link = await dbContext.Set().FirstOrDefaultAsync(l => l.Hash == hash); + var account = link is null ? null : await Members.FindAsync(dbContext, link.AccountId); + + if (link is null || !link.IsUsable(DateTime.UtcNow) || account is null || account.Disabled) + throw new SWValidationException("LINK_INVALID", Invalid); + + return (link, account); + } +} + +/// Who a link is for, so its page can greet them and say what it does. Changes nothing. +[HandlerName("passwordLink")] +[Unprotect] +public class PasswordLink(BitweenDbContext dbContext) : ICommandHandler +{ + public async Task Handle(PasswordLinkModel request) + { + var (link, account) = await PasswordLinks.FindAsync(dbContext, request.Token); + return new PasswordLinkInfoModel + { + Purpose = link.Purpose.ToString(), + Name = account.DisplayName, + Email = account.Email, + ExpiresOn = link.ExpiresOn + }; + } +} + +/// +/// Sets the password a link was sent for. The link stops working, and so does every other one the +/// member had, and every session they had open. +/// +[HandlerName("usePasswordLink")] +[Unprotect] +public class UsePasswordLink(BitweenDbContext dbContext, SignInThrottle throttle, AccountMail mail) + : ICommandHandler +{ + public async Task Handle(UsePasswordLinkModel request) + { + var (link, account) = await PasswordLinks.FindAsync(dbContext, request.Token); + + await using (var transaction = await dbContext.Database.BeginTransactionAsync()) + { + // Spent first, and only once: of two requests with the same link at the same moment, one + // deletes it and the other finds nothing to delete, rather than both setting a password. + if (await dbContext.Set().Where(l => l.Id == link.Id).ExecuteDeleteAsync() == 0) + throw new SWValidationException("LINK_INVALID", PasswordLinks.Invalid); + + account.SetPassword(request.Password); + // Locked out after wrong guesses is usually why someone resets; the new password is the way back in. + account.Unlock(); + await dbContext.SaveChangesAsync(); + + await dbContext.Set().Where(l => l.AccountId == account.Id).ExecuteDeleteAsync(); + // A session opened with the old password outlives the change unless its refresh tokens go. + await dbContext.Set().Where(t => t.AccountId == account.Id).ExecuteDeleteAsync(); + await transaction.CommitAsync(); + } + + throttle.ClearAccount(account.Id); + + // An invite is the member choosing their first password: nothing to warn them about. + if (link.Purpose == AccountLinkPurpose.PasswordReset) + mail.SendInBackground(account.Email, AccountEmails.PasswordChanged( + AccountEmails.ChangedBy.ResetLink, account.Email, mail.PublicBase, mail.ResetAvailable)); + + return new { account.Email }; + } + + private class Validate : AbstractValidator + { + public Validate() + { + RuleFor(i => i.Password).Password(); + } + } +} diff --git a/SW.Bitween.Api/Resources/Accounts/Profile.cs b/SW.Bitween.Api/Resources/Accounts/Profile.cs index 263a6c94..020aa018 100644 --- a/SW.Bitween.Api/Resources/Accounts/Profile.cs +++ b/SW.Bitween.Api/Resources/Accounts/Profile.cs @@ -9,7 +9,7 @@ namespace SW.Bitween.Resources.Accounts; /// -/// Who am I and what may I do. Deliberately ungated — every signed-in caller needs it, and the +/// Who am I and what may I do. Deliberately ungated - every signed-in caller needs it, and the /// permissions it returns are what the UI uses to decide which pages and actions to show. /// [HandlerName("profile")] diff --git a/SW.Bitween.Api/Resources/Accounts/RemoveAccount.cs b/SW.Bitween.Api/Resources/Accounts/RemoveAccount.cs index bda39d90..97fc3177 100644 --- a/SW.Bitween.Api/Resources/Accounts/RemoveAccount.cs +++ b/SW.Bitween.Api/Resources/Accounts/RemoveAccount.cs @@ -13,7 +13,7 @@ public async Task Handle(int key, RemoveAccountModel request) { await requestContext.EnsurePermission(dbContext, Model.Permissions.Users.Delete); - var account = await dbContext.Set().FindAsync(key); + var account = await Members.FindAsync(dbContext, key); if (account is null) throw new SWValidationException("ACCOUNT_NOT_FOUND", $"Account with {key} was not found"); @@ -22,6 +22,7 @@ public async Task Handle(int key, RemoveAccountModel request) throw new SWValidationException("CANNOT_REMOVE_SELF", "You can't remove your own account."); await Administrators.EnsureNotTheLast(dbContext, key); + await GrantLimits.EnsureCallerOutranks(dbContext, requestContext, key); dbContext.Remove(account); await dbContext.SaveChangesAsync(); diff --git a/SW.Bitween.Api/Resources/Accounts/Search.cs b/SW.Bitween.Api/Resources/Accounts/Search.cs index 084e1776..d06af19d 100644 --- a/SW.Bitween.Api/Resources/Accounts/Search.cs +++ b/SW.Bitween.Api/Resources/Accounts/Search.cs @@ -32,6 +32,10 @@ public async Task Handle(SearchMembersModel request) .ToDictionaryAsync(i => i.Id, i => i.DisplayName); } + // The member list is people; service accounts have their own (lookup keeps them, so a + // change one made still shows who made it). + query = query.Where(a => a.Kind == AccountKind.Person); + var count = await query.CountAsync(); var accounts = await query.OrderBy(i => i.CreatedOn) @@ -45,15 +49,38 @@ public async Task Handle(SearchMembersModel request) Id = a.Id, Disabled = a.Disabled, Role = a.Role.ToString(), - LockoutEnd = a.LockoutEnd + LockoutEnd = a.LockoutEnd, + LastSignInOn = a.LastSignInOn, + MicrosoftLinked = a.MicrosoftIdentity != null, + HasPassword = a.Password != null && a.Password != "" }) .ToListAsync(); var accountIds = accounts.Select(a => a.Id).ToList(); var rolesByAccount = await AccountRoles.For(dbContext, accountIds); + // The latest invite of each member still to choose a password. Expired ones too: the + // page says it expired, and offers another. + var waiting = accounts.Where(a => !a.HasPassword).Select(a => a.Id).ToList(); + var invites = waiting.Count == 0 + ? new List() + : await dbContext.Set().AsNoTracking() + .Where(l => waiting.Contains(l.AccountId) && l.Purpose == AccountLinkPurpose.Invite) + .ToListAsync(); + foreach (var account in accounts) + { account.Roles = rolesByAccount.GetValueOrDefault(account.Id, []); + account.Invite = invites.Where(l => l.AccountId == account.Id) + .OrderByDescending(l => l.CreatedOn) + .Select(l => new AccountInviteModel + { + ExpiresOn = l.ExpiresOn, + EmailedOn = l.EmailedOn, + EmailError = l.EmailError + }) + .FirstOrDefault(); + } return new { diff --git a/SW.Bitween.Api/Resources/Accounts/ServiceTokenSignIn.cs b/SW.Bitween.Api/Resources/Accounts/ServiceTokenSignIn.cs new file mode 100644 index 00000000..594b585d --- /dev/null +++ b/SW.Bitween.Api/Resources/Accounts/ServiceTokenSignIn.cs @@ -0,0 +1,80 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using SW.Bitween.Domain.Accounts; +using SW.Bitween.Model; +using SW.HttpExtensions; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Accounts; + +/// +/// Where a service account signs in: its token, traded for an ordinary short session. Nothing else +/// accepts a token, so everything the session does is checked the way a member's is: permissions +/// from its roles on every call, disabling it ends it at once. +/// +/// +/// No refresh token: the caller holds the token and trades it again when the session runs out. +/// Tokens that match nothing count against the address they came from, as wrong passwords do: that +/// is what guessing looks like. A real token that has expired or been revoked doesn't; that's a job +/// nobody has updated yet, and locking its address out would stop every other job beside it. +/// +[HandlerName("servicetoken")] +[Unprotect] +public class ServiceTokenSignIn(BitweenDbContext dbContext, JwtTokenParameters jwtTokenParameters, BitweenOptions options, + IHttpContextAccessor httpContextAccessor, SignInThrottle throttle, ILogger logger) + : ICommandHandler +{ + /// What the throttle counts unknown tokens against: no account has this id. + internal const int NoAccount = 0; + + const string Refused = "This token isn't valid: it's unknown, expired or revoked, or its account is disabled."; + + public async Task Handle(ServiceTokenSignInModel request) + { + var now = DateTime.UtcNow; + var address = httpContextAccessor.HttpContext?.Connection.RemoteIpAddress?.ToString() ?? "unknown"; + + var lockedUntil = throttle.LockedUntil(NoAccount, address, now); + if (lockedUntil is not null) + { + var minutes = (int)Math.Ceiling((lockedUntil.Value - now).TotalMinutes); + throw new SWException($"Too many invalid tokens from this address. Try again in {minutes} minute{(minutes == 1 ? "" : "s")}."); + } + + var value = request?.Token?.Trim(); + var hash = ServiceToken.LooksLikeOne(value) ? ServiceToken.HashOf(value) : null; + var token = hash is null + ? null + : await dbContext.Set().AsNoTracking().SingleOrDefaultAsync(t => t.Hash == hash); + var account = token is null + ? null + : await dbContext.Set().AsNoTracking().SingleOrDefaultAsync(a => a.Id == token.AccountId); + + if (token is null || !token.IsActive(now) || account is null || account.Deleted || account.Disabled || !account.IsService) + { + if (token is null) throttle.RegisterFailure(NoAccount, address, now); + logger.LogWarning("Service token sign-in refused from {Address}: {Reason}.", address, + token is null ? "no such token" : !token.IsActive(now) ? $"token {token.Shown} expired or revoked" : "account disabled"); + throw new SWException(Refused); + } + + // Straight to the row: who used it last is not a change anyone needs in the audit trail. + await dbContext.Set().Where(t => t.Id == token.Id) + .ExecuteUpdateAsync(u => u.SetProperty(t => t.LastUsedOn, now).SetProperty(t => t.LastUsedFrom, address)); + await dbContext.Set().Where(a => a.Id == account.Id) + .ExecuteUpdateAsync(u => u.SetProperty(a => a.LastSignInOn, now)); + + var lifetime = TimeSpan.FromMinutes(options.JwtExpiryMinutes); + return new + { + Jwt = account.CreateJwt(LoginMethod.ApiKey, jwtTokenParameters, lifetime), + ExpiresIn = (int)lifetime.TotalSeconds, + account.Id, + Name = account.DisplayName, + }; + } +} diff --git a/SW.Bitween.Api/Resources/Accounts/SetDisabled.cs b/SW.Bitween.Api/Resources/Accounts/SetDisabled.cs index b88375d6..be332e23 100644 --- a/SW.Bitween.Api/Resources/Accounts/SetDisabled.cs +++ b/SW.Bitween.Api/Resources/Accounts/SetDisabled.cs @@ -1,3 +1,5 @@ +using Microsoft.EntityFrameworkCore; +using System.Linq; using System; using System.Threading.Tasks; using SW.Bitween.Domain.Accounts; @@ -8,7 +10,7 @@ namespace SW.Bitween.Resources.Accounts; /// /// Suspends or restores an account. A disabled account keeps its roles and history but can't sign -/// in — see the Disabled check in the login handler. +/// in - see the Disabled check in the login handler. /// [HandlerName("setDisabled")] public class SetDisabled(BitweenDbContext dbContext, RequestContext requestContext) @@ -18,10 +20,12 @@ public async Task Handle(int key, SetAccountDisabledModel request) { await requestContext.EnsurePermission(dbContext, Model.Permissions.Users.Edit); - var account = await dbContext.Set().FindAsync(key); + var account = await Members.FindAsync(dbContext, key); if (account is null) throw new SWValidationException("ACCOUNT_NOT_FOUND", $"No account exists with the id {key}"); + await GrantLimits.EnsureCallerOutranks(dbContext, requestContext, key); + if (request.Disabled) { if (key == Convert.ToInt32(requestContext.GetNameIdentifier())) @@ -33,6 +37,13 @@ public async Task Handle(int key, SetAccountDisabledModel request) account.SetDisabled(request.Disabled); await dbContext.SaveChangesAsync(); + // A refresh token re-issues a Jwt without asking for the password, so a disabled account's + // sessions have to go with it - the same reasoning as SetPassword. + if (request.Disabled) + await dbContext.Set() + .Where(t => t.AccountId == account.Id) + .ExecuteDeleteAsync(); + return null; } } diff --git a/SW.Bitween.Api/Resources/Accounts/SetPassword.cs b/SW.Bitween.Api/Resources/Accounts/SetPassword.cs index d019c85b..85495b28 100644 --- a/SW.Bitween.Api/Resources/Accounts/SetPassword.cs +++ b/SW.Bitween.Api/Resources/Accounts/SetPassword.cs @@ -10,12 +10,12 @@ namespace SW.Bitween.Resources.Accounts; /// -/// Sets a member's password on their behalf. Bitween has no outbound mail, so there's no -/// self-service reset — without this, anyone who forgets their password is locked out for good. +/// Sets a member's password on their behalf: for an instance without a mail server, where there is +/// no self-service reset, and for anyone who can't wait for one. /// Changing your own password goes through ChangePassword, which asks for the current one. /// [HandlerName("setPassword")] -public class SetPassword(BitweenDbContext dbContext, RequestContext requestContext) +public class SetPassword(BitweenDbContext dbContext, RequestContext requestContext, Services.AccountMail mail) : ICommandHandler { public async Task Handle(int key, SetAccountPasswordModel request) @@ -26,14 +26,17 @@ public async Task Handle(int key, SetAccountPasswordModel request) throw new SWValidationException("USE_CHANGE_PASSWORD", "Use Change password to set your own, so the current one is still required."); - var account = await dbContext.Set().FindAsync(key); + var account = await Members.FindAsync(dbContext, key); if (account is null) throw new SWValidationException("ACCOUNT_NOT_FOUND", $"No account exists with the id {key}"); + // Setting someone's password is signing in as them, so it needs everything they hold. + await GrantLimits.EnsureCallerOutranks(dbContext, requestContext, key); + account.SetPassword(request.Password); await dbContext.SaveChangesAsync(); - // Signing in with a refresh token skips password verification entirely — it looks the + // Signing in with a refresh token skips password verification entirely - it looks the // account up by id and issues a fresh token from whatever state it is now in. So a session // opened with the old password outlives the change unless the tokens go with it, and // "change the password" would not actually remove whoever you changed it because of. @@ -41,6 +44,9 @@ await dbContext.Set() .Where(t => t.AccountId == account.Id) .ExecuteDeleteAsync(); + mail.SendInBackground(account.Email, Services.AccountEmails.PasswordChanged( + Services.AccountEmails.ChangedBy.Administrator, account.Email, mail.SignedInBase, mail.ResetAvailable)); + return null; } diff --git a/SW.Bitween.Api/Resources/Accounts/SetRoles.cs b/SW.Bitween.Api/Resources/Accounts/SetRoles.cs index 693be882..f032dc91 100644 --- a/SW.Bitween.Api/Resources/Accounts/SetRoles.cs +++ b/SW.Bitween.Api/Resources/Accounts/SetRoles.cs @@ -16,13 +16,16 @@ public async Task Handle(int key, SetAccountRolesModel request) { await requestContext.EnsurePermission(dbContext, Model.Permissions.Users.Edit); - var account = await dbContext.Set().FindAsync(key); + var account = await Members.FindAsync(dbContext, key); if (account is null) throw new SWValidationException("ACCOUNT_NOT_FOUND", $"No account exists with the id {key}"); var roleIds = (request.RoleIds ?? []).Distinct().ToList(); - // Don't let the last administrator be demoted — including by themselves. Otherwise an + await GrantLimits.EnsureCallerOutranks(dbContext, requestContext, key); + await GrantLimits.EnsureCallerHoldsRoles(dbContext, requestContext, roleIds); + + // Don't let the last administrator be demoted - including by themselves. Otherwise an // instance ends up with nobody able to manage members or roles. if (!roleIds.Contains(Role.AdministratorId)) await Administrators.EnsureNotTheLast(dbContext, key); diff --git a/SW.Bitween.Api/Resources/Accounts/Unlock.cs b/SW.Bitween.Api/Resources/Accounts/Unlock.cs index 829d5829..07a2ccde 100644 --- a/SW.Bitween.Api/Resources/Accounts/Unlock.cs +++ b/SW.Bitween.Api/Resources/Accounts/Unlock.cs @@ -6,20 +6,24 @@ namespace SW.Bitween.Resources.Accounts; [HandlerName("unlock")] -public class Unlock(BitweenDbContext dbContext, RequestContext requestContext) +public class Unlock(BitweenDbContext dbContext, RequestContext requestContext, SignInThrottle throttle) : ICommandHandler { public async Task Handle(int key, UnlockAccountModel request) { await requestContext.EnsurePermission(dbContext, Model.Permissions.Users.Edit); - var account = await dbContext.Set().FindAsync(key); + var account = await Members.FindAsync(dbContext, key); if (account is null) throw new SWValidationException("ACCOUNT_NOT_FOUND", $"No account exists with the id {key}"); account.Unlock(); await dbContext.SaveChangesAsync(); + // The per-address locks too. They are held per replica, so on a multi-replica deployment + // this lifts the ones on the replica that served it; the rest expire on their own. + throttle.ClearAccount(key); + return null; } } diff --git a/SW.Bitween.Api/Resources/Accounts/Update.cs b/SW.Bitween.Api/Resources/Accounts/Update.cs index 636e34c0..3be5412f 100644 --- a/SW.Bitween.Api/Resources/Accounts/Update.cs +++ b/SW.Bitween.Api/Resources/Accounts/Update.cs @@ -17,7 +17,7 @@ public async Task Handle(int key, UpdateAccountModel request) if (key != loggedInUserId) await requestContext.EnsurePermission(dbContext, Model.Permissions.Users.Edit); - var account = await dbContext.Set().FindAsync(key); + var account = await Members.FindAsync(dbContext, key); if (account is null) throw new SWValidationException("ACCOUNT_NOT_FOUND", $"No account exists with the id {key}"); @@ -28,6 +28,8 @@ public async Task Handle(int key, UpdateAccountModel request) { await requestContext.EnsurePermission(dbContext, Model.Permissions.Users.Edit); var role = (AccountRole)request.Role; + await GrantLimits.EnsureCallerOutranks(dbContext, requestContext, key); + await GrantLimits.EnsureCallerHoldsRoles(dbContext, requestContext, [BuiltInRoleFor(role)]); account.SetRole(role); await AccountRoles.Set(dbContext, key, [BuiltInRoleFor(role)]); } diff --git a/SW.Bitween.Api/Resources/AdapterDrafts/Handlers.cs b/SW.Bitween.Api/Resources/AdapterDrafts/Handlers.cs new file mode 100644 index 00000000..16901613 --- /dev/null +++ b/SW.Bitween.Api/Resources/AdapterDrafts/Handlers.cs @@ -0,0 +1,178 @@ +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain; +using SW.Bitween.Services.Adapters; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.AdapterDrafts; + +static class Drafts +{ + public static object Summary(AdapterDraft d) => new + { + d.Id, d.AdapterId, d.Language, d.Kind, d.BaseVersion, d.FilesHash, + d.CreatedOn, d.CreatedBy, d.ModifiedOn, d.ModifiedBy, + }; + + public static object Full(AdapterDraft d) => new + { + d.Id, d.AdapterId, d.Language, d.Kind, d.BaseVersion, d.FilesHash, + d.CreatedOn, d.CreatedBy, d.ModifiedOn, d.ModifiedBy, + d.Files, + }; + + public static async Task Find(BitweenDbContext db, int key) => + await db.Set().FindAsync(key) ?? throw new SWNotFoundException($"Adapter draft {key}"); +} + +/// Every draft in the editor, newest first. +public class Search(BitweenDbContext dbContext, RequestContext requestContext) : IQueryHandler +{ + public async Task Handle(AdapterDraftSearch request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.AdapterSource.Edit); + var drafts = await dbContext.Set().AsNoTracking() + .Where(d => request.AdapterId == null || d.AdapterId == request.AdapterId) + .OrderByDescending(d => d.ModifiedOn ?? d.CreatedOn) + .ToListAsync(); + return drafts.Select(Drafts.Summary); + } +} + +public class AdapterDraftSearch +{ + public string AdapterId { get; set; } +} + +public class Get(BitweenDbContext dbContext, RequestContext requestContext) : IGetHandler +{ + public async Task Handle(int key) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.AdapterSource.Edit); + return Drafts.Full(await Drafts.Find(dbContext, key)); + } +} + +/// Starts a draft: a new adapter from the template, or a published version's source. +public class Create(BitweenDbContext dbContext, RequestContext requestContext, AdapterWorkshop workshop) + : ICommandHandler +{ + public async Task Handle(AdapterDraftCreate request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.AdapterSource.Edit); + var draft = !string.IsNullOrWhiteSpace(request.FromAdapterId) + ? await workshop.FromVersionAsync(request.FromAdapterId.ToLowerInvariant(), request.FromVersion) + : await workshop.NewAsync(request.Name, request.Language, request.Kind ?? "handler", request.AdapterId); + dbContext.Add(draft); + await dbContext.SaveChangesAsync(); + return draft.Id; + } +} + +/// Saves the draft's files. +public class Update(BitweenDbContext dbContext, RequestContext requestContext) : ICommandHandler +{ + public async Task Handle(int key, AdapterDraftUpdate request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.AdapterSource.Edit); + var draft = await Drafts.Find(dbContext, key); + if (!string.IsNullOrEmpty(request.BaseHash) && request.BaseHash != draft.FilesHash) + throw new SWValidationException("DRAFT_CHANGED", + $"{draft.ModifiedBy ?? draft.CreatedBy ?? "Someone"} saved this draft at " + + $"{(draft.ModifiedOn ?? draft.CreatedOn):yyyy-MM-dd HH:mm} UTC, after you opened it. " + + "Saving now would replace their changes."); + draft.SetFiles(request.Files); + await dbContext.SaveChangesAsync(); + return Drafts.Summary(draft); + } +} + +public class Delete(BitweenDbContext dbContext, RequestContext requestContext) : IDeleteHandler +{ + public async Task Handle(int key) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.AdapterSource.Edit); + dbContext.Remove(await Drafts.Find(dbContext, key)); + await dbContext.SaveChangesAsync(); + return null; + } +} + +/// Builds the draft and checks it against its contracts, as serverless test would. +[HandlerName("build")] +public class Build(BitweenDbContext dbContext, RequestContext requestContext, AdapterWorkshop workshop) + : ICommandHandler +{ + public async Task Handle(int key, AdapterDraftRun request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.AdapterSource.Edit); + var draft = await Drafts.Find(dbContext, key); + var result = await workshop.BuildAsync(draft, request?.Settings, check: request?.BuildOnly != true); + return Describe(result); + } + + internal static object Describe(WorkshopBuild result) => new + { + result.Succeeded, + result.Conforms, + result.Problems, + result.Warnings, + result.Checks, + Settings = result.Manifest?.Properties.Select(p => new { p.Name, p.Description, p.Required, p.Secret, p.Default, p.Type }), + Commands = result.Manifest == null ? null : CommandsOf(result.Manifest), + result.Manifest?.Lifecycle, + result.Manifest?.Kinds, + }; + + // The commands a kind's contract names; the try panel offers these. + static string[] CommandsOf(Serverless.Contract.Catalog.AdapterManifest manifest) => + manifest.Kinds.SelectMany(k => k switch + { + "handler" or "mapper" => new[] { "Handle" }, + "validator" => ["Validate"], + "receiver" => ["Initialize", "ListFiles", "GetFile", "DeleteFile", "Finalize"], + _ => [], + }).Distinct().ToArray(); +} + +/// Builds the draft and calls one of its commands with the settings and input given. +[HandlerName("try")] +public class Try(BitweenDbContext dbContext, RequestContext requestContext, AdapterWorkshop workshop) + : ICommandHandler +{ + public async Task Handle(int key, AdapterDraftRun request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.AdapterSource.Edit); + var draft = await Drafts.Find(dbContext, key); + return await workshop.TryAsync(draft, request?.Settings, request?.Command, request?.Input); + } +} + +/// +/// Publishes the draft as a new version, once it builds and passes its contract checks. Not made +/// current: what runs changes only when someone promotes it, or pins it on a subscription. +/// +[HandlerName("publish")] +public class Publish(BitweenDbContext dbContext, RequestContext requestContext, AdapterWorkshop workshop) + : ICommandHandler +{ + public async Task Handle(int key, AdapterDraftPublish request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.AdapterSource.Operate); + var draft = await Drafts.Find(dbContext, key); + var who = requestContext.GetNameIdentifier(); + // The catalog says who published each version; a name reads better there than an id. + var account = int.TryParse(who, out var accountId) + ? await dbContext.Set().AsNoTracking().FirstOrDefaultAsync(a => a.Id == accountId) + : null; + var (version, build) = await workshop.PublishAsync(draft, request?.Version, request?.ReleaseNotes, request?.Settings, + account?.DisplayName ?? who ?? "Bitween"); + if (version == null) return new { Published = false, Build = Build.Describe(build) }; + + draft.Published(version); + dbContext.Add(new AdapterRelease(draft.AdapterId, version, AdapterRelease.PublishedAction, draft.Id, who)); + await dbContext.SaveChangesAsync(); + return new { Published = true, Version = version, Build = Build.Describe(build) }; + } +} diff --git a/SW.Bitween.Api/Resources/AdapterDrafts/Models.cs b/SW.Bitween.Api/Resources/AdapterDrafts/Models.cs new file mode 100644 index 00000000..05169ffe --- /dev/null +++ b/SW.Bitween.Api/Resources/AdapterDrafts/Models.cs @@ -0,0 +1,49 @@ +using System.Collections.Generic; + +namespace SW.Bitween.Resources.AdapterDrafts; + +public class AdapterDraftCreate +{ + /// A new adapter: its name, language and kind, and optionally its id. + public string Name { get; set; } + public string Language { get; set; } + public string Kind { get; set; } + public string AdapterId { get; set; } + + /// Or a published version to start from: set both, and leave the rest. + public string FromAdapterId { get; set; } + public string FromVersion { get; set; } +} + +public class AdapterDraftUpdate +{ + public Dictionary Files { get; set; } + + /// + /// The draft's FilesHash when the editor opened it or last saved. A save over someone else's is + /// refused rather than silently discarding theirs; left out, the save goes through, as it + /// always did. + /// + public string BaseHash { get; set; } +} + +/// Settings to build, check or try the draft with. Used for the call and never stored. +public class AdapterDraftRun +{ + public Dictionary Settings { get; set; } + + /// Build only, without the contract checks. + public bool BuildOnly { get; set; } + + /// For try: the command, and its input as JSON or text. + public string Command { get; set; } + public string Input { get; set; } +} + +public class AdapterDraftPublish +{ + /// major, minor, patch, or an explicit version; patch when empty. + public string Version { get; set; } + public string ReleaseNotes { get; set; } + public Dictionary Settings { get; set; } +} diff --git a/SW.Bitween.Api/Resources/Adapters/AdapterListing.cs b/SW.Bitween.Api/Resources/Adapters/AdapterListing.cs index af0a9c67..d9f5e762 100644 --- a/SW.Bitween.Api/Resources/Adapters/AdapterListing.cs +++ b/SW.Bitween.Api/Resources/Adapters/AdapterListing.cs @@ -10,10 +10,17 @@ namespace SW.Bitween.Resources.Adapters; /// The adapter id, as a subscription stores it. /// In-process, so it has no published versions of its own. /// -/// Published version files, as paths relative to the remote adapter root — the trailing segment is +/// Published version files, as paths relative to the remote adapter root - the trailing segment is /// the version number. /// -public record AdapterEntry(string Key, bool Native, IReadOnlyList VersionPaths); +/// +/// What the adapter says about itself - the current package's manifest, or a native adapter's. Null +/// for a package published before manifests existed. +/// +/// The published adapter's catalog entry; null when it has none. +public record AdapterEntry(string Key, bool Native, IReadOnlyList VersionPaths, + SW.Serverless.Contract.Catalog.AdapterManifest Manifest = null, + SW.Serverless.Contract.Catalog.AdapterCatalogEntry Catalog = null); /// /// The list of adapters of one kind: the in-process ones plus whatever is published to storage. @@ -29,8 +36,17 @@ public class AdapterListing( ICloudFilesService cloudFilesService, NativeAdapterDiscoveryService nativeAdapterDiscovery, BitweenDbContext dbContext, - SW.Serverless.AdapterInstaller adapterInstaller) + SW.Serverless.AdapterInstaller adapterInstaller, + Services.Adapters.AdapterCatalog catalog, + Services.Adapters.NativeAdapterManifests nativeManifests) { + /// + /// How many adapters' metadata the lists may be reading from storage at once, across the whole + /// process: a screen asks for its four kinds together, each reads every other kind's adapters, + /// and storage services limit their callers. + /// + private static readonly System.Threading.SemaphoreSlim MetadataReads = new(16, 16); + /// The plural, lowercase kind: receivers, handlers, … /// Native adapters first, then the published ones. public async Task> List(string prefix) @@ -38,7 +54,7 @@ public async Task> List(string prefix) var index = serverlessOptions.AdapterRemotePath.Length + 1; var native = (await nativeAdapterDiscovery.GetNativeAdapters(prefix).ExceptRetiring(dbContext)) - .Select(key => new AdapterEntry(key, true, [])) + .Select(key => new AdapterEntry(key, true, [], nativeManifests.Get(key))) .ToList(); var files = (await ListByKindAsync(prefix)) @@ -59,7 +75,58 @@ public async Task> List(string prefix) .Select(v => v.Key[index..]) .ToList())); - return native.Concat(published).ToList(); + // The catalog adds what a package listing cannot: the current version, the history, and + // the presentation. An adapter without an entry is listed exactly as before. + var withCatalog = await Task.WhenAll(published.Select(async entry => + { + var found = await catalog.GetAsync(entry.Key); + return found == null ? entry : entry with { Manifest = found.Manifest, Catalog = found }; + })); + + var listed = withCatalog.Select(e => e.Key).ToHashSet(System.StringComparer.OrdinalIgnoreCase); + var catalogOnly = await CatalogOnlyAsync(prefix, listed); + + return native.Concat(withCatalog).Concat(catalogOnly).ToList(); + } + + /// + /// Adapters that are in the catalog but have no package at adapters/{id}, which is all + /// the listing above looks at. An adapter in another runtime than .NET is never put there - + /// older hosts would start it with dotnet - and neither is a .NET version published without + /// being made current. Each is listed under the kinds its manifests declare, or the old + /// naming convention, and only while it has a version that can still be used. + /// + private async Task> CatalogOnlyAsync(string prefix, ISet listed) + { + var kind = prefix?.TrimEnd('s') ?? ""; + var root = serverlessOptions.AdapterRemotePath; + var catalogRoot = SW.Serverless.Contract.Catalog.AdapterCatalogPaths.CatalogRoot(root); + + IEnumerable files; + try + { + files = await cloudFilesService.ListAsync(catalogRoot); + } + catch + { + // As everywhere the catalog is read: absent rather than fatal. + return []; + } + + var ids = files + .Where(f => f.Key.EndsWith(".json", System.StringComparison.OrdinalIgnoreCase)) + .Select(f => f.Key[catalogRoot.Length..^".json".Length]) + .Where(id => id.Length > 0 && !id.Contains('/') && !listed.Contains(id)) + .ToList(); + + var entries = await Task.WhenAll(ids.Select(id => catalog.GetAsync(id))); + return entries + .Where(e => e != null && e.Versions.Any(v => !v.Withdrawn)) + .Where(e => e.Id.StartsWith($"infolink6.{prefix}.", System.StringComparison.OrdinalIgnoreCase) || + (kind.Length > 0 && e.Versions.Append(new() { Manifest = e.Manifest }) + .Any(v => v.Manifest?.Kinds?.Contains(kind, System.StringComparer.OrdinalIgnoreCase) == true))) + .Select(e => new AdapterEntry(e.Id, false, [], e.Manifest ?? e.Versions.Last(v => !v.Withdrawn).Manifest, e)) + .ToList(); } /// @@ -69,7 +136,7 @@ public async Task> List(string prefix) /// Two ways an adapter says what it is for, and both are honoured. The Kind stamped on it at /// publish time is the real answer: it comes from the code rather than from whoever typed the /// id, it is the only way a third party's adapter can be found at all, and it lets an adapter - /// be reclassified without being renamed — a rename is not free, because every subscription + /// be reclassified without being renamed - a rename is not free, because every subscription /// stores the id. /// /// The infolink6.<kind>s. prefix is the old convention, and everything published before the @@ -83,39 +150,53 @@ private async Task> ListByKindAsync(string prefix) var byConvention = (await cloudFilesService.ListAsync($"{root}/infolink6.{prefix}")).ToList(); var named = byConvention.Select(i => i.Key).ToHashSet(System.StringComparer.OrdinalIgnoreCase); - // The plural the UI asks with — "handlers" — against the singular an adapter declares. + // The plural the UI asks with - "handlers" - against the singular an adapter declares. var kind = prefix?.TrimEnd('s') ?? ""; if (string.IsNullOrWhiteSpace(kind)) return byConvention; - foreach (var item in await cloudFilesService.ListAsync($"{root}/")) - { - if (item.Size <= 0 || named.Contains(item.Key)) continue; - - var declared = await DeclaredKindsAsync(item.Key, root); - if (declared.Contains(kind, System.StringComparer.OrdinalIgnoreCase)) - byConvention.Add(item); - } + var others = (await cloudFilesService.ListAsync($"{root}/")) + .Where(item => item.Size > 0 && !named.Contains(item.Key)) + .ToList(); + // Side by side: once the installer's metadata cache runs out, five minutes by default, this + // reads every other kind's adapters from storage, and one after another that took half a + // minute for a few dozen. Each adapter once, however many of its versions are listed. + var declared = (await Task.WhenAll(others.Select(i => AdapterIdOf(i.Key, root)).Distinct(System.StringComparer.Ordinal) + .Select(async adapterId => + { + await MetadataReads.WaitAsync(); + try { return (Id: adapterId, Kinds: await DeclaredKindsAsync(adapterId)); } + finally { MetadataReads.Release(); } + }))) + .ToDictionary(d => d.Id, d => d.Kinds, System.StringComparer.Ordinal); + + byConvention.AddRange(others.Where(item => + declared[AdapterIdOf(item.Key, root)].Contains(kind, System.StringComparer.OrdinalIgnoreCase))); return byConvention; } + /// The adapter a listed file is a package of. + private static string AdapterIdOf(string key, string root) + { + var adapterId = key.StartsWith($"{root}/", System.StringComparison.OrdinalIgnoreCase) + ? key[(root.Length + 1)..] + : key; + + // Versioned uploads keep the adapter id one segment up from the version. + return Semver.IsVersionNumber(adapterId.Split('/').Last()) + ? string.Join('/', adapterId.Split('/')[..^1]) + : adapterId; + } + /// /// The kinds one adapter declared. Metadata reads are cached by the installer, and an adapter /// whose metadata cannot be read declares nothing rather than taking the whole catalogue down - /// with it — the list is what an operator needs to configure anything at all. + /// with it - the list is what an operator needs to configure anything at all. /// - private async Task DeclaredKindsAsync(string key, string root) + private async Task DeclaredKindsAsync(string adapterId) { try { - var adapterId = key.StartsWith($"{root}/", System.StringComparison.OrdinalIgnoreCase) - ? key[(root.Length + 1)..] - : key; - - // Versioned uploads keep the adapter id one segment up from the version. - if (Semver.IsVersionNumber(adapterId.Split('/').Last())) - adapterId = string.Join('/', adapterId.Split('/')[..^1]); - var metadata = await adapterInstaller.GetMetadataAsync(adapterId); if (metadata?.AdapterValues == null) return []; diff --git a/SW.Bitween.Api/Resources/Adapters/Catalog.cs b/SW.Bitween.Api/Resources/Adapters/Catalog.cs index 6f3d3fb9..6c192631 100644 --- a/SW.Bitween.Api/Resources/Adapters/Catalog.cs +++ b/SW.Bitween.Api/Resources/Adapters/Catalog.cs @@ -16,7 +16,7 @@ namespace SW.Bitween.Resources.Adapters; /// answers with the adapters alone, which left a caller that needs to /// draw a form per adapter to ask once per row. On a screen listing /// all four kinds that is around ninety requests, six of which a browser will run at a time, so the -/// last of them waits behind fifteen rounds of queueing — and every one of those requests carries +/// last of them waits behind fifteen rounds of queueing - and every one of those requests carries /// its own permission check. Answering the whole kind at once makes it four requests for the screen. /// /// @@ -39,9 +39,12 @@ public async Task Handle(AdapterSearchRequest request) // Asked for all at once and left unthrottled on purpose. The native ones answer by // reflection and should not be made to queue, and how many published adapters may be - // running at a time is capped in ServerlessAdapterDescriber — process-wide, which is the + // running at a time is capped in ServerlessAdapterDescriber - process-wide, which is the // only place it can be, since this handler knows nothing of the other requests in flight. - var described = await Task.WhenAll(adapters.Select(async a => (a.Key, Values: await Describe(a.Key)))); + // Not at all for a caller after versions alone: a published adapter is described by starting + // it, which on a cold cache takes half a minute for a kind. + var described = request.StartupValues == false ? [] + : await Task.WhenAll(adapters.Select(async a => (a.Key, Values: await Describe(a.Key)))); var byKey = described.ToDictionary(d => d.Key, d => d.Values); return adapters.Select(a => new @@ -51,8 +54,49 @@ public async Task Handle(AdapterSearchRequest request) // Just the version numbers. VersionPaths carries each one as a path, which is what the // older shape passed through and what made a version read as an object rather than // "1.2.3" to anything trying to label it. - Versions = a.VersionPaths.Select(v => v.Split('/').Last()).ToList(), - StartupValues = byKey[a.Key] + // Old-layout version files, and the catalog's versions that can still be pinned - so the + // version label an older client shows is right for adapters published either way. + Versions = a.VersionPaths.Select(v => v.Split('/').Last()) + .Concat(a.Catalog?.Versions.Where(v => !v.Withdrawn).Select(v => v.Version) ?? []) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(v => System.Version.TryParse(v.Split('-')[0], out var parsed) ? parsed : new System.Version()) + .ToList(), + StartupValues = byKey.GetValueOrDefault(a.Key), + + // Additive: a client that predates the catalog ignores all of these. Every one is null + // or empty for an adapter published before manifests existed. + CurrentVersion = a.Catalog?.Current, + VersionHistory = a.Catalog?.Versions + .Select(v => new + { + v.Version, + v.PublishedOn, + v.PublishedBy, + v.Withdrawn, + v.Manifest?.ReleaseNotes, + MinBitweenVersion = v.Manifest?.Compatibility?.MinVersionOf(Services.Adapters.AdapterCatalog.Application), + // Whether the package carries its source, so a client offers to show it + // only where there is something to show. + HasSource = v.Manifest?.Source?.Files is { Count: > 0 }, + // Which the editor can open: Python and Node versions with their source. + v.Manifest?.Runtime + }) + .ToList(), + DisplayName = a.Manifest?.DisplayName, + a.Manifest?.Summary, + a.Manifest?.Description, + Publisher = a.Manifest?.Publisher?.Name, + PublisherUrl = a.Manifest?.Publisher?.Url, + a.Manifest?.License, + a.Manifest?.Homepage, + a.Manifest?.Repository, + Tags = a.Manifest?.Tags ?? [], + Categories = a.Manifest?.Categories ?? [], + a.Manifest?.Language, + a.Manifest?.Runtime, + a.Manifest?.Lifecycle, + Icon = a.Catalog?.IconDataUri, + MinBitweenVersion = a.Manifest?.Compatibility?.MinVersionOf(Services.Adapters.AdapterCatalog.Application) }); } @@ -64,7 +108,7 @@ private async Task> Describe(string adapterId) } catch (Exception) { - // One adapter that cannot be described — its runtime is missing locally, say — must not + // One adapter that cannot be described - its runtime is missing locally, say - must not // blank out the rest of the catalogue, including the native ones that resolved // perfectly well. It comes back with no properties, as it did when the caller was // asking row by row and swallowing the failure itself. diff --git a/SW.Bitween.Api/Resources/Adapters/GetProperties.cs b/SW.Bitween.Api/Resources/Adapters/GetProperties.cs index 8cc74b15..3056c347 100644 --- a/SW.Bitween.Api/Resources/Adapters/GetProperties.cs +++ b/SW.Bitween.Api/Resources/Adapters/GetProperties.cs @@ -37,9 +37,9 @@ async public Task Handle(string key) return _nativeAdapterDiscovery.GetExpectedStartupValues(decodedKey); } - // Handle serverless adapters. The native branch above returns a different shape — - // each key's default rather than a "key (default)" label — so it is left as it was. - var expected = AdapterStartupValues.WithoutSecretDefaults(await startupValues.Describe(decodedKey)); + // Handle serverless adapters. The native branch above returns a different shape - + // each key's default rather than a "key (default)" label - so it is left as it was. + var expected = AdapterStartupValues.WithoutSecretDefaults(await startupValues.DescribeOrNotFound(decodedKey)); return expected .ToList() diff --git a/SW.Bitween.Api/Resources/Adapters/GetStartupValues.cs b/SW.Bitween.Api/Resources/Adapters/GetStartupValues.cs index f8a120b1..a6f387b5 100644 --- a/SW.Bitween.Api/Resources/Adapters/GetStartupValues.cs +++ b/SW.Bitween.Api/Resources/Adapters/GetStartupValues.cs @@ -25,7 +25,7 @@ public async Task> Handle(string key) await requestContext.EnsurePermission(dbContext, Model.Permissions.Subscriptions.View); return AdapterStartupValues.WithoutSecretDefaults( - await startupValues.Describe(System.Uri.UnescapeDataString(key))); + await startupValues.DescribeOrNotFound(System.Uri.UnescapeDataString(key))); } } } diff --git a/SW.Bitween.Api/Resources/Adapters/Promote.cs b/SW.Bitween.Api/Resources/Adapters/Promote.cs new file mode 100644 index 00000000..aa234027 --- /dev/null +++ b/SW.Bitween.Api/Resources/Adapters/Promote.cs @@ -0,0 +1,34 @@ +using System.Threading.Tasks; +using SW.Bitween.Domain; +using SW.Bitween.Services.Adapters; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Adapters; + +public class AdapterPromoteRequest +{ + public string AdapterId { get; set; } + public string Version { get; set; } +} + +/// +/// Makes a published version the one that runs wherever nothing is pinned - rolling back is +/// promoting an older one. What serverless promote does, from Bitween, and in the audit trail. +/// +[HandlerName("promote")] +public class Promote(BitweenDbContext dbContext, RequestContext requestContext, AdapterWorkshop workshop) + : ICommandHandler +{ + public async Task Handle(AdapterPromoteRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.AdapterSource.Operate); + var adapterId = request.AdapterId?.ToLowerInvariant(); + if (string.IsNullOrWhiteSpace(adapterId) || NativeAdapterDiscoveryService.IsNative(adapterId)) + throw new SWValidationException("AdapterId", "Only a published adapter has versions to promote."); + + await workshop.PromoteAsync(adapterId, request.Version); + dbContext.Add(new AdapterRelease(adapterId, request.Version, AdapterRelease.PromotedAction, null, requestContext.GetNameIdentifier())); + await dbContext.SaveChangesAsync(); + return new { AdapterId = adapterId, Current = request.Version }; + } +} diff --git a/SW.Bitween.Api/Resources/Adapters/RetiringAdapters.cs b/SW.Bitween.Api/Resources/Adapters/RetiringAdapters.cs new file mode 100644 index 00000000..58e42ef9 --- /dev/null +++ b/SW.Bitween.Api/Resources/Adapters/RetiringAdapters.cs @@ -0,0 +1,89 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain; +using SW.Bitween.NativeAdapters; +using SW.Bitween.NativeAdapters.RebexFtpReceiver; +using SW.Bitween.NativeAdapters.RebexFtpUploadHandler; +using SW.Bitween.NativeAdapters.RebexPop3Receiver; + +namespace SW.Bitween.Resources.Adapters; + +/// +/// Built-in adapters on their way out, offered only while something still runs on them: the +/// Scriban mapper the native mapper replaced, and the Rebex adapters, which moved to the +/// marketplace as s9.rebex.ftp.receiver, s9.rebex.ftp.upload and +/// s9.rebex.pop3.receiver. +/// +/// +/// +/// This is how a built-in adapter retires. A subscription already using it is untouched - it keeps +/// running, and it keeps its own editor - but the adapter stops being offered as a choice once the +/// last subscription has moved across. The list then shrinks by itself as the migration finishes, +/// rather than when someone remembers to delete the adapter. +/// +/// +/// One-way on purpose: past that point there is no picking it again. Which is why the question of +/// what counts as "in use" is answered generously - an inactive subscription still holds its +/// settings and can be switched back on, so it counts, and so does a notification channel, a +/// notifier or a retry alert naming a handler. Withholding an adapter a subscription is still saved with would leave that +/// subscription's own adapter missing from the list its picker draws from, which reads as no +/// adapter chosen at all. +/// +/// +/// The same idea already exists a level down: withholds +/// the Rebex-backed adapters while no license key is set, so that a picker never offers something +/// that could only fail. This withholds ones that would only lengthen a migration. +/// +/// +internal static class RetiringAdapters +{ + /// The old mapper's id, which is the class name the discovery service reports. + public const string Mapper = nameof(NativeJSONMapper); + + /// Each retiring adapter's id, and whether anything this database holds still names it. + private static readonly Dictionary>> Retiring = + new(StringComparer.OrdinalIgnoreCase) + { + [Mapper] = (db, id) => db.Set().AnyAsync(s => s.MapperId != null && s.MapperId.ToLower() == id), + [nameof(NativeRebexFtpReceiver)] = ReceiverInUse, + [nameof(NativeRebexPop3Receiver)] = ReceiverInUse, + [nameof(NativeRebexFtpUploadHandler)] = HandlerInUse, + }; + + /// + /// without the retiring ones nothing still names. + /// + /// + /// The database is only asked about a retiring adapter that is in the list to begin with, so + /// the kinds with none, validators, cost nothing. + /// + public static async Task> ExceptRetiring( + this IEnumerable adapters, BitweenDbContext dbContext) + { + var listed = adapters.ToList(); + + foreach (var adapter in listed.Where(Retiring.ContainsKey).ToList()) + { + // Compared lowered rather than as stored: the lookup that resolves an adapter at run + // time is itself case-insensitive, so a row whose casing drifted is still a + // subscription running on it, and answering "no" to that would take it away. + if (!await Retiring[adapter](dbContext, adapter.ToLowerInvariant())) + listed.Remove(adapter); + } + + return listed; + } + + private static Task ReceiverInUse(BitweenDbContext db, string id) => + db.Set().AnyAsync(s => s.ReceiverId != null && s.ReceiverId.ToLower() == id); + + private static async Task HandlerInUse(BitweenDbContext db, string id) => + await db.Set().AnyAsync(s => s.HandlerId != null && s.HandlerId.ToLower() == id) + || await db.Set().AnyAsync(c => c.HandlerId.ToLower() == id) + || await db.Set().AnyAsync(n => n.HandlerId != null && n.HandlerId.ToLower() == id) + || await db.Set().AnyAsync(p => p.AlertHandlerId != null && p.AlertHandlerId.ToLower() == id) + || await db.Set().AnyAsync(o => o.AlertHandlerId != null && o.AlertHandlerId.ToLower() == id); +} diff --git a/SW.Bitween.Api/Resources/Adapters/RetiringMapper.cs b/SW.Bitween.Api/Resources/Adapters/RetiringMapper.cs deleted file mode 100644 index c6b45e10..00000000 --- a/SW.Bitween.Api/Resources/Adapters/RetiringMapper.cs +++ /dev/null @@ -1,69 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.EntityFrameworkCore; -using SW.Bitween.Domain; -using SW.Bitween.NativeAdapters; - -namespace SW.Bitween.Resources.Adapters; - -/// -/// The Scriban mapper the native mapper replaced, offered only while something still runs on it. -/// -/// -/// -/// This is how the old mapper retires. A subscription already using it is untouched — it keeps -/// running, and it keeps its own editor — but the mapper stops being offered as a choice once the -/// last subscription has moved across. The list of mappers then shrinks by itself as the migration -/// finishes, rather than when someone remembers to delete the adapter. -/// -/// -/// One-way on purpose: past that point there is no picking it again. Which is why the question of -/// what counts as "in use" is answered generously — an inactive subscription still holds its -/// template and can be switched back on, so it counts. Withholding a mapper a subscription is -/// still saved with would leave that subscription's own mapper missing from the list its picker -/// draws from, which reads as no mapper chosen at all. -/// -/// -/// The same idea already exists a level down: withholds -/// the Rebex-backed adapters while no license key is set, so that a picker never offers something -/// that could only fail. This withholds one that would only lengthen a migration. -/// -/// -internal static class RetiringMapper -{ - /// The adapter id, which is the class name the discovery service reports. - public const string Id = nameof(NativeJSONMapper); - - private static readonly string Lowered = Id.ToLowerInvariant(); - - /// - /// without the retiring mapper, unless a subscription still names it. - /// - /// - /// The database is only asked when the retiring mapper is in the list to begin with, so the - /// other adapter kinds — receivers, handlers, validators — cost nothing. - /// - public static async Task> ExceptRetiring( - this IEnumerable adapters, BitweenDbContext dbContext) - { - var listed = adapters.ToList(); - - if (!listed.Any(Matches)) return listed; - - // Compared lowered rather than as stored: the lookup that resolves a mapper at run time is - // itself case-insensitive, so a row whose casing drifted is still a subscription running on - // this mapper, and answering "no" to that would take the mapper away from it. - var stillInUse = await dbContext.Set() - .AnyAsync(s => s.MapperId != null && s.MapperId.ToLower() == Lowered); - - if (stillInUse) return listed; - - listed.RemoveAll(Matches); - return listed; - } - - private static bool Matches(string adapter) => - adapter.Equals(Id, StringComparison.OrdinalIgnoreCase); -} diff --git a/SW.Bitween.Api/Resources/Adapters/Source.cs b/SW.Bitween.Api/Resources/Adapters/Source.cs new file mode 100644 index 00000000..d7d76f8e --- /dev/null +++ b/SW.Bitween.Api/Resources/Adapters/Source.cs @@ -0,0 +1,56 @@ +using System.Threading.Tasks; +using SW.Bitween.Domain; +using SW.Bitween.Services.Adapters; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Adapters; + +public class AdapterSourceRequest +{ + public string AdapterId { get; set; } + public string Version { get; set; } + + /// The file, relative to the source folder; for only. + public string Path { get; set; } +} + +/// +/// The files a published version's source holds, with their hashes, so a client can show the tree +/// and tell which files two versions differ in without reading any. +/// +[HandlerName("source")] +public class Source( + AdapterSourceReader reader, + BitweenDbContext dbContext, + RequestContext requestContext) : IQueryHandler +{ + public async Task Handle(AdapterSourceRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.AdapterSource.View); + + var listing = await reader.ListAsync(request.AdapterId?.ToLowerInvariant(), request.Version) + ?? throw new SWNotFoundException($"{request.AdapterId} {request.Version}"); + return listing; + } +} + +/// One source file of a published version, checked against its manifest's hash. Each read is audited. +[HandlerName("sourcefile")] +public class SourceFile( + AdapterSourceReader reader, + BitweenDbContext dbContext, + RequestContext requestContext) : IQueryHandler +{ + public async Task Handle(AdapterSourceRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.AdapterSource.View); + + var adapterId = request.AdapterId?.ToLowerInvariant(); + var file = await reader.ReadAsync(adapterId, request.Version, request.Path) + ?? throw new SWNotFoundException($"{request.AdapterId} {request.Version} {request.Path}"); + + dbContext.Add(new AdapterSourceAccess(adapterId, request.Version, request.Path, requestContext.GetNameIdentifier())); + await dbContext.SaveChangesAsync(); + return file; + } +} diff --git a/SW.Bitween.Api/Resources/Adapters/Versions.cs b/SW.Bitween.Api/Resources/Adapters/Versions.cs new file mode 100644 index 00000000..5affd34c --- /dev/null +++ b/SW.Bitween.Api/Resources/Adapters/Versions.cs @@ -0,0 +1,43 @@ +using System.Threading.Tasks; +using SW.Bitween.Domain; +using SW.Bitween.Services.Adapters; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Adapters; + +public class AdapterVersionsRequest +{ + public string AdapterId { get; set; } +} + +/// Every published version of one adapter, with which is current: bitween adapter versions. +[HandlerName("versions")] +public class Versions(BitweenDbContext dbContext, RequestContext requestContext, AdapterWorkshop workshop) + : IQueryHandler +{ + public async Task Handle(AdapterVersionsRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Subscriptions.View); + if (string.IsNullOrWhiteSpace(request.AdapterId)) throw new SWValidationException("AdapterId", "Say which adapter."); + return await workshop.VersionsAsync(request.AdapterId.ToLowerInvariant()); + } +} + +/// Takes a published version out of use: bitween adapter withdraw. In the audit trail. +[HandlerName("withdraw")] +public class Withdraw(BitweenDbContext dbContext, RequestContext requestContext, AdapterWorkshop workshop) + : ICommandHandler +{ + public async Task Handle(AdapterPromoteRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.AdapterSource.Operate); + var adapterId = request.AdapterId?.ToLowerInvariant(); + if (string.IsNullOrWhiteSpace(adapterId) || NativeAdapterDiscoveryService.IsNative(adapterId)) + throw new SWValidationException("AdapterId", "Only a published adapter has versions to withdraw."); + + await workshop.WithdrawAsync(adapterId, request.Version); + dbContext.Add(new AdapterRelease(adapterId, request.Version, AdapterRelease.WithdrawnAction, null, requestContext.GetNameIdentifier())); + await dbContext.SaveChangesAsync(); + return new { AdapterId = adapterId, Withdrawn = request.Version }; + } +} diff --git a/SW.Bitween.Api/Resources/ApiGateways/AddPartner.cs b/SW.Bitween.Api/Resources/ApiGateways/AddPartner.cs index c5156d49..6917cc0a 100644 --- a/SW.Bitween.Api/Resources/ApiGateways/AddPartner.cs +++ b/SW.Bitween.Api/Resources/ApiGateways/AddPartner.cs @@ -37,7 +37,7 @@ public async Task Handle(int gatewayId, ApiGatewayPartnerCreate model) if (model.NewIntegration != null) { - // Staged, not saved — the attachment takes its foreign key from the subscription EF + // Staged, not saved - the attachment takes its foreign key from the subscription EF // is tracking, so the pair lands on the one SaveChangesAsync below or not at all. // Nothing to check for a duplicate against: an integration that does not exist yet // cannot already be attached. @@ -74,7 +74,7 @@ public async Task Handle(int gatewayId, ApiGatewayPartnerCreate model) dbContext.Add(partnerLink); await dbContext.SaveChangesAsync(); // Attaching an existing integration changes nothing the cache holds, but staging a new - // one above creates a Subscription — and unconditional is what AddRoute does. + // one above creates a Subscription - and unconditional is what AddRoute does. await cache.BroadcastRevoke(); return null; diff --git a/SW.Bitween.Api/Resources/ApiGateways/Create.cs b/SW.Bitween.Api/Resources/ApiGateways/Create.cs index 6758b6f1..45f8e91c 100644 --- a/SW.Bitween.Api/Resources/ApiGateways/Create.cs +++ b/SW.Bitween.Api/Resources/ApiGateways/Create.cs @@ -19,7 +19,8 @@ public async Task Handle(ApiGatewayCreate model) { Name = model.Name, UrlName = model.UrlName, - Inactive = model.Inactive + Inactive = model.Inactive, + FailureAnswer = model.FailureAnswer ?? GatewayFailureAnswer.Accurate, }; if (model.Authentication != null) GatewayAuthentication.Apply(entity, model.Authentication); diff --git a/SW.Bitween.Api/Resources/ApiGateways/GatewayUrlName.cs b/SW.Bitween.Api/Resources/ApiGateways/GatewayUrlName.cs index 0250fb0c..4b91b215 100644 --- a/SW.Bitween.Api/Resources/ApiGateways/GatewayUrlName.cs +++ b/SW.Bitween.Api/Resources/ApiGateways/GatewayUrlName.cs @@ -8,7 +8,7 @@ namespace SW.Bitween.Resources.ApiGateways; /// -/// The url name is the path partners call — /api/gateway/{urlName}/sync — so anything +/// The url name is the path partners call - /api/gateway/{urlName}/sync - so anything /// needing escaping there makes a gateway that reads as configured and cannot be reached. A /// space is the one that actually happens: it saves, the endpoint shown on the page is the one /// the partner copies, and the call 404s with nothing on screen to explain it. @@ -30,7 +30,7 @@ public static void Validate(string urlName) if (!Allowed().IsMatch(urlName)) throw new SWValidationException("GATEWAY_URL_NAME_INVALID", $"'{urlName}' cannot be used in a URL. Use lowercase letters, digits, hyphens " + - "and underscores, with / between parts — no spaces, and no part starting or " + + "and underscores, with / between parts - no spaces, and no part starting or " + "ending with a separator."); // The call ends in /sync or /async, and it is the last segment that says which. A name @@ -38,7 +38,7 @@ public static void Validate(string urlName) var last = urlName[(urlName.LastIndexOf('/') + 1)..]; if (last is "sync" or "async") throw new SWValidationException("GATEWAY_URL_NAME_INVALID", - $"'{urlName}' cannot end in '{last}' — partners add /sync or /async after it."); + $"'{urlName}' cannot end in '{last}' - partners add /sync or /async after it."); } /// @@ -46,7 +46,7 @@ public static void Validate(string urlName) /// updating, so saving a gateway without touching its url name isn't a collision with itself. /// /// - /// The column is uniquely indexed, so this was already refused — as a constraint violation + /// The column is uniquely indexed, so this was already refused - as a constraint violation /// that reached the screen as "Request failed (500)", with nothing to say the name was taken /// or which gateway has it. /// diff --git a/SW.Bitween.Api/Resources/ApiGateways/Get.cs b/SW.Bitween.Api/Resources/ApiGateways/Get.cs index 81a7e6d6..d486cbc9 100644 --- a/SW.Bitween.Api/Resources/ApiGateways/Get.cs +++ b/SW.Bitween.Api/Resources/ApiGateways/Get.cs @@ -31,6 +31,7 @@ public async Task Handle(int key) Name = gateway.Name, UrlName = gateway.UrlName, Inactive = gateway.Inactive, + FailureAnswer = gateway.FailureAnswer, Authentication = GatewayAuthentication.Read(gateway), DefaultKeyHeader = options.PartnerKeyHeader, PartnersCount = gateway.Partners.Count, diff --git a/SW.Bitween.Api/Resources/ApiGateways/Rejections.cs b/SW.Bitween.Api/Resources/ApiGateways/Rejections.cs new file mode 100644 index 00000000..e00a93e7 --- /dev/null +++ b/SW.Bitween.Api/Resources/ApiGateways/Rejections.cs @@ -0,0 +1,46 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain.Gateway; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.ApiGateways; + +/// +/// The calls to this gateway turned away, and why: an unknown or missing key, a partner not +/// attached, the gateway switched off, no subscription behind it, too many calls. Kept in memory on +/// the node that answers, since it started (see ); another node keeps +/// its own. +/// +[HandlerName("rejections")] +public class Rejections(BitweenDbContext dbContext, RequestContext requestContext, GatewayActivity activity) + : IGetHandler +{ + public async Task Handle(int key) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.ApiGateways.View); + var urlName = await dbContext.Set().AsNoTracking().Where(g => g.Id == key) + .Select(g => g.UrlName).SingleOrDefaultAsync() ?? throw new SWNotFoundException(key.ToString()); + + var (counts, recent) = activity.RefusalsOf(urlName); + + // Every node's, kept past restarts: the hourly metrics count each refusal under the reason. + var scopeKey = urlName.Trim('/').ToLowerInvariant(); + var weekAgo = DateTime.UtcNow.AddDays(-7); + var lastSevenDays = await dbContext.Set().AsNoTracking() + .Where(e => e.Scope == Domain.Metrics.ErrorStats.GatewayScope && e.ScopeKey == scopeKey && e.Hour >= weekAgo) + .GroupBy(e => e.Sample) + .Select(g => new { Reason = g.Key, Count = g.Sum(x => (long)x.Count) }) + .ToDictionaryAsync(x => x.Reason ?? "", x => x.Count); + + return new + { + Node = Services.Cluster.NodeHeartbeat.NodeName, + activity.Since, + Counts = counts, + LastSevenDays = lastSevenDays, + Recent = recent.Select(r => new { r.On, r.Reason, r.Status, r.Address }), + }; + } +} diff --git a/SW.Bitween.Api/Resources/ApiGateways/Search.cs b/SW.Bitween.Api/Resources/ApiGateways/Search.cs index ee384c82..3c6ac06a 100644 --- a/SW.Bitween.Api/Resources/ApiGateways/Search.cs +++ b/SW.Bitween.Api/Resources/ApiGateways/Search.cs @@ -5,10 +5,11 @@ using Microsoft.EntityFrameworkCore; using SW.Bitween.Domain.Gateway; using SW.Bitween.Model; +using SW.Bitween.Services.Mirror; namespace SW.Bitween.Resources.ApiGateways { - public class Search(BitweenDbContext dbContext, RequestContext requestContext) : ISearchyHandler + public class Search(BitweenDbContext dbContext, RequestContext requestContext, MirrorService mirror) : ISearchyHandler { public async Task Handle(SearchyRequest searchyRequest, bool lookup = false, string searchPhrase = null) { @@ -16,6 +17,7 @@ public async Task Handle(SearchyRequest searchyRequest, bool lookup = fa // the full list is the data, so that's what the view permission covers. if (!lookup) await requestContext.EnsurePermission(dbContext, Model.Permissions.ApiGateways.View); + await mirror.FilterAsync(searchyRequest, "apiGateways", requestContext, dbContext); var query = from gateway in dbContext.Set() select new ApiGatewayRow @@ -24,6 +26,7 @@ public async Task Handle(SearchyRequest searchyRequest, bool lookup = fa Name = gateway.Name, UrlName = gateway.UrlName, Inactive = gateway.Inactive, + FailureAnswer = gateway.FailureAnswer, PartnersCount = gateway.Partners.Count }; diff --git a/SW.Bitween.Api/Resources/ApiGateways/SearchAttachments.cs b/SW.Bitween.Api/Resources/ApiGateways/SearchAttachments.cs index 50b46212..c67d0dab 100644 --- a/SW.Bitween.Api/Resources/ApiGateways/SearchAttachments.cs +++ b/SW.Bitween.Api/Resources/ApiGateways/SearchAttachments.cs @@ -8,7 +8,7 @@ namespace SW.Bitween.Resources.ApiGateways { /// - /// Paged, searched view of one gateway's attachments — the full list lives on + /// Paged, searched view of one gateway's attachments - the full list lives on /// for callers that need every attached partner id (e.g. the /// attach-partner picker's exclude list), this is only for the gateway page's own table. /// diff --git a/SW.Bitween.Api/Resources/ApiGateways/Update.cs b/SW.Bitween.Api/Resources/ApiGateways/Update.cs index b7a3ada1..cc855af7 100644 --- a/SW.Bitween.Api/Resources/ApiGateways/Update.cs +++ b/SW.Bitween.Api/Resources/ApiGateways/Update.cs @@ -28,6 +28,7 @@ public async Task Handle(int key, ApiGatewayUpdate model) entity.Name = model.Name; entity.UrlName = model.UrlName; entity.Inactive = model.Inactive; + if (model.FailureAnswer != null) entity.FailureAnswer = model.FailureAnswer.Value; if (model.Authentication != null) GatewayAuthentication.Apply(entity, model.Authentication); diff --git a/SW.Bitween.Api/Resources/Audit/Search.cs b/SW.Bitween.Api/Resources/Audit/Search.cs index 55f021b4..69da9cff 100644 --- a/SW.Bitween.Api/Resources/Audit/Search.cs +++ b/SW.Bitween.Api/Resources/Audit/Search.cs @@ -12,14 +12,14 @@ namespace SW.Bitween.Resources.Audit; /// -/// The audit trail, newest first. Serves both the global history page and the per-entity timeline — +/// The audit trail, newest first. Serves both the global history page and the per-entity timeline - /// the latter is this same query with and /// set, which is what the composite index covers. /// public class Search(BitweenDbContext dbContext, RequestContext requestContext) : IQueryHandler { - /// Rows per page ceiling — each row carries its changes as JSON. + /// Rows per page ceiling - each row carries its changes as JSON. const int MaxLimit = 200; public async Task Handle(SearchAuditModel request) diff --git a/SW.Bitween.Api/Resources/BitweenDocs/Get.cs b/SW.Bitween.Api/Resources/BitweenDocs/Get.cs index 8b655910..1c3495a8 100644 --- a/SW.Bitween.Api/Resources/BitweenDocs/Get.cs +++ b/SW.Bitween.Api/Resources/BitweenDocs/Get.cs @@ -6,7 +6,7 @@ namespace SW.Bitween.Resources.BitweenDocs; /// /// An exchange file's content, for the exchange drawer. Signed-in members with the exchanges view -/// only, and only keys that are exactly one of an exchange's files — it used to read any key in the +/// only, and only keys that are exactly one of an exchange's files - it used to read any key in the /// bucket for anyone. /// public class Get(BitweenDbContext dbContext, RequestContext requestContext, XchangeService xchangeService) diff --git a/SW.Bitween.Api/Resources/BusGateways/Create.cs b/SW.Bitween.Api/Resources/BusGateways/Create.cs index 938de652..cfe99cd1 100644 --- a/SW.Bitween.Api/Resources/BusGateways/Create.cs +++ b/SW.Bitween.Api/Resources/BusGateways/Create.cs @@ -42,7 +42,7 @@ public async Task Handle(BusGatewayCreate model) /// /// Shared by Create and Update. An endpoint is what the supervisor turns into the adapter's /// consume list, so an external gateway without one is a gateway that can never receive - /// anything — and it would fail silently, which is the worst way for it to fail. + /// anything - and it would fail silently, which is the worst way for it to fail. /// internal static async Task EnsureDataSourceAsync(BitweenDbContext dbContext, BusGatewayCreate model) { @@ -56,8 +56,8 @@ internal static async Task EnsureDataSourceAsync(BitweenDbContext dbContext, Bus if (dataSource == null) throw new SWNotFoundException($"DataSource with Id {model.DataSourceId} not found"); - // A data source is not only a broker connection — a resident adapter holding a database - // session is one too — and only a broker has queues to consume. Reading this from the + // A data source is not only a broker connection - a resident adapter holding a database + // session is one too - and only a broker has queues to consume. Reading this from the // kind rather than from the adapter id keeps the rule true for providers nobody has // written yet. if (dataSource.Kind != Domain.DataSources.DataSourceKind.Broker) @@ -67,7 +67,7 @@ internal static async Task EnsureDataSourceAsync(BitweenDbContext dbContext, Bus if (string.IsNullOrWhiteSpace(model.Endpoint)) throw new SWException( - "An external bus gateway needs an endpoint — the queue, topic or subscription " + "An external bus gateway needs an endpoint - the queue, topic or subscription " + "on that data source it reads from."); // Two gateways consuming one endpoint on one data source would both be offered every diff --git a/SW.Bitween.Api/Resources/BusGateways/Get.cs b/SW.Bitween.Api/Resources/BusGateways/Get.cs index b25d2955..65f5a48a 100644 --- a/SW.Bitween.Api/Resources/BusGateways/Get.cs +++ b/SW.Bitween.Api/Resources/BusGateways/Get.cs @@ -35,7 +35,7 @@ public async Task Handle(int key) : await dbContext.Set() .AsNoTracking() .Where(d => d.Id == gateway.DataSourceId) - .Select(d => new { d.Name, d.LastKnownState }) + .Select(d => new { d.Name, d.LastKnownState, d.SecretProperties }) .FirstOrDefaultAsync(); return new BusGatewayRow @@ -49,7 +49,9 @@ public async Task Handle(int key) DataSourceName = dataSource?.Name, DataSourceState = dataSource?.LastKnownState, Endpoint = gateway.Endpoint, - EndpointProperties = gateway.EndpointProperties ?? new(), + // Masked as the data source's own properties are: what it declares secret, and any + // well-known credential name. Saving the sentinel back keeps the stored value. + EndpointProperties = DataSources.Secrets.Mask(gateway.EndpointProperties, dataSource?.SecretProperties), RoutesCount = gateway.Routes.Count, Routes = gateway.Routes.Select(r => new BusGatewayRouteDto { diff --git a/SW.Bitween.Api/Resources/BusGateways/Search.cs b/SW.Bitween.Api/Resources/BusGateways/Search.cs index a2de4774..eb0d55d0 100644 --- a/SW.Bitween.Api/Resources/BusGateways/Search.cs +++ b/SW.Bitween.Api/Resources/BusGateways/Search.cs @@ -2,6 +2,7 @@ using SW.Bitween.Domain; using SW.Bitween.Domain.Gateway; using SW.Bitween.Model; +using SW.Bitween.Services.Mirror; using SW.EfCoreExtensions; using SW.PrimitiveTypes; using System.Linq; @@ -9,7 +10,7 @@ namespace SW.Bitween.Resources.BusGateways { - public class Search(BitweenDbContext dbContext, RequestContext requestContext) : ISearchyHandler + public class Search(BitweenDbContext dbContext, RequestContext requestContext, MirrorService mirror) : ISearchyHandler { public async Task Handle(SearchyRequest searchyRequest, bool lookup = false, string searchPhrase = null) { @@ -17,6 +18,7 @@ public async Task Handle(SearchyRequest searchyRequest, bool lookup = fa // the full list is the data, so that's what the view permission covers. if (!lookup) await requestContext.EnsurePermission(dbContext, Model.Permissions.BusGateways.View); + await mirror.FilterAsync(searchyRequest, "busGateways", requestContext, dbContext); var documents = dbContext.Set(); var dataSources = dbContext.Set(); @@ -32,7 +34,7 @@ public async Task Handle(SearchyRequest searchyRequest, bool lookup = fa .Select(d => d.Name).FirstOrDefault(), // Null name means the internal bus, which is what the list column - // reads — an operator should be able to tell at a glance which of + // reads - an operator should be able to tell at a glance which of // their gateways reach outside. DataSourceId = gateway.DataSourceId, DataSourceName = dataSources.Where(d => d.Id == gateway.DataSourceId) diff --git a/SW.Bitween.Api/Resources/BusGateways/Update.cs b/SW.Bitween.Api/Resources/BusGateways/Update.cs index 1dcbd5b6..10f035ec 100644 --- a/SW.Bitween.Api/Resources/BusGateways/Update.cs +++ b/SW.Bitween.Api/Resources/BusGateways/Update.cs @@ -24,14 +24,15 @@ public async Task Handle(int key, BusGatewayUpdate model) await EnsureEndpointFreeAsync(_dbContext, key, model); - // The bound document is still fixed at creation — routes' subscriptions belong to it — + // The bound document is still fixed at creation - routes' subscriptions belong to it - // but where the messages COME from is exactly the thing an operator needs to change // without rebuilding the gateway and all of its routes. entity.Name = model.Name; entity.Inactive = model.Inactive; entity.DataSourceId = model.DataSourceId; entity.Endpoint = model.DataSourceId == null ? null : model.Endpoint; - entity.EndpointProperties = model.EndpointProperties ?? new(); + // A secret the form sent back masked keeps what is stored; one never stored is dropped. + entity.EndpointProperties = DataSources.Secrets.Merge(entity.EndpointProperties, model.EndpointProperties ?? new()); await _dbContext.SaveChangesAsync(); await cache.BroadcastRevoke(); @@ -51,8 +52,8 @@ private static async Task EnsureEndpointFreeAsync( if (dataSource == null) throw new SWNotFoundException($"DataSource with Id {model.DataSourceId} not found"); - // A data source is not only a broker connection — a resident adapter holding a database - // session is one too — and only a broker has queues to consume. Reading this from the + // A data source is not only a broker connection - a resident adapter holding a database + // session is one too - and only a broker has queues to consume. Reading this from the // kind rather than from the adapter id keeps the rule true for providers nobody has // written yet. if (dataSource.Kind != Domain.DataSources.DataSourceKind.Broker) @@ -62,7 +63,7 @@ private static async Task EnsureEndpointFreeAsync( if (string.IsNullOrWhiteSpace(model.Endpoint)) throw new SWException( - "An external bus gateway needs an endpoint — the queue, topic or subscription " + "An external bus gateway needs an endpoint - the queue, topic or subscription " + "on that data source it reads from."); var endpointTaken = await dbContext.Set() diff --git a/SW.Bitween.Api/Resources/Cluster/Nodes.cs b/SW.Bitween.Api/Resources/Cluster/Nodes.cs new file mode 100644 index 00000000..623b605b --- /dev/null +++ b/SW.Bitween.Api/Resources/Cluster/Nodes.cs @@ -0,0 +1,53 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain.Cluster; +using SW.Bitween.Domain.DataSources; +using SW.Bitween.Services.Cluster; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Cluster; + +/// +/// The nodes of this Bitween, as each last described itself (see ): +/// whether it is still there, whether it runs data sources, which adapter runtimes it has, and the +/// leases and exclusive data sources it holds. +/// +[HandlerName("nodes")] +public class Nodes(BitweenDbContext dbContext, RequestContext requestContext) : IQueryHandler +{ + public async Task Handle() + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Settings.View); + + var now = DateTime.UtcNow; + var nodes = await dbContext.Set().AsNoTracking().OrderBy(n => n.Id).ToListAsync(); + var leases = await dbContext.Set().AsNoTracking().ToListAsync(); + var held = await dbContext.Set().AsNoTracking() + .Where(d => d.OwnedByNode != null) + .Select(d => new { d.Id, d.Name, d.OwnedByNode }) + .ToListAsync(); + + return new + { + AnsweredBy = NodeHeartbeat.NodeName, + GoneAfterSeconds = (int)NodeHeartbeat.GoneAfter.TotalSeconds, + Nodes = nodes.Select(n => new + { + Name = n.Id, + n.Host, + n.StartedOn, + n.LastSeenOn, + Online = now - n.LastSeenOn <= NodeHeartbeat.GoneAfter, + n.Version, + n.DataSources, + Runtimes = (n.Runtimes ?? "").Split(',', StringSplitOptions.RemoveEmptyEntries), + Leases = leases.Where(l => l.OwnerNode == n.Id).Select(l => new { Resource = l.Id, l.Term, l.AcquiredOn }), + // Recorded as " (term N)" by the supervisor that took the source. + HoldsDataSources = held.Where(d => d.OwnedByNode == n.Id || d.OwnedByNode.StartsWith(n.Id + " ")) + .Select(d => new { d.Id, d.Name }), + }), + }; + } +} diff --git a/SW.Bitween.Api/Resources/Connections/Authorize.cs b/SW.Bitween.Api/Resources/Connections/Authorize.cs new file mode 100644 index 00000000..64452aa9 --- /dev/null +++ b/SW.Bitween.Api/Resources/Connections/Authorize.cs @@ -0,0 +1,18 @@ +using System.Threading.Tasks; +using SW.Bitween.Model; +using SW.Bitween.Services.Connections; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Connections; + +/// Starts approving an authorization code connection: the provider's page to send the browser to. +[HandlerName("authorize")] +public class Authorize(BitweenDbContext dbContext, RequestContext requestContext, ConnectionTokens tokens) + : ICommandHandler +{ + public async Task Handle(int key, ConnectionAuthorize request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.GlobalValues.Edit); + return new ConnectionAuthorizeResult { Url = await tokens.StartAuthorizationAsync(key) }; + } +} diff --git a/SW.Bitween.Api/Resources/Connections/Complete.cs b/SW.Bitween.Api/Resources/Connections/Complete.cs new file mode 100644 index 00000000..0f16549c --- /dev/null +++ b/SW.Bitween.Api/Resources/Connections/Complete.cs @@ -0,0 +1,20 @@ +using System.Threading.Tasks; +using SW.Bitween.Model; +using SW.Bitween.Services.Connections; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Connections; + +/// Finishes an approval with what the provider sent the browser back to /oauth-callback with. +[HandlerName("complete")] +public class Complete(BitweenDbContext dbContext, RequestContext requestContext, ConnectionTokens tokens) + : ICommandHandler +{ + public async Task Handle(ConnectionComplete request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.GlobalValues.Edit); + var actor = requestContext.User?.Identity?.Name ?? requestContext.GetNameIdentifier(); + var (id, name) = await tokens.CompleteAuthorizationAsync(request.Code, request.State, actor); + return new { Id = id, Name = name }; + } +} diff --git a/SW.Bitween.Api/Resources/Connections/ConnectionRows.cs b/SW.Bitween.Api/Resources/Connections/ConnectionRows.cs new file mode 100644 index 00000000..3c1a39e1 --- /dev/null +++ b/SW.Bitween.Api/Resources/Connections/ConnectionRows.cs @@ -0,0 +1,123 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using SW.Bitween.Domain.Connections; +using SW.Bitween.Model; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Connections; + +/// Turning a connection into what the API shows, and an edit into a connection. +internal static class ConnectionRows +{ + public const string Private = "__private__"; + static readonly Regex NamePattern = new("^[A-Za-z0-9_-]{1,100}$", RegexOptions.Compiled); + + public static ConnectionRow Row(Connection c) => new() + { + Id = c.Id, + Name = c.Name, + Grant = c.Grant.ToString(), + AuthorizeUrl = c.AuthorizeUrl, + TokenUrl = c.TokenUrl, + ClientId = c.ClientId, + ClientSecret = string.IsNullOrEmpty(c.ClientSecret) ? null : Private, + Scopes = c.Scopes, + AuthorizeParameters = c.AuthorizeParameters, + TokenAuth = c.TokenAuth, + Username = c.Username, + Password = string.IsNullOrEmpty(c.Password) ? null : Private, + Status = c.Status.ToString(), + LastError = c.LastError, + AccessTokenExpiresOn = c.AccessTokenExpiresOn, + HasRefreshToken = !string.IsNullOrEmpty(c.RefreshToken), + AuthorizedBy = c.AuthorizedBy, + AuthorizedOn = c.AuthorizedOn, + Reference = $"{{{{connections.{c.Name}.accessToken}}}}", + }; + + /// + /// Applies an edit. Changing who the client is, or where its tokens come from, forgets the + /// tokens: they were the old client's, and using them for the new one would be wrong either way. + /// + public static void Apply(Connection c, ConnectionSave save) + { + var problems = new List(); + if (string.IsNullOrWhiteSpace(save.Name) || !NamePattern.IsMatch(save.Name.Trim())) + problems.Add("A name is letters, digits, '_' and '-', up to 100 of them; settings write it as {{connections.NAME.accessToken}}."); + if (!Enum.TryParse(save.Grant, true, out var grant) || !Enum.IsDefined(grant)) + problems.Add("Grant is ClientCredentials, AuthorizationCode or Password."); + if (!IsHttps(save.TokenUrl)) problems.Add("The token URL must be an https address (http only on this machine, for testing)."); + if (grant == ConnectionGrant.AuthorizationCode && !IsHttps(save.AuthorizeUrl)) + problems.Add("An authorization code connection needs the provider's authorize URL, an https address (http only on this machine, for testing)."); + if (string.IsNullOrWhiteSpace(save.ClientId)) problems.Add("The client ID is required."); + if (grant == ConnectionGrant.Password && string.IsNullOrWhiteSpace(save.Username)) problems.Add("A password connection needs the username."); + var tokenAuth = string.IsNullOrWhiteSpace(save.TokenAuth) ? "post" : save.TokenAuth.Trim().ToLowerInvariant(); + if (tokenAuth is not ("post" or "basic")) problems.Add("Token authentication is post or basic."); + if (problems.Count > 0) throw new SWValidationException("CONNECTION_INVALID", string.Join(" ", problems)); + + var clientChanged = c.Id != 0 && (c.Grant != grant || !Same(c.ClientId, save.ClientId) || !Same(c.TokenUrl, save.TokenUrl) + || !Same(c.AuthorizeUrl, save.AuthorizeUrl) || !Same(c.Scopes, save.Scopes) + || !Same(c.Username, save.Username)); + + c.Name = save.Name.Trim(); + c.Grant = grant; + c.AuthorizeUrl = Trimmed(save.AuthorizeUrl); + c.TokenUrl = Trimmed(save.TokenUrl); + c.ClientId = Trimmed(save.ClientId); + c.Scopes = Trimmed(save.Scopes); + c.AuthorizeParameters = Trimmed(save.AuthorizeParameters); + c.TokenAuth = tokenAuth; + c.Username = Trimmed(save.Username); + if (save.ClientSecret is not null and not Private) + { + clientChanged |= c.Id != 0 && save.ClientSecret != c.ClientSecret; + c.ClientSecret = save.ClientSecret.Length == 0 ? null : save.ClientSecret; + } + if (save.Password is not null and not Private) + { + clientChanged |= c.Id != 0 && save.Password != c.Password; + c.Password = save.Password.Length == 0 ? null : save.Password; + } + + if (c.Id == 0) + c.Status = grant == ConnectionGrant.AuthorizationCode ? ConnectionStatus.NeedsAuthorizing : ConnectionStatus.New; + else if (clientChanged) + c.ForgetTokens(); + } + + /// + /// https, or http to this machine itself: a provider played locally, to try a kit, has no + /// certificate, and nothing sent to the loopback address leaves the machine. + /// + static bool IsHttps(string url) => + Uri.TryCreate(url?.Trim(), UriKind.Absolute, out var uri) && + (uri.Scheme == Uri.UriSchemeHttps || (uri.Scheme == Uri.UriSchemeHttp && uri.IsLoopback)); + + static bool Same(string a, string b) => string.Equals(Trimmed(a), Trimmed(b), StringComparison.Ordinal); + + static string Trimmed(string value) => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + + /// Where a connection is referred to, by name, as a sentence; empty when nowhere. + public static IEnumerable UsesOf(string name, BitweenDbContext db) + { + var lowered = name.ToLowerInvariant(); + bool Uses(IEnumerable> bags) => + bags.Any(b => b != null && Services.Connections.ConnectionTokens.NamesIn(b.Values).Contains(lowered)); + + foreach (var s in db.Set().AsEnumerable()) + if (Uses([s.HandlerProperties, s.MapperProperties, s.ReceiverProperties, s.ValidatorProperties])) + yield return $"subscription '{s.Name}'"; + foreach (var c in db.Set().AsEnumerable()) + if (Uses([c.HandlerProperties])) yield return $"notification channel '{c.Name}'"; + foreach (var n in db.Set().AsEnumerable()) + if (Uses([n.HandlerProperties])) yield return $"notifier '{n.Name}'"; + foreach (var p in db.Set().AsEnumerable()) + if (Uses([p.AlertHandlerProperties])) yield return $"retry policy '{p.Name}'"; + foreach (var p in db.Set().AsEnumerable()) + if (Uses([p.AdapterProperties])) yield return $"partner '{p.Name}'"; + foreach (var g in db.Set().AsEnumerable()) + if (Uses([g.Values])) yield return $"global values '{g.Id}'"; + } +} diff --git a/SW.Bitween.Api/Resources/Connections/Create.cs b/SW.Bitween.Api/Resources/Connections/Create.cs new file mode 100644 index 00000000..ab9cdd37 --- /dev/null +++ b/SW.Bitween.Api/Resources/Connections/Create.cs @@ -0,0 +1,23 @@ +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain.Connections; +using SW.Bitween.Model; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Connections; + +public class Create(BitweenDbContext dbContext, RequestContext requestContext) : ICommandHandler +{ + public async Task Handle(ConnectionSave request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.GlobalValues.Create); + var connection = new Connection(); + ConnectionRows.Apply(connection, request); + var lowered = connection.Name.ToLowerInvariant(); + if (await dbContext.Set().AnyAsync(c => c.Name.ToLower() == lowered)) + throw new SWValidationException("CONNECTION_NAME_TAKEN", $"There is already a connection named '{connection.Name}'."); + dbContext.Add(connection); + await dbContext.SaveChangesAsync(); + return ConnectionRows.Row(connection); + } +} diff --git a/SW.Bitween.Api/Resources/Connections/Delete.cs b/SW.Bitween.Api/Resources/Connections/Delete.cs new file mode 100644 index 00000000..835e8c6c --- /dev/null +++ b/SW.Bitween.Api/Resources/Connections/Delete.cs @@ -0,0 +1,24 @@ +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain.Connections; +using SW.Bitween.Model; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Connections; + +public class Delete(BitweenDbContext dbContext, RequestContext requestContext) : IDeleteHandler +{ + public async Task Handle(int key) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.GlobalValues.Delete); + var connection = await dbContext.Set().SingleOrDefaultAsync(c => c.Id == key) + ?? throw new SWNotFoundException(key.ToString()); + var uses = ConnectionRows.UsesOf(connection.Name, dbContext).ToList(); + if (uses.Count > 0) + throw new SWValidationException("CONNECTION_IN_USE", $"'{connection.Name}' is still used by {string.Join(", ", uses)}."); + dbContext.Remove(connection); + await dbContext.SaveChangesAsync(); + return null; + } +} diff --git a/SW.Bitween.Api/Resources/Connections/Get.cs b/SW.Bitween.Api/Resources/Connections/Get.cs new file mode 100644 index 00000000..67107d96 --- /dev/null +++ b/SW.Bitween.Api/Resources/Connections/Get.cs @@ -0,0 +1,17 @@ +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain.Connections; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Connections; + +public class Get(BitweenDbContext dbContext, RequestContext requestContext) : IGetHandler +{ + public async Task Handle(int key) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.GlobalValues.View); + var connection = await dbContext.Set().AsNoTracking().SingleOrDefaultAsync(c => c.Id == key) + ?? throw new SWNotFoundException(key.ToString()); + return ConnectionRows.Row(connection); + } +} diff --git a/SW.Bitween.Api/Resources/Connections/Search.cs b/SW.Bitween.Api/Resources/Connections/Search.cs new file mode 100644 index 00000000..ba1d1cbf --- /dev/null +++ b/SW.Bitween.Api/Resources/Connections/Search.cs @@ -0,0 +1,23 @@ +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain.Connections; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Connections; + +/// Every connection, as the Connections page and the reference picker list them. +public class Search(BitweenDbContext dbContext, RequestContext requestContext) : ISearchyHandler +{ + public async Task Handle(SearchyRequest searchyRequest, bool lookup = false, string searchPhrase = null) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.GlobalValues.View); + var connections = await dbContext.Set().AsNoTracking().OrderBy(c => c.Name).ToListAsync(); + if (lookup) return connections.ToDictionary(c => c.Id, c => c.Name); + return new SearchyResponse + { + Result = connections.Select(ConnectionRows.Row).ToList(), + TotalCount = connections.Count, + }; + } +} diff --git a/SW.Bitween.Api/Resources/Connections/Test.cs b/SW.Bitween.Api/Resources/Connections/Test.cs new file mode 100644 index 00000000..bd772491 --- /dev/null +++ b/SW.Bitween.Api/Resources/Connections/Test.cs @@ -0,0 +1,18 @@ +using System.Threading.Tasks; +using SW.Bitween.Model; +using SW.Bitween.Services.Connections; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Connections; + +/// Renews the token now and shows how it went: Connected, or the provider's reason. +[HandlerName("test")] +public class Test(BitweenDbContext dbContext, RequestContext requestContext, ConnectionTokens tokens) + : ICommandHandler +{ + public async Task Handle(int key, ConnectionTest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.GlobalValues.Edit); + return ConnectionRows.Row(await tokens.TestAsync(key)); + } +} diff --git a/SW.Bitween.Api/Resources/Connections/Update.cs b/SW.Bitween.Api/Resources/Connections/Update.cs new file mode 100644 index 00000000..b851d7c3 --- /dev/null +++ b/SW.Bitween.Api/Resources/Connections/Update.cs @@ -0,0 +1,33 @@ +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain.Connections; +using SW.Bitween.Model; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Connections; + +public class Update(BitweenDbContext dbContext, RequestContext requestContext) : ICommandHandler +{ + public async Task Handle(int key, ConnectionSave request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.GlobalValues.Edit); + var connection = await dbContext.Set().SingleOrDefaultAsync(c => c.Id == key) + ?? throw new SWNotFoundException(key.ToString()); + var before = connection.Name; + ConnectionRows.Apply(connection, request); + + if (!string.Equals(before, connection.Name, System.StringComparison.OrdinalIgnoreCase)) + { + var lowered = connection.Name.ToLowerInvariant(); + if (await dbContext.Set().AnyAsync(c => c.Id != key && c.Name.ToLower() == lowered)) + throw new SWValidationException("CONNECTION_NAME_TAKEN", $"There is already a connection named '{connection.Name}'."); + // Settings refer to it by name, so a rename would quietly break every one of them. + var uses = string.Join(", ", ConnectionRows.UsesOf(before, dbContext)); + if (uses.Length > 0) + throw new SWValidationException("CONNECTION_IN_USE", $"'{before}' can't be renamed while settings refer to it: {uses}."); + } + + await dbContext.SaveChangesAsync(); + return ConnectionRows.Row(connection); + } +} diff --git a/SW.Bitween.Api/Resources/Dashboard/RetrySummary.cs b/SW.Bitween.Api/Resources/Dashboard/RetrySummary.cs index 62ba0cbd..d59ad8b9 100644 --- a/SW.Bitween.Api/Resources/Dashboard/RetrySummary.cs +++ b/SW.Bitween.Api/Resources/Dashboard/RetrySummary.cs @@ -14,7 +14,7 @@ namespace SW.Bitween.Resources.Dashboard; /// anything at all. /// /// -/// Both in one handler because both belong to the same panel — a repeat offender means little +/// Both in one handler because both belong to the same panel - a repeat offender means little /// without knowing whether retries generally work here. Nine attempts against the same connection /// error is a configuration problem someone has to fix; nine attempts where most retries do /// eventually succeed is bad luck. @@ -65,8 +65,8 @@ group result by result.Success && !result.ResponseBad into worked var succeeded = retries.Where(r => r.Worked).Sum(r => r.Count); var finished = retries.Sum(r => r.Count); - // Chains that keep failing: a failure nothing has been retried from — so it is where that - // chain has got to — which is itself a retry, so the chain is at least two attempts long. + // Chains that keep failing: a failure nothing has been retried from - so it is where that + // chain has got to - which is itself a retry, so the chain is at least two attempts long. var candidates = await ( from xchange in _dbContext.Set() join result in _dbContext.Set() on xchange.Id equals result.Id @@ -125,7 +125,7 @@ orderby xchange.StartedOn descending /// How many attempts each chain has taken, counting the original as one. /// /// - /// A level of every chain per query rather than a walk per chain — five hundred chains four + /// A level of every chain per query rather than a walk per chain - five hundred chains four /// attempts deep is four queries, not two thousand. XchangeResult.AttemptNumber holds /// this already, but only for failures a retry policy's group matched, so a chain retried by /// hand has none and it cannot be read from there. diff --git a/SW.Bitween.Api/Resources/DataSourceStatements/Create.cs b/SW.Bitween.Api/Resources/DataSourceStatements/Create.cs index 06ac374c..bcb3ca55 100644 --- a/SW.Bitween.Api/Resources/DataSourceStatements/Create.cs +++ b/SW.Bitween.Api/Resources/DataSourceStatements/Create.cs @@ -34,7 +34,7 @@ await requestContext.EnsurePermission(dbContext, throw new SWNotFoundException($"DataSource with id '{dataSourceId}' was not found"); // Only a relational source runs SQL. Refused rather than stored, because a statement on a - // broker is configuration nothing will ever read — and silently keeping it is how people + // broker is configuration nothing will ever read - and silently keeping it is how people // conclude the feature is broken. if (dataSource.Kind != DataSourceKind.Relational) throw new SWException( @@ -66,7 +66,7 @@ await requestContext.EnsurePermission(dbContext, /// /// Refuses SQL the database itself will not accept, while the person who wrote it is still - /// looking at it. The check prepares the statement — parsed and planned, never run. + /// looking at it. The check prepares the statement - parsed and planned, never run. /// /// Silent when the adapter cannot answer: a connection that is down is a fact about the /// connection, and blocking someone from saving a fix because the thing they are fixing it @@ -85,7 +85,7 @@ internal static async Task EnsureSqlIsValid( /// /// Case-insensitively unique within the data source. The database index enforces uniqueness but - /// its collation is provider-specific, so the case rule is applied here — where it can also say + /// its collation is provider-specific, so the case rule is applied here - where it can also say /// which statement it collided with, rather than surfacing as a constraint violation. /// internal static async Task EnsureNameIsFree(BitweenDbContext dbContext, int dataSourceId, diff --git a/SW.Bitween.Api/Resources/DataSourceStatements/Delete.cs b/SW.Bitween.Api/Resources/DataSourceStatements/Delete.cs index 79d04b63..3f9ea096 100644 --- a/SW.Bitween.Api/Resources/DataSourceStatements/Delete.cs +++ b/SW.Bitween.Api/Resources/DataSourceStatements/Delete.cs @@ -9,7 +9,7 @@ namespace SW.Bitween.Resources.DataSourceStatements; /// /// Refused while anything names it, and the refusal lists what. A statement deleted out from under -/// a live subscription does not fail at delete time — it fails on the next message, as "not a +/// a live subscription does not fail at delete time - it fails on the next message, as "not a /// statement this data source defines", somewhere nobody is watching. /// public class Delete(BitweenDbContext dbContext, RequestContext requestContext, diff --git a/SW.Bitween.Api/Resources/DataSourceStatements/Search.cs b/SW.Bitween.Api/Resources/DataSourceStatements/Search.cs index 36b057d3..8a09bed9 100644 --- a/SW.Bitween.Api/Resources/DataSourceStatements/Search.cs +++ b/SW.Bitween.Api/Resources/DataSourceStatements/Search.cs @@ -15,7 +15,7 @@ namespace SW.Bitween.Resources.DataSourceStatements; /// Every statement, with the count that matters: how many subscriptions name it. /// /// Zero is the whole point. Without it, SQL nobody uses accumulates forever because nobody can -/// prove it is safe to remove — which is exactly what happened while these lived in one JSON blob. +/// prove it is safe to remove - which is exactly what happened while these lived in one JSON blob. /// public class Search(BitweenDbContext dbContext, RequestContext requestContext, StatementUsageReader usage) : ISearchyHandler diff --git a/SW.Bitween.Api/Resources/DataSourceStatements/Update.cs b/SW.Bitween.Api/Resources/DataSourceStatements/Update.cs index 403da43b..9fff3292 100644 --- a/SW.Bitween.Api/Resources/DataSourceStatements/Update.cs +++ b/SW.Bitween.Api/Resources/DataSourceStatements/Update.cs @@ -23,7 +23,7 @@ public async Task Handle(int key, DataSourceStatementUpdate model) await Create.EnsureNameIsFree(dbContext, entity.DataSourceId, model.Name, existingId: key); // A rename breaks every subscription naming the old one, and there is no way to fix that - // from here — the subscription's properties are its own. So it is refused while anything + // from here - the subscription's properties are its own. So it is refused while anything // still points at it, and the message says what to change first. var renaming = !string.Equals(entity.Name, model.Name?.Trim(), System.StringComparison.OrdinalIgnoreCase); @@ -41,7 +41,7 @@ public async Task Handle(int key, DataSourceStatementUpdate model) } // Only when it actually changed: re-checking untouched SQL would refuse a rename, or a - // change of owner, because of a table someone dropped last week — a fault worth surfacing + // change of owner, because of a table someone dropped last week - a fault worth surfacing // but not here, and not as a block on an unrelated edit. // True when the SQL did not change: there was nothing to check, which is not the same as // "could not check" and should not warn as though it were. diff --git a/SW.Bitween.Api/Resources/DataSourceStatements/Usage.cs b/SW.Bitween.Api/Resources/DataSourceStatements/Usage.cs index ec1af7fd..1e24ebc1 100644 --- a/SW.Bitween.Api/Resources/DataSourceStatements/Usage.cs +++ b/SW.Bitween.Api/Resources/DataSourceStatements/Usage.cs @@ -9,7 +9,7 @@ namespace SW.Bitween.Resources.DataSourceStatements; /// -/// Not just how many, but WHICH — with the slot and the operation, so "can I change this?" is +/// Not just how many, but WHICH - with the slot and the operation, so "can I change this?" is /// answerable without opening every subscription bound to the connection. /// [HandlerName("usage")] diff --git a/SW.Bitween.Api/Resources/DataSources/Create.cs b/SW.Bitween.Api/Resources/DataSources/Create.cs index 13d7acfb..dbe00995 100644 --- a/SW.Bitween.Api/Resources/DataSources/Create.cs +++ b/SW.Bitween.Api/Resources/DataSources/Create.cs @@ -23,8 +23,8 @@ public async Task Handle(DataSourceCreate model) if (nameTaken) throw new SWException($"A data source named '{model.Name}' already exists."); - // Nothing is stored behind a sentinel on a create, so any that arrives — from a data source - // copied out of Get, say — is dropped rather than saved as the literal string. + // Nothing is stored behind a sentinel on a create, so any that arrives - from a data source + // copied out of Get, say - is dropped rather than saved as the literal string. var properties = Secrets.Merge(null, model.Properties); var entity = new DataSource @@ -50,8 +50,8 @@ public async Task Handle(DataSourceCreate model) /// /// Enforced here rather than only in the validator, because the validator runs in the HTTP - /// pipeline and nothing else does: a caller reaching the handler another way — the supervisor's - /// own tests, a future internal caller — would otherwise save a combination that can never + /// pipeline and nothing else does: a caller reaching the handler another way - the supervisor's + /// own tests, a future internal caller - would otherwise save a combination that can never /// work. A soft ceiling above the hard one is unreachable: the runtime fails the allocation /// first, so the graceful recycle it was configured for never happens. /// @@ -64,7 +64,7 @@ internal static void EnsureCeilingsAreUsable(int softMb, int hardMb, if (cpuPercent < 0 || cpuSamples < 0) throw new SWException("A CPU ceiling cannot be negative. Use 0 for the host default."); - // Above 100 is not a ceiling, because the figure is a share of the whole node — nothing can + // Above 100 is not a ceiling, because the figure is a share of the whole node - nothing can // ever exceed it, so the limit would silently never fire. if (cpuPercent > 100) throw new SWException( @@ -99,7 +99,7 @@ public Validate() RuleFor(i => i.Name).NotEmpty().MaximumLength(200); RuleFor(i => i.AdapterId).NotEmpty().MaximumLength(200); - // Zero is meaningful — it turns deduplication off — so only a negative is rejected. + // Zero is meaningful - it turns deduplication off - so only a negative is rejected. RuleFor(i => i.DeduplicationWindowDays).GreaterThanOrEqualTo(0); // Zero means "leave the host default alone" for both. diff --git a/SW.Bitween.Api/Resources/DataSources/Delete.cs b/SW.Bitween.Api/Resources/DataSources/Delete.cs index 03cb3022..3bffa6c6 100644 --- a/SW.Bitween.Api/Resources/DataSources/Delete.cs +++ b/SW.Bitween.Api/Resources/DataSources/Delete.cs @@ -15,7 +15,7 @@ public async Task Handle(int key) { await requestContext.EnsurePermission(dbContext, Model.Permissions.DataSources.Delete); - // The database refuses this anyway — the foreign key restricts — but a raw constraint + // The database refuses this anyway - the foreign key restricts - but a raw constraint // violation tells an operator nothing about which gateway is in the way. var gateways = await dbContext.Set() .Where(gateway => gateway.DataSourceId == key) @@ -29,7 +29,7 @@ public async Task Handle(int key) string.Join(", ", gateways) + ". Point them at the internal bus, or delete them, first."); - // Subscription.DataSourceId restricts too, and was the one left to the database — the + // Subscription.DataSourceId restricts too, and was the one left to the database - the // commoner case of the two, since a data source is normally read by an integration long // before any gateway feeds off it. var subscriptions = await dbContext.Set() diff --git a/SW.Bitween.Api/Resources/DataSources/Inspect.cs b/SW.Bitween.Api/Resources/DataSources/Inspect.cs index d46b4956..ce9ce174 100644 --- a/SW.Bitween.Api/Resources/DataSources/Inspect.cs +++ b/SW.Bitween.Api/Resources/DataSources/Inspect.cs @@ -11,13 +11,13 @@ namespace SW.Bitween.Resources.DataSources; /// -/// Asks the RUNNING adapter what it can see right now — the broker's topology, or the +/// Asks the RUNNING adapter what it can see right now - the broker's topology, or the /// database's catalog. /// /// Deliberately not the same shape as . A test starts a throwaway instance, /// because its whole job is to answer "would these settings work" before anything depends on them. /// Discover and GetStats are questions about the connection that is actually serving traffic, so -/// they go to that instance — starting a second one would answer about a connection nobody is +/// they go to that instance - starting a second one would answer about a connection nobody is /// using, and on a broker that authorises per-connection it might not even answer the same way. /// /// Both are read-only: neither consumes, acknowledges or publishes anything. @@ -33,7 +33,7 @@ public class Inspect(BitweenDbContext dbContext, RequestContext requestContext, /// /// A database adapter widens this: Describe says what the engine and this login can do, and /// Discover walks the catalog. Both are read-only. Query, Execute, Call, Batch and BulkLoad are - /// deliberately absent and must stay absent — a View-level read must not become a way to run + /// deliberately absent and must stay absent - a View-level read must not become a way to run /// SQL against a customer's database by naming it in a request body. private static readonly string[] Allowed = ["Discover", "GetStats", "Describe"]; @@ -65,10 +65,10 @@ public async Task Handle(int key, DataSourceInspectRequest request) // opposite reactions. An exclusive broker connection is held by one node, so this // is the normal answer everywhere else and nothing is wrong. A pooled database // connection is held by every node that runs work, so "not here" means it is not - // running at all — which is a fault worth chasing. + // running at all - which is a fault worth chasing. Error = kind == DataSourceKind.Relational ? "This node is not running the adapter for this data source. A connection " - + "pool is held by every node, so this means it has not started — check the " + + "pool is held by every node, so this means it has not started - check the " + "live connection panel for why." : "This node is not running the adapter for this data source, so it has " + "nothing to ask. A broker connection is exclusive: only the node holding " diff --git a/SW.Bitween.Api/Resources/DataSources/Search.cs b/SW.Bitween.Api/Resources/DataSources/Search.cs index d9ed2365..d01f8b69 100644 --- a/SW.Bitween.Api/Resources/DataSources/Search.cs +++ b/SW.Bitween.Api/Resources/DataSources/Search.cs @@ -50,7 +50,7 @@ public async Task Handle(SearchyRequest searchyRequest, bool lookup = fa query = query.AsNoTracking(); // Properties are deliberately absent from the list: it is a table of connections, and - // sending every credential — even masked — to render a row is not worth the exposure. + // sending every credential - even masked - to render a row is not worth the exposure. if (lookup) return await query.Search(searchyRequest.Conditions) diff --git a/SW.Bitween.Api/Resources/DataSources/Secrets.cs b/SW.Bitween.Api/Resources/DataSources/Secrets.cs index c7208fa1..0e2a2b61 100644 --- a/SW.Bitween.Api/Resources/DataSources/Secrets.cs +++ b/SW.Bitween.Api/Resources/DataSources/Secrets.cs @@ -9,8 +9,8 @@ namespace SW.Bitween.Resources.DataSources; /// /// This does NOT ask the adapter which of its startup values are marked secure, the way /// does for subscription properties. Describing an adapter -/// means starting it, and a bus provider is resident — started once and held for the life of the -/// node — so asking would either start a second copy of a connection that is meant to be exclusive +/// means starting it, and a bus provider is resident - started once and held for the life of the +/// node - so asking would either start a second copy of a connection that is meant to be exclusive /// or block on one that is already running. The data source names its own secrets instead, which /// also lets an operator protect a field the adapter author never thought to mark. /// @@ -35,7 +35,7 @@ public static bool IsSecret(string name, IEnumerable declared) => /// /// A copy with every secret replaced. An empty value is left alone, so "not set" stays - /// distinguishable from "set but hidden" — the difference between a form that is finished and + /// distinguishable from "set but hidden" - the difference between a form that is finished and /// one that is not. /// public static Dictionary Mask( @@ -52,7 +52,7 @@ public static Dictionary Mask( /// /// Resolves sentinels against what is stored. A sentinel with nothing behind it is dropped - /// rather than saved literally — otherwise a data source copied from a response would + /// rather than saved literally - otherwise a data source copied from a response would /// authenticate with the string "__private__". /// public static Dictionary Merge( @@ -73,7 +73,7 @@ public static Dictionary Merge( /// /// The secret names to record. Whatever the caller declared, plus anything matching a - /// well-known credential name — so the list stays true even when the form did not tick it. + /// well-known credential name - so the list stays true even when the form did not tick it. /// public static List Declare( IReadOnlyDictionary properties, IEnumerable declared) diff --git a/SW.Bitween.Api/Resources/DataSources/Telemetry.cs b/SW.Bitween.Api/Resources/DataSources/Telemetry.cs index fbc71434..b8697e69 100644 --- a/SW.Bitween.Api/Resources/DataSources/Telemetry.cs +++ b/SW.Bitween.Api/Resources/DataSources/Telemetry.cs @@ -14,7 +14,7 @@ namespace SW.Bitween.Resources.DataSources; /// /// The data source row carries a summary written back on the reconcile loop, which is up to thirty /// seconds old and deliberately small. This reads the heartbeat directly, so an operator watching a -/// queue drain sees it drain — and gets the two things the row has never carried: the per-queue +/// queue drain sees it drain - and gets the two things the row has never carried: the per-queue /// depths the adapter can see, and the host-observed process figures that keep working when the /// adapter is wedged and reporting nothing at all. /// @@ -47,8 +47,8 @@ public async Task Handle(int key) LastHeartbeatOn = dataSource.LastHeartbeatOn, }; - // Registered only when BusProvidersEnabled, so a node with the feature off answers - // "not here" rather than failing to resolve a service. + // A node with BusProvidersEnabled off runs no data source, so it finds no instance and + // answers "not here". var instance = adapters?.Describe() .FirstOrDefault(h => h.InstanceKey == key.ToString()); diff --git a/SW.Bitween.Api/Resources/DataSources/Test.cs b/SW.Bitween.Api/Resources/DataSources/Test.cs index 4f303918..f7835f33 100644 --- a/SW.Bitween.Api/Resources/DataSources/Test.cs +++ b/SW.Bitween.Api/Resources/DataSources/Test.cs @@ -27,20 +27,20 @@ namespace SW.Bitween.Resources.DataSources; /// never drained, and it is stopped again before this returns. /// [HandlerName("test")] -public class Test(BitweenDbContext dbContext, RequestContext requestContext, +public class Test(BitweenDbContext dbContext, RequestContext requestContext, BitweenOptions bitweenOptions, IResidentAdapterHost adapters = null) : ICommandHandler { public async Task Handle(int key, DataSourceTestRequest request) { await requestContext.EnsurePermission(dbContext, Model.Permissions.DataSources.Operate); - // Registered only when BusProvidersEnabled, so say which switch is off rather than - // failing to resolve a service the operator has never heard of. The switch is named for - // brokers because it predates database sources, so the message says what it now gates - // rather than repeating a name that means nothing to someone configuring PostgreSQL. - if (adapters == null) + // The host is registered on every node - Python and Node adapters run on it - but data + // sources connect only where BusProvidersEnabled turns them on, so say which switch is off. + // It is named for brokers because it predates database sources, so the message says what + // it now gates rather than repeating a name that means nothing to someone configuring PostgreSQL. + if (adapters == null || !bitweenOptions.BusProvidersEnabled) return Failed("Resident data source providers are turned off on this node, so nothing " - + "can connect from here. Turn on Bitween:BusProvidersEnabled — the " + + "can connect from here. Turn on Bitween:BusProvidersEnabled - the " + "switch is older than database sources and still carries the bus name."); var dataSource = await dbContext.Set().AsNoTracking() @@ -65,7 +65,7 @@ public async Task Handle(int key, DataSourceTestRequest request) startupValues["Consume"] = "false"; // Composed here as well as in the supervisor, because statements are rows rather than a - // property on the data source — and without this the test quietly stopped covering them. + // property on the data source - and without this the test quietly stopped covering them. // Preparing every statement against the live schema is most of what the button is FOR: it // is where a typo or a dropped column is caught, and a test that silently checks nothing // still reports success. @@ -100,7 +100,7 @@ public async Task Handle(int key, DataSourceTestRequest request) // A broker that cannot be reached throws on start, before any stage runs. That is a // result, not a server error: the operator asked whether it works, and it does not. // - // What Bitween observes in that case is only "Adapter stream closed" — true, and no + // What Bitween observes in that case is only "Adapter stream closed" - true, and no // use at all. The adapter printed the actual reason before it died, so that is what // this answers with; without it, the one control meant to save reading a log is the // control that sends you to read one. diff --git a/SW.Bitween.Api/Resources/DataSources/Update.cs b/SW.Bitween.Api/Resources/DataSources/Update.cs index 0c38bcea..7dac3d40 100644 --- a/SW.Bitween.Api/Resources/DataSources/Update.cs +++ b/SW.Bitween.Api/Resources/DataSources/Update.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Threading.Tasks; using FluentValidation; using Microsoft.EntityFrameworkCore; @@ -28,12 +29,17 @@ public async Task Handle(int key, DataSourceUpdate model) // Secrets came out of Get masked, so put them back from what is stored. A form that only // changed the prefetch must not overwrite the password with a row of dots. - var properties = Secrets.Merge(entity.Properties, model.Properties); + var keep = AdapterSecretProperties.MayKeepStoredSecrets(entity.AdapterId, model.AdapterId, + entity.Properties, model.Properties ?? new Dictionary()); + var properties = Secrets.Merge(keep ? entity.Properties : null, model.Properties); entity.Name = model.Name; entity.AdapterId = model.AdapterId; entity.Kind = Create.ParseKind(model.Kind); - entity.Placement = Create.ParsePlacement(model.Placement); + // Left out means unchanged. It used to mean Auto, so every client that didn't send it, + // the admin UI included, put an Exclusive or PerNode source back to Auto on any save. + if (!string.IsNullOrWhiteSpace(model.Placement)) + entity.Placement = Create.ParsePlacement(model.Placement); entity.Properties = properties; entity.SecretProperties = Secrets.Declare(properties, model.SecretProperties); entity.Inactive = model.Inactive; diff --git a/SW.Bitween.Api/Resources/DelayedRetries/RunNow.cs b/SW.Bitween.Api/Resources/DelayedRetries/RunNow.cs index 4bb54d94..2e3d325a 100644 --- a/SW.Bitween.Api/Resources/DelayedRetries/RunNow.cs +++ b/SW.Bitween.Api/Resources/DelayedRetries/RunNow.cs @@ -12,7 +12,7 @@ public class RunNow(BitweenDbContext dbContext, RequestContext requestContext, { public async Task Handle(string key, DelayedRetryRunNow request) { - // What's being operated on is an exchange, not the policy that scheduled the retry — + // What's being operated on is an exchange, not the policy that scheduled the retry - // and this is the same page the UI gates on exchange permissions. await requestContext.EnsurePermission(dbContext, Model.Permissions.Exchanges.Operate); @@ -34,6 +34,7 @@ public async Task Handle(string key, DelayedRetryRunNow request) : "This retry could not be carried out: the exchange it belonged to no longer exists."); } + dbContext.Record(requestContext, OperatorAction.RunRetryNow, key); await dbContext.SaveChangesAsync(); return null; } diff --git a/SW.Bitween.Api/Resources/DelayedRetries/Search.cs b/SW.Bitween.Api/Resources/DelayedRetries/Search.cs index 80996549..53c79d7f 100644 --- a/SW.Bitween.Api/Resources/DelayedRetries/Search.cs +++ b/SW.Bitween.Api/Resources/DelayedRetries/Search.cs @@ -21,7 +21,7 @@ public async Task Handle(SearchyRequest searchyRequest, bool lookup = fa join xchange in dbContext.Set() on delayedRetry.Id equals xchange.Id join result in dbContext.Set() on xchange.Id equals result.Id into xr from result in xr.DefaultIfEmpty() - // Same left-join Xchanges/Search.cs uses — the UI lists a pending retry by + // Same left-join Xchanges/Search.cs uses - the UI lists a pending retry by // what it carries, not by its id, so the properties have to come along. join promoted in dbContext.Set() on xchange.Id equals promoted.Id into xp from promoted in xp.DefaultIfEmpty() @@ -32,6 +32,7 @@ from subscriber in xs.DefaultIfEmpty() { Id = delayedRetry.Id, On = delayedRetry.On, + Held = delayedRetry.On >= DelayedRetry.HeldOn, SubscriptionId = xchange.SubscriptionId, SubscriptionName = subscriber.Name, DocumentId = xchange.DocumentId, diff --git a/SW.Bitween.Api/Resources/Documents/BusMessageTypeNames.cs b/SW.Bitween.Api/Resources/Documents/BusMessageTypeNames.cs index ee7c020c..2b06f91e 100644 --- a/SW.Bitween.Api/Resources/Documents/BusMessageTypeNames.cs +++ b/SW.Bitween.Api/Resources/Documents/BusMessageTypeNames.cs @@ -15,7 +15,7 @@ public static class BusMessageTypeNames /// Refuses a name another type holds. Compared lower-cased, because that is how the bus /// compares them: both BasicPublisher and ConsumerDefinition derive the routing key with /// ToLower(), so "Foo" and "foo" are one message on the wire. ToLower() rather than a - /// provider-specific collation — this runs on Postgres, MySql and MsSql. + /// provider-specific collation - this runs on Postgres, MySql and MsSql. /// /// A type paused on the bus holds its name too: its queue is still collecting messages for /// it, and a second type on the same name would consume them. Said as such, because diff --git a/SW.Bitween.Api/Resources/Documents/Create.cs b/SW.Bitween.Api/Resources/Documents/Create.cs index 29e470ed..8a8bcd9c 100644 --- a/SW.Bitween.Api/Resources/Documents/Create.cs +++ b/SW.Bitween.Api/Resources/Documents/Create.cs @@ -20,7 +20,7 @@ public async Task Handle(DocumentCreate model) await requestContext.EnsurePermission(dbContext, Model.Permissions.Documents.Create); // Same check Update makes, and compared the same way. Without it two types - // could be created under one name, and then neither could be saved again — + // could be created under one name, and then neither could be saved again - // Update refuses the name it already has. Ignoring case, because a list // holding both "Invoice" and "invoice" reads as a mistake, not a choice. var wantedName = (model.Name ?? string.Empty).ToLower(); @@ -38,17 +38,20 @@ public async Task Handle(DocumentCreate model) await BusMessageTypeNames.EnsureFree(dbContext, model.BusMessageTypeName); PromotedPropertyValidation.Check(model.PromotedProperties, model.DocumentFormat); + await DocumentSchema.EnsureUsable(model.DocumentFormat, model.ValidationSchema); var entity = new Document(code, model.Name, model.DocumentFormat) { BusEnabled = model.BusEnabled, BusMessageTypeName = model.BusEnabled ? model.BusMessageTypeName : null, DuplicateInterval = model.DuplicateInterval, + DropDuplicates = model.DropDuplicates, DisregardsUnfilteredMessages = model.DisregardsUnfilteredMessages, }; if (model.PromotedProperties != null) entity.SetDictionaries(model.PromotedProperties.ToDictionary()); + entity.SetValidationSchema(model.ValidationSchema); dbContext.Add(entity); await dbContext.SaveChangesAsync(); // Routing resolves an information type by name off the cache, so a new one is diff --git a/SW.Bitween.Api/Resources/Documents/Delete.cs b/SW.Bitween.Api/Resources/Documents/Delete.cs index 0154a41c..037fc9c9 100644 --- a/SW.Bitween.Api/Resources/Documents/Delete.cs +++ b/SW.Bitween.Api/Resources/Documents/Delete.cs @@ -21,11 +21,11 @@ async public Task Handle(int key) await requestContext.EnsurePermission(dbContext, Model.Permissions.Documents.Delete); // Seeded by the model and referenced by every aggregation, so it is not the user's - // to remove — the same footing as the system partner. + // to remove - the same footing as the system partner. if (key == Document.AggregationDocumentId) throw new SWException("The built-in Aggregation Document can't be deleted."); - // Configuration pointing at this type is a real block — a subscription or a bus + // Configuration pointing at this type is a real block - a subscription or a bus // gateway left behind would name a type that no longer exists. Said plainly, because // the database says it as a foreign-key violation, which reached the screen as a bare // 500 naming a constraint. @@ -42,13 +42,13 @@ async public Task Handle(int key) // Exchanges are the deliberate exception. They are this type's history rather than // configuration depending on it, and history should not be able to strand a type - // nobody uses any more — so they go with it. + // nobody uses any more - so they go with it. await using var transaction = await dbContext.Database.BeginTransactionAsync(); var xchangeIds = dbContext.Set().Where(x => x.DocumentId == key).Select(x => x.Id); // Scheduled retries are keyed by exchange id but have no relationship configured, so - // nothing clears them on their own — they would be left pointing at exchanges that no + // nothing clears them on their own - they would be left pointing at exchanges that no // longer exist. Results, aggregations and promoted properties all cascade from the // exchange itself and need no help here. await dbContext.Set().Where(d => xchangeIds.Contains(d.Id)).ExecuteDeleteAsync(); diff --git a/SW.Bitween.Api/Resources/Documents/Get.cs b/SW.Bitween.Api/Resources/Documents/Get.cs index be295311..e7010a42 100644 --- a/SW.Bitween.Api/Resources/Documents/Get.cs +++ b/SW.Bitween.Api/Resources/Documents/Get.cs @@ -21,9 +21,9 @@ public async Task Handle(int key) await requestContext.EnsurePermission(dbContext, Model.Permissions.Documents.View); // DocumentRow rather than DocumentUpdate: it is the read shape, and RetiredOn belongs - // on it rather than on the model used to write a type — retiring is its own command, + // on it rather than on the model used to write a type - retiring is its own command, // not a field an update can set. - return await dbContext.Set().Search("Id", key).Select(document => new DocumentRow + var row = await dbContext.Set().Search("Id", key).Select(document => new DocumentRow { Id = document.Id, Code = document.Code, @@ -31,13 +31,19 @@ public async Task Handle(int key) BusEnabled = document.BusEnabled, BusMessageTypeName = document.BusMessageTypeName, DuplicateInterval = document.DuplicateInterval, + DropDuplicates = document.DropDuplicates, PromotedProperties = document.PromotedProperties.ToKeyAndValueCollection(), + ValidationSchema = document.ValidationSchema, DocumentFormat = document.DocumentFormat, DisregardsUnfilteredMessages = document.DisregardsUnfilteredMessages ?? false, RetiredOn = document.RetiredOn, UsedByCount = dbContext.Set() .Count(subscription => subscription.DocumentId == document.Id) }).SingleOrDefaultAsync(); + + if (row != null) + (row.DuplicatesLastWeek, row.DuplicatesDroppedLastWeek) = await new DuplicateArrivals(dbContext).CountAsync(key); + return row; } } } \ No newline at end of file diff --git a/SW.Bitween.Api/Resources/Documents/GetProperties.cs b/SW.Bitween.Api/Resources/Documents/GetProperties.cs index 13659dcb..ab9086a9 100644 --- a/SW.Bitween.Api/Resources/Documents/GetProperties.cs +++ b/SW.Bitween.Api/Resources/Documents/GetProperties.cs @@ -18,7 +18,7 @@ async public Task Handle(int key) { await requestContext.EnsurePermission(dbContext, Model.Permissions.Documents.View); - var document = await dbContext.FindAsync(key); + var document = await dbContext.FindAsync(key) ?? throw new SWNotFoundException(key.ToString()); return document.PromotedProperties.ToDictionary(k => k.Key, v => v.Key); } } diff --git a/SW.Bitween.Api/Resources/Documents/PromotedPropertyValidation.cs b/SW.Bitween.Api/Resources/Documents/PromotedPropertyValidation.cs index 1463c74e..c692f138 100644 --- a/SW.Bitween.Api/Resources/Documents/PromotedPropertyValidation.cs +++ b/SW.Bitween.Api/Resources/Documents/PromotedPropertyValidation.cs @@ -44,7 +44,7 @@ public static void Check(ICollection promotedProperties, DocumentFo // // The '$' branch is checked rather than trusted. Treating any leading '$' as // proof of a JSONPath let "$", "$." and "$.." through, none of which select - // anything — they saved cleanly and failed only once a message arrived. A step + // anything - they saved cleanly and failed only once a message arrived. A step // is a '.'/'..' followed by a name, or a bracket; names stay deliberately // permissive (anything but a delimiter) so paths that already work keep working. var valid = trimmed.StartsWith("$") diff --git a/SW.Bitween.Api/Resources/Documents/Retire.cs b/SW.Bitween.Api/Resources/Documents/Retire.cs index 9c0624b0..083b4078 100644 --- a/SW.Bitween.Api/Resources/Documents/Retire.cs +++ b/SW.Bitween.Api/Resources/Documents/Retire.cs @@ -34,7 +34,7 @@ public async Task Handle(int key, DocumentRetire request) if (entity.RetiredOn == null) { // Configuration still pointing at it would be left running against a type the UI - // no longer offers — which reads as the type having been removed when it has not. + // no longer offers - which reads as the type having been removed when it has not. // Said before retiring rather than after, so nothing half-happens. if (await dbContext.Set().AnyAsync(s => s.DocumentId == key)) throw new SWException( @@ -53,7 +53,7 @@ public async Task Handle(int key, DocumentRetire request) await dbContext.SaveChangesAsync(); // The receiving path resolves types through the cache, so a retired type would keep - // being offered — and keep accepting work — for the rest of the cache's window. + // being offered - and keep accepting work - for the rest of the cache's window. await cache.BroadcastRevoke(); return new { entity.Id, entity.RetiredOn }; diff --git a/SW.Bitween.Api/Resources/Documents/Search.cs b/SW.Bitween.Api/Resources/Documents/Search.cs index 335e9018..3ed97b6b 100644 --- a/SW.Bitween.Api/Resources/Documents/Search.cs +++ b/SW.Bitween.Api/Resources/Documents/Search.cs @@ -33,7 +33,9 @@ async public Task Handle(SearchyRequest searchyRequest, bool lookup = fa BusMessageTypeName = document.BusMessageTypeName, BusEnabled = document.BusEnabled, DuplicateInterval = document.DuplicateInterval, + DropDuplicates = document.DropDuplicates, PromotedProperties = document.PromotedProperties.ToKeyAndValueCollection(), + ValidationSchema = document.ValidationSchema, DocumentFormat = document.DocumentFormat, RetiredOn = document.RetiredOn, // A correlated count, so the "used by" column the UI shows costs one diff --git a/SW.Bitween.Api/Resources/Documents/Update.cs b/SW.Bitween.Api/Resources/Documents/Update.cs index fd63cdb5..55dba241 100644 --- a/SW.Bitween.Api/Resources/Documents/Update.cs +++ b/SW.Bitween.Api/Resources/Documents/Update.cs @@ -55,24 +55,29 @@ public async Task Handle(int key, DocumentUpdate model) await BusMessageTypeNames.EnsureFree(dbContext, model.BusMessageTypeName, key); PromotedPropertyValidation.Check(model.PromotedProperties, model.DocumentFormat); + // Checked against the format being saved, so switching a type from JSON to XML can't + // leave a JSON Schema on it. + var schema = model.ValidationSchema ?? entity.ValidationSchema; + await DocumentSchema.EnsureUsable(model.DocumentFormat, schema); var oldBusMessageTypeName = entity.BusMessageTypeName; // An absent list means none, the same as it does for retry policy groups. - // Left implicit it threw ArgumentNullException — a 500 for a request the + // Left implicit it threw ArgumentNullException - a 500 for a request the // API had simply never decided the meaning of. entity.SetDictionaries((model.PromotedProperties ?? []).ToDictionary()); - // Name/Code have private setters — SetProperties only writes public-setter + // Name/Code have private setters - SetProperties only writes public-setter // properties, so it silently no-ops on these two (verified empirically). entity.SetName(model.Name); entity.SetCode(code); // The route key is what identifies the type; the body carries an Id too, and // SetProperties copies it straight onto the tracked entity. A caller that omits it - // sends 0, which EF rejects as an attempt to change a primary key — a 500 for a + // sends 0, which EF rejects as an attempt to change a primary key - a 500 for a // request that was perfectly well formed. Normalising it here makes the copy a no-op // whatever the body said. model.Id = key; dbContext.Entry(entity).SetProperties(model); + entity.SetValidationSchema(schema); await dbContext.SaveChangesAsync(); await BitweenCache.BroadcastRevoke(); diff --git a/SW.Bitween.Api/Resources/GlobalAdapterValuesSets/Delete.cs b/SW.Bitween.Api/Resources/GlobalAdapterValuesSets/Delete.cs index cabbbb81..cae8de7c 100644 --- a/SW.Bitween.Api/Resources/GlobalAdapterValuesSets/Delete.cs +++ b/SW.Bitween.Api/Resources/GlobalAdapterValuesSets/Delete.cs @@ -1,4 +1,8 @@ +using System; +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; using SW.Bitween.Domain; using SW.Bitween.Model; using SW.PrimitiveTypes; @@ -17,10 +21,52 @@ public async Task Handle(string key, DeleteGlobalAdapterValuesSetModel _ if (entity is null) throw new SWValidationException("NOT_FOUND", $"GlobalAdapterValuesSet with id {key} was not found"); + await EnsureNothingReferencesIt(key); + dbContext.Remove(entity); await dbContext.SaveChangesAsync(); await cache.BroadcastRevoke(); return null; } + + /// Refuses while a subscription or a notification channel names the set in a token. + /// + /// A token whose set is gone is no error anywhere: it reaches the adapter as written, and the + /// adapter fails at its next run on a password of {{globals.smtp.password}}, with + /// nothing pointing back here. Read in memory because the properties are encrypted at rest, + /// so the database can't search them. + /// + private async Task EnsureNothingReferencesIt(string setId) + { + bool References(params IReadOnlyDictionary[] properties) => + StartupValuesFiller.ReferencesIn(properties.Where(p => p != null).SelectMany(p => p.Values)) + .Globals.Any(g => g.SetId.Equals(setId, StringComparison.OrdinalIgnoreCase)); + + var subscriptions = (await dbContext.Set().AsNoTracking() + .Select(s => new { s.Name, s.HandlerProperties, s.MapperProperties, s.ReceiverProperties, s.ValidatorProperties }) + .ToListAsync()) + .Where(s => References(s.HandlerProperties, s.MapperProperties, s.ReceiverProperties, s.ValidatorProperties)) + .Select(s => s.Name); + + var channels = (await dbContext.Set().AsNoTracking() + .Select(c => new { c.Name, c.HandlerProperties }) + .ToListAsync()) + .Where(c => References(c.HandlerProperties)) + .Select(c => $"the channel {c.Name}"); + + var users = subscriptions.Concat(channels).ToList(); + if (users.Count == 0) return; + + throw new SWValidationException("GLOBAL_VALUES_IN_USE", + $"'{setId}' is still referenced by {Describe(users)}. Change those references first."); + } + + /// A few names and how many more - a set can be used by dozens of subscriptions. + private static string Describe(List names) => names.Count switch + { + 1 => names[0], + <= 3 => $"{string.Join(", ", names.Take(names.Count - 1))} and {names[^1]}", + _ => $"{string.Join(", ", names.Take(3))} and {names.Count - 3} more" + }; } } diff --git a/SW.Bitween.Api/Resources/GlobalAdapterValuesSets/Search.cs b/SW.Bitween.Api/Resources/GlobalAdapterValuesSets/Search.cs index 937d473d..4ab1347d 100644 --- a/SW.Bitween.Api/Resources/GlobalAdapterValuesSets/Search.cs +++ b/SW.Bitween.Api/Resources/GlobalAdapterValuesSets/Search.cs @@ -35,7 +35,7 @@ public async Task Handle(SearchyRequest searchyRequest, bool lookup = fa var result = await query.Search(searchyRequest.Conditions, searchyRequest.Sorts, searchyRequest.PageSize, searchyRequest.PageIndex).ToListAsync(); - // The keys stay — the reference picker offers {{globals.set.key}} from this list, and a + // The keys stay - the reference picker offers {{globals.set.key}} from this list, and a // secret you cannot point at is no use to anyone. Only the values behind them go. foreach (var row in result) row.Values = AdapterSecretProperties.Mask(row.Values, row.SecretProperties); diff --git a/SW.Bitween.Api/Resources/GlobalAdapterValuesSets/Update.cs b/SW.Bitween.Api/Resources/GlobalAdapterValuesSets/Update.cs index 1ae74cee..f87dd985 100644 --- a/SW.Bitween.Api/Resources/GlobalAdapterValuesSets/Update.cs +++ b/SW.Bitween.Api/Resources/GlobalAdapterValuesSets/Update.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using FluentValidation; @@ -21,7 +22,9 @@ public async Task Handle(string key, GlobalAdapterValuesSetUpdate reques entity.Name = request.Name; // A secret the form did not touch comes back as the sentinel; restore it from storage // rather than saving the mask. - entity.Values = AdapterSecretProperties.Merge(entity.Values, request.Values); + entity.Values = AdapterSecretProperties.Merge(entity.Values, request.Values, + AdapterSecretProperties.MayKeepStoredSecrets(null, null, entity.Values, + request.Values ?? new Dictionary())); entity.SecretProperties = request.SecretProperties?.ToList() ?? []; await dbContext.SaveChangesAsync(); diff --git a/SW.Bitween.Api/Resources/Mappers/Preview.cs b/SW.Bitween.Api/Resources/Mappers/Preview.cs index baaa51a7..1f0cd91f 100644 --- a/SW.Bitween.Api/Resources/Mappers/Preview.cs +++ b/SW.Bitween.Api/Resources/Mappers/Preview.cs @@ -45,8 +45,10 @@ public async Task Handle(MapperPreviewRequest request) var jObj = parsed as JObject; var jArr = parsed as JArray; + // Declared secrets are masked: the template is the caller's own, so it can print any + // value it is given. JObject? partnerObj = partner?.AdapterProperties?.Count > 0 - ? JObject.FromObject(partner.AdapterProperties) + ? JObject.FromObject(AdapterSecretProperties.Mask(partner.AdapterProperties, partner.SecretProperties)) : null; JObject? globalsObj = null; @@ -55,7 +57,7 @@ public async Task Handle(MapperPreviewRequest request) { globalsObj = new JObject(); foreach (var set in nonEmptySets) - globalsObj[set.Id] = JObject.FromObject(set.Values); + globalsObj[set.Id] = JObject.FromObject(AdapterSecretProperties.Mask(set.Values, set.SecretProperties)); } if (jObj != null && (partnerObj != null || globalsObj != null)) diff --git a/SW.Bitween.Api/Resources/MappingPreviews/Preview.cs b/SW.Bitween.Api/Resources/MappingPreviews/Preview.cs index 4d34e577..659e972b 100644 --- a/SW.Bitween.Api/Resources/MappingPreviews/Preview.cs +++ b/SW.Bitween.Api/Resources/MappingPreviews/Preview.cs @@ -35,7 +35,7 @@ public class MappingPreviewResponse /// public List RuleErrors { get; set; } = new(); - /// Set when the mapping could not be attempted at all — unreadable document or rules. + /// Set when the mapping could not be attempted at all - unreadable document or rules. public string? Error { get; set; } } @@ -50,7 +50,7 @@ public class MappingPreviewError /// /// /// -/// Runs the same three steps the exchange pipeline runs — read, map, write — against the same +/// Runs the same three steps the exchange pipeline runs - read, map, write - against the same /// context factory. That is deliberate and it is the point: the old preview generated its own /// template and assembled its own partner values, and the two drifted, so a mapping that previewed /// correctly could fail in production. @@ -60,11 +60,17 @@ public class MappingPreviewError /// yet. /// /// -public class Preview(MappingContextFactory contextFactory) +public class Preview(RequestContext requestContext, BitweenDbContext dbContext, + MappingContextFactory contextFactory) : ICommandHandler { public async Task Handle(MappingPreviewRequest request) { + // The editor opens while creating a subscription as well as while editing one, so either + // right will do. Without a guard any signed-in token could map any partner's values. + await requestContext.EnsurePermission(dbContext, + Model.Permissions.Subscriptions.Create, Model.Permissions.Subscriptions.Edit); + MappingRules rules; try { @@ -83,10 +89,10 @@ public async Task Handle(MappingPreviewRequest request) $"Bitween understands up to version {MappingRules.CurrentVersion}.", }; - if (!DocumentFormats.TryGet(rules.SourceFormat, out var source, rules.SourceCsv)) + if (SourceFormat(rules, request.SourceDocument) is not { } source) return new MappingPreviewResponse { Error = DocumentFormats.Unsupported(rules.SourceFormat, "source") }; - if (!DocumentFormats.TryGet(rules.TargetFormat, out var target, rules.TargetCsv)) + if (TargetFormat(rules) is not { } target) return new MappingPreviewResponse { Error = DocumentFormats.Unsupported(rules.TargetFormat, "target") }; ValueNode input; @@ -99,7 +105,7 @@ public async Task Handle(MappingPreviewRequest request) return new MappingPreviewResponse { Error = ex.Message }; } - var context = await contextFactory.Build(request.PartnerId); + var context = await contextFactory.Build(request.PartnerId, maskSecrets: true); try { @@ -129,4 +135,36 @@ public async Task Handle(MappingPreviewRequest request) return new MappingPreviewResponse { Error = ex.Message }; } } + + /// What reads an editor's source sample, or null for a format no reader answers to. + internal static IDocumentFormat? SourceFormat(MappingRules rules, string? sample) + { + if (!DocumentFormats.TryGet(rules.SourceFormat, out var source, rules.SourceOptions())) return null; + + // A workbook can't be pasted into the editor, so its sample is cells copied from Excel, + // which arrive as tab-separated text. Read that way when it isn't a workbook in base64; + // the exchange pipeline only ever reads the workbook itself. + return source is ExcelFormat && !LooksLikeBase64(sample ?? "") + ? new CsvFormat(new CsvOptions { Delimiter = "\t", HasHeader = rules.SourceExcel?.HasHeader != false }) + : source; + } + + /// What writes the preview's output, and reads an editor's output sample. + internal static IDocumentFormat? TargetFormat(MappingRules rules) + { + if (!DocumentFormats.TryGet(rules.TargetFormat, out var target, rules.TargetOptions())) return null; + + // And a workbook can't be shown, so the preview shows its rows the same way. + return target is ExcelFormat + ? new CsvFormat(new CsvOptions { Delimiter = "\t", HasHeader = rules.TargetExcel?.HasHeader != false }) + : target; + } + + static bool LooksLikeBase64(string text) + { + var trimmed = (text ?? "").Trim(); + if (trimmed.Length == 0 || trimmed.Length % 4 != 0) return false; + Span buffer = new byte[trimmed.Length]; + return Convert.TryFromBase64String(trimmed, buffer, out _); + } } diff --git a/SW.Bitween.Api/Resources/MappingSuggestions/Suggest.cs b/SW.Bitween.Api/Resources/MappingSuggestions/Suggest.cs new file mode 100644 index 00000000..6511d7f9 --- /dev/null +++ b/SW.Bitween.Api/Resources/MappingSuggestions/Suggest.cs @@ -0,0 +1,240 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Newtonsoft.Json; +using SW.Bitween.NativeAdapters.Mapper; +using SW.Bitween.NativeAdapters.Mapper.Formats; +using SW.Bitween.Resources.MappingPreviews; +using SW.Bitween.Services.Ai; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.MappingSuggestions; + +public class MappingSuggestionRequest +{ + /// + /// The rules in the editor, serialised. Their formats are always kept; with + /// they are also what gets fixed. + /// + public string MappingRules { get; set; } = ""; + + public string SourceDocument { get; set; } = ""; + + /// What should become. + public string TargetDocument { get; set; } = ""; + + /// Whose partner values the check maps against, as for a preview. + public int? PartnerId { get; set; } + + /// Fix the editor's rules where they miss the output sample, rather than write new ones. + public bool Refine { get; set; } + + /// + /// Notes for the model from the person building the mapping, for what the samples don't show - + /// codes a lookup needs that the sample doesn't happen to contain, say. + /// + public string? Instructions { get; set; } +} + +public class MappingSuggestionResponse +{ + /// The rules, serialised the way the editor stores them. + public string MappingRules { get; set; } = ""; + + /// The source sample mapped with these rules - a preview, run before answering. + public string? OutputDocument { get; set; } + + public string? ContentType { get; set; } + + public List RuleErrors { get; set; } = new(); + + /// Set when the rules could not be run against the sample at all. + public string? Error { get; set; } + + /// Where the output differs from the output sample, at most . + public List Differences { get; set; } = new(); + + /// Whether the rules turn the source sample into the output sample exactly. + public bool MatchesTarget { get; set; } + + /// The model that wrote the rules, so the editor can say which. + public string? Model { get; set; } +} + +/// +/// Asks the model Settings, AI names to write mapping rules from a source sample and an output +/// sample, and checks them before answering. +/// +/// +/// +/// The check is a preview - the same read, map and write the pipeline runs - and a comparison with +/// the output sample, so the editor can say whether the rules do what was asked instead of the +/// person having to find out. +/// +/// +/// One model call per request, and a second only when the first answer isn't readable rules at all. +/// Fixing differences is the editor's next request, with : +/// a model thinking about a large document can take most of a minute, and two of those in one +/// request would outlast the proxies in front of many deployments. +/// +/// +public class Suggest(RequestContext requestContext, BitweenDbContext dbContext, + MappingContextFactory contextFactory, AiModels models) + : ICommandHandler +{ + /// + /// Four times the 300-line sample generation was tested with, which answered in two seconds: + /// room for a real document, without a pasted export becoming a very large and slow prompt. + /// + internal const int MaxSampleLength = 200_000; + + internal const int MaxInstructionsLength = 4_000; + + public async Task Handle(MappingSuggestionRequest request) + { + // The same right as the preview it runs: the editor opens while creating and while editing. + await requestContext.EnsurePermission(dbContext, + Model.Permissions.Subscriptions.Create, Model.Permissions.Subscriptions.Edit); + + // Before anything else costs time or tokens: whatever is sent here goes to a paid provider. + if (request.SourceDocument?.Length > MaxSampleLength || request.TargetDocument?.Length > MaxSampleLength) + throw new SWValidationException("AI_SAMPLES", + $"Samples are limited to {MaxSampleLength / 1000:N0} KB each. A few records are enough to learn a mapping from."); + if (request.Instructions?.Length > MaxInstructionsLength) + throw new SWValidationException("AI_SAMPLES", $"Instructions are limited to {MaxInstructionsLength:N0} characters."); + + var model = models.Current(); + + MappingRules editor; + try + { + editor = JsonConvert.DeserializeObject(request.MappingRules ?? "") ?? new MappingRules(); + } + catch (JsonException ex) + { + throw new SWValidationException("MAPPING_RULES", $"The mapping rules could not be read: {ex.Message}"); + } + if (MappingRulesPrompt.NullIn(editor) is { } missing) + throw new SWValidationException("MAPPING_RULES", $"The mapping rules could not be read: {missing}"); + + if (string.IsNullOrWhiteSpace(request.SourceDocument)) + throw new SWValidationException("AI_SAMPLES", "Paste a sample of the source document first."); + if (string.IsNullOrWhiteSpace(request.TargetDocument)) + throw new SWValidationException("AI_SAMPLES", "Paste a sample of the output document first."); + + // Both samples are read before the model is asked anything: a sample that isn't valid in + // its format would only come back as a failed check, after paying for a call. + var source = Preview.SourceFormat(editor, request.SourceDocument) + ?? throw new SWValidationException("MAPPING_RULES", DocumentFormats.Unsupported(editor.SourceFormat, "source")); + var target = Preview.TargetFormat(editor) + ?? throw new SWValidationException("MAPPING_RULES", DocumentFormats.Unsupported(editor.TargetFormat, "target")); + var input = Read(source, request.SourceDocument, "source"); + var expected = Read(target, request.TargetDocument, "output"); + + (MappingRules, IReadOnlyList)? previous = null; + if (request.Refine) + { + var tried = await Check(editor, request, input, target, expected); + if (tried.MatchesTarget) return tried; + previous = (editor, Problems(tried)); + } + + // The same masked context the preview maps against, so a name offered here is one the + // check can resolve. Only its names leave the server. + var names = MappingRulesPrompt.NamesOf(await contextFactory.Build(request.PartnerId, maskSecrets: true)); + var message = MappingRulesPrompt.Message(editor, request.SourceDocument, request.TargetDocument, + names, request.Instructions, previous); + MappingRules rules; + try + { + rules = MappingRulesPrompt.Parse(await Ask(model, message), editor); + } + catch (FormatException first) + { + // Once more, told why: what went wrong is something a model can act on. + var again = $"{message}\nYour previous answer could not be used: {first.Message}\n" + + "Answer again with only the rules JSON."; + try + { + rules = MappingRulesPrompt.Parse(await Ask(model, again), editor); + } + catch (FormatException second) + { + throw new SWValidationException("AI_UNUSABLE_ANSWER", + $"The model's answer wasn't usable mapping rules ({second.Message}). Try again."); + } + } + + var checkedRules = await Check(rules, request, input, target, expected); + checkedRules.Model = models.ModelName; + return checkedRules; + } + + static Task Ask(IAiModel model, string message) => + model.Complete(new AiPrompt(MappingRulesPrompt.Instructions, message, Json: true)); + + static ValueNode Read(IDocumentFormat format, string sample, string which) + { + try + { + return format.Read(sample); + } + catch (DocumentFormatException ex) + { + throw new SWValidationException("AI_SAMPLES", $"The {which} sample isn't valid {format.Id}: {ex.Message}"); + } + } + + /// Previews the rules against the source sample and compares the result with the output sample. + async Task Check(MappingRules rules, MappingSuggestionRequest request, + ValueNode input, IDocumentFormat target, ValueNode expected) + { + var serialized = MappingRulesPrompt.Serialize(rules); + var preview = await new Preview(requestContext, dbContext, contextFactory).Handle(new MappingPreviewRequest + { + MappingRules = serialized, + SourceDocument = request.SourceDocument, + PartnerId = request.PartnerId, + }); + + var response = new MappingSuggestionResponse + { + MappingRules = serialized, + OutputDocument = preview.OutputDocument, + ContentType = preview.ContentType, + RuleErrors = preview.RuleErrors, + Error = preview.Error, + }; + + // Read back with the format that wrote it, so the comparison is between two trees rather + // than two texts that differ only in whitespace or key order. + if (preview.OutputDocument is not null) + { + try + { + response.Differences = DocumentDifferences.Between(expected, target.Read(preview.OutputDocument)); + } + catch (DocumentFormatException ex) + { + response.Differences.Add(new("(root)", $"the output could not be read back as {target.Id}: {ex.Message}")); + } + } + + // A value frozen from the source matches this sample and no other, which the comparison + // can't see: it is a difference all the same. + response.Differences.AddRange(FrozenValues.In(rules, input)); + + response.MatchesTarget = preview.OutputDocument is not null && preview.Error is null + && preview.RuleErrors.Count == 0 && response.Differences.Count == 0; + return response; + } + + static List Problems(MappingSuggestionResponse tried) => + [ + .. tried.Error is null ? [] : new[] { tried.Error }, + .. tried.RuleErrors.Select(e => $"The rule for {e.Target} failed: {e.Reason}"), + .. tried.Differences.Select(d => d.ToString()), + ]; +} diff --git a/SW.Bitween.Api/Resources/Marketplace/Marketplace.cs b/SW.Bitween.Api/Resources/Marketplace/Marketplace.cs new file mode 100644 index 00000000..637f4b3d --- /dev/null +++ b/SW.Bitween.Api/Resources/Marketplace/Marketplace.cs @@ -0,0 +1,294 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using System; +using SW.Bitween.Domain; +using SW.Bitween.Services; +using SW.Bitween.Services.Adapters; +using SW.Bitween.Services.Marketplace; +using SW.PrimitiveTypes; +using SW.Serverless.Contract.Catalog; + +namespace SW.Bitween.Resources.Marketplace; + +public class MarketplaceSearchRequest +{ + public string Q { get; set; } + public string Tier { get; set; } +} + +public class MarketplacePackageRequest +{ + public string Id { get; set; } +} + +public class MarketplaceInstallRequest +{ + public string Id { get; set; } + public string Version { get; set; } + public bool Current { get; set; } +} + +/// What this Bitween already has of a registry package. +public record InstalledState(bool Installed, bool FromRegistry, string Current, IReadOnlyList Versions); + +/// +/// The Marketplace tab: what the registry lists, with what this Bitween has of each. Turned off, +/// or with no registry address, it says so and fetches nothing. +/// +[HandlerName("search")] +public class Search(BitweenDbContext dbContext, RequestContext requestContext, RegistryClient registry, AdapterCatalog catalog) + : IQueryHandler +{ + public async Task Handle(MarketplaceSearchRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Subscriptions.View); + if (!registry.Configured) return new { Configured = false, registry.Disabled }; + + var found = await registry.SearchAsync(request.Q, request.Tier); + var result = new List(); + foreach (var card in found.Result) + result.Add(new { card, installed = await Installed.Of(catalog, card.Id) }); + return new { Configured = true, Registry = registry.BaseUrl, Result = result, found.TotalCount }; + } +} + +/// One registry package with its versions, and what this Bitween has of it. +[HandlerName("package")] +public class Package(BitweenDbContext dbContext, RequestContext requestContext, RegistryClient registry, AdapterCatalog catalog) + : IQueryHandler +{ + public async Task Handle(MarketplacePackageRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Subscriptions.View); + if (string.IsNullOrWhiteSpace(request.Id)) throw new SWValidationException("Id", "Say which package."); + + var id = request.Id.Trim().ToLowerInvariant(); + var package = await registry.PackageAsync(id); + var latest = package.Card.Latest == null ? null : await registry.VersionAsync(id, package.Card.Latest); + return new { package, latest = latest?.Manifest, installed = await Installed.Of(catalog, id) }; + } +} + +/// Installs a version from the registry, after checking its signatures. In the audit trail. +[HandlerName("install")] +public class Install(BitweenDbContext dbContext, RequestContext requestContext, MarketplaceInstaller installer) + : ICommandHandler +{ + public async Task Handle(MarketplaceInstallRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.AdapterSource.Operate); + var who = requestContext.GetNameIdentifier(); + + var installed = await installer.InstallAsync(request.Id, request.Version, request.Current, who); + dbContext.Add(new AdapterRelease(installed.AdapterId, installed.Version, AdapterRelease.InstalledAction, null, who)); + if (installed.Current) + dbContext.Add(new AdapterRelease(installed.AdapterId, installed.Version, AdapterRelease.PromotedAction, null, who)); + await dbContext.SaveChangesAsync(); + return installed; + } +} + +public class MarketplaceKitRequest +{ + public string Id { get; set; } + public string Version { get; set; } +} + +public class MarketplaceKitInstallRequest +{ + public string Id { get; set; } + public string Version { get; set; } + + /// What this installation is called; the kit's id unless given, another name for a second one. + public string Name { get; set; } + + public Dictionary Parameters { get; set; } +} + +/// A kit's connectors and the parameters its install form asks for. +[HandlerName("kit")] +public class Kit(BitweenDbContext dbContext, RequestContext requestContext, KitInstaller kits) + : IQueryHandler +{ + public async Task Handle(MarketplaceKitRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Subscriptions.View); + return await kits.DescribeAsync(request.Id, request.Version); + } +} + +/// What installing a kit would do, changing nothing. +[HandlerName("kitplan")] +public class KitPlan(BitweenDbContext dbContext, RequestContext requestContext, KitInstaller kits) + : ICommandHandler +{ + public async Task Handle(MarketplaceKitInstallRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.AdapterSource.Operate); + return await kits.PlanAsync(request.Id, request.Version, request.Name, request.Parameters); + } +} + +/// +/// Installs a kit: its connectors, then its configuration with the parameters filled in, as the caller, +/// so each change needs the caller's own permissions and is in the trail under their name. +/// +[HandlerName("kitinstall")] +public class KitInstall(BitweenDbContext dbContext, RequestContext requestContext, KitInstaller kits) + : ICommandHandler +{ + public async Task Handle(MarketplaceKitInstallRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.AdapterSource.Operate); + var run = await kits.InstallAsync(request.Id, request.Version, request.Name, request.Parameters); + if (run.Applied) + await dbContext.RecordNowAsync(requestContext, Domain.OperatorAction.InstallKit, $"{request.Id} {request.Version}", + $"installed as {run.Name}, {run.Changes} change(s)"); + return run; + } +} + +public class MarketplaceUpdatesRequest +{ +} + +/// +/// What the last registry check found for the adapters installed from it: newer versions with their +/// release notes, and installed versions the registry has revoked. Read from what was stored, so it +/// answers with the registry unreachable. +/// +[HandlerName("updates")] +public class Updates(BitweenDbContext dbContext, RequestContext requestContext, RegistryUpdates updates) + : IQueryHandler +{ + public async Task Handle(MarketplaceUpdatesRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Subscriptions.View); + return await updates.ReadAsync(); + } +} + +/// Asks the registry now, rather than waiting for the daily check. +[HandlerName("check")] +public class Check(BitweenDbContext dbContext, RequestContext requestContext, RegistryUpdates updates) + : ICommandHandler +{ + public async Task Handle(MarketplaceUpdatesRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.AdapterSource.Operate); + return await updates.CheckAsync(); + } +} + +public class MarketplaceRegisterRequest +{ + /// The registration code Simplify9 gave, bwreg_... + public string Code { get; set; } + + /// What to call this Bitween in the registry; its address by default. + public string DisplayName { get; set; } +} + +/// +/// Registers this Bitween with the registry: the code becomes an instance id and a token, saved as +/// settings, and the licences are fetched. Needed only for private or paid connectors and usage reports. +/// +[HandlerName("register")] +public class Register(BitweenDbContext dbContext, RequestContext requestContext, RegistryClient registry, BitweenOptions options, + SettingsService settings, IInfolinkCache cache, RegistryLicences licences) + : ICommandHandler +{ + public async Task Handle(MarketplaceRegisterRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Settings.Edit); + if (string.IsNullOrWhiteSpace(request?.Code)) + throw new SWValidationException("Code", "Paste the registration code Simplify9 gave."); + var token = SettingsCatalog.Find("Bitween.RegistryInstanceToken"); + if (!settings.CanStore(token)) + throw new SWValidationException("SETTING_ENCRYPTION_UNAVAILABLE", + $"The instance token is a secret and can only be stored once {BitweenOptions.ConfigurationSection}:{nameof(BitweenOptions.SettingsEncryptionKey)} is configured."); + + var registration = await registry.RegisterAsync(request.Code.Trim(), + string.IsNullOrWhiteSpace(request.DisplayName) ? options.PublicUrl ?? "Bitween" : request.DisplayName.Trim(), + BitweenInfo.Version(options).ToString()); + + // Stored and applied the way Settings, Update does, so the next request already sends the token. + foreach (var (definition, value) in new[] { (SettingsCatalog.Find("Bitween.RegistryInstanceId"), registration.InstanceId), (token, registration.Token) }) + { + var stored = await dbContext.Set().FindAsync(definition.Key); + var toStore = settings.ToStored(definition, value); + if (stored is null) dbContext.Add(new Setting { Id = definition.Key, Value = toStore }); + else stored.Value = toStore; + settings.Apply(definition, value); + } + await dbContext.SaveChangesAsync(); + await cache.BroadcastRevoke(); + + var summary = await licences.RefreshAsync(); + return new { registration.InstanceId, registration.DisplayName, registration.CustomerName, Licences = summary }; + } +} + +public class MarketplaceLicencesRequest +{ +} + +/// The licences this Bitween holds, as last fetched; nothing is asked of the registry. +[HandlerName("licences")] +public class Licences(BitweenDbContext dbContext, RequestContext requestContext, RegistryLicences licences) + : IQueryHandler +{ + public async Task Handle(MarketplaceLicencesRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Subscriptions.View); + return await licences.ReadAsync(); + } +} + +/// Fetches the licences from the registry now. +[HandlerName("refreshlicences")] +public class RefreshLicences(BitweenDbContext dbContext, RequestContext requestContext, RegistryLicences licences) + : ICommandHandler +{ + public async Task Handle(MarketplaceLicencesRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.AdapterSource.Operate); + return await licences.RefreshAsync(); + } +} + +public class MarketplaceUsageRequest +{ + /// The day to report, yyyy-MM-dd; yesterday by default. + public string Day { get; set; } + + /// True sends it; false only shows what would be sent. + public bool Send { get; set; } +} + +/// What the daily usage report says, and sends it when asked. +[HandlerName("usage")] +public class Usage(BitweenDbContext dbContext, RequestContext requestContext, UsageReporter usage) + : ICommandHandler +{ + public async Task Handle(MarketplaceUsageRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Settings.Edit); + var day = string.IsNullOrWhiteSpace(request?.Day) ? DateOnly.FromDateTime(DateTime.UtcNow).AddDays(-1) + : DateOnly.TryParseExact(request.Day, "yyyy-MM-dd", out var d) ? d : throw new SWValidationException("Day", "Give the day as yyyy-MM-dd."); + return request?.Send == true ? await usage.SendAsync(day) : await usage.PreviewAsync(day); + } +} + +static class Installed +{ + public static async Task Of(AdapterCatalog catalog, string id) + { + var entry = await catalog.GetAsync(id); + if (entry == null) return new InstalledState(false, false, null, []); + var fromRegistry = entry.Extensions?.ContainsKey(MarketplaceInstaller.OriginExtension) == true; + var versions = entry.Versions.Where(v => !v.Withdrawn).Select(v => v.Version).ToList(); + return new InstalledState(true, fromRegistry, entry.Current, versions); + } +} diff --git a/SW.Bitween.Api/Resources/Metrics/AdapterDetail.cs b/SW.Bitween.Api/Resources/Metrics/AdapterDetail.cs new file mode 100644 index 00000000..00361706 --- /dev/null +++ b/SW.Bitween.Api/Resources/Metrics/AdapterDetail.cs @@ -0,0 +1,94 @@ +using System.Collections.Generic; +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Caching.Memory; +using SW.Bitween.Domain; +using SW.Bitween.Domain.Metrics; +using SW.Bitween.Model; +using SW.Bitween.Services.Metrics; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Metrics; + +/// +/// One adapter's use in detail: by version, by node, by role and runtime, by day; its most common +/// errors; and the subscriptions that use it with what they moved through it. +/// +[HandlerName("adapter")] +public class AdapterDetail(BitweenDbContext db, RequestContext requestContext, IMemoryCache cache) + : IQueryHandler +{ + public async Task Handle(AdapterMetricsRequest request) + { + await requestContext.EnsurePermission(db, Model.Permissions.Subscriptions.View); + if (string.IsNullOrWhiteSpace(request?.Id)) throw new SWValidationException("Id", "Say which adapter."); + var id = request.Id.Trim().ToLowerInvariant(); + var days = MetricsQueries.Days(request.Days); + return await cache.Cached($"adapter.{id}.{days}", () => Compute(id, days)); + } + + async Task Compute(string id, int days) + { + var to = MetricsRecorder.HourOf(DateTime.UtcNow).AddHours(1); + var from = to.AddDays(-days); + var rows = MetricsQueries.Adapters(db, from, to).Where(s => s.AdapterId == id); + + var total = (await rows.GroupBy(_ => 1).SumBy()).FirstOrDefault() ?? new AdapterTotals(); + var versions = await rows.GroupBy(s => s.Version).SumBy(); + var nodes = await rows.GroupBy(s => s.Node).SumBy(); + var roles = await rows.GroupBy(s => new { s.Role, s.Runtime }).SumBy(); + var daily = await rows.GroupBy(s => s.Hour.Date).SumBy(); + + var errorRows = db.Set().AsNoTracking() + .Where(e => e.Scope == ErrorStats.AdapterScope && e.ScopeKey == id && e.Hour >= from && e.Hour < to); + var errors = await errorRows.GroupBy(e => e.Fingerprint) + .Select(g => new { Fingerprint = g.Key, Count = g.Sum(x => (long)x.Count), LastSeenOn = g.Max(x => x.LastSeenOn) }) + .OrderByDescending(e => e.Count).Take(10).ToListAsync(); + var fingerprints = errors.Select(e => e.Fingerprint).ToList(); + var samples = (await errorRows.Where(e => fingerprints.Contains(e.Fingerprint)) + .Select(e => new { e.Fingerprint, e.Sample, e.LastSeenOn }).ToListAsync()) + .GroupBy(e => e.Fingerprint).ToDictionary(g => g.Key, g => g.OrderByDescending(e => e.LastSeenOn).First().Sample); + + // The subscriptions configured with it now, and what went through them in the window. + var users = await db.Set().AsNoTracking() + .Where(s => s.HandlerId.ToLower() == id || s.MapperId.ToLower() == id || + s.ReceiverId.ToLower() == id || s.ValidatorId.ToLower() == id) + .Select(s => new { s.Id, s.Name, s.Type, s.HandlerVersion, s.MapperVersion }) + .ToListAsync(); + var userIds = users.Select(u => u.Id).ToList(); + var through = (await MetricsQueries.Exchanges(db, from, to).Where(s => userIds.Contains(s.SubscriptionId)) + .GroupBy(s => s.SubscriptionId).SumBy()).ToDictionary(t => t.Key); + + return new + { + Id = id, + Days = days, + Since = await MetricsQueries.AdapterSinceAsync(db), + Total = Shape(total), + Versions = versions.OrderByDescending(v => v.Calls).Select(v => new { Version = v.Key, Totals = Shape(v) }), + Nodes = nodes.OrderByDescending(n => n.Calls).Select(n => new { Node = n.Key, Totals = Shape(n) }), + Roles = roles.Select(r => new { r.Key.Role, r.Key.Runtime, Totals = Shape(r) }), + Daily = daily.OrderBy(d => d.Key).Select(d => new { Day = MetricsQueries.Utc(d.Key), d.Calls, d.Failures, d.P95Ms }), + Errors = errors.Select(e => new + { + e.Fingerprint, e.Count, LastSeenOn = MetricsQueries.Utc(e.LastSeenOn), Sample = samples.GetValueOrDefault(e.Fingerprint), + }), + Subscriptions = users.Select(u => + { + var t = through.GetValueOrDefault(u.Id); + return new + { + SubscriptionId = u.Id, u.Name, Type = u.Type.ToString(), + Exchanges = t?.Total ?? 0, Failed = t?.Failed ?? 0, Bytes = (t?.BytesIn ?? 0) + (t?.BytesOut ?? 0), + }; + }).OrderByDescending(u => u.Exchanges), + }; + } + + static object Shape(AdapterTotals t) => new + { + t.Calls, t.Failures, t.FailureRate, t.Timeouts, t.LimitStops, t.Crashes, t.P50Ms, t.P95Ms, MaxMs = t.DurationMaxMs, + }; +} diff --git a/SW.Bitween.Api/Resources/Metrics/Adapters.cs b/SW.Bitween.Api/Resources/Metrics/Adapters.cs new file mode 100644 index 00000000..10434631 --- /dev/null +++ b/SW.Bitween.Api/Resources/Metrics/Adapters.cs @@ -0,0 +1,59 @@ +using System.Collections.Generic; +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Caching.Memory; +using SW.Bitween.Model; +using SW.Bitween.Services.Metrics; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Metrics; + +/// Every adapter's use over the window: calls, failures by kind, timings, calls by day, last call. +[HandlerName("adapters")] +public class Adapters(BitweenDbContext db, RequestContext requestContext, IMemoryCache cache) + : IQueryHandler +{ + public async Task Handle(MetricsWindowRequest request) + { + await requestContext.EnsurePermission(db, Model.Permissions.Subscriptions.View); + var days = MetricsQueries.Days(request?.Days); + return await cache.Cached($"adapters.{days}", () => Compute(days)); + } + + async Task Compute(int days) + { + var to = MetricsRecorder.HourOf(DateTime.UtcNow).AddHours(1); + var from = to.AddDays(-days); + var totals = await MetricsQueries.Adapters(db, from, to).GroupBy(s => s.AdapterId).SumBy(); + var last = await MetricsQueries.Adapters(db, to.AddDays(-30), to) + .GroupBy(s => s.AdapterId).Select(g => new { g.Key, Last = g.Max(x => x.Hour) }) + .ToDictionaryAsync(x => x.Key, x => MetricsQueries.Utc(x.Last)); + var dailyRows = await MetricsQueries.Adapters(db, from, to) + .GroupBy(s => new { s.AdapterId, Day = s.Hour.Date }) + .Select(g => new { g.Key.AdapterId, g.Key.Day, Calls = g.Sum(x => (long)x.Calls), Failures = g.Sum(x => (long)x.Failures) }) + .ToListAsync(); + var firstDay = from.Date; + + return new + { + Since = await MetricsQueries.AdapterSinceAsync(db), + Days = days, + Adapters = totals.Select(t => new + { + Id = t.Key, t.Calls, t.Failures, t.FailureRate, t.Timeouts, t.LimitStops, t.Crashes, t.P50Ms, t.P95Ms, + MaxMs = t.DurationMaxMs, + LastCalledOn = last.TryGetValue(t.Key, out var l) ? l : (DateTime?)null, + Daily = Enumerable.Range(0, days).Select(i => + { + var day = firstDay.AddDays(i); + var r = dailyRows.Where(d => d.AdapterId == t.Key && d.Day.Date == day).ToList(); + return new { Day = MetricsQueries.Utc(day), Calls = r.Sum(x => x.Calls), Failures = r.Sum(x => x.Failures) }; + }), + }), + // Adapters with no calls in the window but some in the last 30 days, for "last called". + Quiet = last.Where(l => totals.All(t => t.Key != l.Key)).Select(l => new { Id = l.Key, LastCalledOn = l.Value }), + }; + } +} diff --git a/SW.Bitween.Api/Resources/Metrics/Insights.cs b/SW.Bitween.Api/Resources/Metrics/Insights.cs new file mode 100644 index 00000000..9f7c6b24 --- /dev/null +++ b/SW.Bitween.Api/Resources/Metrics/Insights.cs @@ -0,0 +1,141 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Caching.Memory; +using SW.Bitween.Domain; +using SW.Bitween.Domain.Accounts; +using SW.Bitween.Domain.Gateway; +using SW.Bitween.Services.Adapters; +using SW.Bitween.Services.Metrics; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Metrics; + +/// +/// Housekeeping: configuration that is switched off, unused or out of date, each with a link to +/// it. Nothing here is wrong as such; it is what to look at when tidying up an instance. +/// +[HandlerName("insights")] +public class Insights(BitweenDbContext db, RequestContext requestContext, IMemoryCache cache, AdapterCatalog catalog) + : IQueryHandler +{ + const int Listed = 10; + + public record Item(string Label, string Href); + + public record Insight(string Kind, string Severity, string Title, string Detail, int Count, IEnumerable Items); + + public async Task Handle() + { + await requestContext.EnsurePermission(db, Model.Permissions.Dashboard.View); + return await cache.Cached("insights", Compute); + } + + async Task Compute() + { + var to = MetricsRecorder.HourOf(DateTime.UtcNow).AddHours(1); + var since = await MetricsQueries.SinceAsync(db); + var insights = new List(); + + var subscriptions = await db.Set().AsNoTracking() + .Select(s => new + { + s.Id, s.Name, s.Inactive, s.PausedOn, s.PausedAutomatically, s.RetryPolicyId, s.WorkGroupId, s.PartnerId, + s.HandlerId, s.HandlerVersion, s.MapperId, s.MapperVersion, s.QuietSince, + }).ToListAsync(); + Item Link(int id, string name) => new(name, $"/subscriptions/{id}"); + + // Set by QuietSubscriptionsJob: expected traffic that hasn't come. Longest quiet first. + var quiet = subscriptions.Where(s => !s.Inactive && s.PausedOn == null && s.QuietSince != null) + .OrderBy(s => s.QuietSince).ToList(); + if (quiet.Count > 0) + insights.Add(new("quiet-subscriptions", "warn", "Subscriptions gone quiet", + "Expected traffic hasn't arrived. Check the partner or the source they read from.", + quiet.Count, quiet.Take(Listed).Select(s => new Item($"{s.Name} (since {s.QuietSince:yyyy-MM-dd HH:mm} UTC)", + $"/subscriptions/{s.Id}")))); + + var paused = subscriptions.Where(s => !s.Inactive && s.PausedOn != null).ToList(); + if (paused.Count > 0) + insights.Add(new("paused", "warn", "Paused subscriptions", + $"{paused.Count(s => s.PausedAutomatically)} paused themselves after repeated failures.", + paused.Count, paused.Take(Listed).Select(s => Link(s.Id, s.Name)))); + + var inactive = subscriptions.Where(s => s.Inactive).ToList(); + if (inactive.Count > 0) + insights.Add(new("inactive", "info", "Switched-off subscriptions", "Kept, but not running.", + inactive.Count, inactive.Take(Listed).Select(s => Link(s.Id, s.Name)))); + + // Quiet for 30 days - only once the metrics reach that far back. + if (since != null && since <= to.AddDays(-30)) + { + var active = (await MetricsQueries.Exchanges(db, to.AddDays(-30), to).Where(s => s.Total > 0) + .Select(s => s.SubscriptionId).Distinct().ToListAsync()).ToHashSet(); + var unused = subscriptions.Where(s => !s.Inactive && s.PausedOn == null && !active.Contains(s.Id)).ToList(); + if (unused.Count > 0) + insights.Add(new("unused-subscriptions", "info", "Subscriptions without traffic for 30 days", + "Running, but nothing has gone through them.", unused.Count, unused.Take(Listed).Select(s => Link(s.Id, s.Name)))); + } + + // Pinned to a version that has since been withdrawn: it keeps running, but nobody can pin it again. + var withdrawn = new List(); + foreach (var s in subscriptions.Where(s => s.HandlerVersion != null || s.MapperVersion != null)) + foreach (var (id, version) in new[] { (s.HandlerId, s.HandlerVersion), (s.MapperId, s.MapperVersion) }) + if (id != null && version != null && (await catalog.GetAsync(id))?.Find(version)?.Withdrawn == true) + withdrawn.Add(new($"{s.Name} (pins {id} v{version})", $"/subscriptions/{s.Id}")); + if (withdrawn.Count > 0) + insights.Add(new("withdrawn-pins", "warn", "Pinned to a withdrawn version", + "Still runs it, but the version is withdrawn. Move it to a current version.", withdrawn.Count, withdrawn.Take(Listed))); + + var usedPolicies = subscriptions.Where(s => s.RetryPolicyId != null).Select(s => s.RetryPolicyId.Value).ToHashSet(); + var policies = await db.Set().AsNoTracking().Select(p => new { p.Id, p.Name }).ToListAsync(); + var unusedPolicies = policies.Where(p => !usedPolicies.Contains(p.Id)).ToList(); + if (unusedPolicies.Count > 0) + insights.Add(new("unused-retry-policies", "info", "Retry policies nothing uses", "No subscription is assigned them.", + unusedPolicies.Count, unusedPolicies.Take(Listed).Select(p => new Item(p.Name, $"/retry-policies/{p.Id}")))); + + var usedGroups = subscriptions.Where(s => s.WorkGroupId != null).Select(s => s.WorkGroupId.Value).ToHashSet(); + var groups = await db.Set().AsNoTracking().Select(g => new { g.Id, g.Name }).ToListAsync(); + var unusedGroups = groups.Where(g => !usedGroups.Contains(g.Id)).ToList(); + if (unusedGroups.Count > 0) + insights.Add(new("unused-work-groups", "info", "Work groups nothing uses", "Their queues are kept for no subscription.", + unusedGroups.Count, unusedGroups.Take(Listed).Select(g => new Item(g.Name, $"/work-groups/{g.Id}")))); + + var linked = subscriptions.Where(s => s.PartnerId != null).Select(s => s.PartnerId.Value) + .Concat(await db.Set().AsNoTracking().Select(a => a.PartnerId).ToListAsync()) + .Concat(await db.Set().AsNoTracking().Where(r => r.PartnerId != null).Select(r => r.PartnerId.Value).ToListAsync()) + .ToHashSet(); + var partners = await db.Set().AsNoTracking().Where(p => p.Id != Partner.SystemId) + .Select(p => new { p.Id, p.Name }).ToListAsync(); + var loose = partners.Where(p => !linked.Contains(p.Id)).ToList(); + if (loose.Count > 0) + insights.Add(new("unlinked-partners", "info", "Partners not connected to anything", + "Not attached to a gateway, a route or a subscription.", loose.Count, + loose.Take(Listed).Select(p => new Item(p.Name, $"/partners/{p.Id}")))); + + // Whatever signs in with one of these stops working the day it runs out, so it's worth knowing + // two weeks ahead. Only a token still in use counts; one already replaced is expected to lapse. + var now = DateTime.UtcNow; + var soon = now.AddDays(14); + var expiring = await (from t in db.Set().AsNoTracking() + join a in db.Set().AsNoTracking() on t.AccountId equals a.Id + where t.RevokedOn == null && t.ExpiresOn > now && t.ExpiresOn <= soon && !a.Disabled + orderby t.ExpiresOn + select new { a.Id, Account = a.DisplayName, t.Name, t.ExpiresOn }) + .ToListAsync(); + if (expiring.Count > 0) + insights.Add(new("expiring-service-tokens", "warn", "Service tokens expiring within two weeks", + "Whatever uses them stops signing in when they do. Issue a replacement, then revoke the old one.", + expiring.Count, + expiring.Take(Listed).Select(t => new Item($"{t.Account}: {t.Name} ({t.ExpiresOn:yyyy-MM-dd})", + $"/team/service-accounts/{t.Id}")))); + + var changedSettings = await db.Set().AsNoTracking().CountAsync(); + if (changedSettings > 0) + insights.Add(new("changed-settings", "info", "Settings changed from their defaults", + "What is particular about this instance.", changedSettings, [new Item("Settings", "/settings")])); + + return new { Since = since, Insights = insights }; + } +} diff --git a/SW.Bitween.Api/Resources/Metrics/Nodes.cs b/SW.Bitween.Api/Resources/Metrics/Nodes.cs new file mode 100644 index 00000000..57e60bec --- /dev/null +++ b/SW.Bitween.Api/Resources/Metrics/Nodes.cs @@ -0,0 +1,77 @@ +using System.Collections.Generic; +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Caching.Memory; +using Newtonsoft.Json.Linq; +using SW.Bitween.Domain.Cluster; +using SW.Bitween.Services.Cluster; +using SW.Bitween.Services.Metrics; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Metrics; + +/// +/// What each node has done in the last day - exchanges, adapter calls and how they failed - and +/// which resident adapter instances it is running now, with what they use. A node whose failures +/// stand out from the rest is the one to look at. +/// +[HandlerName("nodes")] +public class Nodes(BitweenDbContext db, RequestContext requestContext, IMemoryCache cache) : IQueryHandler +{ + public async Task Handle() + { + await requestContext.EnsurePermission(db, Model.Permissions.Settings.View, Model.Permissions.Monitoring.View); + return await cache.Cached("nodes", Compute); + } + + async Task Compute() + { + var now = DateTime.UtcNow; + var to = MetricsRecorder.HourOf(now).AddHours(1); + var from = to.AddHours(-24); + var exchanges = (await MetricsQueries.Exchanges(db, from, to).GroupBy(s => s.Node).SumBy()).ToDictionary(t => t.Key); + var adapters = (await MetricsQueries.Adapters(db, from, to).GroupBy(s => s.Node).SumBy()).ToDictionary(t => t.Key); + var nodes = await db.Set().AsNoTracking().OrderBy(n => n.Id).ToListAsync(); + + return new + { + Nodes = nodes.Select(n => + { + var e = exchanges.GetValueOrDefault(n.Id.Length <= 100 ? n.Id : n.Id[..100]); + var a = adapters.GetValueOrDefault(n.Id.Length <= 100 ? n.Id : n.Id[..100]); + return new + { + n.Id, + n.Host, + Alive = now - n.LastSeenOn < NodeHeartbeat.GoneAfter, + LastSeenOn = MetricsQueries.Utc(n.LastSeenOn), + Exchanges = e?.Total ?? 0, + Failed = e?.Failed ?? 0, + SuccessRate = e?.SuccessRate, + P95Ms = e?.P95Ms, + AdapterCalls = a?.Calls ?? 0, + AdapterFailures = a?.Failures ?? 0, + Timeouts = a?.Timeouts ?? 0, + LimitStops = a?.LimitStops ?? 0, + Crashes = a?.Crashes ?? 0, + Resident = Parse(n.ResidentHealth), + }; + }), + }; + } + + static JToken Parse(string json) + { + if (string.IsNullOrWhiteSpace(json)) return new JArray(); + try + { + return JToken.Parse(json); + } + catch + { + return new JArray(); + } + } +} diff --git a/SW.Bitween.Api/Resources/Metrics/Overview.cs b/SW.Bitween.Api/Resources/Metrics/Overview.cs new file mode 100644 index 00000000..c21609ea --- /dev/null +++ b/SW.Bitween.Api/Resources/Metrics/Overview.cs @@ -0,0 +1,107 @@ +using System.Collections.Generic; +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Caching.Memory; +using SW.Bitween.Domain; +using SW.Bitween.Domain.Metrics; +using SW.Bitween.Services.Metrics; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Metrics; + +/// +/// The dashboard's numbers: traffic by hour against the same hours a week ago, success and +/// end-to-end time, volume by day, the worst and busiest of everything, subscriptions gone quiet, +/// and the broker's backlog over the day. All from the hourly rows; nothing counts exchanges. +/// +[HandlerName("overview")] +public class Overview(BitweenDbContext db, RequestContext requestContext, IMemoryCache cache) : IQueryHandler +{ + public async Task Handle() + { + await requestContext.EnsurePermission(db, Model.Permissions.Dashboard.View); + return await cache.Cached("overview", Compute); + } + + async Task Compute() + { + var now = DateTime.UtcNow; + var to = MetricsRecorder.HourOf(now).AddHours(1); + + var hourly = await MetricsQueries.Exchanges(db, to.AddHours(-48), to).GroupBy(s => s.Hour).SumBy(); + var weekAgo = await MetricsQueries.Exchanges(db, to.AddHours(-48).AddDays(-7), to.AddDays(-7)).GroupBy(s => s.Hour).SumBy(); + var last24 = await Total(to.AddHours(-24), to); + var previous24 = await Total(to.AddHours(-48), to.AddHours(-24)); + var last7 = await Total(to.AddDays(-7), to); + var daily = await MetricsQueries.Exchanges(db, to.AddDays(-30), to).GroupBy(s => s.Hour.Date).SumBy(); + + var subscriptionNames = await db.Set().AsNoTracking().ToDictionaryAsync(s => s.Id, s => s.Name); + var partnerNames = await db.Set().AsNoTracking().ToDictionaryAsync(p => p.Id, p => p.Name); + + var bySubscription = await MetricsQueries.Exchanges(db, to.AddHours(-24), to).Where(s => s.SubscriptionId > 0) + .GroupBy(s => s.SubscriptionId).SumBy(); + var byWeekSubscription = await MetricsQueries.Exchanges(db, to.AddDays(-7), to).Where(s => s.SubscriptionId > 0) + .GroupBy(s => s.SubscriptionId).SumBy(); + var byPartner = await MetricsQueries.Exchanges(db, to.AddHours(-24), to).Where(s => s.PartnerId > 0) + .GroupBy(s => s.PartnerId).SumBy(); + var byAdapter = await MetricsQueries.Adapters(db, to.AddHours(-24), to).GroupBy(s => s.AdapterId).SumBy(); + + var health = await SubscriptionHealth.ComputeAsync(db, 7, now); + var backlog = await db.Set().AsNoTracking().Where(s => s.At >= now.AddHours(-24)) + .OrderBy(s => s.At).ToListAsync(); + + return new + { + Since = await MetricsQueries.SinceAsync(db), + Hourly = hourly.OrderBy(h => h.Key).Select(h => new + { + Hour = MetricsQueries.Utc(h.Key), h.Total, h.Succeeded, h.BadResponse, h.Failed, + }), + WeekAgo = weekAgo.OrderBy(h => h.Key).Select(h => new { Hour = MetricsQueries.Utc(h.Key.AddDays(7)), h.Total }), + Last24h = Summary(last24), + Previous24h = Summary(previous24), + Last7d = Summary(last7), + Daily = daily.OrderBy(d => d.Key).Select(d => new + { + Day = MetricsQueries.Utc(d.Key), d.Total, d.Succeeded, d.BadResponse, d.Failed, Bytes = d.BytesIn + d.BytesOut, + }), + TopFailing = bySubscription.Where(s => s.Failed > 0).OrderByDescending(s => s.Failed).Take(5).Select(s => new + { + SubscriptionId = s.Key, Name = subscriptionNames.GetValueOrDefault(s.Key), s.Failed, s.Total, + }), + BusiestSubscriptions = byWeekSubscription.OrderByDescending(s => s.Total).Take(5).Select(s => new + { + SubscriptionId = s.Key, Name = subscriptionNames.GetValueOrDefault(s.Key), s.Total, s.Failed, s.BadResponse, + }), + BusiestPartners = byPartner.OrderByDescending(p => p.Total).Take(5).Select(p => new + { + PartnerId = p.Key, Name = partnerNames.GetValueOrDefault(p.Key), p.Total, p.Failed, + }), + SlowestAdapters = byAdapter.Where(a => a.Calls >= 5).OrderByDescending(a => a.P95Ms).Take(5).Select(a => new + { + AdapterId = a.Key, a.Calls, a.P95Ms, a.FailureRate, + }), + Silent = health.Where(h => h.Status == "silent").OrderByDescending(h => h.QuietHours).Select(h => new + { + h.SubscriptionId, h.Name, h.Type, h.LastExchangeHour, h.QuietHours, h.UsualGapHours, + }), + Failing = health.Count(h => h.Status == "failing"), + Degraded = health.Count(h => h.Status == "degraded"), + Backlog = backlog.Select(s => new + { + At = MetricsQueries.Utc(s.At), s.QueueDepth, s.RetryBacklog, s.DeadLetterBacklog, + }), + }; + } + + async Task Total(DateTime from, DateTime to) => + (await MetricsQueries.Exchanges(db, from, to).GroupBy(_ => 1).SumBy()).FirstOrDefault() ?? new ExchangeTotals(); + + static object Summary(ExchangeTotals t) => new + { + t.Total, t.Succeeded, t.BadResponse, t.Failed, t.Retries, t.ManualRetries, t.SuccessRate, t.P50Ms, t.P95Ms, + Bytes = t.BytesIn + t.BytesOut, + }; +} diff --git a/SW.Bitween.Api/Resources/Metrics/Partners.cs b/SW.Bitween.Api/Resources/Metrics/Partners.cs new file mode 100644 index 00000000..b1337a76 --- /dev/null +++ b/SW.Bitween.Api/Resources/Metrics/Partners.cs @@ -0,0 +1,59 @@ +using System.Collections.Generic; +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Caching.Memory; +using SW.Bitween.Domain; +using SW.Bitween.Model; +using SW.Bitween.Services.Metrics; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Metrics; + +/// Every partner's traffic over the window, when it was last seen, and when each of its keys was last used. +[HandlerName("partners")] +public class Partners(BitweenDbContext db, RequestContext requestContext, IMemoryCache cache) + : IQueryHandler +{ + public async Task Handle(MetricsWindowRequest request) + { + await requestContext.EnsurePermission(db, Model.Permissions.Partners.View); + var days = MetricsQueries.Days(request?.Days); + return await cache.Cached($"partners.{days}", () => Compute(days)); + } + + async Task Compute(int days) + { + var to = MetricsRecorder.HourOf(DateTime.UtcNow).AddHours(1); + var from = to.AddDays(-days); + var totals = (await MetricsQueries.Exchanges(db, from, to).Where(s => s.PartnerId > 0).GroupBy(s => s.PartnerId).SumBy()) + .ToDictionary(t => t.Key); + var last = await MetricsQueries.Exchanges(db, to.AddDays(-30), to).Where(s => s.PartnerId > 0 && s.Total > 0) + .GroupBy(s => s.PartnerId).Select(g => new { g.Key, Last = g.Max(x => x.Hour) }) + .ToDictionaryAsync(x => x.Key, x => MetricsQueries.Utc(x.Last)); + var keys = await db.Set().AsNoTracking() + .GroupBy(k => k.PartnerId).Select(g => new { g.Key, Last = g.Max(k => k.LastUsedOn) }) + .ToDictionaryAsync(x => x.Key, x => MetricsQueries.Utc(x.Last)); + var partners = await db.Set().AsNoTracking().Where(p => p.Id != Partner.SystemId) + .Select(p => new { p.Id, p.Name }).ToListAsync(); + + return new + { + Since = await MetricsQueries.SinceAsync(db), + Days = days, + Partners = partners.Select(p => + { + var t = totals.GetValueOrDefault(p.Id); + return new + { + PartnerId = p.Id, p.Name, + Exchanges = t?.Total ?? 0, Failed = t?.Failed ?? 0, BadResponse = t?.BadResponse ?? 0, + SuccessRate = t?.SuccessRate, Bytes = (t?.BytesIn ?? 0) + (t?.BytesOut ?? 0), + LastExchangeHour = last.TryGetValue(p.Id, out var l) ? l : (DateTime?)null, + KeyLastUsedOn = keys.TryGetValue(p.Id, out var k) ? k : (DateTime?)null, + }; + }).OrderByDescending(p => p.Exchanges), + }; + } +} diff --git a/SW.Bitween.Api/Resources/Metrics/SubscriptionDetail.cs b/SW.Bitween.Api/Resources/Metrics/SubscriptionDetail.cs new file mode 100644 index 00000000..498d80ac --- /dev/null +++ b/SW.Bitween.Api/Resources/Metrics/SubscriptionDetail.cs @@ -0,0 +1,78 @@ +using System.Collections.Generic; +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Caching.Memory; +using SW.Bitween.Domain.Metrics; +using SW.Bitween.Model; +using SW.Bitween.Services.Metrics; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Metrics; + +/// +/// One subscription's numbers for its page: its health, traffic by hour over two days and by day +/// over the window, where failures came from, retries, nodes, and its most common errors. +/// +[HandlerName("subscription")] +public class SubscriptionDetail(BitweenDbContext db, RequestContext requestContext, IMemoryCache cache) + : IQueryHandler +{ + public async Task Handle(SubscriptionMetricsRequest request) + { + await requestContext.EnsurePermission(db, Model.Permissions.Subscriptions.View); + var days = MetricsQueries.Days(request?.Days); + var id = request?.Id ?? 0; + return await cache.Cached($"subscription.{id}.{days}", () => Compute(id, days)); + } + + async Task Compute(int id, int days) + { + var now = DateTime.UtcNow; + var to = MetricsRecorder.HourOf(now).AddHours(1); + var from = to.AddDays(-days); + var rows = MetricsQueries.Exchanges(db, from, to).Where(s => s.SubscriptionId == id); + + var total = (await rows.GroupBy(_ => 1).SumBy()).FirstOrDefault() ?? new ExchangeTotals(); + var hourly = await MetricsQueries.Exchanges(db, to.AddHours(-48), to).Where(s => s.SubscriptionId == id) + .GroupBy(s => s.Hour).SumBy(); + var daily = await rows.GroupBy(s => s.Hour.Date).SumBy(); + var nodes = await rows.GroupBy(s => s.Node).SumBy(); + var health = (await SubscriptionHealth.ComputeAsync(db, days, now)).FirstOrDefault(h => h.SubscriptionId == id); + + var key = id.ToString(); + var errorRows = db.Set().AsNoTracking() + .Where(e => e.Scope == ErrorStats.SubscriptionScope && e.ScopeKey == key && e.Hour >= from && e.Hour < to); + var errors = await errorRows.GroupBy(e => e.Fingerprint) + .Select(g => new { Fingerprint = g.Key, Count = g.Sum(x => (long)x.Count), LastSeenOn = g.Max(x => x.LastSeenOn) }) + .OrderByDescending(e => e.Count).Take(10).ToListAsync(); + var fingerprints = errors.Select(e => e.Fingerprint).ToList(); + var samples = (await errorRows.Where(e => fingerprints.Contains(e.Fingerprint)) + .Select(e => new { e.Fingerprint, e.Sample, e.LastSeenOn }).ToListAsync()) + .GroupBy(e => e.Fingerprint).ToDictionary(g => g.Key, g => g.OrderByDescending(e => e.LastSeenOn).First().Sample); + + return new + { + Id = id, + Days = days, + Since = await MetricsQueries.SinceAsync(db), + Status = health?.Status, + health?.LastExchangeHour, + health?.QuietHours, + health?.UsualGapHours, + Total = new + { + total.Total, total.Succeeded, total.BadResponse, total.Failed, total.FailedAtMapper, total.FailedAtHandler, + total.Retries, total.ManualRetries, total.SuccessRate, total.P50Ms, total.P95Ms, Bytes = total.BytesIn + total.BytesOut, + }, + Hourly = hourly.OrderBy(h => h.Key).Select(h => new { Hour = MetricsQueries.Utc(h.Key), h.Total, h.Failed, h.BadResponse }), + Daily = daily.OrderBy(d => d.Key).Select(d => new { Day = MetricsQueries.Utc(d.Key), d.Total, d.Failed, d.P95Ms }), + Nodes = nodes.OrderByDescending(n => n.Total).Select(n => new { Node = n.Key, n.Total, n.Failed }), + Errors = errors.Select(e => new + { + e.Fingerprint, e.Count, LastSeenOn = MetricsQueries.Utc(e.LastSeenOn), Sample = samples.GetValueOrDefault(e.Fingerprint), + }), + }; + } +} diff --git a/SW.Bitween.Api/Resources/Metrics/Subscriptions.cs b/SW.Bitween.Api/Resources/Metrics/Subscriptions.cs new file mode 100644 index 00000000..2ab1c8bc --- /dev/null +++ b/SW.Bitween.Api/Resources/Metrics/Subscriptions.cs @@ -0,0 +1,38 @@ +using System.Collections.Generic; +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Caching.Memory; +using SW.Bitween.Model; +using SW.Bitween.Services.Metrics; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Metrics; + +/// Every subscription's health over the window: status, success, timing, retries, traffic by day. +[HandlerName("subscriptions")] +public class Subscriptions(BitweenDbContext db, RequestContext requestContext, IMemoryCache cache) + : IQueryHandler +{ + public async Task Handle(MetricsWindowRequest request) + { + await requestContext.EnsurePermission(db, Model.Permissions.Subscriptions.View); + var days = MetricsQueries.Days(request?.Days); + return await cache.Cached($"subscriptions.{days}", async () => + { + var rows = await SubscriptionHealth.ComputeAsync(db, days, DateTime.UtcNow); + return (object)new + { + Since = await MetricsQueries.SinceAsync(db), + Days = days, + Subscriptions = rows.Select(r => new + { + r.SubscriptionId, r.Name, r.Type, r.Status, r.SuccessRate, r.P95Ms, r.LastExchangeHour, + r.QuietHours, r.UsualGapHours, r.Daily, + r.Totals.Total, r.Totals.Failed, r.Totals.BadResponse, r.Totals.Retries, r.Totals.ManualRetries, + Bytes = r.Totals.BytesIn + r.Totals.BytesOut, + }), + }; + }); + } +} diff --git a/SW.Bitween.Api/Resources/Mirror/Mirror.cs b/SW.Bitween.Api/Resources/Mirror/Mirror.cs new file mode 100644 index 00000000..d0861a27 --- /dev/null +++ b/SW.Bitween.Api/Resources/Mirror/Mirror.cs @@ -0,0 +1,330 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Services.Mirror; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Mirror; + +/// Whether this Bitween is a mirror and how it reaches prod, without calling prod: what pages ask before showing anything of prod's. +public class Get(BitweenDbContext dbContext, RequestContext requestContext, BitweenOptions options) : IQueryHandler +{ + public async Task Handle() + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.View); + var m = options.Mirror; + var signIn = !string.IsNullOrWhiteSpace(m.SourceToken) ? "token" + : !string.IsNullOrWhiteSpace(m.SourceEmail) && !string.IsNullOrEmpty(m.SourcePassword) ? "member" + : null; + return new + { + m.Target, + m.SourceUrl, + SignIn = signIn, + Configured = !string.IsNullOrWhiteSpace(m.SourceUrl) && signIn != null, + }; + } +} + +/// Why the mirror's work can't start while other work runs. +static class MirrorBusy +{ + /// Null when nothing else runs, or the work under way is the same work, which then carries on. + public static string Why(MirrorProgress running, string kind, string subject) => + running == null || (running.Kind == kind && running.Subject == subject) ? null + : $"{MirrorWork.Describe(running)} is under way. Start this once it finishes."; +} + +public class MirrorSubscriptionRequest +{ + /// The subscription, by its id on this staging. + public int Id { get; set; } + + /// Its name, which is what prod's is found by: one this staging doesn't have is refused. + internal static async Task NameAsync(BitweenDbContext dbContext, MirrorSubscriptionRequest request) => + await dbContext.Subscriptions().AsNoTracking().Where(s => s.Id == request.Id).Select(s => s.Name).FirstOrDefaultAsync() + ?? throw new SWNotFoundException($"There's no subscription {request?.Id} on this staging."); +} + +/// +/// One of this staging's subscriptions beside prod's of the same name: it and each item it needs to run +/// as on prod, the same, faked, different field by field or not on staging; what bringing it would do; and +/// what only a person can do first. Prod is only read; one that can't be read answers { error }. +/// +[HandlerName("subscription")] +public class Subscription(BitweenDbContext dbContext, RequestContext requestContext, MirrorService mirror) + : IQueryHandler +{ + public async Task Handle(MirrorSubscriptionRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.View); + var name = await MirrorSubscriptionRequest.NameAsync(dbContext, request); + try + { + return await mirror.SubscriptionAsync(name); + } + catch (MirrorRefusal refusal) + { + return new { Error = refusal.Message }; + } + } +} + +/// +/// A subscription's test with prod's traffic: prod's latest arrivals it takes part in, how they last ran +/// on staging, and how work for it under way is going. Followed by the page while it runs. +/// +[HandlerName("test")] +public class Test(BitweenDbContext dbContext, RequestContext requestContext, MirrorService mirror, MirrorWork work) + : IQueryHandler +{ + public async Task Handle(MirrorSubscriptionRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.View); + return await mirror.TestAsync(await MirrorSubscriptionRequest.NameAsync(dbContext, request), work); + } +} + +/// +/// Starts taking prod's latest arrivals a subscription takes part in, faked, in the background; they +/// replace the ones taken before. Only on a Bitween its deployment marks as a mirror, and not while other +/// work of the mirror runs. +/// +[HandlerName("testtake")] +public class TestTake(BitweenDbContext dbContext, RequestContext requestContext, BitweenOptions options, MirrorService mirror, MirrorWork work) + : ICommandHandler +{ + public async Task Handle(MirrorSubscriptionRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.Operate); + if (!options.Mirror.Target) throw new SWValidationException("MIRROR_NOT_TARGET", MirrorService.NotTarget); + var name = await MirrorSubscriptionRequest.NameAsync(dbContext, request); + if (MirrorBusy.Why(work.Start(MirrorWork.Sampling, (m, progress, ct) => m.TakeTestAsync(name, progress, ct), name), MirrorWork.Sampling, name) is { } busy) + throw new SWValidationException("MIRROR_BUSY", busy); + return await mirror.TestAsync(name, work); + } +} + +/// +/// Starts a subscription's test in the background: each of prod's arrivals sent in on staging the way it +/// came in on prod, and every exchange it led to compared with its twin. Only on a Bitween its deployment +/// marks as a mirror, and not while other work of the mirror runs. +/// +[HandlerName("testrun")] +public class TestRun(BitweenDbContext dbContext, RequestContext requestContext, BitweenOptions options, MirrorService mirror, MirrorWork work) + : ICommandHandler +{ + public async Task Handle(MirrorSubscriptionRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.Operate); + if (!options.Mirror.Target) throw new SWValidationException("MIRROR_NOT_TARGET", MirrorService.NotTarget); + var name = await MirrorSubscriptionRequest.NameAsync(dbContext, request); + var token = mirror.CallerToken(); + if (MirrorBusy.Why(work.Start(MirrorWork.Rehearsing, (m, progress, ct) => m.RunTestAsync(name, progress, token, ct), name), MirrorWork.Rehearsing, name) is { } busy) + throw new SWValidationException("MIRROR_BUSY", busy); + return await mirror.TestAsync(name, work); + } +} + +/// +/// Removes what a subscription's test made on staging: every exchange its runs started, with their files, +/// and how they compared. Prod's arrivals stay, to run again. Not while the test runs. +/// +[HandlerName("testclear")] +public class TestClear(BitweenDbContext dbContext, RequestContext requestContext, BitweenOptions options, MirrorService mirror, MirrorWork work) + : ICommandHandler +{ + public async Task Handle(MirrorSubscriptionRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.Operate); + if (!options.Mirror.Target) throw new SWValidationException("MIRROR_NOT_TARGET", MirrorService.NotTarget); + var name = await MirrorSubscriptionRequest.NameAsync(dbContext, request); + if (work.Running?.Subject == name) + throw new SWValidationException("MIRROR_BUSY", "The test is running. Clear what it made once it finishes."); + await mirror.ClearTestAsync(name); + return await mirror.TestAsync(name, work); + } +} + +/// +/// How many of prod's items there are of each kind a page lists - subscriptions by type, API and bus +/// gateways - so each page offers to bring only what prod has. Read from prod's lists alone. +/// +[HandlerName("prodkinds")] +public class ProdKinds(BitweenDbContext dbContext, RequestContext requestContext, MirrorService mirror) : IQueryHandler +{ + public async Task Handle() + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.View); + return await mirror.ProdKindsAsync(); + } +} + +public class MirrorProdRequest +{ + /// subscriptions, apiGateways or busGateways. + public string Section { get; set; } + + /// Of subscriptions, only those of this type: Receiving, Aggregation or Response, say. + public string Type { get; set; } +} + +/// +/// Prod's items of one kind, busiest first, each with its traffic over the last week and whether staging +/// has it: what a page offers to bring. Prod is only read. +/// +[HandlerName("prod")] +public class Prod(BitweenDbContext dbContext, RequestContext requestContext, MirrorService mirror) : IQueryHandler +{ + public async Task Handle(MirrorProdRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.View); + try + { + return await mirror.ProdListAsync(request?.Section, string.IsNullOrWhiteSpace(request?.Type) ? null : request.Type); + } + catch (MirrorRefusal refusal) + { + return new { Error = refusal.Message }; + } + } +} + +public class MirrorBringRequest +{ + /// subscriptions, apiGateways or busGateways. + public string Section { get; set; } + + /// The picked items, by the name prod has them under (a gateway's address for an API gateway). + public List Keys { get; set; } + + /// Bring them; without it, only say what bringing them would do. + public bool Apply { get; set; } +} + +/// +/// Brings prod's picked items onto this staging with what they need to run as on prod, faked: what staging +/// doesn't have comes as prod has it, what it has stays as it is, and nothing is removed. Without +/// it says what it would do. Only on a Bitween its deployment marks +/// as a mirror. +/// +[HandlerName("bring")] +public class Bring(BitweenDbContext dbContext, RequestContext requestContext, BitweenOptions options, MirrorService mirror) + : ICommandHandler +{ + public async Task Handle(MirrorBringRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.Operate); + if (!options.Mirror.Target) throw new SWValidationException("MIRROR_NOT_TARGET", MirrorService.NotTarget); + try + { + return await mirror.BringAsync(request?.Section, request?.Keys ?? [], request?.Apply ?? false); + } + catch (MirrorRefusal refusal) + { + throw new SWValidationException("MIRROR", refusal.Message); + } + } +} + +public class MirrorListedRequest +{ + /// subscriptions, apiGateways or busGateways. + public string Section { get; set; } +} + +/// +/// This staging's items of one kind that prod has too, by name, each with how its test last ran, and the +/// tests under way and waiting their turn: what a list shows beside its rows. Prod is only read, from its +/// list; one that can't be read answers with why. +/// +[HandlerName("listed")] +public class Listed(BitweenDbContext dbContext, RequestContext requestContext, MirrorService mirror, MirrorWork work) + : IQueryHandler +{ + public async Task Handle(MirrorListedRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.View); + return await mirror.ListedAsync(request?.Section) with { Testing = work.Running?.Subject, Waiting = work.Waiting }; + } +} + +public class MirrorTestManyRequest +{ + /// subscriptions, apiGateways or busGateways. + public string Section { get; set; } + + /// The items, by their ids on this staging. A gateway's test is that of the subscriptions it leads to. + public List Ids { get; set; } +} + +/// +/// Starts testing several of this staging's items with prod's traffic, one subscription after another in +/// the background: for each, prod's latest arrivals are taken unless some are, then sent in. Only those +/// prod has too. Only on a Bitween its deployment marks as a mirror, and not while other work of the +/// mirror runs. +/// +[HandlerName("testmany")] +public class TestMany(BitweenDbContext dbContext, RequestContext requestContext, BitweenOptions options, MirrorService mirror, MirrorWork work) + : ICommandHandler +{ + public async Task Handle(MirrorTestManyRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.Operate); + if (!options.Mirror.Target) throw new SWValidationException("MIRROR_NOT_TARGET", MirrorService.NotTarget); + IReadOnlyList names; + try + { + names = await mirror.SubscriptionsOfAsync(request?.Section, request?.Ids ?? []); + } + catch (MirrorRefusal refusal) + { + throw new SWValidationException("MIRROR", refusal.Message); + } + if (names.Count == 0) + throw new SWValidationException("MIRROR_NONE", "None of these is, or leads to, a subscription prod has too, so there's nothing of prod's to test with."); + if (work.Start(await mirror.TestJobsAsync(names, mirror.CallerToken())) is { } running) + throw new SWValidationException("MIRROR_BUSY", $"{MirrorWork.Describe(running)} is under way. Start this once it finishes."); + return new { Subscriptions = names }; + } +} + +public class MirrorFilledInRequest +{ + /// The item's kind: subscriptions. + public string Section { get; set; } + + /// The item, by its name. + public string Key { get; set; } +} + +/// +/// Says a person has put staging's own value in each secret a bring had to save as a placeholder on the +/// item: Bitween hides a saved secret, so staging can't see it change. Its to-dos close and its tests run. +/// +[HandlerName("filledin")] +public class FilledIn(BitweenDbContext dbContext, RequestContext requestContext, MirrorService mirror) + : ICommandHandler +{ + public async Task Handle(MirrorFilledInRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.Operate); + return new { Closed = await mirror.FilledInAsync(request?.Section, request?.Key) }; + } +} + +public class MirrorTestStopRequest; + +/// Drops the tests waiting their turn; the one under way finishes. How many were dropped. +[HandlerName("teststop")] +public class TestStop(BitweenDbContext dbContext, RequestContext requestContext, MirrorWork work) : ICommandHandler +{ + public async Task Handle(MirrorTestStopRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.Operate); + var waiting = work.Waiting.Count; + work.Stop(); + return new { Dropped = waiting }; + } +} diff --git a/SW.Bitween.Api/Resources/NotificationChannels/ChannelValidation.cs b/SW.Bitween.Api/Resources/NotificationChannels/ChannelValidation.cs new file mode 100644 index 00000000..c19a28c1 --- /dev/null +++ b/SW.Bitween.Api/Resources/NotificationChannels/ChannelValidation.cs @@ -0,0 +1,117 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain; +using SW.Bitween.Model; +using SW.Bitween.NativeAdapters.SlackMessageHandler; +using SW.Bitween.NativeAdapters.SmtpHandler; +using SW.Bitween.NativeAdapters.TeamsMessageHandler; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.NotificationChannels; + +internal static class ChannelValidation +{ + public static async Task EnsureNameIsFree(BitweenDbContext dbContext, string name, int? exceptId = null) + { + if (await dbContext.Set() + .AnyAsync(c => c.Name == name && (exceptId == null || c.Id != exceptId))) + throw new SWValidationException("NOTIFICATION_CHANNEL_NAME_TAKEN", + $"A notification channel called '{name}' already exists."); + } + + /// + /// Refuses mail settings that would send the password over an unencrypted connection. + /// + /// + /// The SMTP adapter refuses this when it sends, which is the guarantee that matters. Checking + /// here means whoever sets the channel up finds out when they save, rather than from a missing + /// notification days later. + /// + public static void EnsureTransportIsSecure(string handlerId, IReadOnlyDictionary properties) + { + if (properties == null || properties.Count == 0) return; + if (!nameof(NativeSmtpHandler).Equals(handlerId, StringComparison.OrdinalIgnoreCase)) return; + + // A hidden password counts as set: the sentinel means one is stored. Only an explicit + // "false" turns encryption off; absent means the adapter's default, which is on. + var password = Value(properties, nameof(SmtpHandlerInput.Password)); + var useTls = Value(properties, nameof(SmtpHandlerInput.UseTls)); + + if (string.IsNullOrWhiteSpace(password)) return; + if (!bool.TryParse(useTls, out var encrypted) || encrypted) return; + + throw new SWValidationException("CHANNEL_PASSWORD_WITHOUT_TLS", + "This channel would send its mail password over an unencrypted connection. " + + "Turn UseTls on, or clear the password if the server does not need one."); + } + + /// + /// Refuses a Teams or Slack webhook that is not a full https address. Both services only hand + /// out https ones, and anything else would carry the message, and the address that lets anyone + /// post to the channel, in the clear. + /// + /// + /// A reference to a global value is let through, as is the mask of a stored address: the value + /// it stands for is checked when it is sent. + /// + public static void EnsureWebhookIsHttps(string handlerId, IReadOnlyDictionary properties) + { + if (!nameof(NativeTeamsMessageHandler).Equals(handlerId, StringComparison.OrdinalIgnoreCase) && + !nameof(NativeSlackMessageHandler).Equals(handlerId, StringComparison.OrdinalIgnoreCase)) + return; + + var url = Value(properties ?? new Dictionary(), nameof(TeamsMessageInput.WebhookUrl)); + if (string.IsNullOrWhiteSpace(url) || url == AdapterSecretProperties.Sentinel || url.Contains("{{")) return; + + if (!Uri.TryCreate(url.Trim(), UriKind.Absolute, out var uri) || uri.Scheme != Uri.UriSchemeHttps) + throw new SWValidationException("CHANNEL_WEBHOOK_NOT_HTTPS", + "The webhook address has to be a full https:// address, copied from Teams or Slack."); + } + + /// + /// Refuses a channel missing a setting its adapter can't send without, so it fails when it is + /// saved, not when it is needed. + /// + /// Checked on what will be stored, so a dropped secret counts as missing. + public static async Task EnsureRequiredPresent(AdapterRequirements requirements, string handlerId, + IReadOnlyDictionary properties) + { + var missing = await requirements.MissingFor(handlerId, + (properties ?? new Dictionary()) + .Select(kv => new KeyAndValue { Key = kv.Key, Value = kv.Value }).ToList()); + + if (missing.Count > 0) + throw new SWValidationException("CHANNEL_SETTINGS_MISSING", + $"This channel can't send without {string.Join(", ", missing)}."); + } + + /// + /// Refuses a subscription's notification rows that could never send: one with no outcome + /// ticked, a channel listed twice, or a channel that doesn't exist. + /// + public static async Task EnsureNotificationsCanSend(BitweenDbContext dbContext, + ICollection notifications) + { + if (notifications.Any(n => !n.OnFailure && !n.OnBadResult && !n.OnSuccess && !n.OnQuiet)) + throw new SWValidationException("NOTIFICATION_NO_OUTCOME", + "A notification has nothing ticked to send on. Tick at least one outcome, or remove it."); + + if (notifications.GroupBy(n => n.ChannelId).Any(g => g.Count() > 1)) + throw new SWValidationException("NOTIFICATION_CHANNEL_REPEATED", + "The same channel is listed twice. Tick every outcome it should send on in one row."); + + var ids = notifications.Select(n => n.ChannelId).ToList(); + var found = await dbContext.Set().Where(c => ids.Contains(c.Id)) + .Select(c => c.Id).ToListAsync(); + var missing = ids.Except(found).ToList(); + if (missing.Count > 0) + throw new SWValidationException("NOTIFICATION_CHANNEL_NOT_FOUND", + $"Notification channel {string.Join(", ", missing)} was not found."); + } + + private static string Value(IReadOnlyDictionary properties, string key) => + properties.FirstOrDefault(kv => kv.Key.Equals(key, StringComparison.OrdinalIgnoreCase)).Value; +} diff --git a/SW.Bitween.Api/Resources/NotificationChannels/Create.cs b/SW.Bitween.Api/Resources/NotificationChannels/Create.cs new file mode 100644 index 00000000..c9778541 --- /dev/null +++ b/SW.Bitween.Api/Resources/NotificationChannels/Create.cs @@ -0,0 +1,46 @@ +using System.Threading.Tasks; +using FluentValidation; +using SW.Bitween.Domain; +using SW.Bitween.Model; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.NotificationChannels; + +public class Create(BitweenDbContext dbContext, RequestContext requestContext, IInfolinkCache cache, + AdapterRequirements adapterRequirements) + : ICommandHandler +{ + public async Task Handle(NotificationChannelCreate request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Notifiers.Create); + await ChannelValidation.EnsureNameIsFree(dbContext, request.Name); + ChannelValidation.EnsureTransportIsSecure(request.HandlerId, request.HandlerProperties); + ChannelValidation.EnsureWebhookIsHttps(request.HandlerId, request.HandlerProperties); + + // A new channel has nothing stored behind a sentinel, so one that arrives (from a channel + // copied out of Get, say) is dropped rather than saved as the literal password. + var properties = AdapterSecretProperties.Merge(null, request.HandlerProperties) ?? []; + await ChannelValidation.EnsureRequiredPresent(adapterRequirements, request.HandlerId, properties); + + var channel = new NotificationChannel + { + Name = request.Name, + HandlerId = request.HandlerId, + HandlerProperties = properties + }; + + dbContext.Add(channel); + await dbContext.SaveChangesAsync(); + await cache.BroadcastRevoke(); + return channel.Id; + } + + private class Validate : AbstractValidator + { + public Validate() + { + RuleFor(c => c.Name).NotEmpty().MaximumLength(100); + RuleFor(c => c.HandlerId).NotEmpty(); + } + } +} diff --git a/SW.Bitween.Api/Resources/NotificationChannels/Delete.cs b/SW.Bitween.Api/Resources/NotificationChannels/Delete.cs new file mode 100644 index 00000000..f74ff092 --- /dev/null +++ b/SW.Bitween.Api/Resources/NotificationChannels/Delete.cs @@ -0,0 +1,34 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using SW.Bitween.Domain; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.NotificationChannels; + +public class Delete(BitweenDbContext dbContext, RequestContext requestContext, IInfolinkCache cache) + : IDeleteHandler +{ + /// + /// Refused while anything sends through the channel. Deleting it anyway would leave those places + /// set to notify and quietly sending nothing, which is worse than being told to move them first. + /// + public async Task Handle(int key) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Notifiers.Delete); + + var channel = await dbContext.FindAsync(key) + ?? throw new SWNotFoundException(key.ToString()); + + var uses = (await NotificationChannelUsage.Find(dbContext)).GetValueOrDefault(key); + if (uses is { Count: > 0 }) + throw new SWValidationException("NOTIFICATION_CHANNEL_IN_USE", + $"This channel is still used by {string.Join(", ", uses.Select(u => u.Name))}. " + + "Point them at another channel first."); + + dbContext.Remove(channel); + await dbContext.SaveChangesAsync(); + await cache.BroadcastRevoke(); + return null; + } +} diff --git a/SW.Bitween.Api/Resources/NotificationChannels/Get.cs b/SW.Bitween.Api/Resources/NotificationChannels/Get.cs new file mode 100644 index 00000000..f1e93210 --- /dev/null +++ b/SW.Bitween.Api/Resources/NotificationChannels/Get.cs @@ -0,0 +1,31 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain; +using SW.Bitween.Model; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.NotificationChannels; + +public class Get(BitweenDbContext dbContext, RequestContext requestContext, AdapterSecretProperties secrets) + : IGetHandler +{ + public async Task Handle(int key) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Notifiers.View); + + var channel = await dbContext.Set().AsNoTracking() + .FirstOrDefaultAsync(c => c.Id == key) + ?? throw new SWNotFoundException(key.ToString()); + + return new NotificationChannelGet + { + Id = channel.Id, + Name = channel.Name, + HandlerId = channel.HandlerId, + // Secrets go out as the sentinel. Update reads it as "keep what is stored". + HandlerProperties = await secrets.Mask(channel.HandlerId, channel.HandlerProperties) ?? [], + UsedBy = (await NotificationChannelUsage.Find(dbContext)).GetValueOrDefault(channel.Id) ?? [] + }; + } +} diff --git a/SW.Bitween.Api/Resources/NotificationChannels/Search.cs b/SW.Bitween.Api/Resources/NotificationChannels/Search.cs new file mode 100644 index 00000000..b2fa8a77 --- /dev/null +++ b/SW.Bitween.Api/Resources/NotificationChannels/Search.cs @@ -0,0 +1,41 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain; +using SW.Bitween.Model; +using SW.EfCoreExtensions; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.NotificationChannels; + +public class Search(BitweenDbContext dbContext, RequestContext requestContext) : ISearchyHandler +{ + public async Task Handle(SearchyRequest searchyRequest, bool lookup = false, string searchPhrase = null) + { + var query = dbContext.Set().AsNoTracking() + .Select(c => new NotificationChannelRow { Id = c.Id, Name = c.Name, HandlerId = c.HandlerId }); + + // Lookup returns only id/name pairs: every page that picks a channel needs the names, + // whoever is editing it. The settings themselves are what the view permission covers. + if (lookup) + return await query.Search(searchyRequest.Conditions) + .ToDictionaryAsync(k => k.Id.ToString(), v => v.Name); + + await requestContext.EnsurePermission(dbContext, Model.Permissions.Notifiers.View); + + var rows = await query.OrderBy(c => c.Name) + .Search(searchyRequest.Conditions, searchyRequest.Sorts, searchyRequest.PageSize, searchyRequest.PageIndex) + .ToListAsync(); + + var uses = await NotificationChannelUsage.Find(dbContext); + foreach (var row in rows) + row.UsedBy = uses.GetValueOrDefault(row.Id) ?? []; + + return new SearchyResponse + { + TotalCount = await query.Search(searchyRequest.Conditions).CountAsync(), + Result = rows + }; + } +} diff --git a/SW.Bitween.Api/Resources/NotificationChannels/Test.cs b/SW.Bitween.Api/Resources/NotificationChannels/Test.cs new file mode 100644 index 00000000..1b49bbb2 --- /dev/null +++ b/SW.Bitween.Api/Resources/NotificationChannels/Test.cs @@ -0,0 +1,32 @@ +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain; +using SW.Bitween.Model; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.NotificationChannels; + +/// +/// Sends a test message through a channel as it is saved, and says whether it went. +/// +/// +/// Edit rather than view permission: it sends a real message to real people, which is something +/// done to the channel, not a look at it. +/// +[HandlerName("test")] +public class Test(BitweenDbContext dbContext, RequestContext requestContext, NotificationChannelSender sender) + : ICommandHandler +{ + public async Task Handle(int key, NotificationChannelTest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Notifiers.Edit); + + var channel = await dbContext.Set().AsNoTracking() + .FirstOrDefaultAsync(c => c.Id == key) + ?? throw new SWNotFoundException(key.ToString()); + + var error = await sender.SendTest(channel); + // The reason alone, not a stack trace: it is shown to whoever pressed the button. + return new NotificationChannelTestResult { Sent = error == null, Error = error?.GetBaseException().Message }; + } +} diff --git a/SW.Bitween.Api/Resources/NotificationChannels/Update.cs b/SW.Bitween.Api/Resources/NotificationChannels/Update.cs new file mode 100644 index 00000000..0e7d34b6 --- /dev/null +++ b/SW.Bitween.Api/Resources/NotificationChannels/Update.cs @@ -0,0 +1,51 @@ +using System.Threading.Tasks; +using FluentValidation; +using SW.Bitween.Domain; +using SW.Bitween.Model; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.NotificationChannels; + +public class Update(BitweenDbContext dbContext, RequestContext requestContext, IInfolinkCache cache, + AdapterRequirements adapterRequirements) + : ICommandHandler +{ + public async Task Handle(int key, NotificationChannelUpdate request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Notifiers.Edit); + + var channel = await dbContext.FindAsync(key) + ?? throw new SWNotFoundException(key.ToString()); + + await ChannelValidation.EnsureNameIsFree(dbContext, request.Name, key); + ChannelValidation.EnsureTransportIsSecure(request.HandlerId, request.HandlerProperties); + ChannelValidation.EnsureWebhookIsHttps(request.HandlerId, request.HandlerProperties); + + // A hidden secret is put back only while the adapter, and where it connects, stay the same. + // Otherwise someone who can edit but not see the password could point the channel at their + // own server and receive it. Re-entering it proves they know it. + var incoming = request.HandlerProperties ?? []; + var keep = AdapterSecretProperties.MayKeepStoredSecrets(channel.HandlerId, request.HandlerId, + channel.HandlerProperties, incoming); + var properties = AdapterSecretProperties.Merge(channel.HandlerProperties, incoming, keep) ?? []; + + await ChannelValidation.EnsureRequiredPresent(adapterRequirements, request.HandlerId, properties); + + channel.Name = request.Name; + channel.HandlerId = request.HandlerId; + channel.HandlerProperties = properties; + + await dbContext.SaveChangesAsync(); + await cache.BroadcastRevoke(); + return null; + } + + private class Validate : AbstractValidator + { + public Validate() + { + RuleFor(c => c.Name).NotEmpty().MaximumLength(100); + RuleFor(c => c.HandlerId).NotEmpty(); + } + } +} diff --git a/SW.Bitween.Api/Resources/Notifications/Search.cs b/SW.Bitween.Api/Resources/Notifications/Search.cs index 70cc0c09..8316ecac 100644 --- a/SW.Bitween.Api/Resources/Notifications/Search.cs +++ b/SW.Bitween.Api/Resources/Notifications/Search.cs @@ -25,18 +25,37 @@ public async Task Handle(SearchyRequest searchyRequest, bool lookup = fa { Id = notification.Id, Success = notification.Success, + Suppressed = notification.Suppressed, Exception = notification.Exception, FinishedOn = notification.FinishedOn, NotifierName = notification.NotifierName, - XchangeId = notification.XchangeId + XchangeId = notification.XchangeId, + ChannelId = notification.ChannelId }; query = query.AsNoTracking(); + var page = await query.OrderByDescending(p => p.FinishedOn) + .Search(searchyRequest.Conditions, searchyRequest.Sorts, searchyRequest.PageSize, searchyRequest.PageIndex) + .ToListAsync(); + + // Which subscription each send was about, for this page alone: joined into the query, it + // was paid for on every row the count went through. A send can outlive its exchange, + // and still happened, so one without stays in the list unnamed. + var xchangeIds = page.Select(p => p.XchangeId).Where(id => id != null).Distinct().ToList(); + var about = await (from xchange in dbContext.Set().AsNoTracking() + where xchangeIds.Contains(xchange.Id) + join subscription in dbContext.Set() on xchange.SubscriptionId equals (int?)subscription.Id + select new { xchange.Id, SubscriptionId = subscription.Id, subscription.Name }) + .ToDictionaryAsync(x => x.Id); + foreach (var row in page) + if (row.XchangeId != null && about.TryGetValue(row.XchangeId, out var subscription)) + (row.SubscriptionId, row.SubscriptionName) = (subscription.SubscriptionId, subscription.Name); + return new SearchyResponse { TotalCount = await query.Search(searchyRequest.Conditions).CountAsync(), - Result = await query.OrderByDescending(p => p.FinishedOn).Search(searchyRequest.Conditions, searchyRequest.Sorts, searchyRequest.PageSize, searchyRequest.PageIndex).ToListAsync() + Result = page }; } } diff --git a/SW.Bitween.Api/Resources/Notifiers/Delete.cs b/SW.Bitween.Api/Resources/Notifiers/Delete.cs index d92cbcb8..e258a227 100644 --- a/SW.Bitween.Api/Resources/Notifiers/Delete.cs +++ b/SW.Bitween.Api/Resources/Notifiers/Delete.cs @@ -10,7 +10,7 @@ public class Delete(BitweenDbContext dbContext, RequestContext requestContext, I { /// /// No reference check, unlike an integration's delete: nothing has a foreign key to a - /// notifier. RunOnSubscriptions points the other way — the notifier names the + /// notifier. RunOnSubscriptions points the other way - the notifier names the /// integrations it watches, so deleting it takes the whole list with it. /// public async Task Handle(int key) diff --git a/SW.Bitween.Api/Resources/Notifiers/Get.cs b/SW.Bitween.Api/Resources/Notifiers/Get.cs index 211ee898..fd9eb8cf 100644 --- a/SW.Bitween.Api/Resources/Notifiers/Get.cs +++ b/SW.Bitween.Api/Resources/Notifiers/Get.cs @@ -8,7 +8,8 @@ namespace SW.Bitween.Resources.Notifiers { - public class Get(BitweenDbContext dbContext, RequestContext requestContext) : IGetHandler + public class Get(BitweenDbContext dbContext, RequestContext requestContext, AdapterSecretProperties secrets) + : IGetHandler { private readonly BitweenDbContext dbContext = dbContext; private readonly RequestContext requestContext = requestContext; @@ -27,15 +28,20 @@ public async Task Handle(int key) .Where(s => notifier.RunOnSubscriptions.Any(sub => sub == s.Id)) .ToListAsync(); } - - + + // The handler's [Secure] values - an SMTP password, an API key - go out as the + // sentinel, as they do for a subscription's handler. Update puts them back. + var handlerProperties = notifier.HandlerProperties is null + ? null + : await secrets.Mask(notifier.HandlerId, notifier.HandlerProperties); + return new { Id = notifier.Id, Name = notifier.Name, Inactive = notifier.Inactive, HandlerId = notifier.HandlerId, - HandlerProperties= notifier.HandlerProperties?.ToKeyAndValueCollection(), + HandlerProperties= handlerProperties?.ToKeyAndValueCollection(), RunOnSuccessfulResult= notifier.RunOnSuccessfulResult, RunOnBadResult = notifier.RunOnBadResult, RunOnFailedResult = notifier.RunOnFailedResult, diff --git a/SW.Bitween.Api/Resources/Notifiers/Update.cs b/SW.Bitween.Api/Resources/Notifiers/Update.cs index ebb9f904..0759da19 100644 --- a/SW.Bitween.Api/Resources/Notifiers/Update.cs +++ b/SW.Bitween.Api/Resources/Notifiers/Update.cs @@ -16,16 +16,27 @@ public async Task Handle(int key, NotifierUpdate request) var notifier = await dbContext.FindAsync(key); + // A sentinel means "keep the stored secret", which only makes sense for the handler it + // was stored for. Switching handlers drops it, so the secret can't follow the notifier + // to a handler that might send it somewhere else. + var newHandlerId = request.HandlerId ?? notifier.HandlerId; + var incoming = (request.HandlerProperties ?? []).ToDictionary(); + var stored = AdapterSecretProperties.MayKeepStoredSecrets(notifier.HandlerId, newHandlerId, + notifier.HandlerProperties, incoming) + ? notifier.HandlerProperties + : null; + notifier.Update(request.Name, request.RunOnSuccessfulResult, request.RunOnBadResult, request.RunOnFailedResult, - request.HandlerId ?? notifier.HandlerId, + newHandlerId, request.Inactive, request.RunOnSubscriptions?.Select(r => r.Id)?.ToArray()); // An absent list means none, as it does for a document's promoted properties // and a retry policy's groups. Left implicit it threw ArgumentNullException. - notifier.SetDictionaries((request.HandlerProperties ?? []).ToDictionary()); + notifier.SetDictionaries( + AdapterSecretProperties.Merge(stored, incoming)); await dbContext.SaveChangesAsync(); await cache.BroadcastRevoke(); diff --git a/SW.Bitween.Api/Resources/Ops/Background.cs b/SW.Bitween.Api/Resources/Ops/Background.cs new file mode 100644 index 00000000..9bd2ca29 --- /dev/null +++ b/SW.Bitween.Api/Resources/Ops/Background.cs @@ -0,0 +1,96 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Quartz; +using Quartz.Impl.Matchers; +using SW.Bitween.Domain; +using SW.PrimitiveTypes; +using SW.Scheduler; + +namespace SW.Bitween.Resources.Ops; + +/// +/// Bitween's own background work: the system jobs (retries, retention, clean-ups) with when each +/// last ran and runs next, and the outbox of messages waiting for the broker. Neither was visible: +/// a job that stopped firing, or an outbox backing up, showed only as its consequences. +/// +[HandlerName("background")] +public class Background(BitweenDbContext dbContext, RequestContext requestContext, + IScheduleRepository scheduleRepo, ISchedulerFactory schedulerFactory) : IQueryHandler +{ + static readonly (Type Type, string Name, string Does)[] SystemJobs = + [ + (typeof(RetryJob), "Automatic retries", "Re-runs failed exchanges whose retry policy says they are due."), + (typeof(ExchangeRetentionJob), "Exchange retention", "Deletes or archives exchanges older than they are kept."), + (typeof(ReceiveAttemptCleanupJob), "Receive attempt clean-up", "Deletes old records of scheduled jobs' receive attempts, and audit entries past Bitween:AuditRetentionDays when that is set."), + (typeof(Services.DataSources.InboundMessagePruneJob), "Broker deduplication clean-up", "Forgets deduplication keys older than each data source keeps them."), + (typeof(Services.Metrics.QueueSnapshotJob), "Queue snapshots", "Records the broker's queue totals every few minutes, for the backlog trend."), + (typeof(Services.Metrics.MetricsPruneJob), "Metrics clean-up", "Deletes hourly metrics and queue snapshots past their keeping."), + (typeof(QuietSubscriptionsJob), "Quiet subscriptions", "Tells the notifiers when a subscription that expects traffic gets none, and when it comes back."), + ]; + + public async Task Handle() + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Monitoring.View, Model.Permissions.Dashboard.View); + + var scheduler = await schedulerFactory.GetScheduler(); + var running = (await scheduler.GetCurrentlyExecutingJobs()).Select(j => j.JobDetail.JobType).ToHashSet(); + var definitions = scheduleRepo.GetJobDefinitions().ToDictionary(d => d.JobType); + + var jobs = new List(); + foreach (var (type, name, does) in SystemJobs) + { + DateTime? next = null, previous = null; + string cron = null; + var states = new List(); + if (definitions.TryGetValue(type, out var definition)) + foreach (var key in await scheduler.GetJobKeys(GroupMatcher.GroupEquals(definition.Group))) + foreach (var trigger in await scheduler.GetTriggersOfJob(key)) + { + states.Add((await scheduler.GetTriggerState(trigger.Key)).ToString()); + var n = trigger.GetNextFireTimeUtc()?.UtcDateTime; + var p = trigger.GetPreviousFireTimeUtc()?.UtcDateTime; + if (n != null && (next == null || n < next)) next = n; + if (p != null && (previous == null || p > previous)) previous = p; + cron ??= (trigger as ICronTrigger)?.CronExpressionString; + } + + jobs.Add(new + { + Name = name, + Does = does, + Cron = cron, + Scheduled = states.Count > 0, + State = states.Count == 0 ? "NotScheduled" : states.Contains("Error") ? "Error" : states.Contains("Paused") ? "Paused" : "Normal", + LastRanOn = previous, + NextRunOn = next, + RunningNow = running.Contains(type), + }); + } + + var outbox = dbContext.Set().AsNoTracking(); + var pending = outbox.Where(m => m.PublishedOn == null); + var since = DateTime.UtcNow.AddHours(-1); + return new + { + Jobs = jobs, + Outbox = new + { + Pending = await pending.CountAsync(), + OldestPendingOn = await pending.OrderBy(m => m.CreatedOn).Select(m => (DateTime?)m.CreatedOn).FirstOrDefaultAsync(), + Failing = await pending.CountAsync(m => m.Attempts > 0), + LastError = await pending.Where(m => m.LastError != null).OrderByDescending(m => m.CreatedOn) + .Select(m => m.LastError).FirstOrDefaultAsync(), + // Each node counts what it published and says so on its heartbeat: published rows are + // deleted now. Rows still marked published are from nodes of an older version, during + // an upgrade, and are counted as before. + PublishedLastHour = await dbContext.Set().AsNoTracking() + .Where(n => n.LastSeenOn >= since) + .SumAsync(n => n.OutboxPublishedLastHour ?? 0) + + await outbox.CountAsync(m => m.PublishedOn >= since), + }, + }; + } +} diff --git a/SW.Bitween.Api/Resources/Ops/BrokerQueues.cs b/SW.Bitween.Api/Resources/Ops/BrokerQueues.cs index fd39b2eb..e6075512 100644 --- a/SW.Bitween.Api/Resources/Ops/BrokerQueues.cs +++ b/SW.Bitween.Api/Resources/Ops/BrokerQueues.cs @@ -15,7 +15,7 @@ namespace SW.Bitween.Resources.Ops; /// /// Bitween's queues as the broker has them, through the RabbitMQ management API: the ones -/// nothing reads any more, and deleting a lane — a main queue with its retry and bad queues. +/// nothing reads any more, and deleting a lane - a main queue with its retry and bad queues. /// public class BrokerQueues(IBusDashboardDataService dashboardDataService, BusOptions busOptions, diff --git a/SW.Bitween.Api/Resources/Ops/Consumers.cs b/SW.Bitween.Api/Resources/Ops/Consumers.cs index 3e2b84ea..46b57d03 100644 --- a/SW.Bitween.Api/Resources/Ops/Consumers.cs +++ b/SW.Bitween.Api/Resources/Ops/Consumers.cs @@ -10,6 +10,8 @@ namespace SW.Bitween.Resources.Ops; /// The bus's consumer health, with each row told what it is for. Without the lane and the /// ids a caller can only see the queue's machine name, and would have to rebuild /// for itself to work out which group a row is. +/// Concurrency is how many of the lane's messages run at once on a node: its parallel +/// exchanges, capped by its prefetch. /// public record ConsumerLaneView( string Name, @@ -30,7 +32,8 @@ public record ConsumerLaneView( double ProcessingRate, double AckRate, bool IsBackpressured, - AlertSeverity HealthStatus); + AlertSeverity HealthStatus, + ushort Concurrency); [HandlerName("Consumers")] public class Consumers(IBusDashboardDataService dashboardDataService, @@ -66,7 +69,11 @@ public async Task Handle() c.ProcessingRate, c.AckRate, c.IsBackpressured, - c.HealthStatus); + c.HealthStatus, + // Prefetch 0 is no limit at the broker. + c.Prefetch == 0 + ? laneResolver.Concurrency(identity) + : System.Math.Min(laneResolver.Concurrency(identity), c.Prefetch)); }).ToArray(); } } diff --git a/SW.Bitween.Api/Resources/Ops/DeadLetterMessages.cs b/SW.Bitween.Api/Resources/Ops/DeadLetterMessages.cs new file mode 100644 index 00000000..33d24c6e --- /dev/null +++ b/SW.Bitween.Api/Resources/Ops/DeadLetterMessages.cs @@ -0,0 +1,31 @@ +using System.Threading.Tasks; +using SW.Bitween.Domain; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Ops; + +public class DeadLetterMessagesQuery +{ + /// A dead-letter queue, as /ops/deadletters names it. + public string Queue { get; set; } + public int Count { get; set; } = 20; +} + +/// +/// What is sitting in one dead-letter queue. The bodies are the messages themselves - a partner's +/// payload, often - so seeing them takes the right to see exchanges too. +/// +[HandlerName("DeadLetterMessages")] +public class DeadLetterMessages(DeadLetterQueues deadLetters, BitweenDbContext dbContext, RequestContext requestContext) + : IQueryHandler +{ + public async Task Handle(DeadLetterMessagesQuery request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Monitoring.View); + await requestContext.EnsurePermission(dbContext, Model.Permissions.Exchanges.View); + + var queue = await deadLetters.Resolve(request.Queue) + ?? throw new SWNotFoundException("Dead-letter queue"); + return await deadLetters.Browse(queue, request.Count); + } +} diff --git a/SW.Bitween.Api/Resources/Ops/DeadLetterQueues.cs b/SW.Bitween.Api/Resources/Ops/DeadLetterQueues.cs new file mode 100644 index 00000000..36c17149 --- /dev/null +++ b/SW.Bitween.Api/Resources/Ops/DeadLetterQueues.cs @@ -0,0 +1,135 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using RabbitMQ.Client; +using RabbitMQ.Client.Exceptions; +using SW.Bus.RabbitMqExtensions; + +namespace SW.Bitween.Resources.Ops; + +/// One message in a dead-letter queue, as the browse shows it. +public record DeadLetterMessageView( + int Position, + string Body, + bool BodyCut, + string LastException, + string CorrelationId, + string RoutingKey, + IReadOnlyList ExceptionHistory); + +/// +/// The messages the bus gave up on - each consumer's .bad queue - read without taking them +/// off, and moved back onto the consumer's own queue to be tried again. +/// +/// +/// A queue name is only ever matched against the bus's own dead-letter list, never handed to the +/// broker as given, so neither call can be pointed at a queue Bitween doesn't own. +/// +public class DeadLetterQueues( + IBusDashboardDataService dashboard, + IErrorQueueReader errorQueueReader, + IConfiguration configuration, + ILogger logger) +{ + public const int MaxBrowse = 50; + public const int MaxRequeue = 1000; + private const int MaxBodyChars = 64 * 1024; + + /// The queue as the bus names it, or null when it isn't one of its dead-letter queues. + public async Task Resolve(string queueName) + { + if (string.IsNullOrWhiteSpace(queueName)) return null; + var known = await dashboard.GetDeadLetterSummaryAsync(); + return known.Select(d => d.DeadLetterQueueName) + .FirstOrDefault(q => string.Equals(q, queueName.Trim(), StringComparison.OrdinalIgnoreCase)); + } + + /// The first messages, oldest first, left where they are. + public async Task> Browse(string queue, int count) + { + var messages = await errorQueueReader.PeekByQueueName(queue, Math.Clamp(count, 1, MaxBrowse)); + return messages.Select((m, i) => + { + var body = m.RawBody ?? ""; + return new DeadLetterMessageView( + i + 1, + body.Length > MaxBodyChars ? body[..MaxBodyChars] : body, + body.Length > MaxBodyChars, + m.LastException?.ToString(), + m.CorrelationId, + m.RoutingKey, + (m.ExceptionHistory ?? []).Select(e => e?.ToString()).ToList()); + }).ToList(); + } + + /// + /// Moves up to messages, oldest first, back onto the queue they failed + /// on, with a fresh retry count. Each one leaves the dead-letter queue only once the broker has + /// confirmed it is on the main queue, so a failure part-way loses nothing. + /// + /// How many were moved. + public async Task RequeueAsync(string queue, int count) + { + var main = Regex.Replace(queue, @"\.bad$", "", RegexOptions.IgnoreCase); + var connectionString = configuration.GetConnectionString("RabbitMQ") + ?? throw new InvalidOperationException("No RabbitMQ connection string is configured."); + var factory = new ConnectionFactory + { + Uri = new Uri(connectionString), + AutomaticRecoveryEnabled = false, + RequestedConnectionTimeout = TimeSpan.FromSeconds(15) + }; + + await using var connection = await factory.CreateConnectionAsync("bitween-dead-letter-requeue"); + // With confirmations tracked, a publish completes only once the broker has the message, + // and throws when it can't route it. + await using var channel = await connection.CreateChannelAsync( + new CreateChannelOptions(publisherConfirmationsEnabled: true, publisherConfirmationTrackingEnabled: true)); + // Published straight to the queue: through its exchange, every other consumer of the same + // message would get it again too. And a queue that isn't there would swallow it. + await channel.QueueDeclarePassiveAsync(main); + + var moved = 0; + while (moved < Math.Clamp(count, 1, MaxRequeue)) + { + var message = await channel.BasicGetAsync(queue, autoAck: false); + if (message is null) break; + + var properties = new BasicProperties(message.BasicProperties); + if (properties.Headers is { } headers) + properties.Headers = headers.Where(h => !IsRetryState(h.Key)) + .ToDictionary(h => h.Key, h => h.Value); + + try + { + using var confirmed = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + await channel.BasicPublishAsync("", main, mandatory: true, properties, message.Body, confirmed.Token); + } + catch (PublishException) + { + // Returned or refused: it stays where it was. + await channel.BasicNackAsync(message.DeliveryTag, multiple: false, requeue: true); + break; + } + + await channel.BasicAckAsync(message.DeliveryTag, multiple: false); + moved++; + } + + logger.LogInformation("Moved {Count} dead-lettered messages from {Queue} back to {Main}", moved, queue, main); + return moved; + } + + /// What the bus counts retries and records failures with: left on, the message would fail straight back. + private static bool IsRetryState(string header) => + header.StartsWith("x-death", StringComparison.OrdinalIgnoreCase) + || header.StartsWith("x-first-death", StringComparison.OrdinalIgnoreCase) + || header.StartsWith("x-last-death", StringComparison.OrdinalIgnoreCase) + || Regex.IsMatch(header, @"^exception\d*$", RegexOptions.IgnoreCase); +} diff --git a/SW.Bitween.Api/Resources/Ops/DeleteUnattendedQueues.cs b/SW.Bitween.Api/Resources/Ops/DeleteUnattendedQueues.cs index bb8e1482..3596ae8c 100644 --- a/SW.Bitween.Api/Resources/Ops/DeleteUnattendedQueues.cs +++ b/SW.Bitween.Api/Resources/Ops/DeleteUnattendedQueues.cs @@ -19,7 +19,7 @@ public record UnattendedQueueSkip(string QueueName, string Reason); public record DeleteUnattendedQueuesResult(List Deleted, List Skipped); /// -/// Deletes unattended lanes — each one's main queue and whichever of retry/bad exist — along with +/// Deletes unattended lanes - each one's main queue and whichever of retry/bad exist - along with /// whatever they hold. Names are only ever matched against a fresh unattended list, never passed to /// the broker as given, so this can't be pointed at a queue something still uses. A lane that can't /// go is skipped with the reason rather than failing the rest. @@ -71,6 +71,9 @@ public async Task Handle(DeleteUnattendedQueuesModel request) } } + if (deleted.Count > 0) + await dbContext.RecordNowAsync(requestContext, OperatorAction.DeleteQueues, string.Join(", ", deleted), + $"{deleted.Count} deleted, {skipped.Count} skipped"); return new DeleteUnattendedQueuesResult(deleted, skipped); } } diff --git a/SW.Bitween.Api/Resources/Ops/LaneResolver.cs b/SW.Bitween.Api/Resources/Ops/LaneResolver.cs index 11bcab58..e37b08f0 100644 --- a/SW.Bitween.Api/Resources/Ops/LaneResolver.cs +++ b/SW.Bitween.Api/Resources/Ops/LaneResolver.cs @@ -9,7 +9,7 @@ namespace SW.Bitween.Resources.Ops; /// /// What a consumed queue is for. The bus names queues after the consumer class and the -/// bus message type — xchangeservice.48low-result — and only Bitween knows that +/// bus message type - xchangeservice.48low-result - and only Bitween knows that /// 48low is work group 48. Resolving it here keeps the answer next to /// , which is the formula's owner; the UI used /// to rebuild that formula in TypeScript and match on the string. @@ -25,7 +25,7 @@ public enum QueueLane /// A work group's -Result lane. Evaluates notifiers. Notifications, - /// A legacy event type, consumed only while that setting is on. + /// A legacy event type, consumed only while its queue still holds something. Legacy, /// Bookkeeping. No integration traffic. @@ -42,7 +42,7 @@ public record LaneIdentity(QueueLane Lane, string Title, int? WorkGroupId, int? /// Decodes the message-type half of a queue name into the thing it belongs to. One database /// read for all rows, so callers should resolve a whole snapshot at once. /// -public class LaneResolver(BitweenDbContext dbContext) +public class LaneResolver(BitweenDbContext dbContext, BitweenOptions options) { /// 's non-lane message types. private static readonly Dictionary Control = new(System.StringComparer.OrdinalIgnoreCase) @@ -50,7 +50,7 @@ public class LaneResolver(BitweenDbContext dbContext) ["SubscriptionUnpausedEvent"] = "Resumed integrations", }; - /// Only declared while Bitween.ConsumeLegacyEventMessages is on. + /// Only read while their queues still hold something; see . private static readonly Dictionary Legacy = new(System.StringComparer.OrdinalIgnoreCase) { ["ApiXchangeCreatedEvent"] = "API exchange created", @@ -79,7 +79,7 @@ public LaneIdentity Resolve(string consumerName, string messageName) string.Equals(d.BusMessageTypeName, messageName, System.StringComparison.OrdinalIgnoreCase)); // A bus message only gets a queue while its information type is bus-enabled, so a - // name that resolves to nothing means the queue outlived it — and queues are never + // name that resolves to nothing means the queue outlived it - and queues are never // deleted. Reported as-is rather than hidden. return new LaneIdentity(QueueLane.FrontDoor, document?.Name ?? messageName, null, document?.Id); } @@ -100,7 +100,7 @@ public LaneIdentity Resolve(string consumerName, string messageName) ? messageName[..^XchangeService.ResultQueueSuffix.Length] : messageName; - // WorkGroup.None — id 0, so it can never collide with a real group. This is the lane + // WorkGroup.None - id 0, so it can never collide with a real group. This is the lane // everything without a work group shares. if (string.Equals(groupName, WorkGroup.None.GetBusMessageName(), System.StringComparison.OrdinalIgnoreCase)) return new LaneIdentity(lane, "Ungrouped", null, null); @@ -110,4 +110,20 @@ public LaneIdentity Resolve(string consumerName, string messageName) return new LaneIdentity(lane, workGroup?.Name ?? messageName, workGroup?.Id, null); } + + /// + /// The parallel exchanges a queue was attached with, by the rule + /// applies: a work lane its group's, + /// a notifications lane 1, everything else the default. Not capped by prefetch. + /// + public ushort Concurrency(LaneIdentity identity) + { + if (identity.Lane == QueueLane.Notifications) return 1; + if (identity.Lane == QueueLane.Work && identity.WorkGroupId is { } id) + { + var workGroup = workGroups.FirstOrDefault(wg => wg.Id == id); + if (workGroup != null) return workGroup.WorkLaneConcurrency(options.BusDefaultQueuePrefetch ?? 12); + } + return options.ResolvedBusDefaultConcurrency; + } } diff --git a/SW.Bitween.Api/Resources/Ops/RequeueDeadLetters.cs b/SW.Bitween.Api/Resources/Ops/RequeueDeadLetters.cs new file mode 100644 index 00000000..0ad8e57f --- /dev/null +++ b/SW.Bitween.Api/Resources/Ops/RequeueDeadLetters.cs @@ -0,0 +1,45 @@ +using System.Threading.Tasks; +using RabbitMQ.Client.Exceptions; +using SW.Bitween.Domain; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Ops; + +public class RequeueDeadLettersModel +{ + /// A dead-letter queue, as /ops/deadletters names it. + public string Queue { get; set; } + + /// How many, oldest first. Defaults to all of them, up to . + public int? Count { get; set; } +} + +public record RequeueDeadLettersResult(int Requeued); + +/// Sends dead-lettered messages back to the consumer that gave up on them, to be tried again. +[HandlerName("RequeueDeadLetters")] +public class RequeueDeadLetters(DeadLetterQueues deadLetters, BitweenDbContext dbContext, RequestContext requestContext) + : ICommandHandler +{ + public async Task Handle(RequeueDeadLettersModel request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Monitoring.Operate); + + var queue = await deadLetters.Resolve(request.Queue) + ?? throw new SWNotFoundException("Dead-letter queue"); + int requeued; + try + { + requeued = await deadLetters.RequeueAsync(queue, request.Count ?? DeadLetterQueues.MaxRequeue); + } + catch (OperationInterruptedException) + { + throw new SWValidationException("NO_MAIN_QUEUE", + "The queue these messages failed on is gone, so there is nowhere to send them back to."); + } + + await dbContext.RecordNowAsync(requestContext, OperatorAction.RequeueDeadLetters, request.Queue, + $"{requeued} message{(requeued == 1 ? "" : "s")} sent back"); + return new RequeueDeadLettersResult(requeued); + } +} diff --git a/SW.Bitween.Api/Resources/Ops/UnattendedQueues.cs b/SW.Bitween.Api/Resources/Ops/UnattendedQueues.cs index dc6f7c84..02d1eb87 100644 --- a/SW.Bitween.Api/Resources/Ops/UnattendedQueues.cs +++ b/SW.Bitween.Api/Resources/Ops/UnattendedQueues.cs @@ -14,8 +14,8 @@ namespace SW.Bitween.Resources.Ops; /// How many queues this lane is (main plus whichever of retry/bad exist). /// /// Consumers the broker reports across the lane's queues, from any instance. Above zero means -/// something still reads it — a listener that outlived its work group, or another instance -/// running different code during a deploy — so it isn't safe to delete. +/// something still reads it - a listener that outlived its work group, or another instance +/// running different code during a deploy - so it isn't safe to delete. /// /// /// Set when the lane is the queue of an information type paused on the bus: the type keeps its @@ -36,7 +36,7 @@ public record UnattendedQueueView( /// Every other Ops endpoint derives its list from the consumer definitions of the running /// process, so it can only ever show queues this instance already knows about. A paused /// information type's queue is one of these; so is anything left behind before deletes and -/// renames took their queues with them, or recreated empty by a client reconnecting — and they +/// renames took their queues with them, or recreated empty by a client reconnecting - and they /// vanish from those endpoints while keeping whatever they still hold. This is the only view /// that asks the broker instead of asking ourselves. /// diff --git a/SW.Bitween.Api/Resources/Partners/Delete.cs b/SW.Bitween.Api/Resources/Partners/Delete.cs index 21e48b6f..34a91448 100644 --- a/SW.Bitween.Api/Resources/Partners/Delete.cs +++ b/SW.Bitween.Api/Resources/Partners/Delete.cs @@ -28,7 +28,7 @@ public async Task Handle(int key) /// Says what still points at the partner, before the database says it less politely. /// /// - /// All three references are RESTRICT, so the delete was already refused — but as a + /// All three references are RESTRICT, so the delete was already refused - but as a /// foreign key violation surfacing as a bare 500, which names a constraint instead of the /// integration holding it and reads like a broken screen rather than a decision. /// Exchanges are deliberately not checked: they carry a partner id with no foreign key @@ -74,7 +74,7 @@ private async Task EnsureNothingPointsAtIt(int key) /// /// How many holders the message names before it stops listing them. A partner carrying - /// forty integrations would otherwise read out all forty into a dialog — longer to + /// forty integrations would otherwise read out all forty into a dialog - longer to /// understand than the short version, and forty rows fetched to build it. /// private const int MaxNamed = 5; diff --git a/SW.Bitween.Api/Resources/Partners/Get.cs b/SW.Bitween.Api/Resources/Partners/Get.cs index 3fdf9acb..d9486334 100644 --- a/SW.Bitween.Api/Resources/Partners/Get.cs +++ b/SW.Bitween.Api/Resources/Partners/Get.cs @@ -8,6 +8,13 @@ namespace SW.Bitween.Resources.Partners { + /// A partner as its page reads it: the update model, and when each key was last used. + public class PartnerDetail : PartnerUpdate + { + /// By key name. A key missing here hasn't had a call through since this was recorded. + public System.Collections.Generic.Dictionary KeysLastUsedOn { get; set; } = []; + } + public class Get(BitweenDbContext dbContext, RequestContext requestContext) : IGetHandler { private readonly BitweenDbContext dbContext = dbContext; @@ -19,14 +26,16 @@ async public Task Handle(int key) var partner = await dbContext.Set().AsNoTracking(). Search("Id", key). - Select(partner => new PartnerUpdate + Select(partner => new PartnerDetail { Name = partner.Name, ApiCredentials = partner.ApiCredentials.Select(cred => new KeyAndValue { Key = cred.Name, - Value = $"{cred.Key.Remove(5)}...(hidden)" + // Keys are stored hashed, so the prefix kept beside the hash is all there is + // to show. A short key used to throw here, too: Remove(5) on fewer than five. + Value = (cred.KeyPrefix ?? "") + "...(hidden)" }).ToList(), Subscriptions = partner.Subscriptions.Select(sub => new SubscriptionSearch @@ -46,8 +55,13 @@ async public Task Handle(int key) // The names come back so the form can draw the locks; the values behind them do not. if (partner != null) + { partner.AdapterProperties = AdapterSecretProperties.Mask(partner.AdapterProperties, partner.SecretProperties); + partner.KeysLastUsedOn = await dbContext.Set().AsNoTracking() + .Where(u => u.PartnerId == key) + .ToDictionaryAsync(u => u.KeyName, u => u.LastUsedOn); + } return partner; } diff --git a/SW.Bitween.Api/Resources/Partners/Search.cs b/SW.Bitween.Api/Resources/Partners/Search.cs index 8a83e5ee..2d1266ed 100644 --- a/SW.Bitween.Api/Resources/Partners/Search.cs +++ b/SW.Bitween.Api/Resources/Partners/Search.cs @@ -29,7 +29,7 @@ async public Task Handle(SearchyRequest searchyRequest, bool lookup = fa Name = subscriber.Name, // Every place the partner is wired in, matching the three groups the // detail page lists. Counting only the direct subscription link left - // "Used by" reading "—" for partners plainly in use through a gateway. + // "Used by" reading "-" for partners plainly in use through a gateway. SubscriptionsCount = subscriber.Subscriptions.Count + dbContext.Set().Count(g => g.PartnerId == subscriber.Id) + dbContext.Set().Count(r => r.PartnerId == subscriber.Id), diff --git a/SW.Bitween.Api/Resources/Partners/Update.cs b/SW.Bitween.Api/Resources/Partners/Update.cs index 9b50e20e..99594af1 100644 --- a/SW.Bitween.Api/Resources/Partners/Update.cs +++ b/SW.Bitween.Api/Resources/Partners/Update.cs @@ -17,7 +17,8 @@ public async Task Handle(int key, PartnerUpdate model) { await requestContext.EnsurePermission(dbContext, Model.Permissions.Partners.Edit); - var entity = await dbContext.FindAsync(key); + var entity = await dbContext.FindAsync(key) + ?? throw new SWNotFoundException(key.ToString()); entity.SetApiCredentials(model.ApiCredentials.Select(kv => new ApiCredential(kv.Key, kv.Value))); // Read while they are still there: SetProperties copies the model's own @@ -30,7 +31,9 @@ public async Task Handle(int key, PartnerUpdate model) // would overwrite its password with a row of dots. After SetProperties, which would // otherwise put the mask straight back. entity.AdapterProperties = - AdapterSecretProperties.Merge(storedProperties, model.AdapterProperties); + AdapterSecretProperties.Merge(storedProperties, model.AdapterProperties, + AdapterSecretProperties.MayKeepStoredSecrets(null, null, storedProperties, + model.AdapterProperties ?? new Dictionary())); entity.SecretProperties = model.SecretProperties?.ToList() ?? []; entity.LoginIdentity = await PartnerLoginIdentity.Validate(dbContext, model.LoginIdentity, key); await dbContext.SaveChangesAsync(); diff --git a/SW.Bitween.Api/Resources/Permissions/Get.cs b/SW.Bitween.Api/Resources/Permissions/Get.cs index da2652f4..fb8a2202 100644 --- a/SW.Bitween.Api/Resources/Permissions/Get.cs +++ b/SW.Bitween.Api/Resources/Permissions/Get.cs @@ -5,7 +5,7 @@ namespace SW.Bitween.Resources.Permissions; /// -/// The permission catalog. Static data — the management UI builds its role matrix from this, so +/// The permission catalog. Static data - the management UI builds its role matrix from this, so /// the grants it offers can't drift from what the handlers actually enforce. /// public class Get : IQueryHandler diff --git a/SW.Bitween.Api/Resources/Retention/Aggregation.cs b/SW.Bitween.Api/Resources/Retention/Aggregation.cs index 1b207945..b48fcb6a 100644 --- a/SW.Bitween.Api/Resources/Retention/Aggregation.cs +++ b/SW.Bitween.Api/Resources/Retention/Aggregation.cs @@ -8,7 +8,7 @@ namespace SW.Bitween.Resources.Retention; /// -/// Whether retention would remove exchanges before an aggregation with these schedules collects them — +/// Whether retention would remove exchanges before an aggregation with these schedules collects them - /// shown on the aggregation as its schedule is edited, before it's saved. Changes nothing. /// [HandlerName("aggregation")] diff --git a/SW.Bitween.Api/Resources/Retention/Preview.cs b/SW.Bitween.Api/Resources/Retention/Preview.cs index 9da8e9b4..e5a4eba3 100644 --- a/SW.Bitween.Api/Resources/Retention/Preview.cs +++ b/SW.Bitween.Api/Resources/Retention/Preview.cs @@ -5,7 +5,7 @@ namespace SW.Bitween.Resources.Retention; /// -/// What retention settings someone is about to save would do, in the same shape as — +/// What retention settings someone is about to save would do, in the same shape as - /// shown before the change is confirmed. Changes nothing. /// [HandlerName("preview")] diff --git a/SW.Bitween.Api/Resources/RetryPolicies/Attempts.cs b/SW.Bitween.Api/Resources/RetryPolicies/Attempts.cs index ca9e76fc..aa7f0544 100644 --- a/SW.Bitween.Api/Resources/RetryPolicies/Attempts.cs +++ b/SW.Bitween.Api/Resources/RetryPolicies/Attempts.cs @@ -9,7 +9,7 @@ namespace SW.Bitween.Resources.RetryPolicies; /// -/// Lists the failures one group caught for one subscription — what a row of +/// Lists the failures one group caught for one subscription - what a row of /// spent its budget on. /// /// @@ -40,7 +40,7 @@ public async Task Handle(int key, RetryGroupAttemptsRequest request) // Both halves of the pair have to belong to the policy in the route. For the subscription // that keeps this from becoming a way to read any subscription's failures through any - // policy id; for the group it is about the answer being readable — an unknown group would + // policy id; for the group it is about the answer being readable - an unknown group would // otherwise report zero failures, which is indistinguishable from a group that genuinely // has none. var policy = await dbContext.Set().AsNoTracking() diff --git a/SW.Bitween.Api/Resources/RetryPolicies/Create.cs b/SW.Bitween.Api/Resources/RetryPolicies/Create.cs index 036a4d80..038544bd 100644 --- a/SW.Bitween.Api/Resources/RetryPolicies/Create.cs +++ b/SW.Bitween.Api/Resources/RetryPolicies/Create.cs @@ -12,20 +12,14 @@ public async Task Handle(RetryPolicyCreate model) { await requestContext.EnsurePermission(dbContext, Model.Permissions.RetryPolicies.Create); RetryGroupValidation.EnsureCanFire(model.Groups); - RetryGroupValidation.EnsureAlertTransportIsSecure( - model.AlertHandlerId, model.AlertHandlerProperties); - - // A new policy has nothing stored behind a sentinel, so any that arrives — from a policy - // copied out of Get, say — is dropped rather than saved as the literal password. - foreach (var group in model.Groups ?? []) - AdapterSecretProperties.MergeInPlace(null, group.AlertHandlerProperties); + await RetryGroupValidation.EnsureAlertChannelsExist(dbContext, + [model.AlertChannelId, ..RetryGroupValidation.AlertChannelIds(model.Groups)]); var entity = new RetryPolicy { Name = model.Name, Groups = model.Groups ?? [], - AlertHandlerId = model.AlertHandlerId, - AlertHandlerProperties = AdapterSecretProperties.Merge(null, model.AlertHandlerProperties) + AlertChannelId = model.AlertChannelId }; dbContext.Add(entity); await dbContext.SaveChangesAsync(); diff --git a/SW.Bitween.Api/Resources/RetryPolicies/Delete.cs b/SW.Bitween.Api/Resources/RetryPolicies/Delete.cs index 7b711359..71036f33 100644 --- a/SW.Bitween.Api/Resources/RetryPolicies/Delete.cs +++ b/SW.Bitween.Api/Resources/RetryPolicies/Delete.cs @@ -23,7 +23,7 @@ public async Task Handle(int key) var policy = await dbContext.FindAsync(key); var groupIds = policy.Groups.Select(g => g.Id).ToList(); - // One change, one commit — same reasoning as Update: a half-done delete leaves rows keyed + // One change, one commit - same reasoning as Update: a half-done delete leaves rows keyed // by groups that no longer exist anywhere, which nothing can then reach. await using var transaction = await dbContext.Database.BeginTransactionAsync(); diff --git a/SW.Bitween.Api/Resources/RetryPolicies/Get.cs b/SW.Bitween.Api/Resources/RetryPolicies/Get.cs index c0b4e915..39b04ce5 100644 --- a/SW.Bitween.Api/Resources/RetryPolicies/Get.cs +++ b/SW.Bitween.Api/Resources/RetryPolicies/Get.cs @@ -8,15 +8,13 @@ namespace SW.Bitween.Resources.RetryPolicies; -public class Get(BitweenDbContext dbContext, RequestContext requestContext, AdapterSecretProperties secrets) +public class Get(BitweenDbContext dbContext, RequestContext requestContext) : IGetHandler { public async Task Handle(int key) { await requestContext.EnsurePermission(dbContext, Model.Permissions.RetryPolicies.View); - // Materialize first: AlertHandlerProperties is a JSON-converted dictionary, and EF cannot - // translate a further .ToDictionary() over it into SQL inside a projection. var policy = await dbContext.Set() .AsNoTracking() .Search("Id", key) @@ -24,18 +22,11 @@ public async Task Handle(int key) if (policy == null) return null; - // Every level that can carry a handler can carry that handler's password, so every level is - // masked. Groups are edited in place by the caller, which is what Update then merges back. - foreach (var group in policy.Groups) - await secrets.MaskInPlace(group.AlertHandlerId, group.AlertHandlerProperties); - return new RetryPolicyUpdate { Name = policy.Name, Groups = policy.Groups, - AlertHandlerId = policy.AlertHandlerId, - AlertHandlerProperties = - await secrets.Mask(policy.AlertHandlerId, policy.AlertHandlerProperties) + AlertChannelId = policy.AlertChannelId }; } } diff --git a/SW.Bitween.Api/Resources/RetryPolicies/ResetUsage.cs b/SW.Bitween.Api/Resources/RetryPolicies/ResetUsage.cs index 10f46b15..5c75b9a9 100644 --- a/SW.Bitween.Api/Resources/RetryPolicies/ResetUsage.cs +++ b/SW.Bitween.Api/Resources/RetryPolicies/ResetUsage.cs @@ -44,6 +44,11 @@ public async Task Handle(int key, RetryPolicyResetUsage request) query = query.Where(u => u.GroupId == request.GroupId.Value); await query.ExecuteDeleteAsync(); + + // A budget put back by hand frees what was held for it, as a success would. + await DelayedRetry.ReleaseHeldAsync(dbContext, request.SubscriptionId is { } one + ? subscriptionIds.Where(id => id == one).ToList() + : subscriptionIds); return null; } } diff --git a/SW.Bitween.Api/Resources/RetryPolicies/RetryGroupValidation.cs b/SW.Bitween.Api/Resources/RetryPolicies/RetryGroupValidation.cs index 2586b33b..053c15ef 100644 --- a/SW.Bitween.Api/Resources/RetryPolicies/RetryGroupValidation.cs +++ b/SW.Bitween.Api/Resources/RetryPolicies/RetryGroupValidation.cs @@ -2,14 +2,16 @@ using System.Collections.Generic; using System.Linq; using SW.Bitween.Model; -using SW.Bitween.NativeAdapters.SmtpHandler; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain; using SW.PrimitiveTypes; namespace SW.Bitween.Resources.RetryPolicies; /// /// Rejects retry groups that could never fire. The evaluator skips such groups silently -/// (matchers must support the result type being evaluated — see +/// (matchers must support the result type being evaluated - see /// ), which reads as "retries just don't work", so the /// misconfiguration is caught at write time instead. /// @@ -35,7 +37,7 @@ public static void EnsureCanFire(IEnumerable groups) $"Group '{group.Name}' applies to {resultType} but none of its matchers can be " + $"evaluated against {resultType} content. {SupportedMatchersFor(resultType)}"); - // Allow with no budget has nothing to work from — no per-message cap, no total, no delay — + // Allow with no budget has nothing to work from - no per-message cap, no total, no delay - // so the evaluator can only refuse it. Caught here because a group saved that way silently // stops retrying, which reads as the whole feature being broken. if (group.Action == RetryAction.Allow && group.Budget == null) @@ -44,65 +46,60 @@ public static void EnsureCanFire(IEnumerable groups) "delay, or change the action to block."); // An overriding level replaces the one above it rather than merging into it, so a group - // set to Send with no handler would silence the policy's alert instead of redirecting it. - if (group.AlertMode == RetryAlertMode.Send && string.IsNullOrWhiteSpace(group.AlertHandlerId)) - throw new SWValidationException("RETRY_GROUP_ALERT_NO_HANDLER", - $"Group '{group.Name}' is set to send its own budget alert but has no handler. " + - "Choose a handler, or set the alert back to inherit."); + // set to Send with no channel would silence the policy's alert instead of redirecting it. + if (group.AlertMode == RetryAlertMode.Send && group.AlertChannelId == null) + throw new SWValidationException("RETRY_GROUP_ALERT_NO_CHANNEL", + $"Group '{group.Name}' is set to send its own budget alert but has no channel. " + + "Choose a notification channel, or set the alert back to inherit."); - EnsureAlertTransportIsSecure(group.AlertHandlerId, group.AlertHandlerProperties); + EnsureNoAlertHandler(group.AlertHandlerId); } } /// - /// Rejects an alert override that claims to send but names nothing to send with — the same trap + /// Rejects an alert override that claims to send but names nothing to send with - the same trap /// as guards at group level. /// - public static void EnsureAlertCanSend(RetryAlertMode mode, string handlerId) + public static void EnsureAlertCanSend(RetryAlertMode mode, int? channelId) { - if (mode == RetryAlertMode.Send && string.IsNullOrWhiteSpace(handlerId)) - throw new SWValidationException("RETRY_ALERT_NO_HANDLER", - "This override is set to send its own budget alert but has no handler. " + - "Choose a handler, or set it back to inherit."); + if (mode == RetryAlertMode.Send && channelId == null) + throw new SWValidationException("RETRY_ALERT_NO_CHANNEL", + "This override is set to send its own budget alert but has no channel. " + + "Choose a notification channel, or set it back to inherit."); } /// - /// Rejects mail alert settings that would hand the password to an unencrypted connection. + /// Refuses the handler a group used to carry. Alerts go through a channel now, and a handler + /// accepted here would be stored and never used. /// - /// - /// - /// The handler refuses this at send time too, which is the guarantee that matters — properties - /// can also arrive straight through the API or out of a global values set. Catching it here is - /// so the person configuring it finds out when they save, rather than from a missing alert and a - /// line in the log days later. - /// - /// - /// Only the mail handler is named, because only it has a password. A general answer belongs in - /// the adapter contract — an adapter saying which of its own settings conflict — not here. - /// - /// - public static void EnsureAlertTransportIsSecure( - string handlerId, IReadOnlyDictionary properties) + private static void EnsureNoAlertHandler(string handlerId) { - if (properties == null || properties.Count == 0) return; - if (!nameof(NativeSmtpHandler).Equals(handlerId, StringComparison.OrdinalIgnoreCase)) return; + if (!string.IsNullOrWhiteSpace(handlerId)) + throw new SWValidationException("RETRY_ALERT_HANDLER_RETIRED", + "Alerts are sent through notification channels now. Set alertChannelId instead of " + + "alertHandlerId."); + } - // A masked password counts as set: the sentinel means one is stored, not that the field is - // empty. Only an explicit "false" turns encryption off — absent means the adapter's own - // default, which is on. - var password = Value(properties, nameof(SmtpHandlerInput.Password)); - var useTls = Value(properties, nameof(SmtpHandlerInput.UseTls)); + /// Refuses a channel id that names no channel. + public static async Task EnsureAlertChannelsExist(BitweenDbContext dbContext, params int?[] channelIds) + { + var wanted = channelIds.Where(id => id != null).Select(id => id!.Value).Distinct().ToList(); + if (wanted.Count == 0) return; - if (string.IsNullOrWhiteSpace(password)) return; - if (!bool.TryParse(useTls, out var encrypted) || encrypted) return; + var found = await dbContext.Set() + .Where(c => wanted.Contains(c.Id)) + .Select(c => c.Id) + .ToListAsync(); - throw new SWValidationException("ALERT_PASSWORD_WITHOUT_TLS", - "This alert would send its mail password over an unencrypted connection. " + - "Turn UseTls on, or clear the password if the relay does not need one."); + var missing = wanted.Except(found).ToList(); + if (missing.Count > 0) + throw new SWValidationException("NOTIFICATION_CHANNEL_NOT_FOUND", + $"Notification channel {string.Join(", ", missing)} was not found."); } - private static string Value(IReadOnlyDictionary properties, string key) => - properties.FirstOrDefault(kv => kv.Key.Equals(key, StringComparison.OrdinalIgnoreCase)).Value; + /// Every channel a set of groups sends through. + public static int?[] AlertChannelIds(IEnumerable groups) => + (groups ?? []).Select(g => g.AlertChannelId).ToArray(); private static string SupportedMatchersFor(XchangeResultType resultType) => resultType switch { diff --git a/SW.Bitween.Api/Resources/RetryPolicies/SaveAlertOverride.cs b/SW.Bitween.Api/Resources/RetryPolicies/SaveAlertOverride.cs index ec15376f..9618f16a 100644 --- a/SW.Bitween.Api/Resources/RetryPolicies/SaveAlertOverride.cs +++ b/SW.Bitween.Api/Resources/RetryPolicies/SaveAlertOverride.cs @@ -1,4 +1,3 @@ -using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; @@ -10,7 +9,7 @@ namespace SW.Bitween.Resources.RetryPolicies; /// -/// Sets, changes or clears where one subscription's alerts go for one group of this policy — the most +/// Sets, changes or clears where one subscription's alerts go for one group of this policy - the most /// specific level of the hierarchy. /// [HandlerName("savealertoverride")] @@ -19,9 +18,8 @@ public class SaveAlertOverride(BitweenDbContext dbContext, RequestContext reques public async Task Handle(int key, RetryAlertOverrideSave request) { await requestContext.EnsurePermission(dbContext, Model.Permissions.RetryPolicies.Edit); - RetryGroupValidation.EnsureAlertCanSend(request.AlertMode, request.AlertHandlerId); - RetryGroupValidation.EnsureAlertTransportIsSecure( - request.AlertHandlerId, request.AlertHandlerProperties); + RetryGroupValidation.EnsureAlertCanSend(request.AlertMode, request.AlertChannelId); + await RetryGroupValidation.EnsureAlertChannelsExist(dbContext, request.AlertChannelId); var policy = await dbContext.Set().AsNoTracking() .FirstOrDefaultAsync(p => p.Id == key); @@ -44,7 +42,7 @@ public async Task Handle(int key, RetryAlertOverrideSave request) && o.GroupId == request.GroupId); // Inherit is the absence of an override, so store nothing rather than a row that does nothing - // — otherwise the routing list would have to explain a row that changes no behaviour. + // - otherwise the routing list would have to explain a row that changes no behaviour. if (request.AlertMode == RetryAlertMode.Inherit) { if (existing != null) dbContext.Remove(existing); @@ -52,21 +50,6 @@ public async Task Handle(int key, RetryAlertOverrideSave request) return null; } - // A masked secret has to be restored from whichever level the caller was shown it at. Usage - // masks two things for a pair: the override's own properties, and the properties of the level - // it currently inherits from. Overriding an inherited alert starts from the second — there is - // no override row yet — so both are offered here, with the override's own winning. - var group = policy.Groups.First(g => g.Id == request.GroupId); - var inherited = RetryAlertResolver.Resolve(existing, group, policy); - - var restoreFrom = new Dictionary(); - foreach (var kv in inherited?.HandlerProperties ?? new Dictionary()) - restoreFrom[kv.Key] = kv.Value; - foreach (var kv in existing?.AlertHandlerProperties ?? new Dictionary()) - restoreFrom[kv.Key] = kv.Value; - - var properties = AdapterSecretProperties.Merge(restoreFrom, request.AlertHandlerProperties); - if (existing == null) { dbContext.Add(new RetryAlertOverride @@ -74,15 +57,13 @@ public async Task Handle(int key, RetryAlertOverrideSave request) SubscriptionId = request.SubscriptionId, GroupId = request.GroupId, AlertMode = request.AlertMode, - AlertHandlerId = request.AlertHandlerId, - AlertHandlerProperties = properties + AlertChannelId = request.AlertChannelId }); } else { existing.AlertMode = request.AlertMode; - existing.AlertHandlerId = request.AlertHandlerId; - existing.AlertHandlerProperties = properties; + existing.AlertChannelId = request.AlertChannelId; } await dbContext.SaveChangesAsync(); diff --git a/SW.Bitween.Api/Resources/RetryPolicies/Test.cs b/SW.Bitween.Api/Resources/RetryPolicies/Test.cs index 3aae6b5c..a9705ffd 100644 --- a/SW.Bitween.Api/Resources/RetryPolicies/Test.cs +++ b/SW.Bitween.Api/Resources/RetryPolicies/Test.cs @@ -21,7 +21,7 @@ public async Task Handle(TestRetryPolicyRequest request) if (request.ResultType == XchangeResultType.Success) throw new SWValidationException("INVALID_RESULT_TYPE", - "Choose Error or Bad result — a successful result is never retried."); + "Choose Error or Bad result - a successful result is never retried."); var policy = new CustomRetryPolicy { Groups = request.Groups ?? [] }; // In-memory budget: a dry-run must not spend any real integration's total. @@ -44,7 +44,7 @@ public async Task Handle(TestRetryPolicyRequest request) // Once blocked, every later attempt for this same message would be blocked // for the same reason (budgets never un-consume, and a Block action or "no - // match" is structural) — no value in simulating further. + // match" is structural) - no value in simulating further. if (!decision.ShouldRetry) break; } diff --git a/SW.Bitween.Api/Resources/RetryPolicies/Update.cs b/SW.Bitween.Api/Resources/RetryPolicies/Update.cs index 3b741918..a11b4a98 100644 --- a/SW.Bitween.Api/Resources/RetryPolicies/Update.cs +++ b/SW.Bitween.Api/Resources/RetryPolicies/Update.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; @@ -14,13 +15,14 @@ public async Task Handle(int key, RetryPolicyUpdate model) { await requestContext.EnsurePermission(dbContext, Model.Permissions.RetryPolicies.Edit); RetryGroupValidation.EnsureCanFire(model.Groups); - RetryGroupValidation.EnsureAlertTransportIsSecure( - model.AlertHandlerId, model.AlertHandlerProperties); + await RetryGroupValidation.EnsureAlertChannelsExist(dbContext, + [model.AlertChannelId, ..RetryGroupValidation.AlertChannelIds(model.Groups)]); - var entity = await dbContext.FindAsync(key); + var entity = await dbContext.FindAsync(key) + ?? throw new SWNotFoundException(key.ToString()); // Spent budget is keyed by group id, so a group removed here would leave usage rows - // that no policy claims — invisible to the usage report and beyond the reach of reset. + // that no policy claims - invisible to the usage report and beyond the reach of reset. var removedGroupIds = entity.Groups .Select(g => g.Id) .Except((model.Groups ?? []).Select(g => g.Id)) @@ -29,25 +31,12 @@ public async Task Handle(int key, RetryPolicyUpdate model) // Dropping the groups and clearing what belonged to them is one change, so it commits as // one: a group no policy claims whose usage and override rows survive is unreachable from // the usage report and from reset alike. Safe to span, because RetryPolicy raises no domain - // events — nothing reaches the bus before the commit. + // events - nothing reaches the bus before the commit. await using var transaction = await dbContext.Database.BeginTransactionAsync(); - // Secrets came out of Get masked, so put them back from what this same level already holds. - // A group matched by id, because a group added in this very save has nothing to restore from. - foreach (var group in model.Groups ?? []) - { - var storedGroup = entity.Groups.FirstOrDefault(g => g.Id == group.Id); - AdapterSecretProperties.MergeInPlace( - storedGroup?.AlertHandlerProperties, group.AlertHandlerProperties); - } - - var storedPolicyProperties = entity.AlertHandlerProperties; - entity.Name = model.Name; entity.Groups = model.Groups ?? []; - entity.AlertHandlerId = model.AlertHandlerId; - entity.AlertHandlerProperties = - AdapterSecretProperties.Merge(storedPolicyProperties, model.AlertHandlerProperties); + entity.AlertChannelId = model.AlertChannelId; await dbContext.SaveChangesAsync(); if (removedGroupIds.Count > 0) diff --git a/SW.Bitween.Api/Resources/RetryPolicies/Usage.cs b/SW.Bitween.Api/Resources/RetryPolicies/Usage.cs index ca69e623..2d3e7e61 100644 --- a/SW.Bitween.Api/Resources/RetryPolicies/Usage.cs +++ b/SW.Bitween.Api/Resources/RetryPolicies/Usage.cs @@ -16,7 +16,7 @@ namespace SW.Bitween.Resources.RetryPolicies; /// /// /// -/// Both halves share the (SubscriptionId, GroupId) key, so they are reported together — the +/// Both halves share the (SubscriptionId, GroupId) key, so they are reported together - the /// question worth asking about an exhausted budget is whether anyone was told about it, and /// splitting that across two reports leaves the caller to join them by eye. /// diff --git a/SW.Bitween.Api/Resources/RetrySuggestions/Suggest.cs b/SW.Bitween.Api/Resources/RetrySuggestions/Suggest.cs new file mode 100644 index 00000000..3e038dd7 --- /dev/null +++ b/SW.Bitween.Api/Resources/RetrySuggestions/Suggest.cs @@ -0,0 +1,169 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain; +using SW.Bitween.Model; +using SW.Bitween.Services.Ai; +using SW.Bitween.Services.Metrics; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.RetrySuggestions; + +public class RetrySuggestionRequest +{ + public int SubscriptionId { get; set; } + + /// How far back to look, 1 to 90 days. + public int Days { get; set; } = 30; +} + +/// One kind of failure, and what the current and the suggested policy each do with it. +public class RetrySuggestionCoverage +{ + /// The error's first line, masked as it was sent. + public string Error { get; set; } = ""; + public int Count { get; set; } + public int Retried { get; set; } + public int FixedByRetry { get; set; } + public string Now { get; set; } = ""; + public string Suggested { get; set; } = ""; +} + +public class RetrySuggestionResponse +{ + public string Name { get; set; } = ""; + public string Summary { get; set; } = ""; + + /// The groups, ready for POST /api/retrypolicies; nothing is saved until then. + public List Groups { get; set; } = new(); + + public List Coverage { get; set; } = new(); + public int Failures { get; set; } + public int Days { get; set; } + public string? Model { get; set; } + + /// The message that went to the provider, after the fixed . + public string Sent { get; set; } = ""; +} + +/// +/// Proposes a retry policy for a subscription from its own recent failures. +/// +/// +/// +/// The provider sees each kind of failure as a masked line with counts and how often a retry fixed +/// it, and the current policy's rules. Never a document or a response body. The suggestion is then +/// checked as a saved policy would be and run against the real failures here, so the coverage +/// shows what changes for each before anyone saves it. +/// +/// +/// It saves nothing: saving the suggestion is creating a policy, so asking needs that right too, +/// and a question nobody could act on isn't sent. Each one is recorded as an operator action, +/// since it sends something about the subscription to a third party. +/// +/// +public class Suggest(RequestContext requestContext, BitweenDbContext dbContext, AiModels models) + : ICommandHandler +{ + public async Task Handle(RetrySuggestionRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Subscriptions.View); + await requestContext.EnsurePermission(dbContext, Model.Permissions.RetryPolicies.Create); + var model = models.Current(); + + var subscription = await dbContext.Set().AsNoTracking() + .Include(s => s.RetryPolicy) + .FirstOrDefaultAsync(s => s.Id == request.SubscriptionId) + ?? throw new SWNotFoundException(request.SubscriptionId.ToString()); + var current = subscription.CustomRetryPolicy?.Groups ?? subscription.RetryPolicy?.Groups; + var days = Math.Clamp(request.Days, 1, 90); + var since = DateTime.UtcNow.AddDays(-days); + + // The newest failures, by the (SubscriptionId, FinishedOn) index; only the start of each + // error, which is where its type and message are. Bad responses are counted apart, so many + // of them can't crowd the failures out. + var inWindow = dbContext.Set().AsNoTracking() + .Where(r => r.SubscriptionId == subscription.Id && r.FinishedOn >= since); + var errors = await inWindow + .Where(r => !r.Success) + .OrderByDescending(r => r.FinishedOn) + .Take(RetryPolicyPrompt.MaxFailures) + .Select(r => new + { + r.Id, r.FinishedOn, + Exception = r.Exception == null ? null : r.Exception.Substring(0, RetryPolicyPrompt.SampleLength), + }) + .ToListAsync(); + if (errors.Count == 0) + throw new SWValidationException("NO_FAILURES", + $"This subscription has no failed exchanges in the last {days} days, so there is nothing to learn a policy from."); + + // Whether each one's retry succeeded: the attempt made from it, by the RetryFor index. + var ids = errors.Select(e => e.Id).ToList(); + var retries = await dbContext.Set().AsNoTracking() + .Where(x => x.RetryFor != null && ids.Contains(x.RetryFor)) + .Join(dbContext.Set(), x => x.Id, r => r.Id, (x, r) => new { x.RetryFor, Fixed = r.Success && !r.ResponseBad }) + .ToListAsync(); + var retriedFrom = retries.GroupBy(r => r.RetryFor!).ToDictionary(g => g.Key, g => g.Any(r => r.Fixed)); + + var groups = errors + .GroupBy(e => ErrorFingerprint.Of(e.Exception ?? "")) + .OrderByDescending(g => g.Count()) + .Take(RetryPolicyPrompt.MaxKinds) + .ToList(); + + // Each kind's newest error in full, as the live retry sees it: a matcher may hit deep in the stack. + var newestIds = groups.Select(g => g.First().Id).ToList(); + var full = await dbContext.Set().AsNoTracking() + .Where(r => newestIds.Contains(r.Id)) + .Select(r => new { r.Id, r.Exception }) + .ToDictionaryAsync(r => r.Id, r => r.Exception); + + var kinds = groups + .Select(g => + { + var newest = g.First(); + return new FailureKind(full.GetValueOrDefault(newest.Id) ?? newest.Exception ?? "", + RetryPolicyPrompt.Mask(ErrorFingerprint.Sample(newest.Exception ?? "")), g.Count(), + g.Count(e => retriedFrom.ContainsKey(e.Id)), g.Count(e => retriedFrom.TryGetValue(e.Id, out var ok) && ok), + g.Min(e => e.FinishedOn), g.Max(e => e.FinishedOn)); + }) + .ToList(); + var badResponses = await inWindow.CountAsync(r => r.Success && r.ResponseBad); + + var describe = $"Subscription kind: {subscription.Type}; mapper: {subscription.MapperId ?? "none"}; handler: {subscription.HandlerId ?? "none"}"; + var message = RetryPolicyPrompt.Message(describe, days, errors.Count, badResponses, kinds, current); + // Recorded before it goes, so a call that fails or times out after sending is on record too. + await dbContext.RecordNowAsync(requestContext, OperatorAction.SuggestRetryPolicy, $"subscription {subscription.Id}", + $"Sent to {models.ModelName}: {errors.Count} failures, errors masked."); + var suggestion = RetryPolicyPrompt.Parse( + await model.Complete(new AiPrompt(RetryPolicyPrompt.Instructions, message, Json: true, MaxTokens: 4000))); + + var coverage = new List(); + foreach (var kind in kinds) + coverage.Add(new RetrySuggestionCoverage + { + Error = kind.Masked, + Count = kind.Count, + Retried = kind.Retried, + FixedByRetry = kind.FixedByRetry, + Now = await RetryPolicyPrompt.Outcome(current, kind.Sample), + Suggested = await RetryPolicyPrompt.Outcome(suggestion.Groups, kind.Sample), + }); + + return new RetrySuggestionResponse + { + Name = suggestion.Name, + Summary = suggestion.Summary, + Groups = suggestion.Groups, + Coverage = coverage, + Failures = errors.Count, + Days = days, + Model = models.ModelName, + Sent = message, + }; + } +} diff --git a/SW.Bitween.Api/Resources/Roles/Create.cs b/SW.Bitween.Api/Resources/Roles/Create.cs index 3827c073..ee9e8b88 100644 --- a/SW.Bitween.Api/Resources/Roles/Create.cs +++ b/SW.Bitween.Api/Resources/Roles/Create.cs @@ -1,3 +1,4 @@ +using SW.Bitween.Resources.Accounts; using System.Threading.Tasks; using FluentValidation; using SW.Bitween.Domain.Accounts; @@ -13,6 +14,7 @@ public async Task Handle(RoleCreate model) await requestContext.EnsurePermission(dbContext, Model.Permissions.Roles.Create); RoleValidation.EnsureKnownPermissions(model.Permissions); + await GrantLimits.EnsureCallerHoldsPermissions(dbContext, requestContext, model.Permissions); await RoleValidation.EnsureNameIsFree(dbContext, model.Name); var role = new Role(model.Name, model.Description, model.Permissions); diff --git a/SW.Bitween.Api/Resources/Roles/Delete.cs b/SW.Bitween.Api/Resources/Roles/Delete.cs index fc4dcf36..9d8c277c 100644 --- a/SW.Bitween.Api/Resources/Roles/Delete.cs +++ b/SW.Bitween.Api/Resources/Roles/Delete.cs @@ -1,3 +1,4 @@ +using SW.Bitween.Resources.Accounts; using System.Linq; using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; @@ -19,6 +20,8 @@ public async Task Handle(int key) throw new SWValidationException("ROLE_IS_BUILT_IN", $"'{role.Name}' is a built-in role and can't be deleted."); + await GrantLimits.EnsureCallerHoldsPermissions(dbContext, requestContext, role.Permissions); + var memberCount = await dbContext.Set().CountAsync(l => l.RoleId == key); if (memberCount > 0) throw new SWValidationException("ROLE_IN_USE", diff --git a/SW.Bitween.Api/Resources/Roles/Update.cs b/SW.Bitween.Api/Resources/Roles/Update.cs index 0714e864..e703f0be 100644 --- a/SW.Bitween.Api/Resources/Roles/Update.cs +++ b/SW.Bitween.Api/Resources/Roles/Update.cs @@ -1,3 +1,5 @@ +using System.Linq; +using SW.Bitween.Resources.Accounts; using System.Threading.Tasks; using FluentValidation; using SW.Bitween.Model; @@ -14,13 +16,18 @@ public async Task Handle(int key, RoleUpdate model) var role = await RoleValidation.Load(dbContext, key); - // Built-in roles are the floor an instance can always fall back to — if Administrator + // Built-in roles are the floor an instance can always fall back to - if Administrator // could be edited, an admin could lock everyone out of members and roles for good. if (role.IsSystem) throw new SWValidationException("ROLE_IS_BUILT_IN", $"'{role.Name}' is a built-in role and can't be changed. Create a role instead."); RoleValidation.EnsureKnownPermissions(model.Permissions); + + // Both what the role grants now and what it would grant: editing a role you couldn't have + // created is as much a step past your own access as creating one. + await GrantLimits.EnsureCallerHoldsPermissions(dbContext, requestContext, + (role.Permissions ?? []).Concat(model.Permissions ?? [])); await RoleValidation.EnsureNameIsFree(dbContext, model.Name, key); role.Update(model.Name, model.Description, model.Permissions); diff --git a/SW.Bitween.Api/Resources/ServiceAccounts/Create.cs b/SW.Bitween.Api/Resources/ServiceAccounts/Create.cs new file mode 100644 index 00000000..547655b2 --- /dev/null +++ b/SW.Bitween.Api/Resources/ServiceAccounts/Create.cs @@ -0,0 +1,41 @@ +using System.Linq; +using System.Threading.Tasks; +using FluentValidation; +using SW.Bitween.Domain.Accounts; +using SW.Bitween.Model; +using SW.Bitween.Resources.Accounts; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.ServiceAccounts; + +/// A service account, with the roles it signs in with. Its tokens are issued separately. +public class Create(BitweenDbContext dbContext, RequestContext requestContext) + : ICommandHandler +{ + public async Task Handle(CreateServiceAccountModel request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.ServiceAccounts.Create); + + var name = request.Name.Trim(); + var roleIds = (request.RoleIds ?? []).Distinct().ToList(); + await ServiceAccountStore.EnsureNameFree(dbContext, name); + // Nobody makes an account that can do more than they can. + await GrantLimits.EnsureCallerHoldsRoles(dbContext, requestContext, roleIds); + + var account = Account.ForService(name, AccountRoles.LegacyRoleFor(roleIds)); + dbContext.Add(account); + await dbContext.SaveChangesAsync(); + + await AccountRoles.Set(dbContext, account.Id, roleIds); + await dbContext.SaveChangesAsync(); + return account.Id; + } + + private class Validate : AbstractValidator + { + public Validate() + { + RuleFor(i => i.Name).NotEmpty().MaximumLength(200); + } + } +} diff --git a/SW.Bitween.Api/Resources/ServiceAccounts/Delete.cs b/SW.Bitween.Api/Resources/ServiceAccounts/Delete.cs new file mode 100644 index 00000000..02359a14 --- /dev/null +++ b/SW.Bitween.Api/Resources/ServiceAccounts/Delete.cs @@ -0,0 +1,29 @@ +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain.Accounts; +using SW.Bitween.Resources.Accounts; +using SW.HttpExtensions; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.ServiceAccounts; + +/// Removes a service account; its tokens are revoked with it. +public class Delete(BitweenDbContext dbContext, RequestContext requestContext) : IDeleteHandler +{ + public async Task Handle(int key) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.ServiceAccounts.Delete); + + var account = await ServiceAccountStore.Load(dbContext, key); + await GrantLimits.EnsureCallerOutranks(dbContext, requestContext, key); + + var by = requestContext.GetNameIdentifier(); + foreach (var token in await dbContext.Set().Where(t => t.AccountId == key && t.RevokedOn == null).ToListAsync()) + token.Revoke(by); + + dbContext.Remove(account); + await dbContext.SaveChangesAsync(); + return null; + } +} diff --git a/SW.Bitween.Api/Resources/ServiceAccounts/Get.cs b/SW.Bitween.Api/Resources/ServiceAccounts/Get.cs new file mode 100644 index 00000000..fb6f9ff5 --- /dev/null +++ b/SW.Bitween.Api/Resources/ServiceAccounts/Get.cs @@ -0,0 +1,17 @@ +using System.Linq; +using System.Threading.Tasks; +using SW.Bitween.Domain.Accounts; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.ServiceAccounts; + +public class Get(BitweenDbContext dbContext, RequestContext requestContext) : IGetHandler +{ + public async Task Handle(int key) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.ServiceAccounts.View); + var found = await ServiceAccountStore.Describe(dbContext, + dbContext.Set().Where(a => a.Kind == AccountKind.Service && a.Id == key)); + return found.FirstOrDefault() ?? throw new SWNotFoundException(key.ToString()); + } +} diff --git a/SW.Bitween.Api/Resources/ServiceAccounts/IssueToken.cs b/SW.Bitween.Api/Resources/ServiceAccounts/IssueToken.cs new file mode 100644 index 00000000..c11ddb59 --- /dev/null +++ b/SW.Bitween.Api/Resources/ServiceAccounts/IssueToken.cs @@ -0,0 +1,54 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using FluentValidation; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain.Accounts; +using SW.Bitween.Model; +using SW.Bitween.Resources.Accounts; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.ServiceAccounts; + +/// +/// A new token for a service account. The answer is the only place its value ever appears; after +/// this, Bitween holds only its hash. +/// +[HandlerName("issueToken")] +public class IssueToken(BitweenDbContext dbContext, RequestContext requestContext) + : ICommandHandler +{ + public async Task Handle(int key, CreateServiceTokenModel request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.ServiceAccounts.Create); + + var account = await ServiceAccountStore.Load(dbContext, key); + // A token can do whatever its account can; issuing one is handing that over. + await GrantLimits.EnsureCallerOutranks(dbContext, requestContext, key); + + var now = DateTime.UtcNow; + var active = await dbContext.Set() + .CountAsync(t => t.AccountId == key && t.RevokedOn == null && t.ExpiresOn > now); + if (active >= ServiceToken.MaxActivePerAccount) + throw new SWValidationException("TOO_MANY_TOKENS", + $"'{account.DisplayName}' already has {active} active tokens. Revoke one before issuing another."); + + var days = request.ExpiresInDays ?? ServiceToken.DefaultLifetimeDays; + var (token, value) = ServiceToken.Issue(key, request.Name.Trim(), now.AddDays(days)); + dbContext.Add(token); + await dbContext.SaveChangesAsync(); + + var issued = ServiceAccountStore.Describe(token, now); + issued.Token = value; + return issued; + } + + private class Validate : AbstractValidator + { + public Validate() + { + RuleFor(i => i.Name).NotEmpty().MaximumLength(100); + RuleFor(i => i.ExpiresInDays).InclusiveBetween(1, ServiceToken.MaxLifetimeDays).When(i => i.ExpiresInDays is not null); + } + } +} diff --git a/SW.Bitween.Api/Resources/ServiceAccounts/RevokeToken.cs b/SW.Bitween.Api/Resources/ServiceAccounts/RevokeToken.cs new file mode 100644 index 00000000..53b9835f --- /dev/null +++ b/SW.Bitween.Api/Resources/ServiceAccounts/RevokeToken.cs @@ -0,0 +1,32 @@ +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain.Accounts; +using SW.Bitween.Model; +using SW.Bitween.Resources.Accounts; +using SW.HttpExtensions; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.ServiceAccounts; + +/// +/// Ends one token. Sessions it was already traded for run out on their own within minutes; to end +/// those at once too, disable the account. +/// +[HandlerName("revokeToken")] +public class RevokeToken(BitweenDbContext dbContext, RequestContext requestContext) + : ICommandHandler +{ + public async Task Handle(int key, RevokeServiceTokenModel request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.ServiceAccounts.Edit); + + await ServiceAccountStore.Load(dbContext, key); + await GrantLimits.EnsureCallerOutranks(dbContext, requestContext, key); + + var token = await dbContext.Set().SingleOrDefaultAsync(t => t.Id == request.TokenId && t.AccountId == key) + ?? throw new SWValidationException("TOKEN_NOT_FOUND", $"This service account has no token with the id {request.TokenId}"); + token.Revoke(requestContext.GetNameIdentifier()); + await dbContext.SaveChangesAsync(); + return null; + } +} diff --git a/SW.Bitween.Api/Resources/ServiceAccounts/Search.cs b/SW.Bitween.Api/Resources/ServiceAccounts/Search.cs new file mode 100644 index 00000000..151045d8 --- /dev/null +++ b/SW.Bitween.Api/Resources/ServiceAccounts/Search.cs @@ -0,0 +1,18 @@ +using System.Linq; +using System.Threading.Tasks; +using SW.Bitween.Domain.Accounts; +using SW.Bitween.Model; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.ServiceAccounts; + +public class Search(BitweenDbContext dbContext, RequestContext requestContext) + : IQueryHandler +{ + public async Task Handle(SearchServiceAccountsModel request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.ServiceAccounts.View); + return await ServiceAccountStore.Describe(dbContext, + dbContext.Set().Where(a => a.Kind == AccountKind.Service)); + } +} diff --git a/SW.Bitween.Api/Resources/ServiceAccounts/ServiceAccountStore.cs b/SW.Bitween.Api/Resources/ServiceAccounts/ServiceAccountStore.cs new file mode 100644 index 00000000..dfd91442 --- /dev/null +++ b/SW.Bitween.Api/Resources/ServiceAccounts/ServiceAccountStore.cs @@ -0,0 +1,71 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain.Accounts; +using SW.Bitween.Model; +using SW.Bitween.Resources.Accounts; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.ServiceAccounts; + +internal static class ServiceAccountStore +{ + public static async Task Load(BitweenDbContext dbContext, int id) + { + var account = await dbContext.Set().FindAsync(id); + if (account is not { IsService: true } || account.Deleted) + throw new SWValidationException("SERVICE_ACCOUNT_NOT_FOUND", $"No service account exists with the id {id}"); + return account; + } + + public static async Task EnsureNameFree(BitweenDbContext dbContext, string name, int? except = null) + { + if (await dbContext.Set().AnyAsync(a => + a.Kind == AccountKind.Service && a.DisplayName == name && (except == null || a.Id != except))) + throw new SWValidationException("SERVICE_ACCOUNT_EXISTS", $"There's already a service account named '{name}'."); + } + + public static string Status(ServiceToken token, DateTime nowUtc) => + token.RevokedOn is not null ? "revoked" : token.ExpiresOn <= nowUtc ? "expired" : "active"; + + public static T Describe(ServiceToken token, DateTime nowUtc) where T : ServiceTokenModel, new() => new() + { + Id = token.Id, + Name = token.Name, + Shown = token.Shown, + CreatedOn = token.CreatedOn, + CreatedBy = token.CreatedBy, + ExpiresOn = token.ExpiresOn, + RevokedOn = token.RevokedOn, + LastUsedOn = token.LastUsedOn, + LastUsedFrom = token.LastUsedFrom, + Status = Status(token, nowUtc), + }; + + public static async Task> Describe(BitweenDbContext dbContext, IQueryable accounts) + { + var now = DateTime.UtcNow; + var list = await accounts.AsNoTracking().OrderBy(a => a.DisplayName).ToListAsync(); + var ids = list.Select(a => a.Id).ToList(); + var roles = await AccountRoles.For(dbContext, ids); + var tokens = (await dbContext.Set().AsNoTracking() + .Where(t => ids.Contains(t.AccountId)).ToListAsync()) + .ToLookup(t => t.AccountId); + + return list.Select(a => new ServiceAccountModel + { + Id = a.Id, + Name = a.DisplayName, + Disabled = a.Disabled, + CreatedOn = a.CreatedOn, + LastSignInOn = a.LastSignInOn, + Roles = roles.GetValueOrDefault(a.Id, []), + // Active first, newest first; ended ones stay listed, for when they were used last. + Tokens = tokens[a.Id] + .OrderBy(t => Status(t, now) == "active" ? 0 : 1).ThenByDescending(t => t.CreatedOn) + .Select(t => Describe(t, now)).ToList(), + }).ToList(); + } +} diff --git a/SW.Bitween.Api/Resources/ServiceAccounts/SetDisabled.cs b/SW.Bitween.Api/Resources/ServiceAccounts/SetDisabled.cs new file mode 100644 index 00000000..e813211b --- /dev/null +++ b/SW.Bitween.Api/Resources/ServiceAccounts/SetDisabled.cs @@ -0,0 +1,27 @@ +using System.Threading.Tasks; +using SW.Bitween.Model; +using SW.Bitween.Resources.Accounts; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.ServiceAccounts; + +/// +/// Stops or restores every token the account holds at once. Its sessions end too: permissions are +/// read per call, and a disabled account holds none. +/// +[HandlerName("setDisabled")] +public class SetDisabled(BitweenDbContext dbContext, RequestContext requestContext) + : ICommandHandler +{ + public async Task Handle(int key, SetServiceAccountDisabledModel request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.ServiceAccounts.Edit); + + var account = await ServiceAccountStore.Load(dbContext, key); + await GrantLimits.EnsureCallerOutranks(dbContext, requestContext, key); + + account.SetDisabled(request.Disabled); + await dbContext.SaveChangesAsync(); + return null; + } +} diff --git a/SW.Bitween.Api/Resources/ServiceAccounts/Update.cs b/SW.Bitween.Api/Resources/ServiceAccounts/Update.cs new file mode 100644 index 00000000..8008892e --- /dev/null +++ b/SW.Bitween.Api/Resources/ServiceAccounts/Update.cs @@ -0,0 +1,39 @@ +using System.Linq; +using System.Threading.Tasks; +using FluentValidation; +using SW.Bitween.Model; +using SW.Bitween.Resources.Accounts; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.ServiceAccounts; + +/// Renames a service account and replaces its roles. +public class Update(BitweenDbContext dbContext, RequestContext requestContext) + : ICommandHandler +{ + public async Task Handle(int key, UpdateServiceAccountModel request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.ServiceAccounts.Edit); + + var account = await ServiceAccountStore.Load(dbContext, key); + var name = request.Name.Trim(); + var roleIds = (request.RoleIds ?? []).Distinct().ToList(); + await ServiceAccountStore.EnsureNameFree(dbContext, name, key); + await GrantLimits.EnsureCallerOutranks(dbContext, requestContext, key); + await GrantLimits.EnsureCallerHoldsRoles(dbContext, requestContext, roleIds); + + account.UpdateProfile(name); + account.SetRole(AccountRoles.LegacyRoleFor(roleIds)); + await AccountRoles.Set(dbContext, key, roleIds); + await dbContext.SaveChangesAsync(); + return null; + } + + private class Validate : AbstractValidator + { + public Validate() + { + RuleFor(i => i.Name).NotEmpty().MaximumLength(200); + } + } +} diff --git a/SW.Bitween.Api/Resources/Settings/About.cs b/SW.Bitween.Api/Resources/Settings/About.cs new file mode 100644 index 00000000..4935f6b2 --- /dev/null +++ b/SW.Bitween.Api/Resources/Settings/About.cs @@ -0,0 +1,157 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using SW.PrimitiveTypes; +using SW.Serverless.Runtimes; + +namespace SW.Bitween.Resources.Settings; + +/// +/// What this Bitween is and how it is set up, as the node that answers sees it: its version, the +/// node, its database, storage and broker, its readiness checks, which adapter runtimes it has, and +/// the settings that shape behaviour but live in configuration rather than on the Settings page. +/// Secrets are never in it, only whether they are set. +/// +[HandlerName("about")] +public class About(BitweenDbContext dbContext, RequestContext requestContext, BitweenOptions options, + IConfiguration configuration, IServiceProvider services) : IQueryHandler +{ + static readonly DateTimeOffset StartedOn = DateTimeOffset.UtcNow; + + public async Task Handle() + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Settings.View); + + var limits = configuration.GetSection("Bitween:RateLimits"); + var runtimes = services.GetService(); + + return new + { + Version = BitweenInfo.Version(options).ToString(), + // The version and commit CI stamped on this build; none for a build it didn't stamp. + Build = BitweenInfo.Build, + Node = new + { + Name = Environment.MachineName, + StartedOn, + Runtime = RuntimeInformation.FrameworkDescription, + Os = RuntimeInformation.OSDescription, + DataSources = options.BusProvidersEnabled, + }, + Database = options.DatabaseType, + Storage = new { Provider = options.StorageProvider, options.AdapterPath, options.DocumentPrefix }, + Broker = new + { + options.QueuePrefix, + ManagementConfigured = !string.IsNullOrWhiteSpace(options.RabbitMqManagementUrl), + }, + Health = await HealthAsync(), + Adapters = new + { + Runtimes = runtimes == null + ? [] + : await Task.WhenAll(new[] { "dotnet", "python", "node" }.Select(async name => + { + var status = await runtimes.StatusAsync(name); + return new { Name = name, status.Available, status.Version, status.Reason }; + })), + Pip = await ToolAsync("pip", "python3", "-m", "pip", "--version"), + Npm = await ToolAsync("npm", "npm", "--version"), + CommandTimeoutSeconds = options.ServerlessCommandTimeout, + Editor = new + { + Dependencies = options.AdapterEditorDependencies, + MemoryMb = options.AdapterEditorMemoryMb, + CpuCores = options.AdapterEditorCpuCores, + }, + }, + Limits = new + { + SignInPerMinute = limits.GetValue("SignInPerMinute", 10), + RequestsPerMinute = limits.GetValue("RequestsPerMinute", 600), + FileLinksPerMinute = limits.GetValue("FileLinksPerMinute", 60000), + options.MaxResponseWaitSeconds, + options.MaxRetryChainDepth, + options.StaleRunAfterMinutes, + options.NotifierQuietMinutes, + }, + Network = new + { + options.PublicUrl, + options.BlockPrivateNetworkAddresses, + TrustedProxies = string.IsNullOrWhiteSpace(options.TrustedProxies) ? 0 : options.TrustedProxies.Split(',', ';').Length, + options.ExposeApiDocs, + CorsOrigins = options.CorsOrigins ?? [], + }, + Retention = new + { + options.ReceiveAttemptRetentionDays, + options.AuditRetentionDays, + options.ReceiveAttemptCleanupCron, + options.InboundMessagePruneCron, + }, + Telemetry = new + { + OpenTelemetry = !string.IsNullOrWhiteSpace(configuration["OTEL_EXPORTER_OTLP_ENDPOINT"] ?? configuration["OpenTelemetry:Endpoint"]), + }, + }; + } + + /// The readiness checks /health/ready runs: the database and the broker. + async Task HealthAsync() + { + var health = services.GetService(); + if (health == null) return Array.Empty(); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + var report = await health.CheckHealthAsync(_ => true, timeout.Token); + return report.Entries.Select(e => new + { + Name = e.Key, + Status = e.Value.Status.ToString(), + DurationMs = (int)e.Value.Duration.TotalMilliseconds, + e.Value.Description, + Error = e.Value.Exception?.Message, + }).ToList(); + } + + static readonly Dictionary>> Tools = new(); + static readonly object ToolsGate = new(); + + /// Whether a tool the adapter editor needs answers, asked once per process. + static Task ToolAsync(string key, string file, params string[] arguments) + { + lock (ToolsGate) + { + if (!Tools.TryGetValue(key, out var lazy)) + Tools[key] = lazy = new Lazy>(() => ProbeAsync(file, arguments)); + return lazy.Value; + } + } + + static async Task ProbeAsync(string file, string[] arguments) + { + try + { + var start = new ProcessStartInfo(file) { RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false }; + foreach (var argument in arguments) start.ArgumentList.Add(argument); + using var process = Process.Start(start)!; + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + var output = await process.StandardOutput.ReadToEndAsync(timeout.Token); + await process.WaitForExitAsync(timeout.Token); + return process.ExitCode == 0 + ? new { Available = true, Version = output.Trim().Split('\n')[0], Reason = (string)null } + : new { Available = false, Version = (string)null, Reason = $"exited with {process.ExitCode}" }; + } + catch (Exception ex) when (ex is System.ComponentModel.Win32Exception or OperationCanceledException or InvalidOperationException) + { + return new { Available = false, Version = (string)null, Reason = $"'{file}' isn't installed on this node" }; + } + } +} diff --git a/SW.Bitween.Api/Resources/Settings/Config.cs b/SW.Bitween.Api/Resources/Settings/Config.cs index c6c5bff9..ff6a66e2 100644 --- a/SW.Bitween.Api/Resources/Settings/Config.cs +++ b/SW.Bitween.Api/Resources/Settings/Config.cs @@ -19,7 +19,7 @@ public class Config(BitweenOptions BitweenOptions, ThemeOptions themeOptions) : /// This endpoint answers before anyone signs in, because the sign-in page brands itself from /// it. A penetration test followed the default GithubLink from here to our public /// repository, and from there to the sample signing key and the default administrator password - /// — the whole chain started with a link nobody had configured. + /// - the whole chain started with a link nobody had configured. /// /// Withheld only at the default: a deployment that sets its own link is publishing its own /// address and gets it back untouched. So the footer keeps working wherever it was meant to, @@ -41,12 +41,22 @@ public async Task Handle() BitweenOptions.MsalRedirectUri, BitweenOptions.MsalTenantId, BitweenOptions.DisableEmailPasswordLogin, + // Whether to offer the button, and its name; the provider's address stays on the server, + // which starts the sign-in. + OidcEnabled = !string.IsNullOrWhiteSpace(BitweenOptions.OidcAuthority) && !string.IsNullOrWhiteSpace(BitweenOptions.OidcClientId), + OidcDisplayName = string.IsNullOrWhiteSpace(BitweenOptions.OidcDisplayName) ? null : BitweenOptions.OidcDisplayName.Trim(), + // Whether Bitween can email members: the Team page offers invites by email, and the + // sign-in page "Forgot password?", only when it can. + MailServerConfigured = BitweenOptions.MailServer.IsSet, + PasswordResetAvailable = Services.AccountMail.ResetAvailableFor(BitweenOptions), + // Whether AI features are offered: the mapping editor's Generate with AI, so far. + AiConfigured = BitweenOptions.Ai.IsSet, IsRabbitMqManagementConfigured = !string.IsNullOrWhiteSpace(BitweenOptions.RabbitMqManagementUrl) && !string.IsNullOrWhiteSpace(BitweenOptions.RabbitMqManagementUsername) && !string.IsNullOrWhiteSpace(BitweenOptions.RabbitMqManagementPassword), Theme = ThemeWithout(withheld), - // The product defaults, so the sign-in page — which has no session and can't read the - // settings list — can tell a brand value someone chose from one nobody has touched. + // The product defaults, so the sign-in page - which has no session and can't read the + // settings list - can tell a brand value someone chose from one nobody has touched. ThemeDefaults = SettingsService.DefaultsUnder("Theme.") .Where(kv => !withheld.Contains(Pascalize(kv.Key))) .ToDictionary(kv => kv.Key, kv => kv.Value) @@ -58,8 +68,8 @@ public async Task Handle() /// /// /// Built by reflection rather than written out property by property: a hand-written projection - /// silently drops whatever is added to next, and the symptom — one - /// brand value that will not apply — looks nothing like its cause. + /// silently drops whatever is added to next, and the symptom - one + /// brand value that will not apply - looks nothing like its cause. /// private Dictionary ThemeWithout(IReadOnlySet withheld) => typeof(ThemeOptions).GetProperties() diff --git a/SW.Bitween.Api/Resources/Settings/Delete.cs b/SW.Bitween.Api/Resources/Settings/Delete.cs index c78f5cde..f411a91c 100644 --- a/SW.Bitween.Api/Resources/Settings/Delete.cs +++ b/SW.Bitween.Api/Resources/Settings/Delete.cs @@ -7,7 +7,7 @@ namespace SW.Bitween.Resources.Settings; /// -/// Resets one setting to the product default — the value the options class ships with. +/// Resets one setting to the product default - the value the options class ships with. /// /// The row is rewritten rather than deleted: a missing row is how startup recognises a key it has /// never imported, so dropping it would let configuration seep back in on the next boot. Reset @@ -47,11 +47,11 @@ public async Task Handle(string key) await dbContext.SaveChangesAsync(); - // Resetting an already-default setting is a no-op, not an error — the UI can fire this for + // Resetting an already-default setting is a no-op, not an error - the UI can fire this for // a staged reset it never saved a value for. settings.Apply(definition, productDefault); - // A reset has to take effect the same way a write does — see Update. + // A reset has to take effect the same way a write does - see Update. if (definition.OnChange is not null) await definition.OnChange(serviceProvider); await cache.BroadcastRevoke(); diff --git a/SW.Bitween.Api/Resources/Settings/Get.cs b/SW.Bitween.Api/Resources/Settings/Get.cs index 84614464..cf8eb422 100644 --- a/SW.Bitween.Api/Resources/Settings/Get.cs +++ b/SW.Bitween.Api/Resources/Settings/Get.cs @@ -11,8 +11,8 @@ namespace SW.Bitween.Resources.Settings; /// /// Every editable setting, in catalog order: the definition from -/// joined with the stored value. Rows normally exist for all of them — startup imports whatever -/// configuration had — so a missing row means the key couldn't be stored yet. +/// joined with the stored value. Rows normally exist for all of them - startup imports whatever +/// configuration had - so a missing row means the key couldn't be stored yet. /// public class Get(BitweenDbContext dbContext, RequestContext requestContext, SettingsService settings) : IQueryHandler @@ -35,7 +35,7 @@ public async Task Handle() // A secret's value is withheld either way round: only whether one is set is public. var value = definition.Secret ? null : hasRow ? stored : productDefault; // A secret has no product default, and its ciphertext couldn't be compared with one - // anyway — so for a secret both "is set" and "is overridden" mean the same thing: + // anyway - so for a secret both "is set" and "is overridden" mean the same thing: // a non-empty value is stored. var secretIsSet = hasRow && !string.IsNullOrEmpty(stored); @@ -49,6 +49,7 @@ public async Task Handle() DefaultValue = definition.Secret ? string.Empty : productDefault, Value = value, Secret = definition.Secret, + Choices = definition.Choices?.ToArray(), Overridden = definition.Secret ? secretIsSet : hasRow && stored != productDefault, HasValue = definition.Secret ? secretIsSet diff --git a/SW.Bitween.Api/Resources/Settings/MyVersion.cs b/SW.Bitween.Api/Resources/Settings/MyVersion.cs index d4512785..1df63474 100644 --- a/SW.Bitween.Api/Resources/Settings/MyVersion.cs +++ b/SW.Bitween.Api/Resources/Settings/MyVersion.cs @@ -3,16 +3,16 @@ namespace SW.Bitween.Resources.Settings; +/// +/// This Bitween's release, as a version object, the shape it always had. It used to be the assembly's own +/// version, which every build says is 1.0.0.0; it is now the release, as the About page shows it. +/// [HandlerName("myversion")] -public class MyVersion: IQueryHandler +public class MyVersion(BitweenOptions options) : IQueryHandler { - public Task Handle() - { - var version = this.GetType().Assembly.GetName().Version; - - return Task.FromResult(new + public Task Handle() => + Task.FromResult(new { - BitweenApiVersion = version + BitweenApiVersion = BitweenInfo.Version(options), }); - } -} \ No newline at end of file +} diff --git a/SW.Bitween.Api/Resources/Settings/Update.cs b/SW.Bitween.Api/Resources/Settings/Update.cs index f47a2f6f..be9c6db0 100644 --- a/SW.Bitween.Api/Resources/Settings/Update.cs +++ b/SW.Bitween.Api/Resources/Settings/Update.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using System.Threading.Tasks; using SW.Bitween.Domain; using SW.Bitween.Model; @@ -11,7 +12,7 @@ namespace SW.Bitween.Resources.Settings; /// Stores a new value for one setting. It's applied to the live options singletons as well as /// stored, so the very next request already sees it; the cache-revoke broadcast carries the change /// to any other instance. Secrets are encrypted before they're written. Only editable settings can -/// be written at all — the environment-owned ones the page also lists are rejected here. +/// be written at all - the environment-owned ones the page also lists are rejected here. /// public class Update( BitweenDbContext dbContext, @@ -36,7 +37,7 @@ public async Task Handle(string key, SettingUpdate request) $"{definition.Label} is a secret and can only be stored once " + $"{BitweenOptions.ConfigurationSection}:{nameof(BitweenOptions.SettingsEncryptionKey)} is configured."); - // Empty is a real value — it's how you clear an optional link or a license key. + // Empty is a real value - it's how you clear an optional link or a license key. var value = request?.Value ?? string.Empty; try @@ -48,10 +49,17 @@ public async Task Handle(string key, SettingUpdate request) throw new SWValidationException("SETTING_INVALID_VALUE", $"{definition.Label}: {ex.Message}"); } + var before = settings.LiveValue(definition); await Store(definition, value); settings.Apply(definition, value); + if (definition.Clears is { } clears && settings.LiveValue(definition) != before) + foreach (var cleared in clears.Select(SettingsCatalog.Find)) + { + await Store(cleared, string.Empty); + settings.Apply(cleared, string.Empty); + } - // Some settings need more than the property assignment above — re-scheduling a job, for + // Some settings need more than the property assignment above - re-scheduling a job, for // instance. Runs after Apply so the hook reads the value that's now in effect. if (definition.OnChange is not null) await definition.OnChange(serviceProvider); diff --git a/SW.Bitween.Api/Resources/SubscriptionCategories/Create.cs b/SW.Bitween.Api/Resources/SubscriptionCategories/Create.cs index dc45345c..5ded1063 100644 --- a/SW.Bitween.Api/Resources/SubscriptionCategories/Create.cs +++ b/SW.Bitween.Api/Resources/SubscriptionCategories/Create.cs @@ -26,7 +26,7 @@ public async Task Handle(CreateSubscriptionCategoryModel request) } /// - /// The code is uniquely indexed, so a repeat was already refused — as a constraint violation + /// The code is uniquely indexed, so a repeat was already refused - as a constraint violation /// surfacing as a bare 500, which says nothing about the code being taken. /// internal static async Task EnsureCodeIsFree(BitweenDbContext dbContext, string code, int? existingId = null) diff --git a/SW.Bitween.Api/Resources/Subscriptions/AggregateNow.cs b/SW.Bitween.Api/Resources/Subscriptions/AggregateNow.cs index 435d4f40..17f4e607 100644 --- a/SW.Bitween.Api/Resources/Subscriptions/AggregateNow.cs +++ b/SW.Bitween.Api/Resources/Subscriptions/AggregateNow.cs @@ -13,7 +13,8 @@ public async Task Handle(int key, SubscriptionAggregateNow request) { await requestContext.EnsurePermission(dbContext, Model.Permissions.Subscriptions.Operate); - var entity = await dbContext.FindAsync(key); + var entity = await dbContext.FindAsync(key) + ?? throw new SWNotFoundException(key.ToString()); entity.SetAggregateNow(); await dbContext.SaveChangesAsync(); diff --git a/SW.Bitween.Api/Resources/Subscriptions/Bookmark.cs b/SW.Bitween.Api/Resources/Subscriptions/Bookmark.cs new file mode 100644 index 00000000..efe1cacd --- /dev/null +++ b/SW.Bitween.Api/Resources/Subscriptions/Bookmark.cs @@ -0,0 +1,49 @@ +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain; +using SW.Bitween.Services; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Subscriptions; + +public class SubscriptionBookmarkRequest +{ + public int SubscriptionId { get; set; } +} + +public class ForgetSubscriptionBookmark +{ +} + +/// Where the subscription's receiver last got to, when it keeps a bookmark; a null value otherwise. +[HandlerName("bookmark")] +public class Bookmark(BitweenDbContext dbContext, RequestContext requestContext, ReceiverBookmarks bookmarks) + : IQueryHandler +{ + public async Task Handle(SubscriptionBookmarkRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Subscriptions.View); + var receiver = await dbContext.Set().AsNoTracking() + .Where(s => s.Id == request.SubscriptionId).Select(s => s.ReceiverId).SingleOrDefaultAsync(); + var state = string.IsNullOrWhiteSpace(receiver) ? null : await bookmarks.GetAsync(request.SubscriptionId, receiver); + return new { state?.Value, UpdatedOn = state?.UpdatedOn }; + } +} + +/// Forgets the bookmark, so the receiver's next run starts from the beginning. +[HandlerName("forgetbookmark")] +public class ForgetBookmark(BitweenDbContext dbContext, RequestContext requestContext, ReceiverBookmarks bookmarks) + : ICommandHandler +{ + public async Task Handle(int key, ForgetSubscriptionBookmark request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Subscriptions.Operate); + if (!await dbContext.Set().AnyAsync(s => s.Id == key)) throw new SWNotFoundException(key.ToString()); + var forgotten = await bookmarks.ForgetAsync(key); + // What the next run takes changes, so who did it is worth the trail. + await dbContext.RecordNowAsync(requestContext, Domain.OperatorAction.ForgetBookmark, $"subscription {key}", + forgotten > 0 ? "the next run starts from the beginning" : "there was no bookmark"); + return new { Forgotten = forgotten > 0 }; + } +} diff --git a/SW.Bitween.Api/Resources/Subscriptions/ClearRunning.cs b/SW.Bitween.Api/Resources/Subscriptions/ClearRunning.cs new file mode 100644 index 00000000..31618bf8 --- /dev/null +++ b/SW.Bitween.Api/Resources/Subscriptions/ClearRunning.cs @@ -0,0 +1,40 @@ +using System.Linq; +using System.Threading.Tasks; +using Quartz; +using SW.Bitween.Domain; +using SW.PrimitiveTypes; +using SW.Scheduler; + +namespace SW.Bitween.Resources.Subscriptions; + +public class SubscriptionClearRunning { } + +/// +/// Clears the running flag of a scheduled job or aggregation whose run died with its process. Until +/// it was cleared, every later run was skipped as "already running" for StaleRunAfterMinutes, and +/// the only other way out was editing the database. Refused while the job really is running here. +/// Saved through the context, so the audit trail says who cleared it. +/// +[HandlerName("clearrunning")] +public class ClearRunning(BitweenDbContext dbContext, RequestContext requestContext, + IScheduleRepository scheduleRepo, ISchedulerFactory schedulerFactory) + : ICommandHandler +{ + public async Task Handle(int key, SubscriptionClearRunning request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Subscriptions.Operate); + + var entity = await dbContext.FindAsync(key) ?? throw new SWNotFoundException(key.ToString()); + if (!entity.IsRunning) return new { entity.Id, Cleared = false }; + + var scheduler = await schedulerFactory.GetScheduler(); + var jobs = scheduleRepo.GetJobDefinitions().ToDictionary(d => d.JobType); + if ((await GetScheduleHealth.CurrentlyRunningSubscriptionIds(scheduler, jobs)).Contains(key)) + throw new SWValidationException("STILL_RUNNING", + "This job is running right now on this node. Wait for it to finish; it clears the flag itself."); + + entity.ClearRunning(); + await dbContext.SaveChangesAsync(); + return new { entity.Id, Cleared = true }; + } +} diff --git a/SW.Bitween.Api/Resources/Subscriptions/Create.cs b/SW.Bitween.Api/Resources/Subscriptions/Create.cs index 2eb3428c..c26dde63 100644 --- a/SW.Bitween.Api/Resources/Subscriptions/Create.cs +++ b/SW.Bitween.Api/Resources/Subscriptions/Create.cs @@ -24,7 +24,7 @@ namespace SW.Bitween.Resources.Subscriptions /// public class Create(BitweenDbContext dbContext, RequestContext requestContext, IInfolinkCache BitweenCache, SubscriptionSchedulerService subScheduler, - AdapterRequirements adapterRequirements) : ICommandHandler + AdapterRequirements adapterRequirements, SubscriptionRevisions revisions) : ICommandHandler { private readonly BitweenDbContext _dbContext = dbContext; @@ -74,15 +74,16 @@ public async Task Handle(SubscriptionCreate model) await BitweenCache.BroadcastRevoke(); await subScheduler.Sync(entity, Array.Empty()); + await revisions.RecordAsync(entity.Id); return entity.Id; } private class Validate : AbstractValidator { - public Validate(BitweenDbContext dbContext, AdapterRequirements adapterRequirements) + public Validate(BitweenDbContext dbContext, AdapterRequirements adapterRequirements, Services.Mirror.MirrorCopyGate mirrorCopy) { // Same rule Documents enforces on BusMessageTypeName. Publishing to a name no - // information type carries is legitimate — the consumer may be another product — + // information type carries is legitimate - the consumer may be another product - // but it still becomes a RabbitMQ routing key, and a name with a space in it // could never be answered by an information type anyway. RuleFor(i => i.ResponseMessageTypeName) @@ -92,7 +93,8 @@ public Validate(BitweenDbContext dbContext, AdapterRequirements adapterRequireme RuleFor(i => i.ResponseSubscriptionId).CustomAsync(async (responseSubId, context, _) => { - var failure = await ResponseRoutingValidation.CheckDestination(dbContext, responseSubId); + var failure = await ResponseRoutingValidation.CheckDestination(dbContext, responseSubId, + recreatingProd: mirrorCopy.RecreatingProd); if (failure != null) context.AddFailure(nameof(SubscriptionCreate.ResponseSubscriptionId), failure); }); @@ -118,31 +120,39 @@ public Validate(BitweenDbContext dbContext, AdapterRequirements adapterRequireme When(i => i.Type == SubscriptionType.Aggregation, () => { RuleFor(i => i.AggregationForId).NotEmpty(); }); - // An adapter that is named has to be usable. Nothing is required to be named — - // create without a pipeline is still legitimate — but a half-configured adapter + // An adapter that is named has to be usable. Nothing is required to be named - + // create without a pipeline is still legitimate - but a half-configured adapter // is the failure this endpoint exists to stop committing. RuleFor(i => i.ReceiverProperties).CustomAsync(async (provided, context, _) => await AddMissing(context, adapterRequirements, - ((SubscriptionCreate)context.InstanceToValidate).ReceiverId, provided)); + SW.Serverless.Contract.Catalog.AdapterCatalogPaths.Ref( + ((SubscriptionCreate)context.InstanceToValidate).ReceiverId, + ((SubscriptionCreate)context.InstanceToValidate).ReceiverVersion), provided)); RuleFor(i => i.ValidatorProperties).CustomAsync(async (provided, context, _) => await AddMissing(context, adapterRequirements, - ((SubscriptionCreate)context.InstanceToValidate).ValidatorId, provided)); + SW.Serverless.Contract.Catalog.AdapterCatalogPaths.Ref( + ((SubscriptionCreate)context.InstanceToValidate).ValidatorId, + ((SubscriptionCreate)context.InstanceToValidate).ValidatorVersion), provided)); RuleFor(i => i.MapperProperties).CustomAsync(async (provided, context, _) => await AddMissing(context, adapterRequirements, - ((SubscriptionCreate)context.InstanceToValidate).MapperId, provided)); + SW.Serverless.Contract.Catalog.AdapterCatalogPaths.Ref( + ((SubscriptionCreate)context.InstanceToValidate).MapperId, + ((SubscriptionCreate)context.InstanceToValidate).MapperVersion), provided)); RuleFor(i => i.HandlerProperties).CustomAsync(async (provided, context, _) => await AddMissing(context, adapterRequirements, - ((SubscriptionCreate)context.InstanceToValidate).HandlerId, provided)); + SW.Serverless.Contract.Catalog.AdapterCatalogPaths.Ref( + ((SubscriptionCreate)context.InstanceToValidate).HandlerId, + ((SubscriptionCreate)context.InstanceToValidate).HandlerVersion), provided)); // Schedules only mean anything on the two scheduled types, and an empty set is // what Subscription.SetSchedules rejects outright. // // The type is the whole condition. Guarding on `Schedules != null` as well let the - // rule be skipped by the one case it most needed to catch — omitting the field - // entirely — so a scheduled job or aggregation saved with no trigger at all and + // rule be skipped by the one case it most needed to catch - omitting the field + // entirely - so a scheduled job or aggregation saved with no trigger at all and // then simply never ran. RuleFor(i => i.Schedules) .NotEmpty() diff --git a/SW.Bitween.Api/Resources/Subscriptions/Delete.cs b/SW.Bitween.Api/Resources/Subscriptions/Delete.cs index 87cea8f4..dfc4b029 100644 --- a/SW.Bitween.Api/Resources/Subscriptions/Delete.cs +++ b/SW.Bitween.Api/Resources/Subscriptions/Delete.cs @@ -9,7 +9,8 @@ namespace SW.Bitween.Resources.Subscriptions { -public class Delete(BitweenDbContext dbContext, RequestContext requestContext, IInfolinkCache cache) +public class Delete(BitweenDbContext dbContext, RequestContext requestContext, IInfolinkCache cache, + SubscriptionSchedulerService scheduler) : IDeleteHandler { public async Task Handle(int key) @@ -18,7 +19,13 @@ public async Task Handle(int key) await EnsureNothingPointsAtIt(key); + // Read before it goes, for its schedules; unscheduled only once the delete has succeeded, + // so a refused delete leaves the subscription running as it was. + var subscription = await dbContext.Set().AsNoTracking().FirstOrDefaultAsync(s => s.Id == key); + await dbContext.DeleteByKeyAsync(key); + if (subscription != null) + await scheduler.UnscheduleAll(subscription); await cache.BroadcastRevoke(); return null; } @@ -27,7 +34,7 @@ public async Task Handle(int key) /// Says what still points at the integration, before the database says it less politely. /// /// - /// All four references are RESTRICT, so the delete was already refused — but as a + /// All four references are RESTRICT, so the delete was already refused - but as a /// foreign key violation surfacing as a 500, which tells the operator nothing about which /// route or gateway is holding it, and reads like a broken screen rather than a decision. /// Exchanges are deliberately not checked: their reference is nullable and history is not diff --git a/SW.Bitween.Api/Resources/Subscriptions/Get.cs b/SW.Bitween.Api/Resources/Subscriptions/Get.cs index 3629eb66..34b3fad4 100644 --- a/SW.Bitween.Api/Resources/Subscriptions/Get.cs +++ b/SW.Bitween.Api/Resources/Subscriptions/Get.cs @@ -27,9 +27,17 @@ public Get(BitweenDbContext dbContext, AdapterStartupValues startupValues, Reque public async Task Handle(int key) { await requestContext.EnsurePermission(dbContext, Model.Permissions.Subscriptions.View); + return await Read(key); + } + /// The subscription as the API answers for it, secrets masked. Also what a revision records. + public async Task Read(int key) + { + // A missing one is a 404, which the page says as "no longer exists"; it used to be a + // NullReferenceException and a 500. var subscriber = - await dbContext.Set().AsNoTracking().Search("Id", key).SingleOrDefaultAsync(); + await dbContext.Set().AsNoTracking().Search("Id", key).SingleOrDefaultAsync() + ?? throw new SWNotFoundException(key.ToString()); var response = new SubscriptionGet @@ -41,6 +49,10 @@ public async Task Handle(int key) Inactive = subscriber.Inactive, MapperId = subscriber.MapperId, ReceiverId = subscriber.ReceiverId, + HandlerVersion = subscriber.HandlerVersion, + MapperVersion = subscriber.MapperVersion, + ReceiverVersion = subscriber.ReceiverVersion, + ValidatorVersion = subscriber.ValidatorVersion, // Which data source every stage of this subscription runs through. Dropped // from this projection once, and the cost was quiet: the UI read every bound @@ -60,11 +72,15 @@ public async Task Handle(int key) RunOnBadResponses = subscriber.RunOnBadResponses, ReceiveOn = subscriber.ReceiveOn, AggregateOn = subscriber.AggregateOn, - ConsecutiveFailures = subscriber.ConsecutiveFailures, + ConsecutiveFailures = await SubscriptionFailures.CountAsync(dbContext, subscriber), LastException = subscriber.LastException, AggregationTarget = subscriber.AggregationTarget, ValidatorId = subscriber.ValidatorId, PausedOn = subscriber.PausedOn, + PausedAutomatically = subscriber.PausedAutomatically, + AutoPauseAfterFailures = subscriber.AutoPauseAfterFailures, + ExpectTraffic = subscriber.ExpectTraffic, + QuietSince = subscriber.QuietSince, MatchExpression = subscriber.MatchExpression, CategoryDescription = subscriber.Category?.Description, CategoryCode = subscriber.Category?.Code, @@ -72,6 +88,7 @@ public async Task Handle(int key) WorkGroupId = subscriber.WorkGroupId, RetryPolicyId = subscriber.RetryPolicyId, CustomRetryPolicy = subscriber.CustomRetryPolicy, + Notifications = subscriber.Notifications ?? [], Schedules = subscriber.Schedules.Select(s => new ScheduleView { Backwards = s.Backwards, diff --git a/SW.Bitween.Api/Resources/Subscriptions/GetLastRuns.cs b/SW.Bitween.Api/Resources/Subscriptions/GetLastRuns.cs index 5d86b6d1..5dadbd80 100644 --- a/SW.Bitween.Api/Resources/Subscriptions/GetLastRuns.cs +++ b/SW.Bitween.Api/Resources/Subscriptions/GetLastRuns.cs @@ -44,7 +44,7 @@ public async Task Handle(SearchSubscriptionLastRunsModel request) // One small indexed query per scheduled subscription. The subscription id // only exists inside the execution's JSON parameter, so there is nothing to - // GROUP BY — but each of these is an ordered top-N, and the alternative + // GROUP BY - but each of these is an ordered top-N, and the alternative // (pulling a whole retention window of every job's executions and grouping // in memory) is far worse on an instance with chatty schedules. // diff --git a/SW.Bitween.Api/Resources/Subscriptions/GetReceiveAttempts.cs b/SW.Bitween.Api/Resources/Subscriptions/GetReceiveAttempts.cs index af5bda0e..e3dac1ba 100644 --- a/SW.Bitween.Api/Resources/Subscriptions/GetReceiveAttempts.cs +++ b/SW.Bitween.Api/Resources/Subscriptions/GetReceiveAttempts.cs @@ -10,7 +10,7 @@ namespace SW.Bitween.Resources.Subscriptions; /// /// Paged, filterable history of one Receiving subscription's own -/// rows — independent of , which reads the scheduler's own (always-succeeds) +/// rows - independent of , which reads the scheduler's own (always-succeeds) /// history instead. /// [HandlerName("receiveattempts")] diff --git a/SW.Bitween.Api/Resources/Subscriptions/GetScheduleHealth.cs b/SW.Bitween.Api/Resources/Subscriptions/GetScheduleHealth.cs index b5ed29b5..c6686167 100644 --- a/SW.Bitween.Api/Resources/Subscriptions/GetScheduleHealth.cs +++ b/SW.Bitween.Api/Resources/Subscriptions/GetScheduleHealth.cs @@ -26,7 +26,7 @@ public class GetScheduleHealth( ISchedulerFactory schedulerFactory) : IQueryHandler { /// - /// Mirrors SW.Scheduler's internal Constants.JobParamsKey — the Quartz data-map + /// Mirrors SW.Scheduler's internal Constants.JobParamsKey - the Quartz data-map /// entry it serializes the job parameter under. Stable: it is persisted in /// qrtz_job_details, so changing it would break existing schedules. /// @@ -96,12 +96,12 @@ public async Task Handle(SearchSubscriptionScheduleHealthModel request) } /// - /// Subscription ids the scheduler is executing right now. This — not the - /// job-execution rows — is what distinguishes a genuinely running job from a + /// Subscription ids the scheduler is executing right now. This - not the + /// job-execution rows - is what distinguishes a genuinely running job from a /// stale IsRunning flag: a killed run never writes its end either, so its /// execution row stays open forever and would look identical. /// - private static async Task> CurrentlyRunningSubscriptionIds( + internal static async Task> CurrentlyRunningSubscriptionIds( IScheduler scheduler, Dictionary jobs) { var groups = jobs.Values.Select(d => d.Group).ToHashSet(); @@ -125,7 +125,7 @@ private static async Task> CurrentlyRunningSubscriptionIds( /// The state worth reporting: anything that stops a fire outranks the ones that don't. private static string WorstState(IReadOnlyCollection states, int scheduleCount) { - // No trigger where a schedule says there should be one — the job simply + // No trigger where a schedule says there should be one - the job simply // isn't registered with the scheduler and nothing will ever fire it. if (states.Count < scheduleCount) return "Missing"; if (states.Count == 0) return "Missing"; diff --git a/SW.Bitween.Api/Resources/Subscriptions/GetSourcePaths.cs b/SW.Bitween.Api/Resources/Subscriptions/GetSourcePaths.cs index 8e9c4f0e..47872cfd 100644 --- a/SW.Bitween.Api/Resources/Subscriptions/GetSourcePaths.cs +++ b/SW.Bitween.Api/Resources/Subscriptions/GetSourcePaths.cs @@ -43,7 +43,7 @@ select x } catch (SWValidationException) { - // Its file is gone — deleted by retention, most likely. Nothing to offer, which is + // Its file is gone - deleted by retention, most likely. Nothing to offer, which is // what the editor shows for a subscription that hasn't run yet. } diff --git a/SW.Bitween.Api/Resources/Subscriptions/History.cs b/SW.Bitween.Api/Resources/Subscriptions/History.cs new file mode 100644 index 00000000..402a772e --- /dev/null +++ b/SW.Bitween.Api/Resources/Subscriptions/History.cs @@ -0,0 +1,92 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Newtonsoft.Json.Linq; +using SW.Bitween.Domain; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Subscriptions; + +public class SubscriptionHistoryRequest +{ + public int SubscriptionId { get; set; } +} + +public class SubscriptionRevisionRequest +{ + public int SubscriptionId { get; set; } + public int Number { get; set; } +} + +/// One save in a subscription's history, and which settings it changed. +public record SubscriptionRevisionRow(int Number, System.DateTime SavedOn, string SavedBy, IReadOnlyList Changed); + +/// +/// A subscription's revisions, newest first: who saved it, when, and which settings each save +/// changed from the one before. Recorded from the first save after history existed. +/// +[HandlerName("history")] +public class History(BitweenDbContext dbContext, RequestContext requestContext) + : IQueryHandler +{ + public async Task Handle(SubscriptionHistoryRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Subscriptions.View); + + var revisions = await dbContext.Set().AsNoTracking() + .Where(r => r.SubscriptionId == request.SubscriptionId) + .OrderBy(r => r.Number) + .ToListAsync(); + + var names = await AccountNames(revisions.Select(r => r.SavedBy)); + var rows = new List(); + JObject previous = null; + foreach (var revision in revisions) + { + var snapshot = JObject.Parse(revision.Snapshot); + var changed = previous == null + ? new List() + : snapshot.Properties().Select(p => p.Name) + .Union(previous.Properties().Select(p => p.Name)) + .Where(name => !JToken.DeepEquals(snapshot[name], previous[name])) + .OrderBy(name => name) + .ToList(); + rows.Add(new SubscriptionRevisionRow(revision.Number, revision.SavedOn, + names.GetValueOrDefault(revision.SavedBy ?? "", revision.SavedBy), changed)); + previous = snapshot; + } + + rows.Reverse(); + return rows; + } + + /// Accounts by their id, as revisions record them, to the name they're shown by. + async Task> AccountNames(IEnumerable ids) + { + var numeric = ids.Distinct().Select(id => int.TryParse(id, out var n) ? n : (int?)null) + .Where(n => n != null).Select(n => n.Value).ToList(); + return await dbContext.Set().AsNoTracking() + .Where(a => numeric.Contains(a.Id)) + .ToDictionaryAsync(a => a.Id.ToString(), a => a.DisplayName); + } +} + +/// +/// One revision's configuration, in the shape the subscription itself is read in. Restoring it is +/// saving it as the subscription, which the admin UI does, so every check a save makes applies. +/// +[HandlerName("revision")] +public class Revision(BitweenDbContext dbContext, RequestContext requestContext) + : IQueryHandler +{ + public async Task Handle(SubscriptionRevisionRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Subscriptions.View); + + var revision = await dbContext.Set().AsNoTracking() + .FirstOrDefaultAsync(r => r.SubscriptionId == request.SubscriptionId && r.Number == request.Number) + ?? throw new SWNotFoundException($"{request.SubscriptionId}/{request.Number}"); + return new { revision.Number, revision.SavedOn, Configuration = JObject.Parse(revision.Snapshot) }; + } +} diff --git a/SW.Bitween.Api/Resources/Subscriptions/InlineIntegration.cs b/SW.Bitween.Api/Resources/Subscriptions/InlineIntegration.cs index 74e4dfb3..3679a26b 100644 --- a/SW.Bitween.Api/Resources/Subscriptions/InlineIntegration.cs +++ b/SW.Bitween.Api/Resources/Subscriptions/InlineIntegration.cs @@ -8,7 +8,7 @@ namespace SW.Bitween.Resources.Subscriptions { /// - /// Turns an integration defined inline — while a gateway route or attachment is being made — + /// Turns an integration defined inline - while a gateway route or attachment is being made - /// into a subscription, ready to be saved alongside whatever points at it. /// /// Nothing here duplicates : the pipeline is applied by the very same @@ -20,7 +20,7 @@ public static class InlineIntegration { /// /// Which integration a gateway link points at. Exactly one of an existing id and an - /// inline definition has to be given — both, or neither, is a mistake worth naming. + /// inline definition has to be given - both, or neither, is a mistake worth naming. /// public static void EnsureExactlyOne(int? subscriptionId, InlineIntegrationCreate inline) { @@ -56,7 +56,7 @@ private static async Task CheckConfiguration( throw new SWValidationException( ResponseRoutingValidation.Code, responseFailure); - // Neither gateway type carries its own partner — a partner reaches them through the + // Neither gateway type carries its own partner - a partner reaches them through the // attachment or the route, which is the very thing being made. if (model.PartnerId.HasValue) throw new SWValidationException("PARTNER_NOT_ALLOWED", @@ -66,10 +66,10 @@ private static async Task CheckConfiguration( // is exactly what committing here would make permanent. foreach (var (kind, adapterId, provided) in new[] { - ("receiver", model.ReceiverId, model.ReceiverProperties), - ("validator", model.ValidatorId, model.ValidatorProperties), - ("mapper", model.MapperId, model.MapperProperties), - ("handler", model.HandlerId, model.HandlerProperties), + ("receiver", SW.Serverless.Contract.Catalog.AdapterCatalogPaths.Ref(model.ReceiverId, model.ReceiverVersion), model.ReceiverProperties), + ("validator", SW.Serverless.Contract.Catalog.AdapterCatalogPaths.Ref(model.ValidatorId, model.ValidatorVersion), model.ValidatorProperties), + ("mapper", SW.Serverless.Contract.Catalog.AdapterCatalogPaths.Ref(model.MapperId, model.MapperVersion), model.MapperProperties), + ("handler", SW.Serverless.Contract.Catalog.AdapterCatalogPaths.Ref(model.HandlerId, model.HandlerVersion), model.HandlerProperties), }) { var missing = await adapterRequirements.MissingFor(adapterId, provided); @@ -110,8 +110,8 @@ public static async Task Stage( // and not the other. await SubscriptionConfigurationApplier.Apply(dbContext, adapterRequirements, entity, model); - // None of these types runs on its own — a GatewayApiCall waits for an attachment, a - // BusGateway for a route, a Response for a subscription to hand it a response — and the + // None of these types runs on its own - a GatewayApiCall waits for an attachment, a + // BusGateway for a route, a Response for a subscription to hand it a response - and the // one being made is in this same transaction. entity.Inactive = false; diff --git a/SW.Bitween.Api/Resources/Subscriptions/Pause.cs b/SW.Bitween.Api/Resources/Subscriptions/Pause.cs index 515ed28e..ebb91c80 100644 --- a/SW.Bitween.Api/Resources/Subscriptions/Pause.cs +++ b/SW.Bitween.Api/Resources/Subscriptions/Pause.cs @@ -15,7 +15,8 @@ public async Task Handle(int key, SubscriptionPause request) { await requestContext.EnsurePermission(dbContext, Model.Permissions.Subscriptions.Operate); - var entity = await dbContext.FindAsync(key); + var entity = await dbContext.FindAsync(key) + ?? throw new SWNotFoundException(key.ToString()); if (entity!.PausedOn == null) entity.Pause(); else diff --git a/SW.Bitween.Api/Resources/Subscriptions/ReceiveNow.cs b/SW.Bitween.Api/Resources/Subscriptions/ReceiveNow.cs index 5c7ff725..41e9ed27 100644 --- a/SW.Bitween.Api/Resources/Subscriptions/ReceiveNow.cs +++ b/SW.Bitween.Api/Resources/Subscriptions/ReceiveNow.cs @@ -13,7 +13,8 @@ async public Task Handle(int key, SubscriptionReceiveNow request) { await requestContext.EnsurePermission(dbContext, Model.Permissions.Subscriptions.Operate); - var entity = await dbContext.FindAsync(key); + var entity = await dbContext.FindAsync(key) + ?? throw new SWNotFoundException(key.ToString()); entity.SetReceiveNow(); await dbContext.SaveChangesAsync(); diff --git a/SW.Bitween.Api/Resources/Subscriptions/ResponseRoutingValidation.cs b/SW.Bitween.Api/Resources/Subscriptions/ResponseRoutingValidation.cs index 508b9685..fd1ae9a6 100644 --- a/SW.Bitween.Api/Resources/Subscriptions/ResponseRoutingValidation.cs +++ b/SW.Bitween.Api/Resources/Subscriptions/ResponseRoutingValidation.cs @@ -12,8 +12,8 @@ namespace SW.Bitween.Resources.Subscriptions; /// Guards what a delivery response may be fed into. /// /// Only a subscription, because it is the only type whose -/// entry point this is. Every other type has a door of its own — a gateway, a schedule, a -/// partner's call — and feeding it a response runs it through a door it does not have. A bus +/// entry point this is. Every other type has a door of its own - a gateway, a schedule, a +/// partner's call - and feeding it a response runs it through a door it does not have. A bus /// gateway route was the sharpest case: it ran with the bus skipped, so none of the other routes /// bound to the same message saw it. ResponseMessageTypeName is the field that actually /// puts a response on the bus. @@ -31,8 +31,12 @@ internal static class ResponseRoutingValidation /// Returns the failure message, or null when the destination is allowed. /// The subscription being saved; null while it is being created. /// What it points at today; null while it is being created. + /// + /// A mirror's copy recreating what prod already has (see ): + /// an Internal or ApiCall target is prod's saved one, left alone there, so it is let through here too. + /// public static async Task CheckDestination(BitweenDbContext dbContext, int? responseSubscriptionId, - int? selfId = null, int? savedTarget = null) + int? selfId = null, int? savedTarget = null, bool recreatingProd = false) { if (responseSubscriptionId is null || responseSubscriptionId == savedTarget) return null; @@ -41,7 +45,8 @@ public static async Task CheckDestination(BitweenDbContext dbContext, in .Select(s => (SubscriptionType?)s.Type) .SingleOrDefaultAsync(); - if (type != SubscriptionType.Response) + var keptLegacy = recreatingProd && type is SubscriptionType.Internal or SubscriptionType.ApiCall; + if (type != SubscriptionType.Response && !keptLegacy) return "A response can only be handed to a response subscription. To reach anything else, " + "publish the response on the bus and let a gateway's routes pick it up."; diff --git a/SW.Bitween.Api/Resources/Subscriptions/RetryUsage.cs b/SW.Bitween.Api/Resources/Subscriptions/RetryUsage.cs index 6505baa0..23a709ef 100644 --- a/SW.Bitween.Api/Resources/Subscriptions/RetryUsage.cs +++ b/SW.Bitween.Api/Resources/Subscriptions/RetryUsage.cs @@ -13,7 +13,7 @@ namespace SW.Bitween.Resources.Subscriptions; /// /// /// The policy-scoped report answers the same question for every subscription sharing a policy, but it -/// can only find subscriptions by policy id — so a subscription carrying an inline +/// can only find subscriptions by policy id - so a subscription carrying an inline /// CustomRetryPolicy is invisible to it while still spending and recording budget. Asking from /// the subscription's side reaches those too. /// @@ -31,7 +31,7 @@ public async Task Handle(int key, RetryPolicyUsageRequest request) if (subscription == null) throw new SWNotFoundException(key.ToString()); // Whichever policy actually applies. An inline one has no row, so the policy level of the - // alert hierarchy simply is not there for it — passed as null, which the resolver expects. + // alert hierarchy simply is not there for it - passed as null, which the resolver expects. var groups = subscription.CustomRetryPolicy?.Groups ?? subscription.RetryPolicy?.Groups ?? []; return await report.Build( diff --git a/SW.Bitween.Api/Resources/Subscriptions/SaveMapper.cs b/SW.Bitween.Api/Resources/Subscriptions/SaveMapper.cs index d7cc4502..4843b9a0 100644 --- a/SW.Bitween.Api/Resources/Subscriptions/SaveMapper.cs +++ b/SW.Bitween.Api/Resources/Subscriptions/SaveMapper.cs @@ -10,12 +10,13 @@ namespace SW.Bitween.Resources.Subscriptions { [HandlerName("savemapper")] public class SaveMapper(BitweenDbContext dbContext, IInfolinkCache BitweenCache, - RequestContext requestContext) : ICommandHandler + RequestContext requestContext, SubscriptionRevisions revisions) : ICommandHandler { public async Task Handle(int key, SubscriptionSaveMapper model) { await requestContext.EnsurePermission(dbContext, Model.Permissions.Subscriptions.Edit); - var entity = await dbContext.FindAsync(key); + var entity = await dbContext.FindAsync(key) + ?? throw new SWNotFoundException(key.ToString()); entity.MapperId = model.MapperId; entity.SetDictionaries( @@ -28,6 +29,7 @@ public async Task Handle(int key, SubscriptionSaveMapper model) await dbContext.SaveChangesAsync(); await BitweenCache.BroadcastRevoke(); + await revisions.RecordAsync(key); return null; } diff --git a/SW.Bitween.Api/Resources/Subscriptions/Search.cs b/SW.Bitween.Api/Resources/Subscriptions/Search.cs index 902a85a9..db9a0bf2 100644 --- a/SW.Bitween.Api/Resources/Subscriptions/Search.cs +++ b/SW.Bitween.Api/Resources/Subscriptions/Search.cs @@ -8,6 +8,7 @@ using Newtonsoft.Json; using SW.Bitween.Domain; using SW.Bitween.Model; +using SW.Bitween.Services.Mirror; using Document = SW.Bitween.Domain.Document; namespace SW.Bitween.Resources.Subscriptions @@ -16,12 +17,16 @@ public class Search : ISearchyHandler { private readonly BitweenDbContext _dbContext; private readonly RequestContext _requestContext; + private readonly AdapterSecretProperties _secrets; + private readonly MirrorService _mirror; private readonly List _edgeCaseProperties; - public Search(BitweenDbContext dbContext, RequestContext requestContext) + public Search(BitweenDbContext dbContext, RequestContext requestContext, AdapterSecretProperties secrets, MirrorService mirror) { _dbContext = dbContext; _requestContext = requestContext; + _secrets = secrets; + _mirror = mirror; _edgeCaseProperties = new List { @@ -38,6 +43,7 @@ public async Task Handle(SearchyRequest searchyRequest, bool lookup = fa // the full list is the data, so that's what the view permission covers. if (!lookup) await _requestContext.EnsurePermission(_dbContext, Model.Permissions.Subscriptions.View); + await _mirror.FilterAsync(searchyRequest, "subscriptions", _requestContext, _dbContext); var query = from subscriber in _dbContext.Set() join document in _dbContext.Set() on subscriber.DocumentId equals document.Id @@ -54,17 +60,34 @@ join document in _dbContext.Set() on subscriber.DocumentId equals docu MapperId = subscriber.MapperId, ValidatorId = subscriber.ValidatorId, ReceiverId = subscriber.ReceiverId, + HandlerVersion = subscriber.HandlerVersion, + MapperVersion = subscriber.MapperVersion, + ReceiverVersion = subscriber.ReceiverVersion, + ValidatorVersion = subscriber.ValidatorVersion, AggregationForId = subscriber.AggregationForId, Temporary = subscriber.Temporary, ReceiveOn = subscriber.ReceiveOn, // The next-fire time of the other scheduled type. Left out, an aggregation - // row had no next run to show at all — ReceiveOn is only ever set for + // row had no next run to show at all - ReceiveOn is only ever set for // Receiving, and the list page reads one field for both. AggregateOn = subscriber.AggregateOn, AggregationTarget = subscriber.AggregationTarget, PausedOn = subscriber.PausedOn, + PausedAutomatically = subscriber.PausedAutomatically, + AutoPauseAfterFailures = subscriber.AutoPauseAfterFailures, + ExpectTraffic = subscriber.ExpectTraffic, + QuietSince = subscriber.QuietSince, IsRunning = subscriber.IsRunning, - ConsecutiveFailures = subscriber.ConsecutiveFailures, + // As SubscriptionFailures counts it: a delivery subscription set to auto-pause + // counts its failures on its results, and a success no longer writes the row + // back to 0, so the latest result says where the run stands. + ConsecutiveFailures = subscriber.AutoPauseAfterFailures == null || subscriber.ConsecutiveFailures == 0 + ? subscriber.ConsecutiveFailures + : _dbContext.Set() + .Where(r => r.SubscriptionId == subscriber.Id) + .OrderByDescending(r => r.FinishedOn) + .Select(r => r.ConsecutiveFailures) + .FirstOrDefault() ?? subscriber.ConsecutiveFailures, LastException = subscriber.LastException, RetryPolicyId = subscriber.RetryPolicyId, MapperProperties = subscriber.MapperProperties.ToKeyAndValueCollection(), @@ -115,9 +138,12 @@ join document in _dbContext.Set() on subscriber.DocumentId equals docu }; } - /// Empties the adapter property collections on every returned row. + /// + /// Empties the adapter property collections on every returned row, keeping only the names + /// of the global values and partner properties they reference. + /// /// - /// Adapter properties hold credentials — a storage key, an OAuth secret, the password for a + /// Adapter properties hold credentials - a storage key, an OAuth secret, the password for a /// partner system. Subscriptions/Get replaces those with a sentinel before answering; /// this list endpoint returned them verbatim, which put every integration's secrets one /// request away from anyone allowed to see the list at all. @@ -125,19 +151,30 @@ join document in _dbContext.Set() on subscriber.DocumentId equals docu /// Emptied here rather than left out of the projection because the projection needs them: /// the rawsubscriptionproperties filter searches inside their values, in memory, over /// the rows this query produced. So they are loaded, used, and then dropped before the - /// response is shaped. Nothing reads them off this endpoint — both UIs configure a + /// response is shaped. Nothing reads them off this endpoint - both UIs configure a /// subscription from Subscriptions/Get, and the list has never shown them. /// /// /// Emptied rather than masked because masking can only hide what an adapter declares - /// [Secure], and the external adapters declare nothing — masking here would have + /// [Secure], and the external adapters declare nothing - masking here would have /// looked like a fix while leaving those integrations exactly as exposed. /// + /// + /// The references are read here, from the same properties, because the admin UI answers + /// "used by" for value sets and partner properties from this list. When it scanned the + /// properties itself, emptying them left every value set looking unused and safe to delete. + /// /// private static void StripAdapterProperties(List rows) { foreach (var row in rows) { + row.References = StartupValuesFiller.ReferencesIn( + new[] { row.HandlerProperties, row.MapperProperties, row.ReceiverProperties, row.ValidatorProperties } + .Where(properties => properties != null) + .SelectMany(properties => properties) + .Select(property => property.Value)); + row.HandlerProperties = []; row.MapperProperties = []; row.ReceiverProperties = []; @@ -149,7 +186,7 @@ private static void StripAdapterProperties(List rows) /// /// A second query rather than part of the projection above: Schedule.On is a /// stored as ticks, and reading .Days/.Hours/ - /// .Minutes off it inside that joined query is what Postgres cannot translate — a + /// .Minutes off it inside that joined query is what Postgres cannot translate - a /// date_part type mismatch. Read flat for the ids actually being returned and shaped in /// memory, which asks nothing of the translator, so the list can finally say when a job /// runs instead of leaving its schedule column blank. @@ -194,14 +231,18 @@ private async Task> SearchWithEdgeCases( { var searchTerm = edgeCaseFilter.ValueString.ToLower(); + if (edgeCaseFilter.Field.ToLower() == "rawsubscriptionproperties") + { + var matched = new List(); + foreach (var row in data) + if (await AnyVisiblePropertyContains(row, searchTerm)) + matched.Add(row); + data = matched; + continue; + } + data = edgeCaseFilter.Field.ToLower() switch { - "rawsubscriptionproperties" => data.Where(i => - i.HandlerProperties.Any(p => p.Value.ToLower().Contains(searchTerm)) || - i.ReceiverProperties.Any(p => p.Value.ToLower().Contains(searchTerm)) || - i.MapperProperties.Any(p => p.Value.ToLower().Contains(searchTerm)) || - i.ValidatorProperties.Any(p => p.Value.ToLower().Contains(searchTerm))) - .ToList(), "rawfiltersproperties" => data.Where(i => i.DocumentFilter.Any(p => p.Value.ToLower().Contains(searchTerm)) || (i.MatchExpression?.ToString()?.Contains(searchTerm) ?? false)) @@ -212,7 +253,7 @@ private async Task> SearchWithEdgeCases( }; } - // Only the page being returned, same as the normal path — the edge-case filters run + // Only the page being returned, same as the normal path - the edge-case filters run // over the whole set in memory, and shaping schedules for all of it would be waste. var page = data.Skip(searchyRequest.PageSize * searchyRequest.PageIndex) .Take(searchyRequest.PageSize).ToList(); @@ -227,6 +268,39 @@ private async Task> SearchWithEdgeCases( } + /// + /// Matches only what Subscriptions/Get would show. Matching the stored values let a + /// viewer recover a masked secret one character at a time: search "a", "ab", "abc" and + /// watch which subscriptions stay in the result. + /// + private async Task AnyVisiblePropertyContains(SubscriptionSearch row, string searchTerm) + { + var bags = new (string AdapterId, ICollection Properties)[] + { + (row.HandlerId, row.HandlerProperties), + (row.ReceiverId, row.ReceiverProperties), + (row.MapperId, row.MapperProperties), + (row.ValidatorId, row.ValidatorProperties), + }; + + foreach (var (adapterId, properties) in bags) + { + if (properties is null || properties.Count == 0) continue; + + var stored = properties + .Where(p => p.Key is not null) + .GroupBy(p => p.Key) + .ToDictionary(g => g.Key, g => g.Last().Value); + var visible = await _secrets.Mask(adapterId, stored); + + if (visible.Values.Any(v => v is not null && v != AdapterSecretProperties.Sentinel && + v.ToLower().Contains(searchTerm))) + return true; + } + + return false; + } + private ICollection HandleSearchyEdgeCases(ICollection filters) { var list = new List(); diff --git a/SW.Bitween.Api/Resources/Subscriptions/SubscriptionConfigurationApplier.cs b/SW.Bitween.Api/Resources/Subscriptions/SubscriptionConfigurationApplier.cs index f4bb1cbe..b71bbbc0 100644 --- a/SW.Bitween.Api/Resources/Subscriptions/SubscriptionConfigurationApplier.cs +++ b/SW.Bitween.Api/Resources/Subscriptions/SubscriptionConfigurationApplier.cs @@ -4,18 +4,20 @@ using Microsoft.EntityFrameworkCore; using SW.Bitween.Domain; using SW.Bitween.Model; +using SW.Bitween.Resources.NotificationChannels; +using SW.Bitween.Resources.RetryPolicies; using SW.PrimitiveTypes; namespace SW.Bitween.Resources.Subscriptions; /// /// Puts a onto a subscription. Create and Update both -/// go through here, so a field cannot be honoured by one and silently ignored by the other — +/// go through here, so a field cannot be honoured by one and silently ignored by the other - /// which is what happened for as long as create couldn't accept a pipeline at all. /// /// Deliberately does not touch name, document, partner or aggregation-for. On create those are /// decided by the constructor, and an Aggregation subscription's DocumentId is -/// rather than anything the caller sent — copying +/// rather than anything the caller sent - copying /// the model over it would corrupt the subscription. /// /// @@ -27,9 +29,56 @@ internal static class SubscriptionConfigurationApplier /// private const string PrivateSentinel = "__private__"; + /// The adapters a subscription held when it was loaded, which its stored secrets belong to. + internal record StoredAdapters(string Handler, string Mapper, string Receiver, string Validator, + string HandlerRef, string MapperRef, string ReceiverRef, string ValidatorRef) + { + public static StoredAdapters Of(Subscription subscription) => new( + subscription.HandlerId, subscription.MapperId, subscription.ReceiverId, subscription.ValidatorId, + subscription.HandlerRef, subscription.MapperRef, subscription.ReceiverRef, subscription.ValidatorRef); + } + + /// + /// Update reads these before its property copy, which already puts the model's adapter ids on + /// the entity; read here instead, the old adapter would be the new one, and a stored secret would + /// be kept for an adapter it was never meant for. Left out, the entity's are used, which is right + /// for one being created. + /// public static async Task Apply(BitweenDbContext dbContext, AdapterRequirements adapterRequirements, - Subscription entity, SubscriptionConfiguration model) + Subscription entity, SubscriptionConfiguration model, StoredAdapters stored = null) { + // Whether a stored secret may be kept depends on the adapter it was stored for. See + // AdapterSecretProperties.MayKeepStoredSecrets. + stored ??= StoredAdapters.Of(entity); + var (oldHandler, oldMapper, oldReceiver, oldValidator) = + (stored.Handler, stored.Mapper, stored.Receiver, stored.Validator); + var (oldHandlerRef, oldMapperRef, oldReceiverRef, oldValidatorRef) = + (stored.HandlerRef, stored.MapperRef, stored.ReceiverRef, stored.ValidatorRef); + + // A pin without an adapter means nothing, and a blank one means "follow current". + static string Pin(string adapterId, string version) => + string.IsNullOrWhiteSpace(adapterId) || string.IsNullOrWhiteSpace(version) ? null : version.Trim(); + + var pins = ( + Handler: Pin(model.HandlerId, model.HandlerVersion), + Mapper: Pin(model.MapperId, model.MapperVersion), + Receiver: Pin(model.ReceiverId, model.ReceiverVersion), + Validator: Pin(model.ValidatorId, model.ValidatorVersion)); + + static bool Changed(string before, string adapterId, string version) => + !string.Equals(before, SW.Serverless.Contract.Catalog.AdapterCatalogPaths.Ref(adapterId, version), + System.StringComparison.OrdinalIgnoreCase); + + await adapterRequirements.EnsureRunnable("handler", model.HandlerId, pins.Handler, Changed(oldHandlerRef, model.HandlerId, pins.Handler)); + await adapterRequirements.EnsureRunnable("mapper", model.MapperId, pins.Mapper, Changed(oldMapperRef, model.MapperId, pins.Mapper)); + await adapterRequirements.EnsureRunnable("receiver", model.ReceiverId, pins.Receiver, Changed(oldReceiverRef, model.ReceiverId, pins.Receiver)); + await adapterRequirements.EnsureRunnable("validator", model.ValidatorId, pins.Validator, Changed(oldValidatorRef, model.ValidatorId, pins.Validator)); + + entity.HandlerVersion = pins.Handler; + entity.MapperVersion = pins.Mapper; + entity.ReceiverVersion = pins.Receiver; + entity.ValidatorVersion = pins.Validator; + entity.ReceiverId = model.ReceiverId; entity.ValidatorId = model.ValidatorId; entity.MapperId = model.MapperId; @@ -51,7 +100,13 @@ public static async Task Apply(BitweenDbContext dbContext, AdapterRequirements a } entity.ResponseMessageTypeName = model.ResponseMessageTypeName; entity.RunOnBadResponses = model.RunOnBadResponses; - // Meaningless for every other type, where it stays at its default — but harmless + if (model.AutoPauseAfterFailures is < 1 or > 1000) + throw new SWValidationException("AutoPauseAfterFailures", + "Pause after between 1 and 1000 failures in a row, or leave it empty to never pause."); + entity.AutoPauseAfterFailures = model.AutoPauseAfterFailures; + // Replaced like the fields around it: an update that leaves it out stops expecting traffic. + entity.SetExpectTraffic(TrafficWindow.Normalize(model.ExpectTraffic)); + // Meaningless for every other type, where it stays at its default - but harmless // there, and applying it unconditionally is what stops create and update disagreeing. entity.AggregationTarget = model.AggregationTarget; @@ -63,11 +118,11 @@ public static async Task Apply(BitweenDbContext dbContext, AdapterRequirements a System.TimeSpan.Parse($"{dto.Days}.{dto.Hours}:{dto.Minutes}:0"), dto.Backwards)).ToList()); entity.SetDictionaries( - MergeWithOriginal(entity.HandlerProperties, model.HandlerProperties), - MergeWithOriginal(entity.MapperProperties, model.MapperProperties), - MergeWithOriginal(entity.ReceiverProperties, model.ReceiverProperties), + MergeWithOriginal(entity.HandlerProperties, model.HandlerProperties, oldHandler, model.HandlerId), + MergeWithOriginal(entity.MapperProperties, model.MapperProperties, oldMapper, model.MapperId), + MergeWithOriginal(entity.ReceiverProperties, model.ReceiverProperties, oldReceiver, model.ReceiverId), model.DocumentFilter?.ToDictionary() ?? new Dictionary(), - MergeWithOriginal(entity.ValidatorProperties, model.ValidatorProperties) + MergeWithOriginal(entity.ValidatorProperties, model.ValidatorProperties, oldValidator, model.ValidatorId) ); entity.SetMatchExpression(model.MatchExpression); @@ -76,13 +131,34 @@ public static async Task Apply(BitweenDbContext dbContext, AdapterRequirements a throw new SWValidationException("RETRY_POLICY_NOT_FOUND", $"Retry policy {model.RetryPolicyId} was not found."); + // Here rather than in the update handler alone, so a subscription created with an inline + // policy is held to the same rules as a shared policy and as an update. + if (model.CustomRetryPolicy != null) + { + RetryGroupValidation.EnsureCanFire(model.CustomRetryPolicy.Groups); + await RetryGroupValidation.EnsureAlertChannelsExist(dbContext, + RetryGroupValidation.AlertChannelIds(model.CustomRetryPolicy.Groups)); + } + entity.SetRetryPolicy(model.RetryPolicyId, model.CustomRetryPolicy); + + // Null leaves them as they are, so a caller that has never heard of notifications, such as + // an older client or a script, does not wipe them by saving something else. + if (model.Notifications != null) + { + await ChannelValidation.EnsureNotificationsCanSend(dbContext, model.Notifications); + entity.SetNotifications(model.Notifications); + } } private static Dictionary MergeWithOriginal( IReadOnlyDictionary original, - ICollection incoming) + ICollection incoming, string originalAdapterId, string adapterId) { + if (!AdapterSecretProperties.MayKeepStoredSecrets(originalAdapterId, adapterId, original, + (incoming ?? []).Select(kv => new KeyValuePair(kv.Key, kv.Value)))) + original = null; + var result = new Dictionary(); foreach (var kv in incoming ?? Enumerable.Empty()) { diff --git a/SW.Bitween.Api/Resources/Subscriptions/SubscriptionFailures.cs b/SW.Bitween.Api/Resources/Subscriptions/SubscriptionFailures.cs new file mode 100644 index 00000000..7ed6753c --- /dev/null +++ b/SW.Bitween.Api/Resources/Subscriptions/SubscriptionFailures.cs @@ -0,0 +1,32 @@ +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain; + +namespace SW.Bitween.Resources.Subscriptions; + +/// +/// Failed deliveries in a row, as a subscription's page shows them. +/// +/// +/// A subscription set to auto-pause counts its failures on its results, and a success no longer +/// writes the subscription row back to 0 (finding 4i of docs/design/processing-performance.md), so +/// its latest result says where the run stands. The row is still written on each failure and set +/// back to 0 on resuming, so 0 there is 0. Receivers and aggregations count their runs on the row, +/// as does anything whose latest result has no count. +/// +static class SubscriptionFailures +{ + public static async Task CountAsync(BitweenDbContext dbContext, Subscription subscription) + { + // 0 on the row is a run started again by resuming it, or no failure since counting began. + if (subscription.AutoPauseAfterFailures == null || subscription.ConsecutiveFailures == 0) + return subscription.ConsecutiveFailures; + var latest = await dbContext.Set().AsNoTracking() + .Where(r => r.SubscriptionId == subscription.Id) + .OrderByDescending(r => r.FinishedOn) + .Select(r => r.ConsecutiveFailures) + .FirstOrDefaultAsync(); + return latest ?? subscription.ConsecutiveFailures; + } +} diff --git a/SW.Bitween.Api/Resources/Subscriptions/SubscriptionRunHistory.cs b/SW.Bitween.Api/Resources/Subscriptions/SubscriptionRunHistory.cs index 4618cfb7..3d25b9ee 100644 --- a/SW.Bitween.Api/Resources/Subscriptions/SubscriptionRunHistory.cs +++ b/SW.Bitween.Api/Resources/Subscriptions/SubscriptionRunHistory.cs @@ -28,12 +28,12 @@ public static IQueryable Query( BitweenDbContext dbContext, IScheduledJobDefinition job, int subscriptionId, DateTime since) { // Executions are keyed per schedule entry, and a manual run gets its own - // generated key entirely — so neither the subscription id nor the set of + // generated key entirely - so neither the subscription id nor the set of // current schedule keys can be matched on JobName. The subscription id is // in the serialized job parameter instead, which covers all of them: // scheduled runs, manual runs, and runs of schedules since edited away. // - // The trailing comma keeps 128 from matching 1289 — safe because + // The trailing comma keeps 128 from matching 1289 - safe because // ReceivingJobParams/AggregationJobParams both carry CronExpression after // SubscriptionId, so it is never the last property. var needle = $"\"subscriptionId\":{subscriptionId},"; diff --git a/SW.Bitween.Api/Resources/Subscriptions/Update.cs b/SW.Bitween.Api/Resources/Subscriptions/Update.cs index e76723f8..26c50029 100644 --- a/SW.Bitween.Api/Resources/Subscriptions/Update.cs +++ b/SW.Bitween.Api/Resources/Subscriptions/Update.cs @@ -8,13 +8,12 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.AspNetCore.Http; -using SW.Bitween.Resources.RetryPolicies; namespace SW.Bitween.Resources.Subscriptions { public class Update(BitweenDbContext dbContext, IInfolinkCache BitweenCache, RequestContext requestContext, SubscriptionSchedulerService subScheduler, - AdapterRequirements adapterRequirements) : ICommandHandler + AdapterRequirements adapterRequirements, SubscriptionRevisions revisions) : ICommandHandler { private readonly BitweenDbContext _dbContext = dbContext; @@ -28,6 +27,8 @@ public async Task Handle(int key, SubscriptionUpdate model) // Capture before SetSchedules replaces the collection. var oldSchedules = entity.Schedules.ToList(); + // And before SetProperties copies the model's adapter ids over the stored ones. + var storedAdapters = SubscriptionConfigurationApplier.StoredAdapters.Of(entity); // Name and the runtime-state fields, which only an update may set. _dbContext.Entry(entity).SetProperties(model); @@ -37,11 +38,8 @@ public async Task Handle(int key, SubscriptionUpdate model) throw new SWValidationException("RETRY_POLICY_NOT_FOUND", $"Retry policy {model.RetryPolicyId} was not found."); - if (model.CustomRetryPolicy != null) - RetryGroupValidation.EnsureCanFire(model.CustomRetryPolicy.Groups); - // Everything a person configures, through the same code the create handler runs. - await SubscriptionConfigurationApplier.Apply(_dbContext, adapterRequirements, entity, model); + await SubscriptionConfigurationApplier.Apply(_dbContext, adapterRequirements, entity, model, storedAdapters); await _dbContext.SaveChangesAsync(); await BitweenCache.BroadcastRevoke(); @@ -49,6 +47,7 @@ public async Task Handle(int key, SubscriptionUpdate model) // Sync Quartz: unschedule removed entries, schedule new/kept ones. await subScheduler.Sync(entity, oldSchedules); + await revisions.RecordAsync(key); return null; } @@ -105,7 +104,8 @@ private ValueTask GetSub(BitweenDbContext dbContext, IHttpContextA return dbContext.FindAsync(subId); } - public Validate(BitweenDbContext dbContext, IHttpContextAccessor httpContextAccessor, AdapterRequirements adapterRequirements) + public Validate(BitweenDbContext dbContext, IHttpContextAccessor httpContextAccessor, AdapterRequirements adapterRequirements, + Services.Mirror.MirrorCopyGate mirrorCopy) { RuleFor(i => i.Name).NotEmpty(); RuleFor(i => i.MatchExpression).Must(ValidateMatch); @@ -120,8 +120,10 @@ public Validate(BitweenDbContext dbContext, IHttpContextAccessor httpContextAcce { RuleFor(i => i.MapperProperties).CustomAsync(async (provided, context, _) => { + var update = (SubscriptionUpdate)context.InstanceToValidate; + // The pinned version's properties, which may differ from the current one's. var missing = await adapterRequirements.MissingFor( - ((SubscriptionUpdate)context.InstanceToValidate).MapperId, provided); + SW.Serverless.Contract.Catalog.AdapterCatalogPaths.Ref(update.MapperId, update.MapperVersion), provided); if (missing.Count > 0) context.AddFailure($"Missing: {string.Join(",", missing)}"); }); @@ -131,8 +133,9 @@ public Validate(BitweenDbContext dbContext, IHttpContextAccessor httpContextAcce { RuleFor(i => i.HandlerProperties).CustomAsync(async (provided, context, _) => { + var update = (SubscriptionUpdate)context.InstanceToValidate; var missing = await adapterRequirements.MissingFor( - ((SubscriptionUpdate)context.InstanceToValidate).HandlerId, provided); + SW.Serverless.Contract.Catalog.AdapterCatalogPaths.Ref(update.HandlerId, update.HandlerVersion), provided); if (missing.Count > 0) context.AddFailure($"Missing: {string.Join(",", missing)}"); }); @@ -152,7 +155,7 @@ public Validate(BitweenDbContext dbContext, IHttpContextAccessor httpContextAcce { var subscription = await GetSub(dbContext, httpContextAccessor); var failure = await ResponseRoutingValidation.CheckDestination(dbContext, responseSubId, - subscription?.Id, subscription?.ResponseSubscriptionId); + subscription?.Id, subscription?.ResponseSubscriptionId, mirrorCopy.RecreatingProd); if (failure != null) context.AddFailure(nameof(SubscriptionUpdate.ResponseSubscriptionId), failure); }); @@ -169,7 +172,8 @@ public Validate(BitweenDbContext dbContext, IHttpContextAccessor httpContextAcce if (model.Schedules == null || !model.Schedules.Any()) context.AddFailure(nameof(model.Schedules), "Schedules are required for Receiving subscriptions"); - var missing = await adapterRequirements.MissingFor(model.ReceiverId, model.ReceiverProperties); + var missing = await adapterRequirements.MissingFor( + SW.Serverless.Contract.Catalog.AdapterCatalogPaths.Ref(model.ReceiverId, model.ReceiverVersion), model.ReceiverProperties); if (missing.Count > 0) context.AddFailure(nameof(model.ReceiverProperties), $"Missing properties: {string.Join(",", missing)}"); } diff --git a/SW.Bitween.Api/Resources/WorkGroups/Create.cs b/SW.Bitween.Api/Resources/WorkGroups/Create.cs index 0b69a163..e3a0fc15 100644 --- a/SW.Bitween.Api/Resources/WorkGroups/Create.cs +++ b/SW.Bitween.Api/Resources/WorkGroups/Create.cs @@ -1,5 +1,6 @@ using System.Threading.Tasks; using FluentValidation; +using Microsoft.EntityFrameworkCore; using SW.Bitween.Domain; using SW.Bitween.Model; using SW.PrimitiveTypes; @@ -12,6 +13,7 @@ public class Create(BitweenDbContext dbContext, RequestContext requestContext,II public async Task Handle(CreateWorkGroupModel request) { await requestContext.EnsurePermission(dbContext, Model.Permissions.WorkGroups.Create); + await EnsureNameIsFree(dbContext, request.Name); var workgroup = new WorkGroup() { @@ -22,7 +24,8 @@ public async Task Handle(CreateWorkGroupModel request) RabbitMqOptions = new ConsumerSettings { Prefetch = request.Options?.RabbitMqOptions?.Prefetch, - Priority = request.Options?.RabbitMqOptions?.Priority + Priority = request.Options?.RabbitMqOptions?.Priority, + Concurrency = request.Options?.RabbitMqOptions?.Concurrency } } }; @@ -36,6 +39,19 @@ public async Task Handle(CreateWorkGroupModel request) }; } + /// + /// Work groups are told apart by name everywhere but the database: the pickers, bitween apply. + /// Two applies running at once each made "Acme ERP" in the lab, split its subscriptions between + /// the two, and every apply after refused to go on until one was removed by hand. + /// + internal static async Task EnsureNameIsFree(BitweenDbContext dbContext, string name, int? except = null) + { + var wanted = name?.Trim().ToLower(); + if (string.IsNullOrEmpty(wanted)) return; + if (await dbContext.Set().AnyAsync(w => w.Id != except && w.Name.Trim().ToLower() == wanted)) + throw new SWValidationException("Name", $"There's already a work group named '{name.Trim()}'."); + } + private class Validate : AbstractValidator { public Validate() @@ -44,6 +60,10 @@ public Validate() .Matches("^\\S+$") .When(i => !string.IsNullOrEmpty(i.BusMessageName)) .WithMessage("Bus message name cannot contain spaces."); + RuleFor(i => i.Options.RabbitMqOptions.Concurrency) + .GreaterThan((ushort)0) + .When(i => i.Options?.RabbitMqOptions?.Concurrency != null) + .WithMessage("Parallel exchanges must be at least 1."); } } } \ No newline at end of file diff --git a/SW.Bitween.Api/Resources/WorkGroups/Delete.cs b/SW.Bitween.Api/Resources/WorkGroups/Delete.cs index 8b60641f..5d955352 100644 --- a/SW.Bitween.Api/Resources/WorkGroups/Delete.cs +++ b/SW.Bitween.Api/Resources/WorkGroups/Delete.cs @@ -25,7 +25,7 @@ public async Task Handle(int key, DeleteWorkGroupModel _) if (await dbContext.Set().AnyAsync(i => i.WorkGroupId.Value == category.Id)) throw new SWValidationException("CANT_BE_DELETED", "Workgroup with Subscriptions cant be deleted"); - // Data source statements run on a work group too, and that foreign key restricts as well — + // Data source statements run on a work group too, and that foreign key restricts as well - // left unchecked it refused the delete as a bare 500 naming the constraint. var statements = await dbContext.Set() .Where(s => s.WorkGroupId == key) diff --git a/SW.Bitween.Api/Resources/WorkGroups/Search.cs b/SW.Bitween.Api/Resources/WorkGroups/Search.cs index 67a791cc..cf3f6817 100644 --- a/SW.Bitween.Api/Resources/WorkGroups/Search.cs +++ b/SW.Bitween.Api/Resources/WorkGroups/Search.cs @@ -79,7 +79,8 @@ public async Task Handle(SearchWorkGroupModel request) RabbitMqOptions = new ConsumerSettings { Prefetch = workGroup.Options?.RabbitMqOptions?.Prefetch, - Priority = workGroup.Options?.RabbitMqOptions?.Priority + Priority = workGroup.Options?.RabbitMqOptions?.Priority, + Concurrency = workGroup.Options?.RabbitMqOptions?.Concurrency } }, ProcessorAckRate = processorsCounts?.AckRate, diff --git a/SW.Bitween.Api/Resources/WorkGroups/Update.cs b/SW.Bitween.Api/Resources/WorkGroups/Update.cs index 531c7bae..881279ff 100644 --- a/SW.Bitween.Api/Resources/WorkGroups/Update.cs +++ b/SW.Bitween.Api/Resources/WorkGroups/Update.cs @@ -18,8 +18,10 @@ public async Task Handle(int key, CreateWorkGroupModel request) throw new SWValidationException("WORK_GROUP_NOT_FOUND", $"Category with id {key} was not found"); // Update binds the same CreateWorkGroupModel type as Create, so Create's - // Validate (IValidator) already runs for this request too — + // Validate (IValidator) already runs for this request too - // CqApiController resolves validators by the request's concrete type. + if (!string.Equals(workGroup.Name?.Trim(), request.Name?.Trim(), System.StringComparison.OrdinalIgnoreCase)) + await Create.EnsureNameIsFree(dbContext, request.Name, key); var oldBusMessageName = workGroup.GetBusMessageName(); workGroup.Name = request.Name; workGroup.BusMessageName = request.BusMessageName; @@ -28,7 +30,8 @@ public async Task Handle(int key, CreateWorkGroupModel request) RabbitMqOptions = new ConsumerSettings { Prefetch = request.Options?.RabbitMqOptions?.Prefetch, - Priority = request.Options?.RabbitMqOptions?.Priority + Priority = request.Options?.RabbitMqOptions?.Priority, + Concurrency = request.Options?.RabbitMqOptions?.Concurrency } }; diff --git a/SW.Bitween.Api/Resources/XchangeExplanations/Explain.cs b/SW.Bitween.Api/Resources/XchangeExplanations/Explain.cs new file mode 100644 index 00000000..306f72af --- /dev/null +++ b/SW.Bitween.Api/Resources/XchangeExplanations/Explain.cs @@ -0,0 +1,230 @@ +#nullable enable +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain; +using SW.Bitween.Model; +using SW.Bitween.Services.Ai; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.XchangeExplanations; + +public class XchangeExplanationRequest +{ + public string XchangeId { get; set; } = ""; +} + +public class XchangeExplanationResponse +{ + public string Summary { get; set; } = ""; + public string Cause { get; set; } = ""; + public string Fix { get; set; } = ""; + + /// The model that wrote it. + public string? Model { get; set; } + + /// Exactly what went to the provider, masked, so anyone can see what left Bitween. + public string Sent { get; set; } = ""; +} + +/// +/// Explains a failed exchange in plain words, from its error, its setup and its documents - with +/// every value in them masked before anything is sent. +/// +/// +/// +/// What is sent, and nothing else: the error; which step failed; the subscription's kind and +/// adapters, and the names - never the values - of their settings; the retry policy's shape and the +/// retry state; the last attempts' errors; and the documents with their values replaced by types. +/// Adapter settings are names only because the exchange holds them with partner and global values +/// already filled in, and some of those are credentials. +/// +/// +/// A view right, like reading the exchange: an explanation changes nothing. Each one is recorded as +/// an operator action all the same, since it sends something about the exchange to a third party. +/// +/// +public class Explain(RequestContext requestContext, BitweenDbContext dbContext, XchangeService xchangeService, + AiModels models) + : ICommandHandler +{ + /// Earlier attempts whose errors are worth showing beside this one. + const int EarlierAttempts = 3; + + /// The largest document masked and sent; a few records are enough to explain a failure. + const int MaxMaskedFileBytes = 1_000_000; + + public async Task Handle(XchangeExplanationRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Exchanges.View); + var model = models.Current(); + + var xchange = await dbContext.Set().AsNoTracking().FirstOrDefaultAsync(x => x.Id == request.XchangeId) + ?? throw new SWNotFoundException(request.XchangeId); + var result = await dbContext.Set().AsNoTracking().FirstOrDefaultAsync(r => r.Id == xchange.Id); + if (result is null) + throw new SWValidationException("NOT_FINISHED", "This exchange hasn't finished yet, so there's nothing to explain."); + if (result.Success && !result.ResponseBad) + throw new SWValidationException("NOT_FAILED", "This exchange didn't fail."); + + var subscription = xchange.SubscriptionId is { } subscriptionId + ? await dbContext.Subscriptions().AsNoTracking().FirstOrDefaultAsync(s => s.Id == subscriptionId) + : null; + var document = await dbContext.Set().AsNoTracking().FirstOrDefaultAsync(d => d.Id == xchange.DocumentId); + var delayed = await dbContext.Set().AsNoTracking().FirstOrDefaultAsync(d => d.Id == xchange.Id); + + var masking = new Masking(); + var sent = new StringBuilder(); + + // Documents first: masking them is what tells the error masking which values to take out. + var files = new List<(string Name, string Text)>(); + await AddFile(files, masking, xchange, XchangeFileType.Input, xchange.InputSize, xchange.InputContentType ?? ContentTypeOf(document)); + await AddFile(files, masking, xchange, XchangeFileType.Output, result.OutputSize, result.OutputContentType); + await AddFile(files, masking, xchange, XchangeFileType.Response, result.ResponseSize, result.ResponseContentType); + + sent.AppendLine("Failed step: " + FailedStep(xchange, result)); + if (subscription is not null) + { + sent.AppendLine($"Subscription kind: {subscription.Type}"); + sent.AppendLine($"Mapper: {subscription.MapperId ?? "none"}; handler: {subscription.HandlerId ?? "none"}"); + } + else + { + sent.AppendLine($"Mapper: {xchange.MapperId ?? "none"}; handler: {xchange.HandlerId ?? "none"} (the subscription no longer exists)"); + } + sent.AppendLine("Handler settings (names only): " + Names(xchange.HandlerProperties)); + sent.AppendLine("Document format: " + (document?.DocumentFormat.ToString() ?? "unknown")); + if (result.ResponseBad) sent.AppendLine("The partner's response was marked as bad data."); + var earlier = await EarlierErrors(xchange); + sent.AppendLine($"Attempt {earlier.Chain + 1} of its retry chain"); + if (!string.IsNullOrWhiteSpace(result.RetryBlockedReason)) + sent.AppendLine("Why it won't be retried: " + masking.Error(result.RetryBlockedReason)); + if (delayed is not null) sent.AppendLine($"A retry is scheduled for {delayed.On:u}."); + sent.AppendLine(RetryPolicy(subscription)); + + sent.AppendLine(); + sent.AppendLine(""); + sent.AppendLine(string.IsNullOrWhiteSpace(result.Exception) ? "(no error text; see the response)" : masking.Error(result.Exception)); + sent.AppendLine(""); + + // Only each earlier error's first line: the rest is a stack trace, nearly always the same one. + var thisError = FirstLine(result.Exception); + foreach (var (attempt, error) in earlier.Errors) + { + var line = FirstLine(error); + sent.AppendLine(line == thisError + ? $"Earlier attempt {attempt}: the same error." + : $"Earlier attempt {attempt}: {masking.Error(line)}"); + } + + foreach (var (name, text) in files) + { + sent.AppendLine(); + sent.AppendLine($"<{name}_document>"); + sent.AppendLine(text); + sent.AppendLine($""); + } + + var message = sent.ToString(); + + // Recorded before it goes, so a call that fails or times out after sending is on record too. + await dbContext.RecordNowAsync(requestContext, OperatorAction.ExplainFailure, xchange.Id, + $"Sent to {models.ModelName}; documents and error masked."); + var explanation = FailurePrompt.Parse(await model.Complete(new AiPrompt(FailurePrompt.Instructions, message, Json: true, MaxTokens: 4000))); + + return new XchangeExplanationResponse + { + Summary = explanation.Summary, + Cause = explanation.Cause, + Fix = explanation.Fix, + Model = models.ModelName, + Sent = message, + }; + } + + async Task AddFile(List<(string, string)> files, Masking masking, Xchange xchange, XchangeFileType type, int size, + string? contentType) + { + if (size <= 0) return; + var name = type.ToString().ToLowerInvariant(); + // Masking reads the whole document before cutting what it sends; past this, it isn't read. + if (size > MaxMaskedFileBytes) + { + files.Add((name, "(not sent: too large to mask)")); + return; + } + string text; + try + { + text = await xchangeService.GetFile(xchange, type); + } + catch (SWValidationException) + { + files.Add((name, "(no longer in storage)")); + return; + } + files.Add((name, masking.Document(text, contentType) + ?? "(not sent: only JSON, XML and CSV documents can be masked)")); + } + + /// Not stored, so worked out: no output from a mapping exchange means mapping failed. + static string FailedStep(Xchange xchange, XchangeResult result) => + result.Success && result.ResponseBad ? "delivered, but the partner's response was bad" + : xchange.MapperId is not null && result.OutputSize <= 0 ? "mapping" + : "delivery (the handler)"; + + static string Names(IReadOnlyDictionary? properties) => + properties is null || properties.Count == 0 + ? "none" + : string.Join(", ", properties.OrderBy(p => p.Key).Select(p => $"{p.Key} ({(string.IsNullOrEmpty(p.Value) ? "empty" : "set")})")); + + static string RetryPolicy(Subscription? subscription) + { + var groups = subscription?.RetryPolicy?.Groups ?? subscription?.CustomRetryPolicy?.Groups; + if (groups is null || groups.Count == 0) return "Retry policy: none"; + return "Retry policy" + (subscription?.RetryPolicy?.Name is { } name ? $" \"{name}\"" : "") + ": " + string.Join("; ", + groups.Select(g => $"{g.Name} ({(g.Enabled ? "on" : "off")}, applies to {string.Join("/", g.AppliesTo)}, {g.Action}" + + (g.Budget is { } b ? $", up to {b.MaxAttemptsTotal} attempts" : "") + ")")); + } + + /// How many attempts came before this one, and the errors of the last few. + /// + /// Counted along the chain rather than read from the result's attempt number, which counts only + /// the retries a policy made - five attempts by hand all say 1. + /// + async Task<(int Chain, List<(string Id, string Error)> Errors)> EarlierErrors(Xchange xchange) + { + var errors = new List<(string, string)>(); + var chain = 0; + var id = xchange.RetryFor; + while (id is not null && chain < MaxChain) + { + var previous = await dbContext.Set().AsNoTracking() + .Where(x => x.Id == id).Select(x => new { x.Id, x.RetryFor }).FirstOrDefaultAsync(); + if (previous is null) break; + chain++; + if (errors.Count < EarlierAttempts) + { + var error = await dbContext.Set().AsNoTracking() + .Where(r => r.Id == previous.Id).Select(r => r.Exception).FirstOrDefaultAsync(); + if (!string.IsNullOrWhiteSpace(error)) errors.Add((previous.Id, error)); + } + id = previous.RetryFor; + } + return (chain, errors); + } + + /// A chain is walked this far at most; a longer one says so as "50+" would, by stopping. + const int MaxChain = 50; + + static string FirstLine(string? text) => (text ?? "").Split('\n')[0].Trim(); + + static string? ContentTypeOf(Document? document) => document?.DocumentFormat switch + { + DocumentFormat.Json => "application/json", + DocumentFormat.Xml => "application/xml", + _ => null, + }; +} diff --git a/SW.Bitween.Api/Resources/XchangeQueries/Ask.cs b/SW.Bitween.Api/Resources/XchangeQueries/Ask.cs new file mode 100644 index 00000000..4839d119 --- /dev/null +++ b/SW.Bitween.Api/Resources/XchangeQueries/Ask.cs @@ -0,0 +1,117 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Domain; +using SW.Bitween.Services.Ai; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.XchangeQueries; + +public class XchangeQueryRequest +{ + public string Question { get; set; } = ""; + + /// The asker's time zone, as the browser names it: "Asia/Amman". UTC when missing or unknown. + public string? TimeZone { get; set; } +} + +public class XchangeQueryResponse +{ + /// The Exchanges page's URL parameters: status=failed&partnerId=12&range=7d. + public string Query { get; set; } = ""; + + /// A sentence for above the result, with {count} where the page's total goes. + public string Answer { get; set; } = ""; + + /// What the filter couldn't say. + public List Notes { get; set; } = new(); + + public string? Model { get; set; } + + /// Exactly what went to the provider. + public string Sent { get; set; } = ""; +} + +/// +/// Turns a plain-English question about exchanges into the Exchanges page's filter. +/// +/// +/// The model is sent the question, the time and names - of partners, subscriptions, information +/// types and promoted-property keys - and nothing about any exchange. The filter it answers with +/// is the page's own, so it runs as every other filter does, with the asker's rights. +/// Each question is recorded as an operator action, since it goes to a third party. +/// +public class Ask(RequestContext requestContext, BitweenDbContext dbContext, AiModels models) + : ICommandHandler +{ + public async Task Handle(XchangeQueryRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Exchanges.View); + + var question = request.Question?.Trim() ?? ""; + if (question.Length == 0) + throw new SWValidationException("QUESTION", "Type a question about your exchanges first."); + if (question.Length > ExchangeQueryPrompt.MaxQuestionLength) + throw new SWValidationException("QUESTION", + $"Questions are limited to {ExchangeQueryPrompt.MaxQuestionLength} characters."); + + var model = models.Current(); + + var zone = TimeZoneOf(request.TimeZone); + var now = TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, zone); + var names = await NamesAsync(); + var message = ExchangeQueryPrompt.Message(question, now, zone.Id, names); + + // Recorded before it goes, so a call that fails or times out after sending is on record too. + await dbContext.RecordNowAsync(requestContext, OperatorAction.AskExchanges, "exchanges", + $"Asked {models.ModelName}: {question}"); + var answer = await model.Complete(new AiPrompt(ExchangeQueryPrompt.Instructions, message, Json: true, MaxTokens: 2000)); + var query = ExchangeQueryPrompt.Parse(answer, names); + + return new XchangeQueryResponse + { + Query = query.Query, + Answer = query.Answer, + Notes = query.Notes.ToList(), + Model = models.ModelName, + Sent = message, + }; + } + + async Task NamesAsync() + { + var partners = await dbContext.Set().AsNoTracking() + .OrderBy(p => p.Name).Select(p => new QueryName(p.Id, p.Name, null)) + .Take(ExchangeQueryPrompt.MaxNames + 1).ToListAsync(); + + var subscriptions = (await dbContext.Set().AsNoTracking() + .OrderBy(s => s.Name) + .Select(s => new { s.Id, s.Name, s.Type, s.PartnerId, s.DocumentId, s.Inactive }) + .Take(ExchangeQueryPrompt.MaxNames + 1).ToListAsync()) + .Select(s => new QueryName(s.Id, s.Name, + $"{s.Type}, partner {(s.PartnerId?.ToString() ?? "none")}, information type {s.DocumentId}{(s.Inactive ? ", inactive" : "")}")) + .ToList(); + + var documents = await dbContext.Set().AsNoTracking() + .OrderBy(d => d.Name).Take(ExchangeQueryPrompt.MaxNames + 1) + .Select(d => new { d.Id, d.Name, d.PromotedProperties }) + .ToListAsync(); + var informationTypes = documents + .Select(d => new QueryName(d.Id, d.Name, d.PromotedProperties is { Count: > 0 } promoted + ? "promotes " + string.Join(", ", promoted.Keys) + : null)) + .ToList(); + var keys = documents.SelectMany(d => d.PromotedProperties?.Keys ?? Enumerable.Empty()) + .Distinct(StringComparer.OrdinalIgnoreCase).Order(StringComparer.OrdinalIgnoreCase).ToList(); + + return new QueryNames(partners, subscriptions, informationTypes, keys); + } + + static TimeZoneInfo TimeZoneOf(string? id) => + !string.IsNullOrWhiteSpace(id) && TimeZoneInfo.TryFindSystemTimeZoneById(id.Trim(), out var zone) + ? zone + : TimeZoneInfo.Utc; +} diff --git a/SW.Bitween.Api/Resources/Xchanges/BulkRetry.cs b/SW.Bitween.Api/Resources/Xchanges/BulkRetry.cs index 68b1ca52..65913c5f 100644 --- a/SW.Bitween.Api/Resources/Xchanges/BulkRetry.cs +++ b/SW.Bitween.Api/Resources/Xchanges/BulkRetry.cs @@ -8,7 +8,7 @@ namespace SW.Bitween.Resources.Xchanges { /// - /// Retries a selection of exchanges — either a hand-picked list of ids or everything a filter + /// Retries a selection of exchanges - either a hand-picked list of ids or everything a filter /// matches. Returns what it did: answers the same question /// beforehand, so the caller can show it and be sure the two agree. /// @@ -32,13 +32,13 @@ public async Task Handle(XchangeBulkRetry request) { await _requestContext.EnsurePermission(_dbContext, Model.Permissions.Exchanges.Operate); - var prepared = await new BulkRetryPlanner(_dbContext).Prepare(request); + var prepared = await new BulkRetryPlanner(_dbContext).Prepare(request, BulkRetryPlanner.Limit); var plan = prepared.Plan; if (plan.OverLimit) throw new SWValidationException("TOO_MANY", $"{plan.Selected:n0} exchanges is more than the {BulkRetryPlanner.Limit} this can retry " + - "in one go. Narrow the filter and retry the rest after."); + $"in one go. Send the preview's targets back in rounds of {BulkRetryPlanner.Limit}."); var retried = 0; foreach (var id in prepared.Targets) @@ -72,7 +72,7 @@ public async Task Handle(XchangeBulkRetry request) } // The tolerant read, not GetFile: a retry re-sends the original input, so an - // exchange whose input has been deleted or expired cannot be retried — and one of + // exchange whose input has been deleted or expired cannot be retried - and one of // those in a selection of five hundred must not take the other 499 with it. var xchangeFile = await _xchangeService.ReadInputFile(xchange); if (xchangeFile == null) @@ -106,7 +106,7 @@ await _xchangeService.CreateXchange(xchange, xchangeFile, subscription?.WorkGrou e.Validations.Any(v => v.Key == "ALREADY_RETRIED")) { // The planner resolves every selection to the end of its chain, so this is not - // reachable by choosing badly — it means someone retried this attempt in the + // reachable by choosing badly - it means someone retried this attempt in the // moment between the plan being worked out and it being carried out. Reported // like the other per-exchange refusals rather than thrown, so one racing // operator cannot cancel another's whole recovery. @@ -121,6 +121,9 @@ await _xchangeService.CreateXchange(xchange, xchangeFile, subscription?.WorkGrou retried++; } + if (retried > 0) + _dbContext.Record(_requestContext, OperatorAction.BulkRetry, $"{retried} exchanges", + $"{retried} retried, {plan.Skipped?.Count ?? 0} skipped"); await _dbContext.SaveChangesAsync(); plan.WillRetry = retried; diff --git a/SW.Bitween.Api/Resources/Xchanges/BulkRetryPlanner.cs b/SW.Bitween.Api/Resources/Xchanges/BulkRetryPlanner.cs index da8a9f5f..8fc46cf0 100644 --- a/SW.Bitween.Api/Resources/Xchanges/BulkRetryPlanner.cs +++ b/SW.Bitween.Api/Resources/Xchanges/BulkRetryPlanner.cs @@ -14,19 +14,30 @@ namespace SW.Bitween.Resources.Xchanges; /// /// /// Shared by the preview and the retry itself so that what someone confirms is what runs. Built by -/// hand rather than injected, like RetryGroupBudget — it is a piece of one request's work, +/// hand rather than injected, like RetryGroupBudget - it is a piece of one request's work, /// not a service. /// internal sealed class BulkRetryPlanner { /// /// The largest selection one request will carry out. Every exchange retried means reading its - /// input back out of storage and writing a new exchange, in sequence, inside the one request — + /// input back out of storage and writing a new exchange, in sequence, inside the one request - /// so the ceiling is about what can finish before something upstream gives up waiting, not - /// about the database. Past it the caller is asked to narrow the filter. + /// about the database. A bigger selection is retried in rounds of this many. /// internal const int Limit = 500; + /// + /// The largest selection a preview works out. Up to it, the plan lists every attempt that will + /// run, and the caller sends them back in rounds of . Past it the caller is + /// asked to narrow the filter. + /// + /// + /// Rounds rather than raising : each round is its own request, so none of them + /// runs long, and a round that fails leaves the ones before it done. + /// + internal const int MaxSelection = 5000; + /// /// Guards the walk to the newest attempt against a cycle in the data, the same way /// does. No honest chain approaches it. @@ -44,24 +55,29 @@ internal sealed class Prepared internal List Targets { get; init; } = new List(); } - internal async Task Prepare(XchangeBulkRetry request) + /// + /// The largest selection to work out: for a retry, + /// for a preview. + /// + internal async Task Prepare(XchangeBulkRetry request, int upTo) { var selection = new XchangeSelection(_dbContext); - var selected = await selection.Resolve(request, Limit); + var selected = await selection.Resolve(request, upTo); - if (selected.Count > Limit) + if (selected.Count > upTo) return new Prepared { Plan = new XchangeBulkRetryPlan { // A hand-picked list is already counted. A filter's selection was cut off at - // Limit + 1 to notice it was too long without reading it all, so that one is - // counted properly now — the caller is told how far past the line it is rather + // upTo + 1 to notice it was too long without reading it all, so that one is + // counted properly now - the caller is told how far past the line it is rather // than just "501", and never a count of rows it did not ask about. Selected = string.IsNullOrWhiteSpace(request.Filter) ? selected.Count : await selection.Count(request), Limit = Limit, + MaxSelection = MaxSelection, OverLimit = true } }; @@ -70,8 +86,10 @@ internal async Task Prepare(XchangeBulkRetry request) var state = await ReadState(newestAttempt.Values.Distinct().ToList()); var reset = request.Reset; - var plan = new XchangeBulkRetryPlan { Selected = selected.Count, Limit = Limit }; + var plan = new XchangeBulkRetryPlan { Selected = selected.Count, Limit = Limit, MaxSelection = MaxSelection }; var targets = new List(); + // The list keeps the order; the set answers "already in it" without walking up to MaxSelection entries. + var targeted = new HashSet(); foreach (var selectedId in selected) { @@ -91,13 +109,14 @@ internal async Task Prepare(XchangeBulkRetry request) continue; } - // Two selections in the same chain — an exchange and its own retry, say — come to the + // Two selections in the same chain - an exchange and its own retry, say - come to the // same attempt, which is retried once. - if (!targets.Contains(targetId)) + if (targeted.Add(targetId)) targets.Add(targetId); } plan.WillRetry = targets.Count; + plan.Targets = targets; plan.Properties = await ReadPromotedProperties(plan); return new Prepared { Plan = plan, Targets = targets }; } @@ -117,7 +136,7 @@ private static string WhyNot(SelectionState state, bool substituted, bool reset) // Only when the caller asked for the subscription's current configuration, which there is // no way to read once the subscription is gone. Checked here as well as at the moment of // retrying, because a plan that promises to retry something the retry then skips makes the - // confirmation worthless — the whole point of showing it is that it is what will happen. + // confirmation worthless - the whole point of showing it is that it is what will happen. if (reset && state.SubscriptionMissing) return $"{subject} cannot have its properties re-resolved: the subscription no longer exists."; @@ -125,8 +144,8 @@ private static string WhyNot(SelectionState state, bool substituted, bool reset) return $"{subject} already has an auto-retry scheduled. Run that now instead."; // An exchange with no result is deliberately not skipped as "still running". That is what - // a broker outage leaves behind — hundreds of exchanges that were never processed and never - // will be — and retrying those in bulk is the main thing a wide selection is for. Refusing + // a broker outage leaves behind - hundreds of exchanges that were never processed and never + // will be - and retrying those in bulk is the main thing a wide selection is for. Refusing // them would take the recovery path away to protect against double-running work that, in // the case anyone actually selects in bulk, is not running at all. @@ -141,7 +160,7 @@ private static string WhyNot(SelectionState state, bool substituted, bool reset) /// would actually run, since the selected one is refused a second retry. /// /// - /// One query per round for the whole selection at once, rather than a walk per exchange — + /// One query per round for the whole selection at once, rather than a walk per exchange - /// five hundred selections down a chain of four is four queries, not two thousand. /// private async Task> FindNewestAttempts(List selected) @@ -150,8 +169,8 @@ private async Task> FindNewestAttempts(List s for (var depth = 0; depth < MaxDepth; depth++) { - // Where each selection has got to so far. Asking about that frontier — rather than - // about one level of the tree, tracking which nodes the walk had already seen — is what + // Where each selection has got to so far. Asking about that frontier - rather than + // about one level of the tree, tracking which nodes the walk had already seen - is what // keeps selections that sit in the same chain consistent with each other. With a seen // set, a selection lagging behind another stopped dead on a node the leader had already // stepped over, and was reported as "already retried, its newest attempt runs instead" @@ -168,7 +187,7 @@ private async Task> FindNewestAttempts(List s // One retry per exchange is the rule, but exchanges retried before it was enforced can // have more than one. Follow the newest, and let the retry tree in the drawer be where - // the fork is shown — a bulk retry has to choose something, and the newest attempt is + // the fork is shown - a bulk retry has to choose something, and the newest attempt is // the one that reflects where the work actually got to. var step = children .GroupBy(c => c.RetryFor) @@ -192,7 +211,7 @@ private async Task> FindNewestAttempts(List s /// /// What the payload promoted, for each exchange the plan names on either side of a - /// substitution or in a skip — so the caller can describe them the way the exchange list does + /// substitution or in a skip - so the caller can describe them the way the exchange list does /// rather than by id alone. /// private async Task>> ReadPromotedProperties( @@ -227,7 +246,7 @@ private sealed class SelectionState public System.DateTime? ScheduledRetryOn { get; set; } /// - /// Covers an exchange that never had a subscription — a document-only one — as well as one + /// Covers an exchange that never had a subscription - a document-only one - as well as one /// whose subscription has since been deleted. Both leave a reset with nothing to read. /// public bool SubscriptionMissing { get; set; } diff --git a/SW.Bitween.Api/Resources/Xchanges/BulkRetryPreview.cs b/SW.Bitween.Api/Resources/Xchanges/BulkRetryPreview.cs index dc77a50f..b9cc69bf 100644 --- a/SW.Bitween.Api/Resources/Xchanges/BulkRetryPreview.cs +++ b/SW.Bitween.Api/Resources/Xchanges/BulkRetryPreview.cs @@ -12,6 +12,11 @@ namespace SW.Bitween.Resources.Xchanges; /// ones that have since succeeded or are still running drop out, and two selections in one chain /// come to the same attempt. Retrying is not undoable, so the caller gets to show all of that and /// be told the number before anyone commits to it. +/// +/// It works out a selection past what one retry carries out, up to +/// , so the caller can retry it in rounds from the plan's +/// targets. +/// /// [HandlerName("bulkretrypreview")] public class BulkRetryPreview : ICommandHandler @@ -31,7 +36,7 @@ public async Task Handle(XchangeBulkRetry request) // describes an action only they can take, so it is gated with the action. await _requestContext.EnsurePermission(_dbContext, Model.Permissions.Exchanges.Operate); - var prepared = await new BulkRetryPlanner(_dbContext).Prepare(request); + var prepared = await new BulkRetryPlanner(_dbContext).Prepare(request, BulkRetryPlanner.MaxSelection); return prepared.Plan; } } diff --git a/SW.Bitween.Api/Resources/Xchanges/Get.cs b/SW.Bitween.Api/Resources/Xchanges/Get.cs index eae6070b..d03eeb95 100644 --- a/SW.Bitween.Api/Resources/Xchanges/Get.cs +++ b/SW.Bitween.Api/Resources/Xchanges/Get.cs @@ -48,7 +48,11 @@ join subscriber in dbContext.Set() on xchange.SubscriptionId equal //OutputUri = xchangeService.GetFileUrl(queryResult.xchange.Id, XchangeFileType.Output), // A sealed Bitween link: the partner follows it with no login, as it did the public // storage URL this used to be, and exchange files are private in storage now. - ResponseUri = xchangeService.FileUrl(queryResult.xchange.Id, queryResult.xchange.FilesPrefix, XchangeFileType.Response), + // No link to a response the delivery got but storage couldn't keep: it would open nothing. + ResponseUri = queryResult.result.NotStoredReason(XchangeFileType.Response) != null + ? null + : xchangeService.FileUrl(queryResult.xchange.Id, queryResult.xchange.FilesPrefix, XchangeFileType.Response), + Warning = queryResult.result.Warning, }; } diff --git a/SW.Bitween.Api/Resources/Xchanges/Retry.cs b/SW.Bitween.Api/Resources/Xchanges/Retry.cs index 7b03b329..7d507c77 100644 --- a/SW.Bitween.Api/Resources/Xchanges/Retry.cs +++ b/SW.Bitween.Api/Resources/Xchanges/Retry.cs @@ -29,7 +29,8 @@ public async Task Handle(string key, XchangeRetry xchangeRetry) throw new SWValidationException("AUTO_RETRY_SCHEDULED", "An auto-retry is already scheduled for this exchange. Use \"Run Now\" to execute it immediately instead of retrying manually."); - var xchange = await dbContext.FindAsync(key); + var xchange = await dbContext.FindAsync(key) + ?? throw new SWNotFoundException(key.ToString()); var inputFileData = await xchangeService.GetFile(xchange, XchangeFileType.Input); var xchangeFile = new XchangeFile(inputFileData, xchange.InputName); var subscription = await dbContext.Subscriptions().FirstOrDefaultAsync(s => s.Id == xchange.SubscriptionId); @@ -46,10 +47,21 @@ public async Task Handle(string key, XchangeRetry xchangeRetry) await xchangeService.CreateXchange(xchange, xchangeFile, subscription?.WorkGroup, manualRetry: true); } - - - await dbContext.SaveChangesAsync(); - + dbContext.Record(requestContext, OperatorAction.Retry, key, + xchangeRetry.Reset ? "with the subscription's current settings" : null); + try + { + await dbContext.SaveChangesAsync(); + } + catch (DbUpdateException) + { + // Another retry of the same exchange was saved a moment before this one. + if (await dbContext.Set().AsNoTracking().AnyAsync(c => c.Id == key)) + throw new SWValidationException("ALREADY_RETRIED", + "This exchange was retried a moment ago by someone or something else. Retry that attempt instead."); + throw; + } + return null; } } diff --git a/SW.Bitween.Api/Resources/Xchanges/RetryTree.cs b/SW.Bitween.Api/Resources/Xchanges/RetryTree.cs index 2ef802e1..36ca75c5 100644 --- a/SW.Bitween.Api/Resources/Xchanges/RetryTree.cs +++ b/SW.Bitween.Api/Resources/Xchanges/RetryTree.cs @@ -15,13 +15,13 @@ namespace SW.Bitween.Resources.Xchanges; /// /// /// Its own endpoint rather than part of the exchange search, because a chain cannot be had from the -/// same query that lists exchanges — it takes a walk of unknown length, and the search is already +/// same query that lists exchanges - it takes a walk of unknown length, and the search is already /// the expensive one. Callers avoid asking altogether for the common case: and come back with every row, and /// an exchange with neither has no chain to fetch. /// /// -/// Walked iteratively, a level at a time, rather than as a recursive query — the three supported +/// Walked iteratively, a level at a time, rather than as a recursive query - the three supported /// databases would each need their own SQL for that, to save queries on a chain that a retry budget /// keeps to single digits in practice. Both walks are indexed lookups on RetryFor. /// @@ -31,7 +31,7 @@ public class RetryTree : IQueryHandler { /// /// How far the walk goes in each direction before giving up and saying so. Far past any real - /// chain — a retry policy's budget is the practical limit — so this exists to keep a cycle in + /// chain - a retry policy's budget is the practical limit - so this exists to keep a cycle in /// the data from becoming an endless loop, not to shorten honest answers. /// private const int MaxDepth = 100; @@ -40,7 +40,7 @@ public class RetryTree : IQueryHandler /// How many attempts one answer carries. Depth alone does not bound the work: an exchange /// retried many times over before the one-retry rule existed has all of those as direct /// children, and every one would be loaded and serialized. Far above any real chain, so - /// reaching it means the data is unusual — and the answer then says it is partial rather than + /// reaching it means the data is unusual - and the answer then says it is partial rather than /// quietly costing the database more the older the exchange is. /// private const int MaxNodes = 500; @@ -64,7 +64,7 @@ public async Task Handle(XchangeRetryTreeRequest request) var truncated = false; // Up to the original. Each step reads one pointer, so the exchange asked about does not - // have to be the newest attempt — opening any of them shows the same chain. + // have to be the newest attempt - opening any of them shows the same chain. var rootId = request.Id; var ancestors = new HashSet { rootId }; for (var depth = 0; depth < MaxDepth; depth++) @@ -75,7 +75,7 @@ public async Task Handle(XchangeRetryTreeRequest request) .FirstOrDefaultAsync(); // Also covers the exchange not existing at all: no row, so no parent, and the walk - // down then finds nothing either — an empty tree rather than a 404, because a caller + // down then finds nothing either - an empty tree rather than a 404, because a caller // asking about an exchange it is looking at wants the chain, not another error to // handle. if (parent == null || !ancestors.Add(parent)) diff --git a/SW.Bitween.Api/Resources/Xchanges/Search.cs b/SW.Bitween.Api/Resources/Xchanges/Search.cs index af8fe0a5..3dc89dcc 100644 --- a/SW.Bitween.Api/Resources/Xchanges/Search.cs +++ b/SW.Bitween.Api/Resources/Xchanges/Search.cs @@ -17,7 +17,7 @@ public class Search(BitweenDbContext dbContext, XchangeService xchangeService, { /// /// Largest exact total the exchange search reports. Beyond it the response carries - /// CountCap + 1, meaning "more than this" — the client renders that as "10,000+". + /// CountCap + 1, meaning "more than this" - the client renders that as "10,000+". /// Kept in step with COUNT_CAP in ClientApp's ExchangesPage.tsx. /// internal const int CountCap = 10_000; @@ -92,7 +92,7 @@ from delayedRetry in drGroup.DefaultIfEmpty() // a left one: an exchange still running, or one that failed before it // produced a result, has no row there. Reading a non-nullable int off // that null threw "Nullable object must have a value" out of the whole - // query — one such exchange 500'd the entire list. + // query - one such exchange 500'd the entire list. InputFileSize = xchange.InputSize, OutputFileSize = result != null ? result.OutputSize : 0, ResponseFileSize = result != null ? result.ResponseSize : 0, @@ -101,10 +101,11 @@ from delayedRetry in drGroup.DefaultIfEmpty() // gateway/bus-route partner, or the subscription's own PartnerId as a // fallback there too) but the column was added later with no backfill, so // pre-migration xchanges have it null even when their subscription carries - // a direct PartnerId — fall back to that for those legacy rows. + // a direct PartnerId - fall back to that for those legacy rows. PartnerId = xchange.PartnerId ?? subscriber.PartnerId, ScheduledRetryOn = delayedRetry != null ? delayedRetry.On : (DateTime?)null, - RetryBlockedReason = result.RetryBlockedReason + RetryBlockedReason = result.RetryBlockedReason, + Warning = result.Warning }; query = query.ApplySpecialFilters(searchyRequest, dbContext); @@ -113,8 +114,8 @@ from delayedRetry in drGroup.DefaultIfEmpty() searchyRequest.Sorts, searchyRequest.PageSize, searchyRequest.PageIndex); // Only id and information type, so the lookup has nothing to hide from a caller without - // the view permission. It returned whole rows before — errors, promoted values, source - // values — to anyone signed in. Capped, because exchanges run to millions and a lookup + // the view permission. It returned whole rows before - errors, promoted values, source + // values - to anyone signed in. Capped, because exchanges run to millions and a lookup // with no page size would otherwise list every one of them. if (lookup) return await s.Take(LookupLimit).ToDictionaryAsync(k => k.Id, v => v.DocumentName); @@ -129,7 +130,7 @@ from delayedRetry in drGroup.DefaultIfEmpty() // Which of these have already been retried, so the client can tell a spent exchange // from a retryable one without asking about each row's chain. Asked separately rather // than as a subquery in the projection above, because TotalCount reuses that query and - // an exact count already costs more than fetching the rows does — this way neither + // an exact count already costs more than fetching the rows does - this way neither // plan changes. One indexed lookup over RetryFor for the page's worth of ids. if (r.Count > 0) { diff --git a/SW.Bitween.Api/Resources/Xchanges/Summary.cs b/SW.Bitween.Api/Resources/Xchanges/Summary.cs new file mode 100644 index 00000000..10a807cf --- /dev/null +++ b/SW.Bitween.Api/Resources/Xchanges/Summary.cs @@ -0,0 +1,189 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Model; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Xchanges; + +/// +/// The exchange list's filter, summarised: how many exchanges are in each status, and, over a date +/// range, when they started. GET /api/xchanges/summary, with the same filter= +/// parameters the search takes. +/// +/// +/// +/// A status filter in the request is ignored. This is what the status tabs above the list show - +/// each status's count under every other filter - so applying the selected tab first would turn +/// the other three into zeros. +/// +/// +/// Filters through , the lean query bulk retry and +/// export select with, rather than the search's own projection: nothing here needs a file link or +/// a promoted value, and the search's projection builds those in C#, which cannot be grouped on. +/// Every special filter the search understands is applied the same way, so a count is never of a +/// different set than the list it sits over. +/// +/// +[HandlerName("summary")] +public class Summary(BitweenDbContext dbContext, RequestContext requestContext) : ISearchyHandler +{ + /// The status numbers reads, in the order they are counted. + private const string Processing = "0", Success = "1", BadResponse = "2", Failed = "3"; + + public async Task Handle(SearchyRequest searchyRequest, bool lookup = false, string searchPhrase = null) + { + // The same read the list itself is: whoever may see the exchanges, or the dashboard that + // summarises them, may see how many there are. + await requestContext.EnsurePermission(dbContext, Model.Permissions.Exchanges.View, Model.Permissions.Dashboard.View); + + // Removed before anything else reads the request, from every condition rather than only + // the first the special filters look at: a status filter left in a later condition would + // reach Searchy as a column, and XchangeRow.StatusFilter is not one the database has. + foreach (var condition in searchyRequest.Conditions) + foreach (var statusFilter in condition.Filters.Where(f => f.Field == "StatusFilter").ToList()) + condition.Filters.Remove(statusFilter); + + var rows = await XchangeFilters.FilteredRowsAsync(dbContext, searchyRequest); + + // Read after FilteredRowsAsync, which is what moves the request's dates to UTC. The + // StartedOn filters are plain columns, so Searchy has applied them and left them in place. + var (rangeStart, rangeEnd) = StartedOnRange(searchyRequest); + + await using var transaction = await dbContext.Database.BeginTransactionAsync(IsolationLevel.ReadUncommitted); + + // One capped count per status rather than a single grouped count: each can stop at the + // cap, as the search's total does, where a grouping has to visit every matching row. Four + // short queries over a broad filter cost far less than one exhaustive one. + var counts = new XchangeStatusCounts + { + Processing = await CountCapped(rows, Processing), + Success = await CountCapped(rows, Success), + BadResponse = await CountCapped(rows, BadResponse), + Failed = await CountCapped(rows, Failed), + }; + + var summary = new XchangeSummary { Counts = counts }; + if (rangeStart is not { } start) + return summary; + + var end = rangeEnd ?? DateTime.UtcNow; + if (end < start) + return summary; + + var bucketMinutes = XchangeHistogram.ChooseBucketMinutes(start, end); + summary.BucketMinutes = bucketMinutes; + summary.Buckets = XchangeHistogram.Fold(start, end, bucketMinutes, + await StartedCounts(rows, bucketMinutes)); + return summary; + } + + private Task CountCapped(IQueryable rows, string status) => + rows.WhereStatus(status, dbContext).Take(Search.CountCap + 1).CountAsync(); + + /// + /// The matching exchanges grouped by when they started, to the precision the bucket size + /// needs, and by the two columns their status is made of. + /// + /// + /// + /// Unlike the counts, this cannot stop early: grouping visits every row in the range. What + /// keeps it affordable is that a histogram is only asked for over a date range, which the index + /// on StartedOn narrows to, and that what comes back is at most one row per minute per status + /// rather than one per exchange. + /// + /// + /// Grouped on the date's parts rather than on a computed bucket number, because Year, Month, + /// Day, Hour and Minute translate on every provider the solution supports, while date + /// arithmetic is spelt differently by each of them. The folding into buckets wider than the + /// grouping happens in . Status is grouped as its two + /// nullable columns, not as the four statuses, for the same reason: a CASE over them would be + /// one more thing to translate four ways. + /// + /// + private static async Task> StartedCounts(IQueryable rows, + int bucketMinutes) + { + if (bucketMinutes >= 1440) + return await rows + .GroupBy(r => new { r.StartedOn.Year, r.StartedOn.Month, r.StartedOn.Day, r.Status, r.ResponseBad }) + .Select(g => new XchangeStartedCount + { + Year = g.Key.Year, + Month = g.Key.Month, + Day = g.Key.Day, + Status = g.Key.Status, + ResponseBad = g.Key.ResponseBad, + Count = g.Count() + }) + .ToListAsync(); + + if (bucketMinutes >= 60) + return await rows + .GroupBy(r => new + { + r.StartedOn.Year, r.StartedOn.Month, r.StartedOn.Day, r.StartedOn.Hour, r.Status, + r.ResponseBad + }) + .Select(g => new XchangeStartedCount + { + Year = g.Key.Year, + Month = g.Key.Month, + Day = g.Key.Day, + Hour = g.Key.Hour, + Status = g.Key.Status, + ResponseBad = g.Key.ResponseBad, + Count = g.Count() + }) + .ToListAsync(); + + return await rows + .GroupBy(r => new + { + r.StartedOn.Year, r.StartedOn.Month, r.StartedOn.Day, r.StartedOn.Hour, r.StartedOn.Minute, + r.Status, r.ResponseBad + }) + .Select(g => new XchangeStartedCount + { + Year = g.Key.Year, + Month = g.Key.Month, + Day = g.Key.Day, + Hour = g.Key.Hour, + Minute = g.Key.Minute, + Status = g.Key.Status, + ResponseBad = g.Key.ResponseBad, + Count = g.Count() + }) + .ToListAsync(); + } + + /// + /// The StartedOn range the request's filters give, as the list sends it: a "from" as + /// StartedOn:6: (greater than or equal) and a "to" as StartedOn:8: (less than or + /// equal). Either may be missing. Two of the same kind narrow it to their overlap, as the + /// filters themselves do. + /// + internal static (DateTime? Start, DateTime? End) StartedOnRange(SearchyRequest searchyRequest) + { + DateTime? start = null, end = null; + foreach (var filter in searchyRequest.Conditions.SelectMany(c => c.Filters) + .Where(f => f.Field == nameof(XchangeRow.StartedOn))) + { + // DatesToUtc sets this only for a value that reads as a date; anything else was never a + // range Searchy could have applied either. + if (filter.ValueDateTime is not { } dateTime) + continue; + + var value = DateTime.SpecifyKind(dateTime, DateTimeKind.Utc); + if (filter.Rule == SearchyRule.GreaterThanOrEquals) + start = start is { } s && s > value ? s : value; + else if (filter.Rule == SearchyRule.LessThanOrEquals) + end = end is { } e && e < value ? e : value; + } + + return (start, end); + } +} diff --git a/SW.Bitween.Api/Resources/Xchanges/Update.cs b/SW.Bitween.Api/Resources/Xchanges/Update.cs index f4ad5001..69afc15c 100644 --- a/SW.Bitween.Api/Resources/Xchanges/Update.cs +++ b/SW.Bitween.Api/Resources/Xchanges/Update.cs @@ -15,14 +15,29 @@ namespace SW.Bitween.Resources.Xchanges { [Unprotect] public class Update(RequestContext requestContext, XchangeService xchangeService, BitweenDbContext dbContext, - BitweenOptions BitweenSettings, IInfolinkCache cache) : ICommandHandler + BitweenOptions BitweenSettings, IInfolinkCache cache, + Microsoft.AspNetCore.Http.IHttpContextAccessor httpContextAccessor = null) : ICommandHandler { public async Task Handle(string documentIdOrName, dynamic request) + { + // Created in a method of its own, so the payload's text and file are garbage before the + // wait. The request itself CqApi holds until this returns. + (object answer, string xchangeId, int waitResponse) = await CreateAsync(documentIdOrName, (object)request); + return waitResponse <= 0 ? answer : await WaitAsync(xchangeId, waitResponse); + } + + /// The answer when there is no wait, else the exchange to wait for and how long. + private async Task<(object Answer, string XchangeId, int WaitResponse)> CreateAsync(string documentIdOrName, + dynamic request) { Document document; - //Inject external request context values into the object - request._ExternalRequestContext = JsonConvert.SerializeObject(requestContext.Values); + // Inject external request context values into the object - minus the caller's + // credentials. This lands in the stored input file, where anyone who can view + // exchanges reads it, and is handed to mappers and handlers, none of which need the + // partner's key to do their job. + request._ExternalRequestContext = JsonConvert.SerializeObject( + requestContext.Values.Where(v => !IsCredential(v.Name, BitweenSettings.PartnerKeyHeader)).ToList()); if (int.TryParse(documentIdOrName, out var documentId)) document = await cache.DocumentByIdAsync(documentId); @@ -44,10 +59,15 @@ public async Task Handle(string documentIdOrName, dynamic request) var sub = subs.SingleOrDefault(); + string body = request.ToString(); + var schemaErrors = await DocumentSchema.Check(document.DocumentFormat, document.ValidationSchema, body); + if (schemaErrors.Count > 0) + throw new SWValidationException("SCHEMA_MISMATCH", string.Join("\n", schemaErrors)); + if (par.Partner.Id == Partner.SystemId && sub is null) { await xchangeService.SubmitFilterXchange(document.Id,new XchangeFile(request.ToString())); - return null; + return (null, null, 0); } if (sub is null) @@ -59,10 +79,11 @@ public async Task Handle(string documentIdOrName, dynamic request) .Where(item => item.Name.ToLower() == "waitresponse") .Select(item => item.Value).FirstOrDefault(); + // No header, no wait - as before. With one, the wait is capped; see ResultWait. var waitResponse = 0; if (int.TryParse(waitResponseHeader, out var waitResponseValue)) { - waitResponse = waitResponseValue <= 0 ? 120 : waitResponseValue; + waitResponse = ResultWait.Clamp(waitResponseValue, BitweenSettings.MaxResponseWaitSeconds); xchangeReferences.Add($"waitresponse: {waitResponse}"); } @@ -70,26 +91,68 @@ public async Task Handle(string documentIdOrName, dynamic request) var globalAdapterValuesSets = await cache.ListGlobalAdapterValuesSetsAsync(); var validatorProperties = sub.ValidatorProperties.ToDictionary().Fill(par.Partner, globalAdapterValuesSets); - await xchangeService.RunValidator(sub.ValidatorId, validatorProperties, xchangeFile); - - var xchangeId = - await xchangeService.SubmitSubscriptionXchange(sub.Id, xchangeFile, xchangeReferences.ToArray()); + await xchangeService.RunValidator(sub.ValidatorRef, validatorProperties, xchangeFile); + + // Same Idempotency-Key from the same partner for the same type: the first call's exchange. + var idempotencyKey = requestContext.Values + .Where(v => v.Type == RequestValueType.HttpHeader && + v.Name.Equals(IdempotencyGuard.Header, StringComparison.OrdinalIgnoreCase)) + .Select(v => v.Value?.Trim()).FirstOrDefault(); + if (idempotencyKey?.Length > IdempotencyGuard.MaxKeyLength) + throw new SWValidationException("IDEMPOTENCY_KEY", + $"{IdempotencyGuard.Header} is longer than {IdempotencyGuard.MaxKeyLength} characters."); + var idempotencyScope = $"legacy:partner:{par.Partner.Id}:document:{document.Id}"; + + var xchangeId = string.IsNullOrEmpty(idempotencyKey) + ? null + : await IdempotencyGuard.FindAsync(dbContext, idempotencyScope, idempotencyKey); + // A mirror's rehearsal calls as prod's partner did, under a correlation of its own, and + // sends what the rehearsal before it sent. + var rehearsal = Services.Mirror.MirrorTraffic.CorrelationFor(BitweenSettings, httpContextAccessor?.HttpContext?.Request.Headers); + // Not for a replayed call: its payload is a repeat by definition, and it is answered already. + var duplicate = xchangeId is null && rehearsal is null + ? await new DuplicateArrivals(dbContext).CheckAsync(document, par.Partner.Id, xchangeFile) + : null; + if (duplicate?.Drop == true) + { + await dbContext.SaveChangesAsync(); + xchangeId = duplicate.OriginalXchangeId; + } + else if (xchangeId is null) + { + if (duplicate != null && duplicate.FitsWith(xchangeReferences)) xchangeReferences.Add(duplicate.Reference); + var subscription = await cache.SubscriptionByIdAsync(sub.Id); + var xchange = await xchangeService.CreateXchange(subscription, xchangeFile, + xchangeReferences.ToArray(), rehearsal ?? Guid.NewGuid().ToString("N")); + if (string.IsNullOrEmpty(idempotencyKey)) + { + await dbContext.SaveChangesAsync(); + xchangeId = xchange.Id; + } + else + { + await IdempotencyGuard.StageAsync(dbContext, idempotencyScope, idempotencyKey, xchange.Id); + (xchangeId, _) = await IdempotencyGuard.SaveAsync(dbContext, idempotencyScope, idempotencyKey, + xchange.Id); + } + } if (waitResponse <= 0) - return new CqApiResult(xchangeId) + return (new CqApiResult(xchangeId) { Status = CqApiResultStatus.Ok - }; + }, xchangeId, waitResponse); + + return (null, xchangeId, waitResponse); + } - var currentFibTerm = 1; - var previousTerm = 1; - while (currentFibTerm <= waitResponse) + private async Task WaitAsync(string xchangeId, int waitResponse) + { + var available = await ResultWait.UntilAsync(() => IsResultAvailable(xchangeId), waitResponse, + httpContextAccessor?.HttpContext?.RequestAborted ?? System.Threading.CancellationToken.None); + + if (available) { - await Task.Delay(TimeSpan.FromSeconds(currentFibTerm)); - var nextTerm = Math.Min(currentFibTerm + previousTerm, 8); - previousTerm = currentFibTerm; - currentFibTerm = nextTerm; - if (!await IsResultAvailable(xchangeId)) continue; var xchangeResult = await dbContext.FindAsync(xchangeId); @@ -130,5 +193,16 @@ private async Task IsResultAvailable(string xchangeId) .AsNoTracking() .AnyAsync(i => i.Id == xchangeId); } + + private static readonly HashSet CredentialNames = new(StringComparer.OrdinalIgnoreCase) + { + "authorization", "proxy-authorization", "cookie", PartnerKeyHeaders.Default, + }; + + private static bool IsCredential(string name, string partnerKeyHeader) => + name is not null && + (CredentialNames.Contains(name) || + (!string.IsNullOrWhiteSpace(partnerKeyHeader) && + name.Equals(partnerKeyHeader.Trim(), StringComparison.OrdinalIgnoreCase))); } } \ No newline at end of file diff --git a/SW.Bitween.Api/Resources/Xchanges/XchangeFilters.cs b/SW.Bitween.Api/Resources/Xchanges/XchangeFilters.cs index 862a84d0..5abd3f1e 100644 --- a/SW.Bitween.Api/Resources/Xchanges/XchangeFilters.cs +++ b/SW.Bitween.Api/Resources/Xchanges/XchangeFilters.cs @@ -1,6 +1,11 @@ +using System; +using System.Collections.Generic; using System.Linq; +using System.Linq.Expressions; using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Query; +using SW.EfCoreExtensions; using SW.Bitween.Domain; using SW.Bitween.Model; using SW.PrimitiveTypes; @@ -20,6 +25,69 @@ namespace SW.Bitween.Resources.Xchanges; /// internal static class XchangeFilters { + /// + /// The exchanges a filter matches, filtered exactly as the search would have filtered them, as + /// lean rows: the columns the list can filter on and nothing else. + /// + /// + /// + /// What a "select all matching" bulk action and the summary both start from. Dates are moved to + /// UTC and a run filter is resolved here, inside the one entry point, so neither caller can + /// forget a step and select a different set than the list showed for the same filter. + /// + /// + /// The projection carries the columns the exchange list can filter on (see + /// buildExchangeQuery in the client) and nothing else, so this stays translatable and + /// composable - the search's own projection cannot be reused for that, as it builds file URLs + /// in C#. Filtering on a column that is not here would silently match nothing, so a new filter + /// on the list needs a column here too. + /// + /// + /// The special filters are removed from as they are applied, + /// as they are for the search; what is left in it afterwards is only what Searchy handled. + /// + /// + internal static async Task> FilteredRowsAsync(BitweenDbContext dbContext, + SearchyRequest searchyRequest) + { + searchyRequest.DatesToUtc(); + // Async, and inside here rather than in the callers, so a "select all matching" over a run + // selects exactly the rows the list showed for it. + await ResolveReceiveAttemptFilterAsync(searchyRequest, dbContext); + + var query = from xchange in dbContext.Set() + join result in dbContext.Set() on xchange.Id equals result.Id into xr + from result in xr.DefaultIfEmpty() + join agg in dbContext.Set() on xchange.Id equals agg.Id into xa + from agg in xa.DefaultIfEmpty() + join promoted in dbContext.Set() on xchange.Id equals promoted.Id into xp + from promoted in xp.DefaultIfEmpty() + join subscriber in dbContext.Set() on xchange.SubscriptionId equals subscriber.Id into xs + from subscriber in xs.DefaultIfEmpty() + select new XchangeRow + { + Id = xchange.Id, + SubscriptionId = xchange.SubscriptionId, + DocumentId = xchange.DocumentId, + StartedOn = xchange.StartedOn, + CorrelationId = xchange.CorrelationId, + Status = result.Success, + ResponseBad = result.ResponseBad, + // For the ErrorContains filter. Only read when that filter is set, so it + // costs nothing to the selections that do not use it. + Exception = result.Exception, + RetryFor = xchange.RetryFor, + AggregationXchangeId = agg.AggregationXchangeId, + PromotedPropertiesRaw = promoted.PropertiesRaw, + // Same fallback the search makes for exchanges written before the column + // existed, so a partner filter selects the same rows it listed. + PartnerId = xchange.PartnerId ?? subscriber.PartnerId + }; + + query = query.ApplySpecialFilters(searchyRequest, dbContext); + return query.AsNoTracking().Search(searchyRequest.Conditions); + } + /// /// Turns a ReceiveAttemptId filter into the Id filter the rest of the pipeline /// already understands, by reading the run's own record of the exchanges it created. @@ -28,11 +96,11 @@ internal static class XchangeFilters /// Separate from , and async, because the run has to be read /// before the query can be built. ReceiveAttempt.ExchangeIds is persisted as a single /// separator-delimited string rather than an array (see BitweenDbContext), so no - /// subquery can reach inside it — and with three database providers in the solution there is + /// subquery can reach inside it - and with three database providers in the solution there is /// no one translation that would. /// /// It rewrites rather than filters here so the result stays identical to the long - /// ?ids=a,b,c URL this replaces, relatives and all — the only thing that changes is that + /// ?ids=a,b,c URL this replaces, relatives and all - the only thing that changes is that /// the ids no longer have to fit in a web address. /// internal static async Task ResolveReceiveAttemptFilterAsync(SearchyRequest searchyRequest, @@ -59,7 +127,7 @@ internal static async Task ResolveReceiveAttemptFilterAsync(SearchyRequest searc $"Run {attemptId} no longer exists. Its history may have been cleaned up."); // A run that created nothing leaves an empty array, and the Contains branch below reads - // that as "matches no row" — which is the honest answer. Dropping the filter instead + // that as "matches no row" - which is the honest answer. Dropping the filter instead // would widen the selection to every exchange, and bulk retry acts on whatever this // selects. condition.Filters.Add(new SearchyFilter @@ -115,46 +183,12 @@ internal static IQueryable ApplySpecialFilters(this IQueryable f.Field == "StatusFilter").ToList(); foreach (var statusFilter in statusFilters) { - switch (statusFilter.Value) - { - case "0": - // "Still running" means no result row exists yet. Asking for it as - // Status == null reads as `x0.success IS NULL` on the left join, and - // Postgres cannot estimate that: it guesses one row, plans every join - // above it for one row, and picks per-row sequential scans of the small - // side tables. Measured on 1M exchanges that was 22.8s for 25 rows. - // NOT EXISTS asks the same question as an anti-join, which it can - // estimate — 34ms. Equivalent because success is NOT NULL, so a result - // row can never itself carry a null status. - query = query.Where(i => - !dbContext.Set().Any(r => r.Id == i.Id)); - break; - case "1": - query = query.Where(i => i.Status == true && i.ResponseBad != true); - break; - - case "2": - query = query.Where(i => i.Status == true && i.ResponseBad == true); - break; - - case "3": - query = query.Where(i => i.Status == false); - break; - - default: - // The filter is removed below whether or not it matched, so falling through - // here used to drop it silently and widen the selection to everything. A - // search returning too much is merely wrong; bulk retry runs over whatever - // this selects, so an unreadable status has to be refused rather than ignored. - throw new SWValidationException("NOT_SUPPORTED", - $"'{statusFilter.Value}' is not an exchange status."); - } - + query = query.WhereStatus(statusFilter.Value, dbContext); condition.Filters.Remove(statusFilter); } // "Only the newest attempt of each chain." An exchange that has been retried is superseded - // by that retry, so a list of failures otherwise counts one piece of work as many rows — + // by that retry, so a list of failures otherwise counts one piece of work as many rows - // a chain retried nine times fills nine of them, none of which is the current state. // // NOT EXISTS rather than a left join for the same reason the still-running status uses it: @@ -162,57 +196,260 @@ internal static IQueryable ApplySpecialFilters(this IQueryable f.Field == "LatestOnly").ToList(); foreach (var latestFilter in latestFilters) { - if (latestFilter.Value?.ToString()?.ToLower() is "true" or "1") + if (IsOn(latestFilter)) query = query.Where(i => !dbContext.Set().Any(r => r.RetryFor == i.Id)); condition.Filters.Remove(latestFilter); } + // "Only retries": attempts made from an earlier exchange, whether a person asked for them or + // a retry policy did. RetryFor is on the exchange itself, so no second table is involved. + var retryFilters = condition.Filters.Where(f => f.Field == "IsRetry").ToList(); + foreach (var retryFilter in retryFilters) + { + if (IsOn(retryFilter)) + query = Negated(retryFilter) + ? query.Where(i => i.RetryFor == null) + : query.Where(i => i.RetryFor != null); + + condition.Filters.Remove(retryFilter); + } + + // "Waiting for a scheduled retry": a DelayedRetry row exists for it. That row is keyed by the + // exchange's own id and deleted when the retry runs, so its presence is the whole question. + // EXISTS rather than reading the search's left join to DelayedRetry, for the same reason + // the still-running status uses NOT EXISTS - and because the lean projection bulk actions + // and the summary filter over does not join DelayedRetry at all. + var scheduledFilters = condition.Filters.Where(f => f.Field == "HasScheduledRetry").ToList(); + foreach (var scheduledFilter in scheduledFilters) + { + if (IsOn(scheduledFilter)) + query = Negated(scheduledFilter) + ? query.Where(i => !dbContext.Set().Any(d => d.Id == i.Id)) + : query.Where(i => dbContext.Set().Any(d => d.Id == i.Id)); + + condition.Filters.Remove(scheduledFilter); + } + + // "Rolled up into an aggregation": the exchange was taken into an aggregation exchange, + // which is what carries its data onwards from there. + var aggregatedFilters = condition.Filters.Where(f => f.Field == "Aggregated").ToList(); + foreach (var aggregatedFilter in aggregatedFilters) + { + if (IsOn(aggregatedFilter)) + query = Negated(aggregatedFilter) + ? query.Where(i => i.AggregationXchangeId == null) + : query.Where(i => i.AggregationXchangeId != null); + + condition.Filters.Remove(aggregatedFilter); + } + + // Text inside the error, case-insensitive. Named apart from the Exception column rather than + // reusing it, because a plain Searchy filter on Exception would compare case as stored, and + // an error message's case is whatever the adapter that threw it chose. Folded on both sides + // like the promoted properties below; a leading-wildcard LIKE could use no index either way. + // An empty value is no filter at all rather than "has any error", which is what Contains("") + // would quietly turn into. + var errorFilters = condition.Filters.Where(f => f.Field == "ErrorContains").ToList(); + foreach (var errorFilter in errorFilters) + { + var value = errorFilter.Value?.ToString(); + if (!string.IsNullOrEmpty(value)) + { + value = value.ToLower(); + // An exchange with no error doesn't contain the text either, so "does not contain" + // keeps it - as "is not" on a subscription keeps the exchanges that have none. + query = Negated(errorFilter) + ? query.Where(i => i.Exception == null || !i.Exception.ToLower().Contains(value)) + : query.Where(i => i.Exception.ToLower().Contains(value)); + } + + condition.Filters.Remove(errorFilter); + } + // The one promoted property the value below has to be found in. Sent apart from the value // because the client used to send them together as "key:value", which only matched a value // that starts with what was typed: "SO-1001" was found by "SO", never by "1001". var keyFilters = condition.Filters.Where(f => f.Field == "PromotedPropertyKey").ToList(); var key = keyFilters.Select(f => f.Value?.ToString()).FirstOrDefault(v => !string.IsNullOrEmpty(v))?.ToLower(); + var propertiesFilters = condition.Filters + .Where(f => f.Field == "PromotedPropertiesRaw").ToList(); + + // The key and the values below are one question - "orderId contains 1001" - so "not" turns + // the whole of it round, never one part: an exchange whose orderId doesn't contain 1001, or + // that has no orderId at all. Asked for one way in one part and the other way in another, + // it has no reading worth guessing at, and bulk retry acts on whatever this selects. + var negations = keyFilters.Concat(propertiesFilters).Select(Negated).Distinct().ToList(); + if (negations.Count > 1) + throw new SWValidationException("NOT_SUPPORTED", + "A promoted property filter is negated as a whole: send its key and its value with the same rule."); + var negate = negations.SingleOrDefault(); + foreach (var keyFilter in keyFilters) condition.Filters.Remove(keyFilter); + foreach (var propertyFilter in propertiesFilters) + condition.Filters.Remove(propertyFilter); + + // An empty value is no filter, as with the error text above. + var match = PropertyMatch(key, propertiesFilters + .Select(f => f.Value?.ToString()) + .Where(v => !string.IsNullOrEmpty(v)) + .Select(v => v.ToLower())); + if (match != null) + query = query.Where(negate ? NoneOrNot(match) : match); + // Any number of properties at once - "orderNo contains 1001" and "city is Paris" - each with + // its key and text in the one filter, as they are stored: PromotedProperty:4:orderNo:1001. + // A key alone asks whether the exchange promoted it. Every one has to hold, and rule 2 turns + // each round on its own. The pair above stays for links and callers made before this; it can + // only ask about one key, since it has no way to say which value goes with which. + var pairFilters = condition.Filters.Where(f => f.Field == "PromotedProperty").ToList(); + foreach (var pairFilter in pairFilters) + { + var text = pairFilter.Value?.ToString(); + if (!string.IsNullOrEmpty(text)) + { + if (pairFilter.Rule is not (SearchyRule.EqualsTo or SearchyRule.Contains or SearchyRule.NotEqualsTo)) + throw new SWValidationException("NOT_SUPPORTED", + "A promoted property is asked for with rule 1 or 4, or rule 2 to turn it round."); + + var colon = text.IndexOf(':'); + var pairKey = (colon < 0 ? text : text[..colon]).ToLower(); + var pairValue = colon < 0 ? "" : text[(colon + 1)..].ToLower(); + if (pairKey.Length == 0) + throw new SWValidationException("NOT_SUPPORTED", + "Name the promoted property: PromotedProperty:4:key:text. Text in any property is PromotedPropertiesRaw."); + + var pair = PropertyMatch(pairKey, pairValue.Length == 0 ? [] : [pairValue]); + query = query.Where(Negated(pairFilter) ? NoneOrNot(pair) : pair); + } + + condition.Filters.Remove(pairFilter); + } + + return query; + } + + /// + /// An exchange whose promoted properties include (when there is one) + /// and contain each of - inside that key's value when there is a key, + /// anywhere when there is not. Both already lower-cased. Null when there is nothing to ask. + /// + private static Expression> PropertyMatch(string key, IEnumerable values) + { // Stored as "key:value,key:value": the key comes first or after a comma, which keeps "id" // from matching "orderId:". Asked of lower(properties_raw) as it is, not with a comma put in // front: Postgres keeps statistics for that expression (its trigram index), and without them // it guessed almost nothing matched, so a common key fetched and sorted every match rather // than reading the newest 25 - 6s against 1M exchanges. var keyTag = "," + key + ":"; + Expression> match = null; if (key != null) - query = query.Where(i => i.PromotedPropertiesRaw.ToLower().StartsWith(key + ":") - || i.PromotedPropertiesRaw.ToLower().Contains(keyTag)); + match = i => i.PromotedPropertiesRaw.ToLower().StartsWith(key + ":") + || i.PromotedPropertiesRaw.ToLower().Contains(keyTag); - var propertiesFilters = condition.Filters - .Where(f => f.Field == "PromotedPropertiesRaw").ToList(); - foreach (var propertyFilter in propertiesFilters) + foreach (var value in values) { - var value = propertyFilter.Value.ToString()!.ToLower(); - // Both sides lower-cased at query time. Promoted values keep the case the // payload had (see FilterService), so the column has to be folded here for // the search to stay case-insensitive. No index is lost: a Contains is a // leading-wildcard LIKE, which the b-tree on this column could never serve. - query = query.Where(i => i.PromotedPropertiesRaw.ToLower().Contains(value)); + match = match.And(i => i.PromotedPropertiesRaw.ToLower().Contains(value)); // With a key, only inside that key's value: what follows ",key:" up to the next comma. // A value that itself holds a comma is only searched up to it, since the stored text // cannot tell that comma from the one before the next key. IndexOf rather than Contains // for the same estimate reason as the key above: Postgres guesses a third of rows pass - // an unknown comparison but under 1% pass an unknown LIKE. + // an unknown comparison but under 1% pass an unknown LIKE. "rest" - what follows the + // key, with a comma added - is written out twice, as a query's `let` would inline it. if (key != null) - query = from i in query - let rest = ("," + i.PromotedPropertiesRaw.ToLower()) - .Substring(("," + i.PromotedPropertiesRaw.ToLower()).IndexOf(keyTag) + keyTag.Length) + "," - where rest.Substring(0, rest.IndexOf(",")).IndexOf(value) >= 0 - select i; - - condition.Filters.Remove(propertyFilter); + match = match.And(i => + (("," + i.PromotedPropertiesRaw.ToLower()) + .Substring(("," + i.PromotedPropertiesRaw.ToLower()).IndexOf(keyTag) + keyTag.Length) + ",") + .Substring(0, (("," + i.PromotedPropertiesRaw.ToLower()) + .Substring(("," + i.PromotedPropertiesRaw.ToLower()).IndexOf(keyTag) + keyTag.Length) + ",") + .IndexOf(",")) + .IndexOf(value) >= 0); } - return query; + return match; + } + + /// A special filter asked for the other way round: rule 2, "not". + private static bool Negated(SearchyFilter filter) => filter.Rule == SearchyRule.NotEqualsTo; + + /// Both, as one predicate - which the property filter needs, to be able to turn it round whole. + private static Expression> And(this Expression> left, + Expression> right) => + left == null + ? right + : Expression.Lambda>( + Expression.AndAlso(left.Body, + ReplacingExpressionVisitor.Replace(right.Parameters[0], left.Parameters[0], right.Body)), + left.Parameters); + + /// + /// The property filter turned round, keeping the exchanges that promoted nothing. Spelt out + /// rather than left to the negation: SQL's NOT of a LIKE over a null is null, which drops the row. + /// + private static Expression> NoneOrNot(Expression> match) + { + var row = match.Parameters[0]; + var raw = Expression.Property(row, nameof(XchangeRow.PromotedPropertiesRaw)); + return Expression.Lambda>( + Expression.OrElse(Expression.Equal(raw, Expression.Constant(null, typeof(string))), Expression.Not(match.Body)), + row); + } + + /// + /// Whether an on/off filter is on. Only "true" or "1" turn one on; anything else leaves it off + /// rather than refusing it, which is what LatestOnly has always done - "select all + /// matching" with no filters set sends LatestOnly:1:false to mean "everything". + /// + private static bool IsOn(SearchyFilter filter) => + filter.Value?.ToString()?.ToLower() is "true" or "1"; + + /// + /// Narrows to one exchange status, by the number the client's + /// StatusFilter sends: 0 still running, 1 success, 2 success with a bad response, 3 failed. + /// + /// + /// Its own method so the summary can count each status under the same definition the list + /// filters by. A count that defined "success" even slightly differently from the tab it sits on + /// would show one number and list another. + /// + internal static IQueryable WhereStatus(this IQueryable query, object status, + BitweenDbContext dbContext) + { + switch (status) + { + case "0": + // "Still running" means no result row exists yet. Asking for it as + // Status == null reads as `x0.success IS NULL` on the left join, and + // Postgres cannot estimate that: it guesses one row, plans every join + // above it for one row, and picks per-row sequential scans of the small + // side tables. Measured on 1M exchanges that was 22.8s for 25 rows. + // NOT EXISTS asks the same question as an anti-join, which it can + // estimate - 34ms. Equivalent because success is NOT NULL, so a result + // row can never itself carry a null status. + return query.Where(i => + !dbContext.Set().Any(r => r.Id == i.Id)); + case "1": + return query.Where(i => i.Status == true && i.ResponseBad != true); + + case "2": + return query.Where(i => i.Status == true && i.ResponseBad == true); + + case "3": + return query.Where(i => i.Status == false); + + default: + // The filter is removed by the caller whether or not it matched, so falling through + // here used to drop it silently and widen the selection to everything. A search + // returning too much is merely wrong; bulk retry runs over whatever this selects, + // so an unreadable status has to be refused rather than ignored. + throw new SWValidationException("NOT_SUPPORTED", + $"'{status}' is not an exchange status."); + } } } diff --git a/SW.Bitween.Api/Resources/Xchanges/XchangeHistogram.cs b/SW.Bitween.Api/Resources/Xchanges/XchangeHistogram.cs new file mode 100644 index 00000000..031e2ffb --- /dev/null +++ b/SW.Bitween.Api/Resources/Xchanges/XchangeHistogram.cs @@ -0,0 +1,145 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using SW.Bitween.Model; + +namespace SW.Bitween.Resources.Xchanges; + +/// +/// How many exchanges started in one slice of time with one status combination, as the summary's +/// grouping query returns it: the date split into parts, because the parts are what every database +/// provider can group on. +/// +/// +/// The parts finer than the grouping asked for are left 0 - a by-day grouping has no hour. +/// +public sealed class XchangeStartedCount +{ + public int Year { get; set; } + public int Month { get; set; } + public int Day { get; set; } + public int Hour { get; set; } + public int Minute { get; set; } + + /// The result's success; null when there is no result yet. + public bool? Status { get; set; } + + public bool? ResponseBad { get; set; } + public int Count { get; set; } +} + +/// +/// The pure half of the summary's histogram: picking a bucket size for a range, and folding the +/// grouped counts into contiguous, zero-filled buckets. No database here, so it can be tested on +/// its own. +/// +public static class XchangeHistogram +{ + /// + /// The bucket sizes a histogram may use, in minutes, smallest first. Each divides a day or is a + /// whole number of days, so buckets line up with the clock: 5-minute buckets start on :00, :05, + /// and so on, never at whatever minute the range happened to start. + /// + public static readonly IReadOnlyList BucketSizes = + [1, 5, 10, 15, 30, 60, 120, 180, 360, 720, 1440, 10080]; + + /// The most buckets a range is divided into, unless even the largest size gives more. + public const int MaxBuckets = 60; + + /// + /// The smallest bucket size that divides the range into at most + /// buckets. A day comes out as 49 half-hours, a week as 57 three-hour buckets, thirty days as 31 + /// days. Fewer is possible when the next size down was only just too fine: the last hour is 61 + /// one-minute buckets once both of its ends are counted, so it gets 13 five-minute ones instead. + /// + /// + /// A range longer than sixty weeks gets weekly buckets anyway, more than sixty of them: a week + /// is the coarsest size there is, and inventing a longer one would no longer line up with + /// anything on a calendar. + /// + public static int ChooseBucketMinutes(DateTime start, DateTime end) + { + foreach (var size in BucketSizes) + if (BucketCount(start, end, size) <= MaxBuckets) + return size; + + return BucketSizes[^1]; + } + + /// + /// How many buckets of cover the range, counting both the one + /// falls in and the one falls in - the end is + /// inclusive, as the list's "to" date is. + /// + public static long BucketCount(DateTime start, DateTime end, int bucketMinutes) + { + if (end < start) + return 0; + + var ticks = BucketTicks(bucketMinutes); + return (Floor(end, bucketMinutes).Ticks - Floor(start, bucketMinutes).Ticks) / ticks + 1; + } + + /// + /// The start of the bucket falls in, in UTC. + /// + /// + /// Counted from , which is midnight, so every size that divides a + /// day floors to a clock boundary; and a Monday, so weekly buckets start on Mondays. + /// + public static DateTime Floor(DateTime time, int bucketMinutes) + { + var ticks = BucketTicks(bucketMinutes); + return new DateTime(time.Ticks - time.Ticks % ticks, DateTimeKind.Utc); + } + + /// + /// Folds the grouped counts into buckets of , from the one + /// falls in to the one falls in, every bucket + /// present whether anything started in it or not - a gap in the list would read as a gap in + /// time to anything drawing it. + /// + /// + /// The status is read from the two columns the way + /// reads it, so the bars add up to the counts beside them: no result is still running (the + /// result's success is never null, so a null here means the row is missing), a success with a + /// bad response is its own status, and any other success is a success. A count that falls + /// outside the range is dropped rather than stretched onto the nearest bucket. + /// + public static List Fold(DateTime start, DateTime end, int bucketMinutes, + IEnumerable counts) + { + var buckets = new List(); + if (end < start) + return buckets; + + var first = Floor(start, bucketMinutes); + var count = BucketCount(start, end, bucketMinutes); + var size = TimeSpan.FromMinutes(bucketMinutes); + for (var i = 0; i < count; i++) + buckets.Add(new XchangeSummaryBucket { Start = first + size * i }); + + var ticks = BucketTicks(bucketMinutes); + foreach (var c in counts) + { + var started = new DateTime(c.Year, c.Month, c.Day, c.Hour, c.Minute, 0, DateTimeKind.Utc); + var index = (Floor(started, bucketMinutes).Ticks - first.Ticks) / ticks; + if (index < 0 || index >= buckets.Count) + continue; + + var bucket = buckets[(int)index]; + if (c.Status == null) + bucket.Processing += c.Count; + else if (c.Status == false) + bucket.Failed += c.Count; + else if (c.ResponseBad == true) + bucket.BadResponse += c.Count; + else + bucket.Success += c.Count; + } + + return buckets; + } + + private static long BucketTicks(int bucketMinutes) => bucketMinutes * TimeSpan.TicksPerMinute; +} diff --git a/SW.Bitween.Api/Resources/Xchanges/XchangeSelection.cs b/SW.Bitween.Api/Resources/Xchanges/XchangeSelection.cs index 4e6a56cc..ef1335d8 100644 --- a/SW.Bitween.Api/Resources/Xchanges/XchangeSelection.cs +++ b/SW.Bitween.Api/Resources/Xchanges/XchangeSelection.cs @@ -2,15 +2,13 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; -using SW.EfCoreExtensions; -using SW.Bitween.Domain; using SW.Bitween.Model; using SW.PrimitiveTypes; namespace SW.Bitween.Resources.Xchanges; /// -/// The exchanges a bulk action on the exchange list covers — a bulk retry, a files export: the rows +/// The exchanges a bulk action on the exchange list covers - a bulk retry, a files export: the rows /// picked by hand, or everything a "select all matching" filter matches minus the rows unticked after. /// /// @@ -56,47 +54,10 @@ internal async Task Count(IXchangeSelection request) /// filtered them. /// /// - /// The projection carries the columns the exchange list can filter on (see - /// buildExchangeQuery in the client) and nothing else, so this stays translatable and - /// composable — the search's own projection cannot be reused for that, as it builds file URLs - /// in C#. Filtering on a column that is not here would silently match nothing, so a new filter - /// on the list needs a column here too. + /// The query itself lives in , shared with the + /// summary so the counts above the list and the set a bulk action acts on are the same rows - + /// read its remarks before filtering on a new column. /// - private async Task> Query(IXchangeSelection request) - { - var searchyRequest = new SearchyRequest(request.Filter); - searchyRequest.DatesToUtc(); - // Async, and inside here rather than in the two callers, so a "select all matching" over a - // run selects exactly the rows the list showed for it. - await XchangeFilters.ResolveReceiveAttemptFilterAsync(searchyRequest, dbContext); - - var query = from xchange in dbContext.Set() - join result in dbContext.Set() on xchange.Id equals result.Id into xr - from result in xr.DefaultIfEmpty() - join agg in dbContext.Set() on xchange.Id equals agg.Id into xa - from agg in xa.DefaultIfEmpty() - join promoted in dbContext.Set() on xchange.Id equals promoted.Id into xp - from promoted in xp.DefaultIfEmpty() - join subscriber in dbContext.Set() on xchange.SubscriptionId equals subscriber.Id into xs - from subscriber in xs.DefaultIfEmpty() - select new XchangeRow - { - Id = xchange.Id, - SubscriptionId = xchange.SubscriptionId, - DocumentId = xchange.DocumentId, - StartedOn = xchange.StartedOn, - CorrelationId = xchange.CorrelationId, - Status = result.Success, - ResponseBad = result.ResponseBad, - RetryFor = xchange.RetryFor, - AggregationXchangeId = agg.AggregationXchangeId, - PromotedPropertiesRaw = promoted.PropertiesRaw, - // Same fallback the search makes for exchanges written before the column - // existed, so a partner filter selects the same rows it listed. - PartnerId = xchange.PartnerId ?? subscriber.PartnerId - }; - - query = query.ApplySpecialFilters(searchyRequest, dbContext); - return query.AsNoTracking().Search(searchyRequest.Conditions); - } + private Task> Query(IXchangeSelection request) => + XchangeFilters.FilteredRowsAsync(dbContext, new SearchyRequest(request.Filter)); } diff --git a/SW.Bitween.Api/SW.Bitween.Api.csproj b/SW.Bitween.Api/SW.Bitween.Api.csproj index cdb3243e..46087794 100644 --- a/SW.Bitween.Api/SW.Bitween.Api.csproj +++ b/SW.Bitween.Api/SW.Bitween.Api.csproj @@ -6,7 +6,10 @@ - + + + + @@ -21,7 +24,10 @@ - + + + + @@ -35,22 +41,33 @@ - - + + + + + - + + + + + + + + diff --git a/SW.Bitween.Api/Services/AESCryptoService.cs b/SW.Bitween.Api/Services/AESCryptoService.cs index f9916092..aa5cf885 100644 --- a/SW.Bitween.Api/Services/AESCryptoService.cs +++ b/SW.Bitween.Api/Services/AESCryptoService.cs @@ -8,8 +8,8 @@ namespace SW.Bitween; public static class AESCryptoService { /// - /// The salt this has always used. It is a constant, and a well known one — the bytes spell - /// "Ivan Medvedev" from an old MSDN sample — so it adds nothing an attacker does not have. + /// The salt this has always used. It is a constant, and a well known one - the bytes spell + /// "Ivan Medvedev" from an old MSDN sample - so it adds nothing an attacker does not have. /// It cannot be changed without making every value already encrypted undecryptable, so it /// stays until there is a migration to move ciphertext to a per-value salt. /// diff --git a/SW.Bitween.Api/Services/AccountMail.cs b/SW.Bitween.Api/Services/AccountMail.cs new file mode 100644 index 00000000..958f5300 --- /dev/null +++ b/SW.Bitween.Api/Services/AccountMail.cs @@ -0,0 +1,171 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using SW.Bitween.NativeAdapters; +using SW.Bitween.NativeAdapters.ChannelMessages; +using SW.Bitween.NativeAdapters.EmailMessageHandler; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Services; + +/// +/// Emails about a person's own account - an invite, a password reset, a password changed, an +/// account locked - sent to that person through the mail server set in Settings. +/// +/// +/// Not through notification channels: those tell a team about its integrations, and a channel can +/// be a shared Teams or Slack room. These go to exactly one person, and carry links that sign in. +/// Laid out by the Email channel's own adapter, so they look like Bitween's other emails. +/// +public class AccountMail(BitweenOptions options, ILogger logger, + IHttpContextAccessor httpContextAccessor, DestinationCircuits circuits = null, ThemeOptions theme = null) +{ + /// Whether there is a mail server to send through. + public bool CanSend => options.MailServer.IsSet; + + /// Whether the sign-in page can offer "Forgot password?". + public bool ResetAvailable => ResetAvailableFor(options); + + /// + /// Needs the public address as well as a mail server. Anyone can ask for a reset, and the link + /// in it must not be built on the address their request came in on: a forged Host header would + /// send the victim a link to the attacker's site, with the victim's reset token in it. + /// + public static bool ResetAvailableFor(BitweenOptions options) => + options.MailServer.IsSet && !string.IsNullOrWhiteSpace(options.PublicUrl) && !options.DisableEmailPasswordLogin; + + /// The public address the chart sets, or null. + public string PublicBase => string.IsNullOrWhiteSpace(options.PublicUrl) ? null : options.PublicUrl.TrimEnd('/'); + + /// + /// Where links point for a request a signed-in member made: the public address, or else the one + /// they are using. Never for a request anyone could make; see . + /// + public string SignedInBase + { + get + { + if (PublicBase is { } publicBase) return publicBase; + var request = httpContextAccessor.HttpContext?.Request; + return request is { Host.HasValue: true } + ? $"{request.Scheme}://{request.Host}{request.PathBase}".TrimEnd('/') + : ""; + } + } + + /// Sends one email; null when it went, else why it didn't. + public async Task SendAsync(string to, ChannelMessage message) + { + if (!CanSend) return "No mail server is set up. Add one in Settings, under Mail server."; + try + { + var email = new NativeEmailMessageHandler(options.MailServer, circuits, theme?.PrimaryColor); + email.InitializeStartupValues(new Dictionary { ["To"] = to }); + await email.Handle(new XchangeFile(JsonConvert.SerializeObject(message))); + return null; + } + catch (Exception ex) + { + logger.LogWarning(ex, "An account email ({Title}) could not be sent: {Error}", + message.Title, ex.GetBaseException().Message); + return ex.GetBaseException().Message; + } + } + + /// + /// Sends without waiting, for an email the request doesn't depend on. Also what keeps a reset + /// from taking longer for an address that has an account than for one that hasn't. + /// + public void SendInBackground(string to, ChannelMessage message) + { + if (!CanSend || string.IsNullOrWhiteSpace(to)) return; + _ = Task.Run(() => SendAsync(to, message)); + } +} + +/// What each account email says. +public static class AccountEmails +{ + public enum ChangedBy + { + Themselves, + Administrator, + ResetLink + } + + public static ChannelMessage Invite(string inviter, string email, string linkBase, string token) => new() + { + Title = "You're invited to Bitween", + Tone = ChannelMessageTone.Info, + Text = $"{inviter} added you to Bitween{On(linkBase)}. Choose your password to sign in. " + + $"The link works once, for {Hours(Domain.Accounts.AccountLink.InviteLifetime)}.", + Facts = [new("Email", email)], + LinkUrl = $"{linkBase}/set-password?token={token}", + LinkLabel = "Choose your password" + }; + + /// For an instance that signs in only with Microsoft or single sign-on: no password to choose. + public static ChannelMessage InviteToSignIn(string inviter, string email, string linkBase) => new() + { + Title = "You're invited to Bitween", + Tone = ChannelMessageTone.Info, + Text = $"{inviter} added you to Bitween{On(linkBase)}. Sign in with your company account.", + Facts = [new("Email", email)], + LinkUrl = string.IsNullOrEmpty(linkBase) ? null : $"{linkBase}/login", + LinkLabel = "Sign in" + }; + + public static ChannelMessage Reset(string email, string linkBase, string token) => new() + { + Title = "Reset your Bitween password", + Tone = ChannelMessageTone.Info, + Text = $"Someone asked to reset the password of {email}{On(linkBase)}. If it was you, choose a new one: " + + $"the link works once, for {Hours(Domain.Accounts.AccountLink.ResetLifetime)}. " + + "If it wasn't, ignore this email. Your password stays as it is.", + LinkUrl = $"{linkBase}/set-password?token={token}", + LinkLabel = "Choose a new password" + }; + + public static ChannelMessage PasswordChanged(ChangedBy by, string email, string linkBase, bool resetAvailable) => new() + { + Title = "Your Bitween password was changed", + Tone = ChannelMessageTone.Warning, + Text = by switch + { + ChangedBy.Administrator => + $"An administrator set a new password for {email}{On(linkBase)}. You've been signed out everywhere. " + + "If you didn't expect this, tell your administrator.", + ChangedBy.ResetLink => + $"The password of {email}{On(linkBase)} was changed with a reset link. You've been signed out everywhere. " + + "If it wasn't you, tell your administrator.", + _ => + $"The password of {email}{On(linkBase)} was just changed. If it wasn't you, " + + (resetAvailable ? "reset it now and tell your administrator." : "tell your administrator now.") + }, + Facts = [new("When", When(DateTime.UtcNow))], + LinkUrl = string.IsNullOrEmpty(linkBase) ? null : resetAvailable ? $"{linkBase}/forgot-password" : $"{linkBase}/login", + LinkLabel = resetAvailable ? "Reset your password" : "Sign in" + }; + + public static ChannelMessage Locked(string email, string linkBase, int attempts, DateTime lockedUntil) => new() + { + Title = "Your Bitween account was locked", + Tone = ChannelMessageTone.Warning, + Text = $"After {attempts} wrong passwords for {email}{On(linkBase)}, signing in with a password is locked " + + "for a while. If this wasn't you, someone may be guessing your password: tell your administrator.", + Facts = [new("Locked until", When(lockedUntil))] + }; + + static string On(string linkBase) => + Uri.TryCreate(linkBase, UriKind.Absolute, out var uri) ? $" at {uri.Host}" : ""; + + static string Hours(TimeSpan lifetime) => + lifetime.TotalHours == 1 ? "1 hour" : $"{lifetime.TotalHours:0} hours"; + + static string When(DateTime utc) => + utc.ToString("d MMM yyyy, HH:mm 'UTC'", CultureInfo.InvariantCulture); +} diff --git a/SW.Bitween.Api/Services/AdapterRequirements.cs b/SW.Bitween.Api/Services/AdapterRequirements.cs index 5a4aa60f..589ebcd3 100644 --- a/SW.Bitween.Api/Services/AdapterRequirements.cs +++ b/SW.Bitween.Api/Services/AdapterRequirements.cs @@ -10,19 +10,58 @@ namespace SW.Bitween; /// Which of an adapter's required startup properties a caller failed to supply. /// /// The answer was written out four times across the subscription validators before this -/// existed — three in Update alone. Create needs the same answer, and six copies of it would +/// existed - three in Update alone. Create needs the same answer, and six copies of it would /// be six places for the rule to drift. Whether the adapter runs in-process or in a serverless /// container is 's problem, not this one's. /// /// -public class AdapterRequirements(AdapterStartupValues startupValues) +public class AdapterRequirements( + AdapterStartupValues startupValues, + Services.Adapters.AdapterCatalog catalog = null, + BitweenOptions bitweenOptions = null) { + /// + /// Refuses a version pin that cannot run: on a native adapter, which has no versions; on a + /// slot with no adapter; on a version that was never published or has been withdrawn. And + /// refuses an adapter - pinned or current - whose manifest asks for a newer Bitween than this + /// one, when the slot is being changed to it. Nothing is checked for a slot left as it was, so + /// saving an unrelated field never fails over a choice made before. + /// + public async Task EnsureRunnable(string slot, string adapterId, string version, bool changed) + { + if (!changed || string.IsNullOrWhiteSpace(adapterId)) return; + + if (!string.IsNullOrWhiteSpace(version)) + { + if (NativeAdapterDiscoveryService.IsNative(adapterId)) + throw new SWValidationException("ADAPTER_VERSION", + $"The {slot} '{adapterId}' is built in and has no versions to pin."); + if (catalog != null && !await catalog.HasVersionAsync(adapterId, version)) + throw new SWValidationException("ADAPTER_VERSION", + $"Version {version} of the {slot} '{adapterId}' is not published, or has been withdrawn."); + } + + if (catalog == null) return; + var manifest = await catalog.ManifestOf(SW.Serverless.Contract.Catalog.AdapterCatalogPaths.Ref(adapterId, version)); + var needs = manifest?.Compatibility?.MinVersionOf(Services.Adapters.AdapterCatalog.Application); + if (!BitweenInfo.Satisfies(needs, bitweenOptions)) + throw new SWValidationException("ADAPTER_NEEDS_NEWER_BITWEEN", + $"The {slot} '{adapterId}'{(string.IsNullOrWhiteSpace(version) ? "" : $" {version}")} needs Bitween {needs} or later; this is {BitweenInfo.Version(bitweenOptions)}."); + } + /// Native (native: prefix) or serverless. Null/blank means nothing is missing. /// What the caller supplied. Blank values count as not supplied. public async Task> MissingFor(string adapterId, ICollection provided) { if (string.IsNullOrEmpty(adapterId)) return Array.Empty(); + // A version that can't be pinned has nothing to describe. Asking for it started the + // package to read its settings, which threw - a 500 from the request validator, which runs + // before EnsureRunnable can refuse the version with a reason. + var (id, version) = SW.Serverless.Contract.Catalog.AdapterCatalogPaths.Split(adapterId); + if (version != null && catalog != null && !await catalog.HasVersionAsync(id, version)) + return Array.Empty(); + var required = (await startupValues.Describe(adapterId)) .Where(p => !p.Value.Optional).Select(p => p.Key); diff --git a/SW.Bitween.Api/Services/AdapterSecretProperties.cs b/SW.Bitween.Api/Services/AdapterSecretProperties.cs index e5f13e35..b33546a5 100644 --- a/SW.Bitween.Api/Services/AdapterSecretProperties.cs +++ b/SW.Bitween.Api/Services/AdapterSecretProperties.cs @@ -8,14 +8,14 @@ namespace SW.Bitween; /// -/// Keeps adapter secrets — an api key, a mail password — out of responses, and puts them back when +/// Keeps adapter secrets - an api key, a mail password - out of responses, and puts them back when /// an unchanged one is saved again. /// /// /// /// An adapter marks a startup value [Secure], which is what /// reports. replaces those values with on the way out, and -/// reads the sentinel on the way back in as "keep what is stored" — so a form +/// reads the sentinel on the way back in as "keep what is stored" - so a form /// that only changed the subject line does not overwrite the password with a row of dots. /// /// @@ -55,8 +55,13 @@ public async Task> Mask( return properties.ToDictionary(kv => kv.Key, _ => Sentinel); } + // Adapters read their settings ignoring case, so a secret saved as "password" is still the + // adapter's Password, and has to be hidden like it. + var byName = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var (name, value) in described) byName.TryAdd(name, value); + return properties.ToDictionary(kv => kv.Key, kv => - described.TryGetValue(kv.Key, out var startupValue) + byName.TryGetValue(kv.Key, out var startupValue) && startupValue.Private && !string.IsNullOrEmpty(kv.Value) ? Sentinel @@ -68,7 +73,7 @@ public async Task> Mask( /// secrets nobody can ask an adapter about. /// /// - /// A partner property or a global value is not a startup value of any one adapter — it is + /// A partner property or a global value is not a startup value of any one adapter - it is /// referenced as {{partner.KEY}} by however many adapters point at it, so there is no /// single adapter to describe. The owner names its own secrets instead, exactly as a data /// source does. Unlike DataSources.Secrets this does not also guess from the key's name: @@ -111,11 +116,63 @@ public static Dictionary Merge( return result; } + /// + /// Whether a sentinel may still mean "keep what is stored". Not once the adapter is a different + /// one, or a property saying where it connects has changed: a secret kept across that goes to + /// wherever the editor pointed it - their own server, say - without their ever having seen it. + /// Re-entering it then proves they know it. + /// + public static bool MayKeepStoredSecrets(string storedAdapterId, string newAdapterId, + IReadOnlyDictionary stored, IEnumerable> incoming) + { + if (!string.Equals(storedAdapterId ?? "", newAdapterId ?? "", StringComparison.OrdinalIgnoreCase)) + return false; + + var before = stored ?? new Dictionary(); + // The same name twice in different case: the check below would see only one of them, and + // the adapter, which reads names ignoring case, could connect with the other. "host" set to + // a new server beside "Host" sent as the sentinel would carry the stored password there. + var named = (incoming ?? []).Where(kv => kv.Key != null).ToList(); + if (named.Select(kv => kv.Key).Distinct(StringComparer.OrdinalIgnoreCase).Count() != named.Count) + return false; + + var after = named + .GroupBy(kv => kv.Key, StringComparer.OrdinalIgnoreCase) + .ToDictionary(g => g.Key, g => g.Last().Value, StringComparer.OrdinalIgnoreCase); + + foreach (var key in before.Keys.Concat(after.Keys).Where(IsDestination).Distinct(StringComparer.OrdinalIgnoreCase)) + { + before.TryGetValue(key, out var was); + after.TryGetValue(key, out var now); + if (now == Sentinel) continue; + if (!string.Equals(was ?? "", now ?? "", StringComparison.Ordinal)) return false; + } + + return true; + } + + private static readonly string[] DestinationWords = + ["url", "uri", "host", "server", "endpoint", "address", "domain", "port", "connectionstring"]; + + /// A property that says where an adapter connects, judged by its name. + public static bool IsDestination(string key) => + key != null && DestinationWords.Any(w => key.Replace("_", "").Replace("-", "") + .Contains(w, StringComparison.OrdinalIgnoreCase)); + + /// + /// , + /// except that when is false a sentinel restores nothing - the + /// secret has to be entered again. + /// + public static Dictionary Merge(IReadOnlyDictionary stored, + IReadOnlyDictionary incoming, bool keepStored) => + Merge(keepStored ? stored : null, incoming); + /// /// applied to the dictionary the caller already holds. /// /// - /// is an immutable value object — every property is init — so a + /// is an immutable value object - every property is init - so a /// group's properties cannot be swapped for a masked copy. Editing the dictionary in place is /// the way to reach them without either loosening that contract or rebuilding each group /// property by property, which would silently drop whatever property is added to it next. diff --git a/SW.Bitween.Api/Services/AdapterStartupValues.cs b/SW.Bitween.Api/Services/AdapterStartupValues.cs index 59f31a40..67c87cca 100644 --- a/SW.Bitween.Api/Services/AdapterStartupValues.cs +++ b/SW.Bitween.Api/Services/AdapterStartupValues.cs @@ -7,7 +7,7 @@ namespace SW.Bitween; /// -/// What startup properties an adapter expects — its key names, which are optional, which are +/// What startup properties an adapter expects - its key names, which are optional, which are /// secret, and their defaults. /// /// @@ -26,18 +26,57 @@ namespace SW.Bitween; public class AdapterStartupValues( NativeAdapterDiscoveryService nativeAdapterDiscovery, ServerlessAdapterDescriber serverlessDescriber, - IServiceProvider serviceProvider) + IServiceProvider serviceProvider, + AdapterCatalog catalog = null) { /// Drops what is remembered about a published adapter. public void Forget(string adapterId) => serverlessDescriber.Forget(adapterId); - /// Native (native prefix) or published. + /// + /// Native (native prefix) or published - or a published adapter pinned to a version, + /// {id}/{version}, whose properties may differ from the current one's. + /// /// Key name to description. Empty when the adapter reports nothing. + /// + /// For a screen asking about one adapter: one that isn't published, or whose package can't be + /// read, is not found (404) with the reason, rather than the 500 it was. + /// + public async Task> DescribeOrNotFound(string adapterId) + { + try + { + return await Describe(adapterId); + } + catch (InvalidOperationException ex) + { + throw new SW.PrimitiveTypes.SWNotFoundException($"Adapter '{adapterId}' can't be described: {ex.Message}"); + } + } + public async Task> Describe(string adapterId) { if (string.IsNullOrWhiteSpace(adapterId)) return new Dictionary(); + // A published package that lists its properties in its manifest is described from that - + // no process to start, and the same answer for a resident adapter as a classic one. A + // package without one, or one that lists none, is asked as it always was. + if (catalog != null && !NativeAdapterDiscoveryService.IsNative(adapterId)) + { + var manifest = await catalog.ManifestOf(adapterId); + if (manifest?.Properties is { Count: > 0 } declared) + return FromManifest(declared); + + // An adapter in another runtime than .NET cannot be asked: it speaks only gRPC, and + // asking means the stdio exchange of .NET adapters, which it would leave waiting for a + // reply until the describer timed out - holding one of its few slots all the while. + // Nor does it need asking. Its manifest was written from what it said when it was + // built, so a manifest that lists nothing means it expects nothing. + if (manifest != null && !string.IsNullOrWhiteSpace(manifest.Runtime) && + !string.Equals(manifest.Runtime, SW.Serverless.Contract.Catalog.AdapterManifest.DotnetRuntime, StringComparison.OrdinalIgnoreCase)) + return FromManifest(manifest.Properties ?? []); + } + // Reflection over an in-process type, so there is nothing here worth caching, and nothing // worth queueing behind the published adapters either. if (adapterId.StartsWith(NativeAdapterDiscoveryService.NativePrefix, StringComparison.OrdinalIgnoreCase)) @@ -50,20 +89,37 @@ public async Task> Describe(string adapterId) // // Every caller of this then failed in its own way and none named a cause: saving a // subscription that used one was refused outright, and the two that mask secrets failed - // closed and returned every property as "__private__" — so a screen showed a masked value + // closed and returned every property as "__private__" - so a screen showed a masked value // where the chosen statement should be, and its dropdown could not match it. // // Nothing is the right answer rather than a shrug. A resident adapter's settings live on // its DATA SOURCE, which describes and masks its own; what a subscription holds for one - // is which statement to run and what to do with it — routing, not secrets. + // is which statement to run and what to do with it - routing, not secrets. if (await ResidentAdapters.IsResidentAsync(serviceProvider, adapterId)) return new Dictionary(); return await serverlessDescriber.Describe(adapterId); } + /// A manifest's properties in the shape every caller of already reads. + public static IDictionary FromManifest( + IEnumerable properties) + { + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var property in properties) + result[property.Name] = new StartupValue + { + Optional = !property.Required, + Default = property.Default, + Type = property.Type ?? SW.Serverless.Contract.Catalog.AdapterProperty.TextType, + Private = property.Secret, + Description = property.Description ?? property.DisplayName + }; + return result; + } + /// - /// The same description with the default withheld from every secret property — the shape to + /// The same description with the default withheld from every secret property - the shape to /// hand to a client. /// /// diff --git a/SW.Bitween.Api/Services/Adapters/AdapterCatalog.cs b/SW.Bitween.Api/Services/Adapters/AdapterCatalog.cs new file mode 100644 index 00000000..189e1c61 --- /dev/null +++ b/SW.Bitween.Api/Services/Adapters/AdapterCatalog.cs @@ -0,0 +1,105 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Caching.Memory; +using SW.PrimitiveTypes; +using SW.Serverless.Contract.Catalog; + +namespace SW.Bitween.Services.Adapters; + +/// +/// The published adapters' catalog - current version, every version with its manifest, the icon - +/// and the manifests of the native ones, behind one lookup. +/// +/// +/// A published adapter has a catalog entry only once it has been published by an installer that +/// writes one. Every answer here is therefore optional, and every caller falls back to what it did +/// before catalogs existed: list the package files, describe the adapter by asking it. +/// +public class AdapterCatalog( + ICloudFilesService cloudFiles, + ServerlessOptions serverlessOptions, + NativeAdapterManifests nativeManifests, + IMemoryCache cache) +{ + /// + /// What adapters call Bitween in their manifest's compatibility.applications - the minimum + /// Bitween they need - and what manifests before that field called minBitweenVersion. + /// + public const string Application = "bitween"; + + // Long enough that a screen listing every kind reads each entry once; short enough that a + // publish shows up without anyone thinking to clear anything. + static readonly TimeSpan CacheFor = TimeSpan.FromSeconds(60); + + AdapterCatalogStore Store => new(cloudFiles, serverlessOptions.AdapterRemotePath); + + static string CacheKey(string adapterId) => $"bitween.adapter-catalog.{adapterId.ToLowerInvariant()}"; + + /// The adapter's catalog entry, or null when it has none - native, or published before catalogs. + public async Task GetAsync(string adapterId) + { + if (string.IsNullOrWhiteSpace(adapterId) || NativeAdapterDiscoveryService.IsNative(adapterId)) return null; + + if (cache.TryGetValue(CacheKey(adapterId), out AdapterCatalogEntry cached)) return cached; + + AdapterCatalogEntry entry; + try + { + entry = await Store.GetAsync(adapterId); + } + catch + { + // A catalog that cannot be read is treated as absent: everything that uses it has a + // fallback, and a storage hiccup must not empty the adapter lists. + return null; + } + + cache.Set(CacheKey(adapterId), entry, CacheFor); + return entry; + } + + public void Forget(string adapterId) + { + if (!string.IsNullOrWhiteSpace(adapterId)) cache.Remove(CacheKey(adapterId)); + } + + /// + /// The manifest of what runs: a native adapter's own, a pinned + /// version's, or the current package's. Null when there is none to read. + /// + public async Task ManifestOf(string adapterRef) + { + if (string.IsNullOrWhiteSpace(adapterRef)) return null; + if (NativeAdapterDiscoveryService.IsNative(adapterRef)) return nativeManifests.Get(adapterRef); + + var (adapterId, version) = AdapterCatalogPaths.Split(adapterRef); + var entry = await GetAsync(adapterId); + if (entry == null) return null; + return version == null ? entry.Manifest : entry.Find(version)?.Manifest; + } + + /// + /// Whether of the adapter can be pinned: published and not withdrawn. + /// Without a catalog entry, published means its package exists. + /// + public async Task HasVersionAsync(string adapterId, string version) + { + if (!AdapterCatalogPaths.IsVersion(version)) return false; + + var entry = await GetAsync(adapterId); + if (entry != null) + return entry.Find(version) is { Withdrawn: false }; + + // Where versions are published now, or where an installer from before the versions prefix put them. + foreach (var key in new[] + { + AdapterCatalogPaths.Version(serverlessOptions.AdapterRemotePath, adapterId, version), + AdapterCatalogPaths.LegacyVersion(serverlessOptions.AdapterRemotePath, adapterId, version) + }) + if ((await cloudFiles.ListAsync(key)).Any(f => string.Equals(f.Key, key, StringComparison.Ordinal) && f.Size > 0)) + return true; + return false; + } +} diff --git a/SW.Bitween.Api/Services/Adapters/AdapterChanges.cs b/SW.Bitween.Api/Services/Adapters/AdapterChanges.cs new file mode 100644 index 00000000..ab118711 --- /dev/null +++ b/SW.Bitween.Api/Services/Adapters/AdapterChanges.cs @@ -0,0 +1,54 @@ +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Services.Adapters; + +/// Told to every node when an adapter's published versions change. +public class AdapterChangedMessage +{ + public string AdapterId { get; set; } +} + +/// +/// Forgets what this node and every other has cached about an adapter: its catalog entry and its +/// described properties. Each node caches both on its own, so forgetting only on the node that +/// published left the others showing the old version and its old properties until their caches +/// expired. The broadcast is best effort, like the configuration cache's: the publish has happened +/// whether or not the bus could be told, and the caches still expire on their own. +/// +public class AdapterChanges(AdapterCatalog catalog, AdapterStartupValues startupValues, + IServiceScopeFactory scopes, ILogger logger) +{ + public async Task ForgetEverywhereAsync(string adapterId) + { + Forget(adapterId); + try + { + using var scope = scopes.CreateScope(); + var broadcast = scope.ServiceProvider.GetService(); + if (broadcast != null) await broadcast.Broadcast(new AdapterChangedMessage { AdapterId = adapterId }); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Couldn't tell the other nodes that {AdapterId} changed; they show it as cached until it expires.", adapterId); + } + } + + public void Forget(string adapterId) + { + catalog.Forget(adapterId); + startupValues.Forget(adapterId); + } +} + +public class AdapterChangedListener(AdapterChanges changes) : IListen +{ + public Task Process(AdapterChangedMessage message) + { + if (!string.IsNullOrWhiteSpace(message?.AdapterId)) changes.Forget(message.AdapterId); + return Task.CompletedTask; + } +} diff --git a/SW.Bitween.Api/Services/Adapters/AdapterFailureReader.cs b/SW.Bitween.Api/Services/Adapters/AdapterFailureReader.cs index 375934e6..17fd70f0 100644 --- a/SW.Bitween.Api/Services/Adapters/AdapterFailureReader.cs +++ b/SW.Bitween.Api/Services/Adapters/AdapterFailureReader.cs @@ -11,14 +11,14 @@ namespace SW.Bitween.Services.Adapters; /// /// Turns what a dying adapter printed into a sentence an operator can act on. /// -/// An adapter that fails during startup — a broker refusing the credentials, TLS attempted against -/// a plaintext port — writes the reason to its own stderr and exits. The host keeps that output, +/// An adapter that fails during startup - a broker refusing the credentials, TLS attempted against +/// a plaintext port - writes the reason to its own stderr and exits. The host keeps that output, /// but nothing carried it anywhere a person looks: the connection test said "Adapter stream /// closed", the data source row's LastException stayed null because that field only ever held what /// a LIVE adapter reported about itself, and the actual exception existed solely in the API log. /// -/// So the two questions an operator asks — "why did the test fail" and "why is this connection -/// down" — could only be answered by tailing a log on the node. This is what makes them +/// So the two questions an operator asks - "why did the test fail" and "why is this connection +/// down" - could only be answered by tailing a log on the node. This is what makes them /// answerable from the screen that asked. /// public static class AdapterFailureReader @@ -30,7 +30,7 @@ public static class AdapterFailureReader /// The most useful thing in the adapter's last output, or null when it said nothing. /// /// Prefers the last error it logged, because an adapter that dies logs the reason last. Falls - /// back to the final line of anything at all — an adapter can also die without using the SDK's + /// back to the final line of anything at all - an adapter can also die without using the SDK's /// logger, and half an answer beats none. /// public static string Summarise(IEnumerable diagnostics) @@ -68,8 +68,8 @@ public static string Explain(string message, IEnumerable diagnostics) /// thread. Waiting for the process to exit is what makes the difference between "Adapter /// stream closed" and the exception that closed it. /// - /// The wait is short and bounded, and an adapter that is still alive — a timeout rather than a - /// crash — is not waited on at all. + /// The wait is short and bounded, and an adapter that is still alive - a timeout rather than a + /// crash - is not waited on at all. /// public static async Task> SettledOutputAsync(ResidentAdapterInstance instance, int graceMilliseconds = 2000) @@ -150,7 +150,7 @@ private static string Clean(string line) .Trim(); // The SDK writes "message#!#exception#!#", so what is left is the message, then the - // exception's own first line — which is the type and text, and is the sentence to lead + // exception's own first line - which is the type and text, and is the sentence to lead // with. Everything past it is a stack trace nobody reads on a data source screen. var parts = text.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); if (parts.Length == 0) return null; @@ -166,7 +166,7 @@ private static string Clean(string line) .Select(l => l[4..].Trim()) .LastOrDefault(l => !string.IsNullOrWhiteSpace(l) && !lead.Contains(l, StringComparison.Ordinal)); - var summary = cause == null ? lead : $"{lead} — caused by {cause}"; + var summary = cause == null ? lead : $"{lead} - caused by {cause}"; if (string.IsNullOrWhiteSpace(summary)) return null; diff --git a/SW.Bitween.Api/Services/Adapters/AdapterInvoker.cs b/SW.Bitween.Api/Services/Adapters/AdapterInvoker.cs index 42a03f3a..3b84b122 100644 --- a/SW.Bitween.Api/Services/Adapters/AdapterInvoker.cs +++ b/SW.Bitween.Api/Services/Adapters/AdapterInvoker.cs @@ -1,7 +1,12 @@ +using System; using System.Collections.Generic; +using System.Diagnostics; +using System.IO; using System.Linq; using System.Threading.Tasks; +using SW.Bitween.Services.Metrics; using SW.PrimitiveTypes; +using SW.Serverless.Contract.Catalog; namespace SW.Bitween.Services.Adapters; @@ -10,9 +15,15 @@ namespace SW.Bitween.Services.Adapters; /// /// The pipeline asks for a mapper, handler, validator or receiver by id and role; which of the /// three runtimes ends up running it is decided here and nowhere else. That is what keeps the -/// choice from being copied into every call site — and what lets a test replace the whole thing. +/// choice from being copied into every call site - and what lets a test replace the whole thing. +/// +/// Every adapter call passes through here, so this is also where each one is counted for the +/// adapter metrics: how long it took, and whether it failed and how - a timeout, a stop for going +/// over a limit, a process that died, or an error of the adapter's own. /// -public class AdapterInvoker(IEnumerable runtimes) : IAdapterInvoker +public class AdapterInvoker(IEnumerable runtimes, MetricsRecorder metrics = null, AdapterCatalog catalog = null, + Connections.ConnectionTokens connections = null, Marketplace.RegistryLicences licences = null) + : IAdapterInvoker { // Order matters: the classic runtime claims everything, so it has to be asked last. private readonly IReadOnlyList _runtimes = runtimes.ToList(); @@ -20,9 +31,34 @@ public class AdapterInvoker(IEnumerable runtimes) : IAdapterInv public async Task BeginAsync(string adapterId, AdapterRole role, IDictionary properties = null, string correlationId = null) { + // Connection references become tokens here, as the adapter starts, and nowhere earlier: an + // exchange stores its settings with the reference in them, so no token is ever saved on + // one, and a retry later still gets a token that works. Every role and runtime passes + // through this, so every adapter can use them. + if (connections != null) properties = await connections.ResolveAsync(properties); + + // A paid connector from the registry runs only on a licence that is live or in grace. Read + // from the catalog's and the licence store's caches, so nothing is fetched on a call. + if (licences != null) await licences.EnsureLicensedAsync(adapterId); + foreach (var runtime in _runtimes) if (await runtime.CanRunAsync(adapterId)) - return await runtime.BeginAsync(adapterId, role, properties, correlationId); + { + if (metrics == null) return TextOutside(runtime, await runtime.BeginAsync(adapterId, role, properties, correlationId)); + + var meter = new Meter(metrics, adapterId, await VersionOf(adapterId), role, RuntimeKind(runtime)); + var clock = Stopwatch.StartNew(); + try + { + return new MeteredSession(TextOutside(runtime, await runtime.BeginAsync(adapterId, role, properties, correlationId)), meter); + } + catch (Exception ex) + { + // A session that couldn't even start is a failed call: a process that wouldn't spawn. + meter.Record(clock.ElapsedMilliseconds, ex); + throw; + } + } throw new BitweenException($"No runtime here knows how to run adapter '{adapterId}'."); } @@ -33,4 +69,107 @@ public async Task InvokeAsync(string adapterId, AdapterRole ro await using var session = await BeginAsync(adapterId, role, properties, correlationId); return await session.InvokeAsync(method, argument); } + + /// + /// The version that runs: the pinned one, or for an unpinned reference the catalog's current + /// version now (read from the catalog's one-minute cache, which the runtimes have just read). + /// + async Task VersionOf(string adapterRef) + { + if (string.IsNullOrWhiteSpace(adapterRef) || NativeAdapterDiscoveryService.IsNative(adapterRef)) return ""; + var (id, version) = AdapterCatalogPaths.Split(adapterRef); + if (version != null) return version; + try + { + return catalog == null ? "" : (await catalog.GetAsync(id))?.Current ?? ""; + } + catch + { + return ""; + } + } + + static string RuntimeKind(IAdapterRuntime runtime) => runtime switch + { + NativeAdapterRuntime => "native", + ResidentAdapterRuntime => "resident", + ClassicAdapterRuntime => "process", + _ => runtime.GetType().Name.ToLowerInvariant(), + }; + + /// How a failed call failed, from what the runtimes throw. + public static AdapterFailure Classify(Exception ex) => ex switch + { + null => AdapterFailure.None, + TimeoutException => AdapterFailure.Timeout, + SW.Serverless.Resident.AdapterStoppedException { LimitExceeded: true } => AdapterFailure.LimitStop, + SW.Serverless.Resident.AdapterStoppedException => AdapterFailure.Crash, + IOException io when io.Message.Contains("stream closed", StringComparison.OrdinalIgnoreCase) => AdapterFailure.Crash, + _ when ex.Message.Contains("Process not started or terminated", StringComparison.OrdinalIgnoreCase) => AdapterFailure.Crash, + _ => AdapterFailure.Error, + }; + + sealed class Meter(MetricsRecorder metrics, string adapterRef, string version, AdapterRole role, string runtime) + { + readonly string _adapterId = AdapterCatalogPaths.Split(adapterRef).Item1; + + public void Record(long ms, Exception ex) => + metrics.AdapterCall(_adapterId, version, role.ToString().ToLowerInvariant(), runtime, ms, Classify(ex), ex?.Message); + } + + /// + /// The session as it should see its arguments: a built-in adapter, run in this process, takes the + /// pipeline's parsed payload; every other runtime is handed the file as text alone, as + /// it always was, so the adapter contract does not change. + /// + static IAdapterSession TextOutside(IAdapterRuntime runtime, IAdapterSession session) => + runtime is NativeAdapterRuntime ? session : new TextSession(session); + + sealed class TextSession(IAdapterSession inner) : IAdapterSession + { + public Task InvokeAsync(string method, object argument = null) => + inner.InvokeAsync(method, NativeAdapters.ParsedXchangeFile.Plain(argument)); + + public Task InvokeAsync(string method, object argument = null) => + inner.InvokeAsync(method, NativeAdapters.ParsedXchangeFile.Plain(argument)); + + public ValueTask DisposeAsync() => inner.DisposeAsync(); + } + + /// A session whose every call is timed and counted. + sealed class MeteredSession(IAdapterSession inner, Meter meter) : IAdapterSession + { + public async Task InvokeAsync(string method, object argument = null) + { + var clock = Stopwatch.StartNew(); + try + { + var result = await inner.InvokeAsync(method, argument); + meter.Record(clock.ElapsedMilliseconds, null); + return result; + } + catch (Exception ex) + { + meter.Record(clock.ElapsedMilliseconds, ex); + throw; + } + } + + public async Task InvokeAsync(string method, object argument = null) + { + var clock = Stopwatch.StartNew(); + try + { + await inner.InvokeAsync(method, argument); + meter.Record(clock.ElapsedMilliseconds, null); + } + catch (Exception ex) + { + meter.Record(clock.ElapsedMilliseconds, ex); + throw; + } + } + + public ValueTask DisposeAsync() => inner.DisposeAsync(); + } } diff --git a/SW.Bitween.Api/Services/Adapters/AdapterSourceReader.cs b/SW.Bitween.Api/Services/Adapters/AdapterSourceReader.cs new file mode 100644 index 00000000..42720129 --- /dev/null +++ b/SW.Bitween.Api/Services/Adapters/AdapterSourceReader.cs @@ -0,0 +1,208 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.Caching.Memory; +using SW.PrimitiveTypes; +using SW.Serverless.Contract.Catalog; + +namespace SW.Bitween.Services.Adapters; + +/// The source a published version carries, as listed in its manifest. +public class AdapterSourceListing +{ + public string AdapterId { get; set; } + public string Version { get; set; } + public string Language { get; set; } + public string Runtime { get; set; } + public string BuildCommand { get; set; } + public List Lockfiles { get; set; } = []; + + /// Path relative to the source folder, and its SHA-256: enough to tell which files two versions differ in. + public List Files { get; set; } = []; +} + +public class AdapterSourceFileInfo +{ + public string Path { get; set; } + public string Sha256 { get; set; } +} + +public class AdapterSourceFile +{ + public string Path { get; set; } + public string Sha256 { get; set; } + public long Size { get; set; } + + /// True when the file is not text, or too large to show; is then null. + public bool Binary { get; set; } + + public string Content { get; set; } +} + +/// +/// Reads the source a published adapter version carries - serverless build puts it in the +/// package under the manifest's source folder, whatever the language - so it can be shown and +/// compared in Bitween. Packages published before that, or with --no-source, carry none, +/// and say so by having no source in their manifest. +/// +/// +/// The list comes from the catalog's manifest and costs no download. A file comes from the +/// package itself, and is checked against the hash its manifest recorded, so what is shown is +/// what was published. A package's source is read once and kept a few minutes: comparing two +/// versions reads several files of each. +/// +public class AdapterSourceReader( + AdapterCatalog catalog, + ICloudFilesService cloudFiles, + ServerlessOptions serverlessOptions, + IMemoryCache cache) +{ + /// Larger than this is shown as binary: a source file this size is almost always generated. + public const int MaxShownBytes = 1024 * 1024; + + /// A package whose source is larger than this is not read into memory at all. + public const long MaxSourceBytes = 50L * 1024 * 1024; + + static readonly TimeSpan CacheFor = TimeSpan.FromMinutes(5); + + /// Null when the adapter or version is unknown; an empty listing when it carries no source. + public async Task ListAsync(string adapterId, string version) + { + var manifest = await ManifestOf(adapterId, version); + if (manifest == null) return null; + + return new AdapterSourceListing + { + AdapterId = adapterId, + Version = version, + Language = manifest.Language, + Runtime = manifest.Runtime, + BuildCommand = manifest.Source?.BuildCommand, + Lockfiles = manifest.Source?.Lockfiles ?? [], + Files = (manifest.Source?.Files ?? []) + .OrderBy(f => f.Key, StringComparer.Ordinal) + .Select(f => new AdapterSourceFileInfo { Path = f.Key, Sha256 = f.Value }) + .ToList() + }; + } + + /// Null when the version, or the file in it, is unknown. + public async Task ReadAsync(string adapterId, string version, string path) + { + var manifest = await ManifestOf(adapterId, version); + // Only what the manifest lists can be asked for, so a path can never reach outside the + // source folder, nor into the package's binaries. + if (manifest?.Source?.Files == null || path == null || !manifest.Source.Files.TryGetValue(path, out var expected)) + return null; + + var files = await SourceOf(adapterId, version, manifest.Source); + if (!files.TryGetValue(path, out var bytes)) + throw new SWException($"Version {version} of '{adapterId}' lists {path} in its manifest, but its package does not carry it."); + + var actual = Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(); + if (!string.Equals(actual, expected, StringComparison.OrdinalIgnoreCase)) + throw new SWException($"{path} in version {version} of '{adapterId}' does not match the hash its manifest recorded; it is not shown."); + + var text = bytes.Length <= MaxShownBytes ? AsText(bytes) : null; + return new AdapterSourceFile + { + Path = path, + Sha256 = actual, + Size = bytes.Length, + Binary = text == null, + Content = text + }; + } + + async Task ManifestOf(string adapterId, string version) + { + if (string.IsNullOrWhiteSpace(adapterId) || !AdapterCatalogPaths.IsVersion(version)) return null; + var entry = await catalog.GetAsync(adapterId); + return entry?.Find(version)?.Manifest; + } + + async Task> SourceOf(string adapterId, string version, AdapterSource source) + { + var key = $"bitween.adapter-source.{adapterId.ToLowerInvariant()}.{version.ToLowerInvariant()}"; + if (cache.TryGetValue(key, out Dictionary cached)) return cached; + + var files = await Download(adapterId, version, source); + cache.Set(key, files, CacheFor); + return files; + } + + async Task> Download(string adapterId, string version, AdapterSource source) + { + var root = serverlessOptions.AdapterRemotePath; + var packageKey = AdapterCatalogPaths.Version(root, adapterId, version); + if (!await Exists(packageKey)) + { + var legacy = AdapterCatalogPaths.LegacyVersion(root, adapterId, version); + if (!await Exists(legacy)) + throw new SWException($"The package of version {version} of '{adapterId}' is not in storage."); + packageKey = legacy; + } + + // A zip is read from its end, so it goes to a temp file first rather than being held whole. + var temp = Path.GetTempFileName(); + try + { + await using (var remote = await cloudFiles.OpenReadAsync(packageKey)) + await using (var local = File.Create(temp)) + await remote.CopyToAsync(local); + + var prefix = (string.IsNullOrWhiteSpace(source.Path) ? AdapterSource.DefaultPath : source.Path).Trim('/') + "/"; + var files = new Dictionary(StringComparer.Ordinal); + long total = 0; + using var zip = ZipFile.OpenRead(temp); + foreach (var entry in zip.Entries) + { + // Zips written on Windows separate with '\'. + var name = entry.FullName.Replace('\\', '/'); + if (!name.StartsWith(prefix, StringComparison.Ordinal) || name.EndsWith('/')) continue; + var relative = name[prefix.Length..]; + if (!source.Files.ContainsKey(relative)) continue; + + total += entry.Length; + if (total > MaxSourceBytes) + throw new SWException($"The source of version {version} of '{adapterId}' is larger than Bitween reads."); + + await using var stream = entry.Open(); + using var buffer = new MemoryStream((int)entry.Length); + await stream.CopyToAsync(buffer); + files[relative] = buffer.ToArray(); + } + + return files; + } + finally + { + try { File.Delete(temp); } catch { } + } + } + + async Task Exists(string key) => + (await cloudFiles.ListAsync(key)).Any(f => string.Equals(f.Key, key, StringComparison.Ordinal) && f.Size > 0); + + /// The file as UTF-8 text, or null when it isn't text. + internal static string AsText(byte[] bytes) + { + // A NUL byte is the reliable tell of a binary file; text in any encoding a source file + // would use has none. + if (Array.IndexOf(bytes, (byte)0) >= 0) return null; + try + { + var text = new UTF8Encoding(false, true).GetString(bytes); + return text.Length > 0 && text[0] == '' ? text[1..] : text; + } + catch (DecoderFallbackException) + { + return null; + } + } +} diff --git a/SW.Bitween.Api/Services/Adapters/AdapterWorkshop.cs b/SW.Bitween.Api/Services/Adapters/AdapterWorkshop.cs new file mode 100644 index 00000000..2ebabd34 --- /dev/null +++ b/SW.Bitween.Api/Services/Adapters/AdapterWorkshop.cs @@ -0,0 +1,458 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using SW.Bitween.Domain; +using SW.PrimitiveTypes; +using SW.Serverless.Contract.Catalog; +using SW.Serverless.Tooling; +using SW.Serverless.Tooling.Building; +using SW.Serverless.Tooling.Conformance; +using SW.Serverless.Tooling.Scaffolding; +using SW.Bitween.Adapters.Tooling; + +namespace SW.Bitween.Services.Adapters; + +public class WorkshopBuild +{ + public bool Succeeded { get; set; } + public List Problems { get; set; } = []; + public List Warnings { get; set; } = []; + + /// What the adapter says it is, once built: its settings and commands, for the try panel's form. + public AdapterManifest Manifest { get; set; } + + public List Checks { get; set; } = []; + public bool Conforms => Succeeded && Checks.All(c => c.Outcome != nameof(CheckOutcome.Failed)); +} + +public record WorkshopCheck(string Name, string Outcome, string Detail); + +public class WorkshopRun +{ + public bool Succeeded { get; set; } + public string Output { get; set; } + public string Error { get; set; } + public List Problems { get; set; } = []; +} + +/// +/// The adapter editor's server side: drafts started from a template or a published version, then +/// built, checked, tried and published with SW.Serverless.Tooling - the same code bitween adapter build, +/// test, run and publish are - so a version published here is the one the CLI would have published. +/// +/// +/// +/// Python and JavaScript/TypeScript only, and their own code only: the image has their runtimes but +/// neither pip nor npm, and a dependency would mean fetching packages from the internet in the +/// middle of a save. A draft that names one is refused with the CLI as the way to add it. +/// +/// +/// Building and trying run the draft's code on this server. That is what the permissions are for: +/// adapter-source.edit to do it at all, adapter-source.operate to publish. At most +/// two run at a time, so the editor can't take the server's processing down with it. +/// +/// +public class AdapterWorkshop( + AdapterCatalog catalog, + AdapterSourceReader sourceReader, + AdapterChanges changes, + ICloudFilesService cloudFiles, + ServerlessOptions serverlessOptions, + BitweenOptions bitweenOptions) +{ + static readonly SemaphoreSlim Running = new(2, 2); + const int CallTimeoutSeconds = 30; + + /// + /// What a draft may use while it is checked or tried here, besides the two at a time and the + /// timeout per call: memory and sustained CPU, from Bitween:AdapterEditorMemoryMb and + /// AdapterEditorCpuCores. A run over either is stopped, and the editor shows which limit it hit. + /// + LocalAdapterLimits Limits => new() + { + MemoryLimitBytes = Math.Max(0, bitweenOptions.AdapterEditorMemoryMb) * 1024L * 1024L, + CpuPercentLimit = Math.Max(0, bitweenOptions.AdapterEditorCpuCores) * 100.0 / Environment.ProcessorCount, + }; + + // ------------------------------------------------------------------ starting a draft + + /// A new adapter, from the template serverless init writes. + public async Task NewAsync(string name, string language, string kind, string adapterId) + { + if (!AdapterDraft.Languages.Contains(language)) + throw new SWValidationException("Language", $"The editor writes {string.Join(", ", AdapterDraft.Languages)} adapters."); + + var id = string.IsNullOrWhiteSpace(adapterId) ? Scaffolder.IdFrom(name ?? "") : adapterId.Trim().ToLowerInvariant(); + if (await catalog.GetAsync(id) != null) + throw new SWValidationException("AdapterId", $"'{id}' is already published; edit one of its versions instead."); + + var parent = Path.Combine(Path.GetTempPath(), "bitween-workshop", Guid.NewGuid().ToString("N")); + try + { + var made = BitweenAdapters.Scaffold(new ScaffoldRequest { Name = name, Id = id, Language = language, Kind = kind, ParentDirectory = parent }); + if (!made.Succeeded) throw new SWValidationException("Name", string.Join("; ", made.Problems)); + var files = made.Files.ToDictionary(f => f.Replace('\\', '/'), f => File.ReadAllText(Path.Combine(made.ProjectDirectory, f))); + return new AdapterDraft(id, language, kind, null, files); + } + finally + { + try { Directory.Delete(parent, true); } catch { } + } + } + + /// A draft of a published Python or Node version, from the source its package carries. + public async Task FromVersionAsync(string adapterId, string version) + { + var listing = await sourceReader.ListAsync(adapterId, version) + ?? throw new SWNotFoundException($"{adapterId} {version}"); + var language = listing.Runtime switch + { + AdapterManifest.PythonRuntime => "python", + AdapterManifest.NodeRuntime => listing.Language == "typescript" ? "typescript" : "node", + _ => throw new SWValidationException("AdapterId", + $"Version {version} of '{adapterId}' runs on {listing.Runtime ?? "dotnet"}; the editor writes Python and JavaScript adapters. Build it with the CLI."), + }; + if (listing.Files.Count == 0) + throw new SWValidationException("Version", $"Version {version} of '{adapterId}' was published without its source."); + + var files = new Dictionary(); + foreach (var file in listing.Files) + { + var read = await sourceReader.ReadAsync(adapterId, version, file.Path); + if (read is { Binary: false }) files[file.Path] = read.Content; + } + + var manifest = await catalog.ManifestOf(AdapterCatalogPaths.Ref(adapterId, version)); + return new AdapterDraft(adapterId, language, manifest?.Kinds.FirstOrDefault(), version, files); + } + + // ------------------------------------------------------------------ building, checking, trying + + /// Builds the draft and, unless only the build is wanted, checks it against its contracts. + public async Task BuildAsync(AdapterDraft draft, IDictionary settings, bool check) + { + await Running.WaitAsync(); + try + { + await using var built = await BuiltAsync(draft); + var result = built.Result; + if (!result.Succeeded || !check) return result; + + var report = await new ConformanceRunner().RunAsync(new ConformanceOptions + { + PackageDirectory = built.PackageDirectory, + Settings = settings ?? new Dictionary(), + CommandTimeoutSeconds = CallTimeoutSeconds, + // A receiver's DeleteFile changes the source it reads; never from here. + AllowDelete = false, + Contracts = { BitweenAdapters.Contract }, + Limits = Limits, + }); + result.Checks = report.Checks.Select(c => new WorkshopCheck(c.Name, c.Outcome.ToString(), c.Detail)).ToList(); + // The checks pass or fail on what the adapter does with the settings given. One it marks + // required but that was left empty was untested, while the result read as a clean pass. + foreach (var missing in (result.Manifest?.Properties ?? []).Where(p => p.Required && + (settings == null || !settings.TryGetValue(p.Name, out var v) || string.IsNullOrWhiteSpace(v)))) + result.Warnings.Add($"{missing.Name} is required but was empty, so the checks ran without it. " + + "A subscription has to give it a value."); + return result; + } + finally + { + Running.Release(); + } + } + + /// Builds the draft and calls one command, as a host would, with the settings given. + public async Task TryAsync(AdapterDraft draft, IDictionary settings, string command, string input) + { + if (string.IsNullOrWhiteSpace(command)) throw new SWValidationException("Command", "Say which command to call."); + await Running.WaitAsync(); + try + { + await using var built = await BuiltAsync(draft); + if (!built.Result.Succeeded) return new WorkshopRun { Problems = built.Result.Problems }; + try + { + await using var host = await LocalAdapterHost.StartAsync(built.PackageDirectory, settings ?? new Dictionary(), + commandTimeoutSeconds: CallTimeoutSeconds, limits: Limits); + return new WorkshopRun { Succeeded = true, Output = await host.CallAsync(command, ReadInput(input)) }; + } + catch (Exception ex) when (ex is not OutOfMemoryException) + { + return new WorkshopRun { Error = ex.GetBaseException().Message }; + } + } + finally + { + Running.Release(); + } + } + + // ------------------------------------------------------------------ publishing + + /// + /// Publishes the draft as a new version, not made current: subscriptions keep running what they + /// run until someone promotes it or pins it. It must build and pass its contract checks first. + /// + public async Task<(string Version, WorkshopBuild Build)> PublishAsync(AdapterDraft draft, string version, string releaseNotes, + IDictionary settings, string publishedBy) + { + await Running.WaitAsync(); + try + { + await using var built = await BuiltAsync(draft); + var result = built.Result; + if (!result.Succeeded) return (null, result); + + var report = await new ConformanceRunner().RunAsync(new ConformanceOptions + { + PackageDirectory = built.PackageDirectory, + Settings = settings ?? new Dictionary(), + CommandTimeoutSeconds = CallTimeoutSeconds, + Contracts = { BitweenAdapters.Contract }, + Limits = Limits, + }); + result.Checks = report.Checks.Select(c => new WorkshopCheck(c.Name, c.Outcome.ToString(), c.Detail)).ToList(); + if (!report.Passed) return (null, result); + + var published = await PackagePublisher.PublishPackageAsync(cloudFiles, new PublishPackageRequest + { + PackagePath = built.ZipPath, + Version = string.IsNullOrWhiteSpace(version) ? "patch" : version.Trim(), + Promote = false, + ReleaseNotes = releaseNotes, + PublishedBy = publishedBy, + RemotePath = serverlessOptions.AdapterRemotePath, + }, _ => { }); + + await changes.ForgetEverywhereAsync(draft.AdapterId); + return (published.Version, result); + } + finally + { + Running.Release(); + } + } + + /// Makes a published version the one that runs where nothing is pinned. + public async Task PromoteAsync(string adapterId, string version) + { + var work = Path.Combine(Path.GetTempPath(), "bitween-workshop", Guid.NewGuid().ToString("N")); + try + { + await new AdapterRepository(cloudFiles, _ => { }, serverlessOptions.AdapterRemotePath).PromoteAsync(adapterId, version, work); + } + finally + { + try { Directory.Delete(work, true); } catch { } + await changes.ForgetEverywhereAsync(adapterId); + } + } + + // ------------------------------------------------------------------ packages built elsewhere + + /// + /// Publishes a package the bitween CLI built - any language, .NET included - as it would publish + /// straight to storage, only through Bitween: its manifest is checked, its version settled, and it + /// is made current only when asked. + /// + public async Task UploadAsync(Stream zip, string version, bool current, string releaseNotes, string publishedBy) + { + var work = Path.Combine(Path.GetTempPath(), "bitween-workshop", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(work); + try + { + var path = Path.Combine(work, "package.zip"); + await using (var file = File.Create(path)) + await zip.CopyToAsync(file); + + PublishResult published; + try + { + published = await PackagePublisher.PublishPackageAsync(cloudFiles, new PublishPackageRequest + { + PackagePath = path, + Version = version, + Promote = current, + ReleaseNotes = releaseNotes, + PublishedBy = publishedBy, + RemotePath = serverlessOptions.AdapterRemotePath, + }, _ => { }); + } + catch (InvalidDataException) + { + throw new SWValidationException("Package", "That isn't a zip: send the package bitween adapter build made."); + } + + await changes.ForgetEverywhereAsync(published.Manifest.Id); + return published; + } + finally + { + try { Directory.Delete(work, true); } catch { } + } + } + + /// Every version of an adapter, with which is current and which are withdrawn. + public Task VersionsAsync(string adapterId) => + new AdapterRepository(cloudFiles, _ => { }, serverlessOptions.AdapterRemotePath).ListVersionsAsync(adapterId); + + /// Takes a version out of use: still listed, never offered for pinning, never made current. + public async Task WithdrawAsync(string adapterId, string version) + { + await new AdapterRepository(cloudFiles, _ => { }, serverlessOptions.AdapterRemotePath).WithdrawAsync(adapterId, version); + await changes.ForgetEverywhereAsync(adapterId); + } + + // ------------------------------------------------------------------ the build itself + + sealed class Built : IAsyncDisposable + { + public string Work { get; init; } + public WorkshopBuild Result { get; init; } + public string PackageDirectory { get; init; } + public string ZipPath { get; init; } + + public ValueTask DisposeAsync() + { + try { Directory.Delete(Work, true); } catch { } + return ValueTask.CompletedTask; + } + } + + async Task BuiltAsync(AdapterDraft draft) + { + var work = Path.Combine(Path.GetTempPath(), "bitween-workshop", Guid.NewGuid().ToString("N")); + var project = Path.Combine(work, "project"); + Directory.CreateDirectory(project); + + var files = draft.Files; + var result = new WorkshopBuild(); + result.Problems.AddRange(Dependencies(files, bitweenOptions.AdapterEditorDependencies)); + if (result.Problems.Count > 0) return new Built { Work = work, Result = result }; + + foreach (var (path, content) in files) + { + // A lock file can name any source, a local folder included; the build resolves from the + // registry instead, from package.json's plain versions. + if (path == "package-lock.json") + { + result.Warnings.Add("package-lock.json isn't used here: the editor installs the versions package.json names, from the registry"); + continue; + } + var target = Path.Combine(project, path); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + await File.WriteAllTextAsync(target, content); + } + + // The draft is for this adapter id whatever its adapter.json says; a typo there would + // otherwise publish someone else's adapter. + var manifestPath = Path.Combine(project, AdapterManifest.FileName); + if (!File.Exists(manifestPath)) + { + result.Problems.Add($"the draft has no {AdapterManifest.FileName}"); + return new Built { Work = work, Result = result }; + } + var authored = JsonNode.Parse(await File.ReadAllTextAsync(manifestPath), + documentOptions: new JsonDocumentOptions { CommentHandling = JsonCommentHandling.Skip, AllowTrailingCommas = true })?.AsObject(); + if (authored == null) + { + result.Problems.Add($"{AdapterManifest.FileName} isn't a JSON object"); + return new Built { Work = work, Result = result }; + } + if ((string)authored["id"] != draft.AdapterId) + { + result.Problems.Add($"{AdapterManifest.FileName} says \"id\": \"{(string)authored["id"]}\", but this draft is for {draft.AdapterId}"); + return new Built { Work = work, Result = result }; + } + + // With platforms that leave this server out, the build installs the requirements here a + // second time, to describe the adapter, and that install may build a package from source: + // running its setup code on the server. The CLI can do that on an author's machine; here, no. + if (files.ContainsKey("requirements.txt") && authored["platforms"] is JsonArray platforms && platforms.Count > 0 && + !platforms.Any(p => string.Equals((string)p, SW.Serverless.Runtimes.AdapterRuntimes.CurrentPlatform, StringComparison.OrdinalIgnoreCase))) + { + result.Problems.Add($"{AdapterManifest.FileName} lists platforms without {SW.Serverless.Runtimes.AdapterRuntimes.CurrentPlatform}, " + + "the one this server runs on; build it with bitween adapter build instead"); + return new Built { Work = work, Result = result }; + } + + var build = await PackageBuilder.BuildAsync(BitweenAdapters.BuildRequest(project, Path.Combine(work, "out"))); + result.Succeeded = build.Succeeded; + result.Problems.AddRange(build.Problems); + result.Warnings.AddRange(build.Warnings); + result.Manifest = build.Manifest; + return new Built { Work = work, Result = result, PackageDirectory = build.PackageDirectory, ZipPath = build.ZipPath }; + } + + static readonly string[] OwnPackages = ["sw-serverless", "sw_serverless", "simplyworks-bitween", "simplyworks_bitween", + "@simplyworks/sw-serverless", "@simplyworks/bitween"]; + + /// + /// What the editor won't build because of the packages a draft names. Without + /// Bitween:AdapterEditorDependencies, any beyond the SDKs. With it, any that isn't a plain name + /// and version from the registry: a URL, a path, a git source or a pip option could make the + /// server fetch from anywhere, or copy its own files into the package. + /// + static IEnumerable Dependencies(IDictionary files, bool allowed) + { + if (files.TryGetValue("requirements.txt", out var requirements)) + { + var lines = requirements.Split('\n').Select(l => l.Trim()) + .Where(l => l.Length > 0 && !l.StartsWith('#')) + .Select(l => l.Split(" #")[0].Trim()) + .ToList(); + var names = lines.Select(l => Regex.Match(l, @"^[A-Za-z0-9_.\-]+").Value) + .Where(n => !OwnPackages.Contains(n, StringComparer.OrdinalIgnoreCase)) + .ToList(); + if (names.Count > 0 && !allowed) + yield return $"requirements.txt names {string.Join(", ", names)}; this Bitween's editor builds adapters that need only the SDK. " + + "Build one with dependencies using bitween adapter build, which vendors them, or turn on Bitween:AdapterEditorDependencies"; + else if (allowed) + foreach (var line in lines.Where(l => !PlainRequirement.IsMatch(l))) + yield return $"requirements.txt: '{line}' isn't a package name and version; the editor fetches only those, from the package index"; + } + + if (files.TryGetValue("package.json", out var packageJson)) + { + JsonObject document = null; + try { document = JsonNode.Parse(packageJson)?.AsObject(); } catch (JsonException) { } + var dependencies = document?["dependencies"]?.AsObject() + .Where(d => !OwnPackages.Contains(d.Key, StringComparer.OrdinalIgnoreCase)).ToList() ?? []; + if (dependencies.Count > 0 && !allowed) + yield return $"package.json depends on {string.Join(", ", dependencies.Select(d => d.Key))}; this Bitween's editor builds adapters that need only the SDK. " + + "Build one with dependencies using bitween adapter build, which installs them, or turn on Bitween:AdapterEditorDependencies"; + else if (allowed) + foreach (var (name, spec) in dependencies) + if (!PlainPackageName.IsMatch(name) || spec is not JsonValue || !PlainVersionRange.IsMatch(spec.ToString())) + yield return $"package.json: \"{name}\": {spec?.ToJsonString()} isn't a package and version range; the editor installs only those, from the registry"; + + if (allowed && document?["optionalDependencies"] is JsonObject { Count: > 0 } || allowed && document?["bundleDependencies"] != null) + yield return "package.json: the editor installs dependencies only, not optional or bundled ones"; + } + } + + // name, optional [extras], optional version specifiers, optional ; marker. No URL, path or option. + static readonly Regex PlainRequirement = new( + @"^[A-Za-z0-9][A-Za-z0-9_.\-]*(\[[A-Za-z0-9_.,\- ]+\])?\s*((===?|!=|~=|<=?|>=?)\s*[A-Za-z0-9_.*+!\-]+\s*(,\s*(===?|!=|~=|<=?|>=?)\s*[A-Za-z0-9_.*+!\-]+\s*)*)?(;[^@:/\\]*)?$"); + + static readonly Regex PlainPackageName = new(@"^(@[a-z0-9][a-z0-9._\-]*/)?[a-z0-9][a-z0-9._\-]*$"); + + // Semver ranges and dist-tags: no "file:", "git+", "http:", "npm:" aliases or paths. + static readonly Regex PlainVersionRange = new(@"^[A-Za-z0-9 .*^~<>=|+\-]*$"); + + /// The try panel's input, as the CLI's run reads --input: JSON when it parses, the text otherwise. + static object ReadInput(string input) + { + if (string.IsNullOrEmpty(input)) return null; + try { return Newtonsoft.Json.Linq.JToken.Parse(input); } + catch (Newtonsoft.Json.JsonException) { return input; } + } +} diff --git a/SW.Bitween.Api/Services/Adapters/IAdapterRuntime.cs b/SW.Bitween.Api/Services/Adapters/IAdapterRuntime.cs index 6d6bf468..50fdde9d 100644 --- a/SW.Bitween.Api/Services/Adapters/IAdapterRuntime.cs +++ b/SW.Bitween.Api/Services/Adapters/IAdapterRuntime.cs @@ -20,14 +20,14 @@ public enum AdapterRole /// A run of calls against ONE adapter instance. /// /// A receiver is why this is a session rather than a single call: Initialize, ListFiles, a GetFile -/// and DeleteFile per item, then Finalize — all of which have to reach the same instance, or +/// and DeleteFile per item, then Finalize - all of which have to reach the same instance, or /// Initialize runs somewhere the listing never sees. /// public interface IAdapterSession : IAsyncDisposable { Task InvokeAsync(string method, object argument = null); - /// For a method that returns nothing — Initialize, DeleteFile, Finalize. + /// For a method that returns nothing - Initialize, DeleteFile, Finalize. Task InvokeAsync(string method, object argument = null); } @@ -42,7 +42,7 @@ public interface IAdapterRuntime { /// /// Whether this runtime is the one for that adapter. Async because deciding can mean reading - /// the adapter's published metadata — cached, but not free. + /// the adapter's published metadata - cached, but not free. /// Task CanRunAsync(string adapterId); @@ -53,7 +53,7 @@ Task BeginAsync(string adapterId, AdapterRole role, /// /// The one thing the pipeline calls to run an adapter. Everything that needs a mapper, handler, /// validator or receiver goes through here, so the choice of runtime is made once and in one -/// place — and so a test can replace the lot. +/// place - and so a test can replace the lot. /// public interface IAdapterInvoker { diff --git a/SW.Bitween.Api/Services/Adapters/NativeAdapterManifests.cs b/SW.Bitween.Api/Services/Adapters/NativeAdapterManifests.cs new file mode 100644 index 00000000..73ab7b39 --- /dev/null +++ b/SW.Bitween.Api/Services/Adapters/NativeAdapterManifests.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using SW.Bitween.NativeAdapters; +using SW.Serverless.Contract.Catalog; + +namespace SW.Bitween.Services.Adapters; + +/// +/// The manifests the native, in-process adapters carry, as embedded adapter.json resources +/// beside their code. They describe and present an adapter the same way a published one's does - +/// display name, summary, publisher, icon, properties - so a catalog or marketplace treats both alike. +/// +/// +/// What a native adapter needs configured is still read from its startup-values type, which is the +/// code the adapter actually runs with; a test keeps the manifests' property lists from drifting +/// away from it. +/// +public class NativeAdapterManifests +{ + readonly Dictionary manifests; + + public NativeAdapterManifests(IEnumerable nativeAdapters) + { + manifests = new Dictionary(StringComparer.OrdinalIgnoreCase); + var assemblies = nativeAdapters.Select(a => a.GetType().Assembly) + .Append(typeof(INativeAdapter).Assembly) + .Distinct(); + + foreach (var assembly in assemblies) + foreach (var (id, manifest) in parsed.GetOrAdd(assembly, a => Read(a).ToList())) + manifests[id] = manifest; + } + + // Parsed once per assembly: this is resolved per request, and the resources never change. + static readonly System.Collections.Concurrent.ConcurrentDictionary> parsed = new(); + + /// The manifest for a native adapter, by its id (its type name); null when it has none. + public AdapterManifest Get(string adapterId) => + adapterId != null && manifests.TryGetValue(adapterId, out var manifest) ? manifest : null; + + public IReadOnlyCollection Ids => manifests.Keys; + + /// Every embedded resource named ….adapter.json, keyed by the manifest's id. + public static IEnumerable<(string Id, AdapterManifest Manifest)> Read(Assembly assembly) + { + foreach (var name in assembly.GetManifestResourceNames() + .Where(n => n.EndsWith("." + AdapterManifest.FileName, StringComparison.OrdinalIgnoreCase))) + { + using var stream = assembly.GetManifestResourceStream(name); + if (stream == null) continue; + using var reader = new StreamReader(stream); + var manifest = AdapterManifest.Parse(reader.ReadToEnd()); + if (!string.IsNullOrWhiteSpace(manifest.Id)) yield return (manifest.Id, manifest); + } + } +} diff --git a/SW.Bitween.Api/Services/Adapters/NativeAdapterRuntime.cs b/SW.Bitween.Api/Services/Adapters/NativeAdapterRuntime.cs index 56797996..b5967667 100644 --- a/SW.Bitween.Api/Services/Adapters/NativeAdapterRuntime.cs +++ b/SW.Bitween.Api/Services/Adapters/NativeAdapterRuntime.cs @@ -8,11 +8,14 @@ namespace SW.Bitween.Services.Adapters; /// -/// Adapters that ship inside Bitween and run in-process. No packaging, no process, no protocol — +/// Adapters that ship inside Bitween and run in-process. No packaging, no process, no protocol - /// the only runtime where "invoke" is a method call. /// -public class NativeAdapterRuntime(NativeAdapterDiscoveryService discovery) : IAdapterRuntime +public class NativeAdapterRuntime(NativeAdapterDiscoveryService discovery, BitweenOptions options = null) : IAdapterRuntime { + /// Bitween:NativeAdapterTimeout, 900 seconds unless set; 0 for none. + TimeSpan? Limit => (options?.NativeAdapterTimeout ?? 900) is > 0 and var seconds ? TimeSpan.FromSeconds(seconds) : null; + public Task CanRunAsync(string adapterId) => Task.FromResult(adapterId != null && adapterId.StartsWith( NativeAdapterDiscoveryService.NativePrefix, StringComparison.OrdinalIgnoreCase)); @@ -33,19 +36,19 @@ public Task BeginAsync(string adapterId, AdapterRole role, _ => discovery.GetNativeHandler(adapterId, settings), }; - return Task.FromResult(new NativeAdapterSession(adapter)); + return Task.FromResult(new NativeAdapterSession(adapter, adapterId, Limit)); } /// /// Dispatches by method name onto the resolved adapter, so the pipeline can treat a native /// adapter exactly like a packaged one. /// - private sealed class NativeAdapterSession(object adapter) : IAdapterSession + private sealed class NativeAdapterSession(object adapter, string adapterId, TimeSpan? limit) : IAdapterSession { public async Task InvokeAsync(string method, object argument = null) => - (TResult)await Dispatch(method, argument); + (TResult)await Within(Dispatch(method, argument), limit, adapter, adapterId, method); - public Task InvokeAsync(string method, object argument = null) => Dispatch(method, argument); + public Task InvokeAsync(string method, object argument = null) => Within(Dispatch(method, argument), limit, adapter, adapterId, method); private async Task Dispatch(string method, object argument) { @@ -60,6 +63,9 @@ private async Task Dispatch(string method, object argument) case (INativeInfolinkValidator v, nameof(IInfolinkValidator.Validate)): return await v.Validate((XchangeFile)argument); + case (IKeepsBookmark b, nameof(IKeepsBookmark.Bookmark)): + return await b.Bookmark(); + case (INativeInfolinkReceiver r, nameof(IInfolinkReceiver.Initialize)): await r.Initialize(); return null; @@ -87,4 +93,27 @@ private async Task Dispatch(string method, object argument) // Nothing to release: a native adapter is an object, not a process or a lease. public ValueTask DisposeAsync() => ValueTask.CompletedTask; } + + /// + /// The call, or a once has passed. An in-process + /// call can't be stopped, only left: the adapter is disposed when it can be, which closes what it + /// holds open (a file server's connection), and the exchange fails as a timeout. The call left behind + /// may still finish, so a retry can deliver twice what a hung partner did in fact receive. + /// + internal static async Task Within(Task call, TimeSpan? limit, object adapter, string adapterId, string method) + { + if (limit is not { } wait) return await call; + try + { + return await call.WaitAsync(wait); + } + catch (TimeoutException) when (!call.IsCompleted) + { + _ = call.ContinueWith(t => _ = t.Exception, TaskContinuationOptions.OnlyOnFaulted); + if (adapter is IDisposable disposable) + try { disposable.Dispose(); } catch { /* releasing only */ } + throw new TimeoutException( + $"{adapterId} did not finish {method} within {wait.TotalSeconds:0} seconds (Bitween:NativeAdapterTimeout), so it was left."); + } + } } diff --git a/SW.Bitween.Api/Services/Adapters/ResidentAdapterRuntime.cs b/SW.Bitween.Api/Services/Adapters/ResidentAdapterRuntime.cs index 15b0e6f2..9b63fd28 100644 --- a/SW.Bitween.Api/Services/Adapters/ResidentAdapterRuntime.cs +++ b/SW.Bitween.Api/Services/Adapters/ResidentAdapterRuntime.cs @@ -14,7 +14,7 @@ namespace SW.Bitween.Services.Adapters; /// /// A packaged adapter that stays running, rented from the pool and invoked over its socket. /// -/// The two packaged lifecycles are published identically — same zip, same key, same metadata bag — +/// The two packaged lifecycles are published identically - same zip, same key, same metadata bag - /// and differ only in their entry point. Running a resident adapter down the classic path does not /// fail cleanly: the host spawns it and then waits out the command timeout for an answer that /// never comes, because the adapter dialled out and is waiting for a host that is not listening. @@ -54,7 +54,10 @@ public async Task CanRunAsync(string adapterId) public async Task BeginAsync(string adapterId, AdapterRole role, IDictionary properties, string correlationId) { - var adapters = serviceProvider.GetService() + // The host is on every node, for Python and Node adapters; running a resident adapter in + // the pipeline is still the opt-in it always was. + var enabled = serviceProvider.GetService()?.BusProvidersEnabled ?? true; + var adapters = (enabled ? serviceProvider.GetService() : null) ?? throw new BitweenException( $"Adapter '{adapterId}' is a resident adapter, but resident adapters are not " + "enabled on this node (Bitween:BusProvidersEnabled). It cannot run here."); @@ -70,7 +73,7 @@ public async Task BeginAsync(string adapterId, AdapterRole role spec.StartupValues[kv.Key] = kv.Value; } - // A subscription bound to a data source runs against THAT connection — the instance the + // A subscription bound to a data source runs against THAT connection - the instance the // supervisor already keeps up for it, holding the pool the data source exists to provide. // Renting instead would start a second process with a second pool, configured from // subscription properties that do not hold the credentials at all. @@ -82,16 +85,16 @@ public async Task BeginAsync(string adapterId, AdapterRole role // The subscription's own adapter properties travel with each CALL, not with the // process: this instance is shared by every subscription bound to the data source, // and its startup values are the data source's. Without this a subscription could - // not say which statement to run — it would be reading whatever the data source + // not say which statement to run - it would be reading whatever the data source // was started with, which is the same answer for all of them. return new RunningInstanceSession(running, spec.StartupValues); // Not here. For an EXCLUSIVE data source that is the normal case on every node but - // one — a broker connection is held by a single node so that a queue is drained once. + // one - a broker connection is held by a single node so that a queue is drained once. // // Exclusivity is about CONSUMING, though, not about connecting. A subscription's // handler runs on whichever node picked up the message, so a delivery that publishes - // to the customer's broker would fail on every node but the owner — which is to say, + // to the customer's broker would fail on every node but the owner - which is to say, // almost always. A publish-only connection of our own is the answer: it sends and // never subscribes, so nothing is consumed twice. if (role is AdapterRole.Handler or AdapterRole.Mapper) @@ -99,8 +102,8 @@ public async Task BeginAsync(string adapterId, AdapterRole role var publishing = await PublishOnlySpecAsync(adapterId, dataSourceId); if (publishing != null) // The same split as the exclusive path: the rented instance's startup values - // are the CONNECTION's, and the slot's own properties — where to publish above - // all — travel with the call. Passing only the spec would send the message to + // are the CONNECTION's, and the slot's own properties - where to publish above + // all - travel with the call. Passing only the spec would send the message to // an endpoint the connection never knew about, which is to say nowhere. return new ResidentAdapterSession( await adapters.RentAsync(publishing), spec.StartupValues); @@ -109,12 +112,12 @@ public async Task BeginAsync(string adapterId, AdapterRole role throw new BitweenException( $"Data source {dataSourceId} is not running on this node, so adapter " + $"'{adapterId}' has no connection to work through. If the data source is " - + "exclusive, another node holds it; if it is per-node, look at its health — " + + "exclusive, another node holds it; if it is per-node, look at its health - " + "the supervisor could not start it here."); } // Rented, not started: the process is already up, so the call costs a round trip rather - // than a launch. Returning the lease is what releases it to the next message — and what + // than a launch. Returning the lease is what releases it to the next message - and what // triggers the reset that clears per-message state between borrowers. return new ResidentAdapterSession(await adapters.RentAsync(spec)); } @@ -127,7 +130,7 @@ public async Task BeginAsync(string adapterId, AdapterRole role /// for the same reason: a second instance that subscribed would drain the customer's queue /// alongside the node that owns it, and the lease exists precisely to stop that. /// - /// Pooled rather than exclusive, so several nodes may hold one at once — which is correct for + /// Pooled rather than exclusive, so several nodes may hold one at once - which is correct for /// publishing and wrong for consuming. The pool key includes a hash of these values, so the /// publish-only instance is a different renter from anything else and cannot be handed the /// consuming one by mistake. @@ -173,7 +176,7 @@ private async Task PublishOnlySpecAsync(string adapterId, string da /// A session against the data source's long-lived instance. Nothing is returned on dispose: /// the instance is not ours to give back, and it must outlive this Xchange to be any use to /// the next one. That also means no per-session reset, so a data source adapter must not keep - /// per-message state in a field — which is the same rule any shared connection follows. + /// per-message state in a field - which is the same rule any shared connection follows. /// private sealed class RunningInstanceSession( ResidentAdapterInstance instance, IDictionary properties) : IAdapterSession diff --git a/SW.Bitween.Api/Services/Adapters/ResidentAdapters.cs b/SW.Bitween.Api/Services/Adapters/ResidentAdapters.cs index 9de1c837..5fae68db 100644 --- a/SW.Bitween.Api/Services/Adapters/ResidentAdapters.cs +++ b/SW.Bitween.Api/Services/Adapters/ResidentAdapters.cs @@ -10,7 +10,7 @@ namespace SW.Bitween.Services.Adapters; /// /// It lives here because two services outside the runtime need to ask, and both got the answer /// wrong in the same way: they probed the adapter by SPAWNING it down the classic stdio path. A -/// resident adapter dials out instead of speaking stdio, so that probe never gets an answer — one +/// resident adapter dials out instead of speaking stdio, so that probe never gets an answer - one /// caller failed as "Received null data" and refused to save the subscription at all, the other /// failed closed and masked every property as a secret. Neither failure named a cause. /// diff --git a/SW.Bitween.Api/Services/AggregationJob.cs b/SW.Bitween.Api/Services/AggregationJob.cs index 4985b41a..eeb1dcaf 100644 --- a/SW.Bitween.Api/Services/AggregationJob.cs +++ b/SW.Bitween.Api/Services/AggregationJob.cs @@ -11,13 +11,20 @@ namespace SW.Bitween; -public record AggregationJobParams(int SubscriptionId, string? CronExpression); +/// The roll-up's correlation id: a mirror's rehearsal names one, so the roll-up runs as its traffic. +/// +/// Only exchanges whose correlation starts with it are rolled up: a mirror's test rolls up what tests ran, +/// never staging's own exchanges. Null rolls up every one waiting. +/// +public record AggregationJobParams(int SubscriptionId, string? CronExpression, string? Correlation = null, string? InputPrefix = null); [ScheduleConfig(AllowConcurrentExecution = false, MisfireInstructions = MisfireInstructions.Skip)] public class AggregationJob( BitweenDbContext dbContext, + RunFlagUpdater runFlagUpdater, XchangeService xchangeService, - ILogger logger) : IScheduledJob + ILogger logger, + BitweenOptions options = null) : IScheduledJob { public async Task Execute(AggregationJobParams jobParams) { @@ -25,11 +32,37 @@ public async Task Execute(AggregationJobParams jobParams) .FirstOrDefaultAsync(s => s.Id == jobParams.SubscriptionId && !s.Inactive); if (aggSub == null) return; + if (Services.Mirror.MirrorTraffic.HoldsScheduledRun(options, aggSub, jobParams.CronExpression)) + { + // Held, but its next run still moves on, so its page doesn't show one in the past. + try { aggSub.SetSchedules(); await dbContext.SaveChangesAsync(); } + catch (BitweenException) { } + return; + } + + // Quartz runs each fire once across the cluster, but each schedule of a subscription and each + // "aggregate now" is a job of its own, so two runs of one aggregation can still overlap, on one + // replica or two. Only one roll-up could be saved (an exchange is rolled up once), so the other + // run failed and was logged as an error. Claimed as a receiver is, the other run steps aside. + if (!await runFlagUpdater.MarkAsRunning(aggSub.Id)) return; + try + { + await RollUpAsync(aggSub, jobParams.Correlation, jobParams.InputPrefix); + } + finally + { + // Released after the roll-up is saved, so a run that starts next sees it. + await runFlagUpdater.MarkAsIdle(aggSub.Id); + } + } + + async Task RollUpAsync(Subscription aggSub, string correlationId, string inputPrefix) + { var startedOn = DateTime.UtcNow; // Decided as the run goes and written once at the end. Recording inside both the try // and the catch wrote two rows for a single run whenever something threw after the - // roll-up was already built — SetSchedules does exactly that on an aggregation that + // roll-up was already built - SetSchedules does exactly that on an aggregation that // somehow has no schedule. var outcome = ReceiveOutcome.NoNewData; string errorMessage = null; @@ -43,9 +76,13 @@ join result in dbContext.Set() on xchange.Id equals result.Id join agg in dbContext.Set() on xchange.Id equals agg.Id into xa from agg in xa.DefaultIfEmpty() where result.Success == true && agg == null && - xchange.SubscriptionId == aggSub.AggregationForId && !aggSub.Inactive + xchange.SubscriptionId == aggSub.AggregationForId && !aggSub.Inactive && + (inputPrefix == null || xchange.CorrelationId.StartsWith(inputPrefix)) + orderby xchange.StartedOn select new { xchange.Id, xchange.FilesPrefix }; + // Oldest first: without an order, a backlog over the limit was rolled up in whatever order + // the database returned, and the same old exchanges could wait run after run. var targetXchangeList = await xchangeQuery.Take(10000).ToListAsync(); if (targetXchangeList.Count > 0) @@ -55,7 +92,7 @@ from agg in xa.DefaultIfEmpty() var urlList = targetXchangeList.Select(x => xchangeService.AdapterFileUrl(x.Id, x.FilesPrefix, aggSub.AggregationTarget)).ToList(); var xchangeAggregationFile = new XchangeFile(JsonConvert.SerializeObject(urlList)); - var aggXchange = await xchangeService.CreateXchange(aggSub, xchangeAggregationFile); + var aggXchange = await xchangeService.CreateXchange(aggSub, xchangeAggregationFile, correlationId: correlationId); dbContext.Add(aggXchange); targetXchangeList.ForEach(x => dbContext.Add(new XchangeAggregation(x.Id, aggXchange.Id))); outcome = ReceiveOutcome.Received; @@ -75,7 +112,7 @@ from agg in xa.DefaultIfEmpty() // that is bookkeeping, and calling the run failed would misreport delivered work. // The error is kept on the attempt either way, and on the integration's health. if (createdExchangeIds.Length == 0) outcome = ReceiveOutcome.Failed; - logger.LogError(ex, "Error processing aggregation for subscription {SubscriptionId}", jobParams.SubscriptionId); + logger.LogError(ex, "Error processing aggregation for subscription {SubscriptionId}", aggSub.Id); } RecordAttempt(aggSub.Id, startedOn, outcome, errorMessage, createdExchangeIds); diff --git a/SW.Bitween.Api/Services/Ai/AiModels.cs b/SW.Bitween.Api/Services/Ai/AiModels.cs new file mode 100644 index 00000000..83007d00 --- /dev/null +++ b/SW.Bitween.Api/Services/Ai/AiModels.cs @@ -0,0 +1,103 @@ +using System; +using System.Net.Http; +using System.Threading.Tasks; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Services.Ai; + +/// One request to a model: standing instructions, one message, and room for the answer. +/// The same on every call of a feature, so a provider can cache it. +/// What this call is about. +/// Whether the answer must be a single JSON object, for providers that can enforce it. +public record AiPrompt(string Instructions, string Message, bool Json = false, int MaxTokens = 16000); + +/// A configured model, as an AI feature sees it: text in, text out. +public interface IAiModel +{ + /// The model's answer. + /// When the provider can't be reached, refuses, or cuts the answer off. + Task Complete(AiPrompt prompt); +} + +/// +/// The model Settings, AI names, built fresh for each call so a changed key or model applies at once. +/// +/// +/// Each provider gets its own official SDK behind , rather than one SDK +/// pretending to be the other: their requests differ in ways a shim hides - Anthropic caches the +/// instructions, OpenAI enforces a JSON answer. +/// +public class AiModels(BitweenOptions options) +{ + // One client per address policy for the process, recycling its connections, as RegistryClient + // does. The SDKs set their own per-request timeout, so this one never cuts them short. + // + // Connections go through the adapters' address guard: the API address is an administrator's + // setting, and an AI call carries the key and the samples, so it gets no more reach than an + // HTTP adapter has - never link-local or cloud metadata, and not private or loopback either + // while Bitween:BlockPrivateNetworkAddresses is on. + static readonly Lazy Open = new(() => Guarded(blockPrivate: false)); + static readonly Lazy Closed = new(() => Guarded(blockPrivate: true)); + + static HttpClient Guarded(bool blockPrivate) => new(new SocketsHttpHandler + { + PooledConnectionLifetime = TimeSpan.FromMinutes(5), + ConnectCallback = SW.Bitween.NativeAdapters.OutboundAddressGuard.ConnectCallback(blockPrivate), + }) + { + Timeout = System.Threading.Timeout.InfiniteTimeSpan, + }; + + /// How long one call may take. Long enough for a model to think about a large document. + internal static readonly TimeSpan CallTimeout = TimeSpan.FromMinutes(3); + + /// For tests: answers in place of the network, as RegistryClient.HandlerOverride does. + internal static Func HandlerOverride { get; set; } + + /// For tests: a model in place of either provider, for features tested without HTTP. + internal static Func ModelOverride { get; set; } + + HttpClient Client => HandlerOverride is { } handler + ? new HttpClient(handler()) { Timeout = System.Threading.Timeout.InfiniteTimeSpan } + : options.BlockPrivateNetworkAddresses ? Closed.Value : Open.Value; + + public bool Configured => options.Ai.IsSet; + + /// The model calls, as Settings names it. + public string ModelName => options.Ai.ResolvedModel; + + /// AI_NOT_CONFIGURED until Settings, AI has a key. + public IAiModel Current() + { + var ai = options.Ai; + if (!ai.IsSet) + throw new SWValidationException("AI_NOT_CONFIGURED", + string.IsNullOrWhiteSpace(ai.ApiKey) + ? "AI features are off until an API key is set under Settings, AI." + : "AI features are off until a model is named under Settings, AI."); + + // Checked here as well as when the setting is saved: configuration binds it without the check. + if (AiOptions.AddressProblem(ai.BaseUrl) is { } problem) + throw new SWValidationException("AI_NOT_CONFIGURED", $"The API address under Settings, AI can't be used: {problem}"); + + if (ModelOverride is { } model) return model(ai); + + var baseUrl = string.IsNullOrWhiteSpace(ai.BaseUrl) ? null : ai.BaseUrl.Trim(); + return ai.IsAnthropic + ? new AnthropicModel(ai.ApiKey.Trim(), ai.ResolvedModel, baseUrl, Client) + : new OpenAiModel(ai.ApiKey.Trim(), ai.ResolvedModel, baseUrl, Client); + } + + // Fixed wording rather than the SDK's exception message: that can carry whatever the far end + // answered, and the far end is an address an administrator typed in. + internal static SWValidationException Unreachable(string provider, string detail) => + new("AI_UNREACHABLE", $"{provider} can't be reached right now: {detail}"); + + internal static SWValidationException Refused(string provider, string detail) => + new("AI_PROVIDER_ERROR", $"{provider} refused the request: {detail}"); + + internal const string NoConnection = + "the connection failed or timed out, or the address isn't one Bitween may connect to."; + + internal static string Answered(int status) => $"it answered HTTP {status}."; +} diff --git a/SW.Bitween.Api/Services/Ai/AiOptions.cs b/SW.Bitween.Api/Services/Ai/AiOptions.cs new file mode 100644 index 00000000..2e55af77 --- /dev/null +++ b/SW.Bitween.Api/Services/Ai/AiOptions.cs @@ -0,0 +1,61 @@ +using System; + +namespace SW.Bitween.Services.Ai; + +/// +/// The AI provider every AI feature calls - generating mapping rules first, more to follow. One +/// instance for the life of the app: Settings changes its properties in place, and each call reads +/// them, so a new key or model applies to the next request. +/// +public class AiOptions +{ + public const string Anthropic = "Anthropic"; + + /// OpenAI, or anything that speaks its API at . + public const string OpenAI = "OpenAI"; + + public static readonly string[] Providers = [Anthropic, OpenAI]; + + /// What an Anthropic provider uses while is empty. + public const string DefaultAnthropicModel = "claude-opus-5-5"; + + public string Provider { get; set; } = Anthropic; + + public string ApiKey { get; set; } + + /// Empty uses with Anthropic. Other providers need one. + public string Model { get; set; } + + /// + /// Where the provider's API is, when it isn't the provider's own - an Azure OpenAI resource, a + /// local Ollama, a company proxy. Empty uses the provider's public endpoint. + /// + public string BaseUrl { get; set; } + + public bool IsAnthropic => string.Equals(Provider?.Trim(), Anthropic, StringComparison.OrdinalIgnoreCase); + + public bool IsOpenAI => string.Equals(Provider?.Trim(), OpenAI, StringComparison.OrdinalIgnoreCase); + + public string ResolvedModel => + !string.IsNullOrWhiteSpace(Model) ? Model.Trim() : IsAnthropic ? DefaultAnthropicModel : null; + + /// + /// Whether AI features can run: a provider Bitween knows, a key, and a model to send it with. A + /// misspelt provider in configuration is off, not quietly sent down the OpenAI path. + /// + public bool IsSet => (IsAnthropic || IsOpenAI) && !string.IsNullOrWhiteSpace(ApiKey) && ResolvedModel is not null; + + /// + /// Why an API address can't be used, or null when it can - or is empty. Plain http only reaches + /// this machine, such as a local Ollama: anywhere else it would send the key in the clear. + /// + public static string AddressProblem(string address) + { + if (string.IsNullOrWhiteSpace(address)) return null; + if (!Uri.TryCreate(address.Trim(), UriKind.Absolute, out var url) || url.Scheme is not ("http" or "https")) + return $"'{address.Trim()}' isn't an http or https address."; + if (url.Scheme == "http" && !url.IsLoopback) + return "plain http is only for this machine (localhost); use https anywhere else, so the key isn't sent in the clear."; + return null; + } +} diff --git a/SW.Bitween.Api/Services/Ai/AnthropicModel.cs b/SW.Bitween.Api/Services/Ai/AnthropicModel.cs new file mode 100644 index 00000000..b7e7b9d9 --- /dev/null +++ b/SW.Bitween.Api/Services/Ai/AnthropicModel.cs @@ -0,0 +1,86 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Threading.Tasks; +using Anthropic; +using Anthropic.Exceptions; +using Anthropic.Models.Messages; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Services.Ai; + +/// Claude, through Anthropic's own SDK. +/// +/// +/// Neither thinking nor effort is set: the model is whatever Settings names, and the models disagree +/// about both - an older one rejects an effort it doesn't know. Each current model's own default +/// applies instead, which for the default, Opus 5.5, is adaptive thinking at medium effort. +/// +/// +/// The instructions are marked for caching: they're the same on every call of a feature, and the +/// larger part of each request. +/// +/// +sealed class AnthropicModel(string apiKey, string model, string baseUrl, HttpClient http) : IAiModel +{ + const string Name = "Anthropic"; + + public async Task Complete(AiPrompt prompt) + { + var client = baseUrl == null + ? new AnthropicClient { ApiKey = apiKey, HttpClient = http, Timeout = AiModels.CallTimeout } + : new AnthropicClient { ApiKey = apiKey, HttpClient = http, Timeout = AiModels.CallTimeout, BaseUrl = baseUrl }; + + var parameters = new MessageCreateParams + { + Model = model, + MaxTokens = prompt.MaxTokens, + System = new List + { + new() { Text = prompt.Instructions, CacheControl = new CacheControlEphemeral() }, + }, + Messages = [new() { Role = Role.User, Content = prompt.Message }], + }; + + Message response; + try + { + response = await client.Messages.Create(parameters); + } + catch (AnthropicUnauthorizedException) + { + throw AiModels.Refused(Name, "the API key under Settings, AI isn't valid."); + } + catch (AnthropicNotFoundException) + { + throw AiModels.Refused(Name, $"there's no model called '{model}'. Check Settings, AI."); + } + catch (AnthropicRateLimitException) + { + throw AiModels.Unreachable(Name, "the account's rate limit was reached. Try again in a minute."); + } + catch (Anthropic5xxException ex) + { + throw AiModels.Unreachable(Name, AiModels.Answered((int)ex.StatusCode)); + } + catch (AnthropicApiException ex) + { + throw AiModels.Refused(Name, AiModels.Answered((int)ex.StatusCode)); + } + catch (Exception ex) when (ex is AnthropicIOException or HttpRequestException or TaskCanceledException) + { + throw AiModels.Unreachable(Name, AiModels.NoConnection); + } + + if (response.StopReason == "refusal") + throw new SWValidationException("AI_REFUSED", + $"The model declined this request{(response.StopDetails?.Explanation is { } why ? ": " + why : ".")}"); + + if (response.StopReason == "max_tokens") + throw new SWValidationException("AI_TRUNCATED", + "The model's answer was too long and was cut off. Try smaller samples."); + + return string.Concat(response.Content.Select(b => b.Value).OfType().Select(t => t.Text)); + } +} diff --git a/SW.Bitween.Api/Services/Ai/DocumentDifferences.cs b/SW.Bitween.Api/Services/Ai/DocumentDifferences.cs new file mode 100644 index 00000000..97b1ce97 --- /dev/null +++ b/SW.Bitween.Api/Services/Ai/DocumentDifferences.cs @@ -0,0 +1,116 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using SW.Bitween.NativeAdapters.Mapper; + +namespace SW.Bitween.Services.Ai; + +/// +/// Where a mapped document differs from the one it should have been, in words a person - or a +/// model asked to fix the rules - can act on. +/// +/// +/// +/// Compared as trees rather than as text, so whitespace, key order and number formatting don't +/// count. Types do: 42 where "42" was wanted is a difference a partner's parser +/// notices. +/// +/// +/// A blank value in the output sample - null, empty or only whitespace - is a placeholder: output +/// samples are often a partner's template with the values left out, and any value the rules write +/// there is an answer, not a difference. The field itself still has to be written. +/// +/// +public static class DocumentDifferences +{ + /// Enough to fix the rules from; a wholly wrong document would otherwise list every field. + public const int Limit = 20; + + public static List Between(ValueNode expected, ValueNode actual) + { + var found = new List(); + Compare(expected, actual, "", found); + return found; + } + + static void Compare(ValueNode? expected, ValueNode? actual, string path, List found) + { + if (found.Count >= Limit) return; + // Named the way the mapper names a failing rule's document, so the editor reads both alike. + var at = path.Length == 0 ? "(root)" : path; + + if (expected is null && actual is null) return; + if (expected is null) + { + found.Add(new(at, $"not in the output sample, but the rules write {Describe(actual!)}")); + return; + } + if (actual is null) + { + found.Add(new(at, $"missing; the output sample has {Describe(expected)}")); + return; + } + if (IsPlaceholder(expected) && actual is ScalarNode) return; + if (expected.Kind != actual.Kind) + { + found.Add(new(at, $"the output sample has {Describe(expected)}, the rules write {Describe(actual)}")); + return; + } + + switch (expected) + { + case ScalarNode e when !Same(e.Value, ((ScalarNode)actual).Value): + found.Add(new(at, $"the output sample has {Describe(e)}, the rules write {Describe(actual)}")); + break; + + case ObjectNode e: + var a = (ObjectNode)actual; + foreach (var key in e.Keys) Compare(e[key], a[key], Join(path, key), found); + foreach (var key in a.Keys.Where(k => e[k] is null)) Compare(null, a[key], Join(path, key), found); + break; + + case ListNode e: + var items = ((ListNode)actual).Items; + if (e.Items.Count != items.Count) + found.Add(new(at, $"the output sample has {e.Items.Count} entries, the rules write {items.Count}")); + for (var i = 0; i < Math.Min(e.Items.Count, items.Count); i++) + Compare(e.Items[i], items[i], $"{path}[{i}]", found); + break; + } + } + + static string Join(string path, string key) => path.Length == 0 ? key : $"{path}.{key}"; + + static bool IsPlaceholder(ValueNode node) => + node is ScalarNode scalar && (scalar.Value is null || scalar.Value is string text && string.IsNullOrWhiteSpace(text)); + + static bool Same(object? expected, object? actual) => (expected, actual) switch + { + (null, null) => true, + (decimal e, decimal a) => e == a, + (string e, string a) => string.Equals(e, a, StringComparison.Ordinal), + (bool e, bool a) => e == a, + _ => false, + }; + + static string Describe(ValueNode node) => node switch + { + ScalarNode { Value: null } => "null", + ScalarNode { Value: string s } => $"the text {Quote(s)}", + ScalarNode { Value: decimal d } => $"the number {d.ToString(CultureInfo.InvariantCulture)}", + ScalarNode { Value: bool b } => b ? "true" : "false", + ObjectNode => "an object", + ListNode l => $"a list of {l.Items.Count}", + _ => "a value", + }; + + static string Quote(string text) => text.Length <= 60 ? $"\"{text}\"" : $"\"{text[..57]}...\""; +} + +/// One place a mapped document differs from its sample: a path such as items[0].sku, and how. +public sealed record MappingDifference(string Target, string Reason) +{ + public override string ToString() => $"{Target}: {Reason}"; +} diff --git a/SW.Bitween.Api/Services/Ai/ExchangeQueryPrompt.cs b/SW.Bitween.Api/Services/Ai/ExchangeQueryPrompt.cs new file mode 100644 index 00000000..69f76896 --- /dev/null +++ b/SW.Bitween.Api/Services/Ai/ExchangeQueryPrompt.cs @@ -0,0 +1,266 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace SW.Bitween.Services.Ai; + +/// A partner, subscription or information type, as the model may see it: an id and a name. +public record QueryName(int Id, string Name, string? Detail = null); + +/// Everything a question may name, and the only things a filter may hold. +public record QueryNames( + IReadOnlyList Partners, + IReadOnlyList Subscriptions, + IReadOnlyList InformationTypes, + IReadOnlyList PropertyKeys); + +/// A question turned into an Exchanges page filter. +/// The page's own URL parameters, so the filter is the page's chips and its link. +/// A sentence for the page to show, with {count} where the page's total goes. +/// What the filter couldn't say: parts of the question, or names that don't exist. +public record ExchangeQuery(string Query, string Answer, IReadOnlyList Notes); + +/// +/// Turns a plain-English question about exchanges into the Exchanges page's filter. +/// +/// +/// +/// The model sees the question, the time, and names: of partners, subscriptions and information +/// types, and promoted-property keys. Never an exchange, a count, or a property's value - the count +/// in the answer is the page's, filled in where the model leaves {count}. +/// +/// +/// The answer is a filter in the model's own small shape rather than a URL, and every part of it is +/// checked against before it becomes one: an id that doesn't exist or a key +/// no information type promotes is dropped and said so, never passed on to match nothing silently. +/// +/// +public static class ExchangeQueryPrompt +{ + /// How many of each kind of name are sent; the rest are left out, alphabetically last. + public const int MaxNames = 2_000; + + public const int MaxQuestionLength = 500; + + const int MaxText = 200; + const int MaxAnswer = 300; + + public const string Instructions = """ + You turn a question about exchanges in Bitween, an integration platform, into a filter for + its Exchanges page. + + An exchange is one document passing through Bitween for one subscription: received, mapped, + then delivered by a handler. A subscription belongs to a partner and carries one information + type (a kind of document, such as shipment orders). An exchange's status is one of: + - "failed": it failed. + - "badResponse": it was delivered, but the partner's response was marked as bad. + - "processing": not finished yet. + - "success": delivered fine. + Promoted properties are values Bitween copies out of a document so it can be found by them, + such as an order number; each has a key. + + You get the question, the current time in the asker's time zone, and the partners, + subscriptions and information types by id and name, with the promoted-property keys. You + never see exchanges or counts. + + Answer with one JSON object and nothing else. Leave out or set to null anything the question + doesn't ask for: + { + "status": "failed" | "badResponse" | "processing" | "success", + "subscriptionIds": [ids], "partnerIds": [ids], "informationTypeIds": [ids], + "properties": [{ "key": "a promoted-property key", "text": "text in its value, or null for any value", "not": false }], + "anyProperty": "text to find in any promoted property", + "error": "text to find in the error message", + "correlationId": "an exact correlation id", + "range": "15m" | "1h" | "24h" | "7d" | "30d" | "today" | "yesterday" | "m" | "h" | "d", + "from": "YYYY-MM-DDTHH:mm", "to": "YYYY-MM-DDTHH:mm", + "latest": true, "retry": true, "scheduled": true, "aggregated": true, + "not": ["subscription" | "partner" | "informationType" | "property" | "error" | "retry" | "scheduled" | "aggregated"], + "answer": "one short sentence describing the result, with {count} where the number goes", + "notUnderstood": "the part of the question the filter can't express, or null" + } + + Rules: + - Use only ids from the lists. Match names loosely - "skynet" is the partner "SkyNet Express" - + and when a name fits a partner, filter by partner rather than listing its subscriptions. + - Several ids of one kind mean any of them. All filters together must all hold. + - "not" turns the named filters round: "not": ["partner"] means every partner except those. + - For a time like "since Monday" or "on 3 October", use from and to in the asker's time zone. + For "the last 2 hours", "today" or "yesterday", use range. Never both. + - "latest": only the latest attempt of each exchange, leaving out ones that were retried. Use + it for "still failing", "unresolved" or "needs attention". + - "retry": the exchange is a retry of an earlier one. "scheduled": a retry is scheduled. + "aggregated": the exchange was gathered into an aggregation. + - For a count question ("how many ..."), still return the filter; the page counts it. + - The answer is shown above the result, so write it as a result, not a reply: + "{count} failed SkyNet exchanges since Monday", never "Here are...". + - If the question isn't about exchanges at all, return only "answer" and "notUnderstood". + """; + + /// The message for one question. + public static string Message(string question, DateTime now, string timeZone, QueryNames names) + { + var message = new StringBuilder(); + // Fixed wording whatever the server's culture: an Arabic one would change the year and the weekday. + message.AppendLine(string.Create(CultureInfo.InvariantCulture, $"Now: {now:yyyy-MM-dd HH:mm}, {now:dddd}, in {timeZone}.")); + message.AppendLine(); + message.AppendLine(""); + message.AppendLine(question.Trim()); + message.AppendLine(""); + List("partners", names.Partners, message); + List("subscriptions", names.Subscriptions, message); + List("information_types", names.InformationTypes, message); + message.AppendLine(); + message.AppendLine(""); + message.AppendLine(names.PropertyKeys.Count == 0 ? "(none)" : string.Join(", ", names.PropertyKeys)); + message.AppendLine(""); + return message.ToString(); + } + + static void List(string tag, IReadOnlyList names, StringBuilder message) + { + message.AppendLine(); + message.AppendLine($"<{tag}>"); + foreach (var name in names.Take(MaxNames)) + message.AppendLine(name.Detail is null ? $"{name.Id}: {name.Name}" : $"{name.Id}: {name.Name} ({name.Detail})"); + if (names.Count == 0) message.AppendLine("(none)"); + if (names.Count > MaxNames) message.AppendLine($"({names.Count - MaxNames} more not listed)"); + message.AppendLine($""); + } + + static readonly string[] Statuses = ["failed", "badResponse", "processing", "success"]; + static readonly string[] Negatable = ["subscription", "partner", "informationType", "property", "error", "retry", "scheduled", "aggregated"]; + static readonly Regex Range = new(@"^(today|yesterday|[1-9]\d{0,3}[mhd])$", RegexOptions.Compiled); + static readonly Regex Bound = new(@"^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2})?$", RegexOptions.Compiled); + + /// The model's answer as the page's filter, keeping only what holds. + public static ExchangeQuery Parse(string answer, QueryNames names) + { + var notes = new List(); + var start = answer.IndexOf('{'); + var end = answer.LastIndexOf('}'); + JObject? json = null; + if (start >= 0 && end > start) + { + try + { + json = JObject.Parse(answer[start..(end + 1)]); + } + catch (JsonException) + { + } + } + if (json is null) + return new ExchangeQuery("", "", ["The AI's answer couldn't be read as a filter. Try asking again."]); + + var query = new List<(string Key, string Value)>(); + var set = new HashSet(); + + if (Text(json, "status") is { } status) + { + var known = Statuses.FirstOrDefault(s => s.Equals(status, StringComparison.OrdinalIgnoreCase)); + if (known is null) notes.Add($"Status \"{status}\" isn't one the page has."); + else query.Add(("status", known)); + } + + Ids(json, "subscriptionIds", "subscriptionId", "subscription", names.Subscriptions, query, set, notes); + Ids(json, "partnerIds", "partnerId", "partner", names.Partners, query, set, notes); + Ids(json, "informationTypeIds", "informationTypeId", "informationType", names.InformationTypes, query, set, notes); + + if (Text(json, "correlationId") is { } correlation) query.Add(("correlationId", correlation)); + + foreach (var property in json["properties"] as JArray ?? []) + { + if (property is not JObject p || Text(p, "key") is not { } key) continue; + var known = names.PropertyKeys.FirstOrDefault(k => k.Equals(key, StringComparison.OrdinalIgnoreCase)); + if (known is null) + { + notes.Add($"No information type promotes a property \"{key}\"."); + continue; + } + var text = Text(p, "text"); + query.Add(("prop", $"{(Flag(p, "not") ? "!" : "")}{known}{(text is null ? "" : $":{text}")}")); + } + if (Text(json, "anyProperty") is { } anyProperty) + { + query.Add(("property", anyProperty)); + set.Add("property"); + } + + if (Text(json, "error") is { } error) + { + query.Add(("error", error)); + set.Add("error"); + } + + foreach (var flag in new[] { "retry", "scheduled", "aggregated" }) + if (Flag(json, flag)) + { + query.Add((flag, "1")); + set.Add(flag); + } + if (Flag(json, "latest")) query.Add(("latest", "1")); + + var range = Text(json, "range")?.ToLowerInvariant(); + var from = Text(json, "from"); + var to = Text(json, "to"); + if (range is not null && Range.IsMatch(range)) + { + query.Add(("range", range)); + } + else + { + if (range is not null) notes.Add($"\"{range}\" isn't a time range the page has."); + if (from is not null && Bound.IsMatch(from)) query.Add(("from", from)); + if (to is not null && Bound.IsMatch(to)) query.Add(("to", to)); + } + + // Only filters that are set can be turned round; the page ignores the rest anyway. + var not = (json["not"] as JArray ?? []) + .Select(n => n.Type == JTokenType.String ? Negatable.FirstOrDefault(k => k.Equals((string?)n, StringComparison.OrdinalIgnoreCase)) : null) + .Where(n => n is not null && set.Contains(n)) + .Distinct().Order().ToList(); + if (not.Count > 0) query.Add(("not", string.Join(",", not))); + + if (Text(json, "notUnderstood") is { } notUnderstood) notes.Insert(0, notUnderstood); + + var sentence = Text(json, "answer", MaxAnswer) ?? ""; + return new ExchangeQuery( + string.Join("&", query.Select(q => $"{Uri.EscapeDataString(q.Key)}={Uri.EscapeDataString(q.Value)}")), + sentence, notes); + } + + static void Ids(JObject json, string field, string parameter, string kind, IReadOnlyList known, + List<(string, string)> query, HashSet set, List notes) + { + var ids = new List(); + foreach (var token in json[field] as JArray ?? []) + { + if (token.Type != JTokenType.Integer) continue; + // As text first: a number too big for an id is just an id that doesn't exist. + var text = token.ToString(Formatting.None); + if (int.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out var id) && known.Any(n => n.Id == id)) ids.Add(id); + else notes.Add($"There is no {Words(kind)} with id {text}."); + } + if (ids.Count == 0) return; + query.Add((parameter, string.Join(",", ids.Distinct()))); + set.Add(kind); + } + + static string Words(string kind) => kind == "informationType" ? "information type" : kind; + + static string? Text(JObject json, string field, int max = MaxText) + { + if (json[field] is not { Type: JTokenType.String } token) return null; + var text = ((string?)token)?.Trim(); + return string.IsNullOrEmpty(text) ? null : text.Length <= max ? text : text[..max]; + } + + static bool Flag(JObject json, string field) => json[field] is { Type: JTokenType.Boolean } token && (bool)token; +} diff --git a/SW.Bitween.Api/Services/Ai/FailurePrompt.cs b/SW.Bitween.Api/Services/Ai/FailurePrompt.cs new file mode 100644 index 00000000..297a24c7 --- /dev/null +++ b/SW.Bitween.Api/Services/Ai/FailurePrompt.cs @@ -0,0 +1,68 @@ +#nullable enable +using System; +using System.Linq; +using Newtonsoft.Json.Linq; + +namespace SW.Bitween.Services.Ai; + +/// An exchange's failure in plain words: what happened, why, and what to change. +public sealed record FailureExplanation(string Summary, string Cause, string Fix); + +/// What a model is told to explain a failed exchange, and how its answer is read. +public static class FailurePrompt +{ + public const string Instructions = """ + You explain why an exchange failed in Bitween, an integration platform, to the person running it. + They know their partners and subscriptions; they may not be developers. + + How Bitween works: a partner or a receiver delivers a document; a subscription picks it up; its + mapper turns the document into what the destination expects; its handler delivers that, often + over HTTP, and may get a response back. A failure is either in mapping or in delivery. Failed + exchanges can be retried by a retry policy, or by hand. + + Every value in the documents and the error has been masked before you see it: text is , + numbers are , booleans are , credentials and tokens are , and long lists show their first entries. Keys, + structure and the error's wording are real. Never guess at a masked value; reason from structure, + names, types and the error. + + Answer with exactly one JSON object and nothing else: + { "summary": "...", "cause": "...", "fix": "..." } + - summary: one sentence, what went wrong, in plain words. + - cause: the most likely reason, two to four sentences, pointing at the field, setting or step + involved. Say how sure you are when the evidence is thin. + - fix: the change most likely to make the next attempt succeed, concrete and short - which setting, + rule or field to change and to what, or who to ask (the partner, an administrator). If a retry + alone would help (a timeout, a 5xx, a rate limit), say so. + """; + + /// The model's answer as an explanation; a plain-text answer becomes the summary. + public static FailureExplanation Parse(string answer) + { + var start = answer.IndexOf('{'); + var end = answer.LastIndexOf('}'); + if (start >= 0 && end > start) + { + try + { + var json = JObject.Parse(answer[start..(end + 1)]); + var summary = Text(json["summary"]); + if (!string.IsNullOrWhiteSpace(summary)) + return new(summary.Trim(), (Text(json["cause"]) ?? "").Trim(), (Text(json["fix"]) ?? "").Trim()); + } + catch (Newtonsoft.Json.JsonException) + { + // Read as prose below. + } + } + return new(answer.Trim(), "", ""); + } + + /// A field as text, whatever the model made it: a fix often comes back as a list of steps. + static string? Text(JToken? token) => token switch + { + null or { Type: JTokenType.Null } => null, + JValue value => value.ToString(), + JArray list => string.Join("\n", list.Select(i => i.Type == JTokenType.String ? (string?)i : i.ToString(Newtonsoft.Json.Formatting.None))), + _ => token.ToString(Newtonsoft.Json.Formatting.None), + }; +} diff --git a/SW.Bitween.Api/Services/Ai/FrozenValues.cs b/SW.Bitween.Api/Services/Ai/FrozenValues.cs new file mode 100644 index 00000000..8a4c92c2 --- /dev/null +++ b/SW.Bitween.Api/Services/Ai/FrozenValues.cs @@ -0,0 +1,102 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using Newtonsoft.Json.Linq; +using SW.Bitween.NativeAdapters.Mapper; + +namespace SW.Bitween.Services.Ai; + +/// +/// Text a rule writes as a constant that the source sample holds as data. +/// +/// +/// +/// A constant the source doesn't have - a currency, a sender code - is what a fixed value is for. +/// One the source does have is a value that changes from document to document, frozen to match the +/// one sample: first + " Haddad" turns every customer into a Haddad, and still matches the +/// output sample, so the comparison alone would call it right. +/// +/// +/// Whole values only, ignoring case and surrounding spaces, and only text of three characters or +/// more: a constant "1" or "N" turns up in most documents by coincidence. +/// +/// +public static class FrozenValues +{ + const int MinLength = 3; + + /// One difference per frozen value, named the way the mapper names the rule's errors. + public static List In(MappingRules rules, ValueNode source) + { + var data = new HashSet(StringComparer.OrdinalIgnoreCase); + Collect(source, data); + var found = new List(); + if (data.Count == 0) return found; + + if (rules.Root is not null) List(rules.Root, "", data, found); + Fields(rules.Fields, rules.Lists, "", data, found); + return found; + } + + static void Fields(IEnumerable fields, IEnumerable lists, string prefix, + HashSet data, List found) + { + foreach (var field in fields) Field(field, Describe(prefix, field.Target), data, found); + foreach (var list in lists) List(list, prefix, data, found); + } + + static void List(ListRule list, string prefix, HashSet data, List found) + { + var key = Describe(prefix, list.Target); + if (list.Item is not null) Field(list.Item, key, data, found); + Fields(list.Fields, list.Lists, key, data, found); + foreach (var entry in list.Fixed.Concat(list.After)) + { + if (entry.Item is not null) Field(entry.Item, key, data, found); + Fields(entry.Fields, entry.Lists, key, data, found); + } + } + + static void Field(FieldRule field, string target, HashSet data, List found) + { + if (field.From.Kind == ValueSourceKind.Fixed && Frozen(field.From.Value, data) is { } value) + found.Add(new(target, $"the fixed value \"{value}\" is in the source sample, so it would be the same for every document; read it from the source")); + + if (field.Transform is { } transform + && transform.Fn is "concat" or "replace" or "defaultIfEmpty" + && transform.Args.TryGetValue(transform.Fn == "defaultIfEmpty" ? "value" : "with", out var arg) + && Frozen(arg, data) is { } text) + found.Add(new(target, $"the {transform.Fn} text \"{text}\" is in the source sample, so it would be the same for every document; read it from the source")); + } + + static string? Frozen(object? value, HashSet data) + { + var text = (value is JValue jvalue ? jvalue.Value : value) as string; + text = text?.Trim(); + return text is { Length: >= MinLength } && data.Contains(text) ? text : null; + } + + static void Collect(ValueNode node, HashSet data) + { + switch (node) + { + case ScalarNode { Value: string text } when text.Trim().Length >= MinLength: + data.Add(text.Trim()); + break; + case ObjectNode obj: + foreach (var (_, child) in obj.Children()) Collect(child, data); + break; + case ListNode list: + foreach (var item in list.Items) Collect(item, data); + break; + } + } + + /// As DocumentMapper describes a rule's target, so the editor marks the right row. + static string Describe(string prefix, List target) + { + var joined = target.Count == 0 ? "(no target)" : string.Join(".", target); + return prefix.Length == 0 ? joined : $"{prefix}[].{joined}"; + } +} diff --git a/SW.Bitween.Api/Services/Ai/MappingRulesPrompt.cs b/SW.Bitween.Api/Services/Ai/MappingRulesPrompt.cs new file mode 100644 index 00000000..0eeb26f4 --- /dev/null +++ b/SW.Bitween.Api/Services/Ai/MappingRulesPrompt.cs @@ -0,0 +1,367 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using Newtonsoft.Json.Serialization; +using SW.Bitween.NativeAdapters.Mapper; + +namespace SW.Bitween.Services.Ai; + +/// +/// What a model is told to write mapping rules, and how its answer becomes rules again. +/// +/// +/// +/// The answer is asked for as plain JSON and read with the mapper's own types, rather than +/// constrained by a schema: the rules nest lists inside lists, which providers' schema-constrained +/// output can't express. Reading it strictly - an unknown property is an error - and then running +/// it against the sample catches more than a schema would. +/// +/// +/// is the same text on every call, so a provider can cache it. Everything +/// that varies goes in the message. +/// +/// +public static class MappingRulesPrompt +{ + public const string Instructions = """ + You write mapping rules for Bitween's rules-based mapper. Given a sample of a source document + and the document that should be produced from it, answer with the rules that turn one into the + other. + + Answer with exactly one JSON object: the complete rules. No prose, no Markdown, no code fence. + + # The rules format + + { + "version": 1, + "sourceFormat": "json", // given in the message; copy it + "targetFormat": "json", // given in the message; copy it + "sourceDateOrder": "yearFirst", // yearFirst | dayFirst | monthFirst + "fields": [ FieldRule, ... ], + "lists": [ ListRule, ... ], + "root": ListRule // optional: makes the whole output an array + } + + Copy any sourceCsv, targetCsv, sourceFixedWidth, targetFixedWidth, sourceExcel and targetExcel + settings the message gives you, unchanged. Use no property this description doesn't name. + + ## FieldRule: writes one value + + { "target": ["customer", "name"], "from": ValueSource, "transform": Transform, "lookup": Lookup, "type": "number" } + + - target: the output path as segments, so a key containing a dot is one segment. Missing + objects along the path are created. + - from (required), then optionally transform, lookup and type, applied in that order. + - type converts the value to "string", "number" or "boolean". Use it whenever the output + wants a different JSON type than the source has, e.g. "42" in the source and 42 in the output. + + ValueSource, by kind: + - { "kind": "path", "path": "order.id" } - a dot path read from the current scope: the + document, or the current list entry inside a list. + - { "kind": "rootPath", "path": "order.id" } - a dot path from the document root, even inside + a list. Use it to copy a header value onto every line. + - { "kind": "fixed", "value": "EUR" } - a literal, keeping its JSON type. + - { "kind": "count" } - how many source entries the enclosing list walked; for trailer records. + - { "kind": "partner", "key": "SenderId" } - a property of the exchange's partner. + - { "kind": "global", "setId": "regions", "key": "default" } - a value from a global value set. + You only ever see these two by name, in the message. Use them only for names it lists, and + only when the output wants a value the source doesn't have and a name plainly matches it. + - "source" also exists, but needs a document you can't see. Don't use it. + + Paths match keys exactly and never step into a list: "lines.0.sku" is not a path. A list + rule is the only way to reach list entries. A missing path gives null, not an error. + + Transform, one per field, exactly these: + - { "fn": "upper" }, { "fn": "lower" }, { "fn": "trim" } + - { "fn": "substring", "start": 0, "length": 3 } - start counts from 0; length optional (to the end) + - { "fn": "replace", "find": "-", "with": "" } + - { "fn": "concat", "with": " Ltd" } - appends text + - { "fn": "round", "decimals": 2 } - halves away from zero + - { "fn": "multiply", "by": 100 }, { "fn": "add", "amount": -1 } + - { "fn": "formatDate", "format": "yyyy-MM-dd" } - reads a date, writes it with a .NET format string + - { "fn": "defaultIfEmpty", "value": "N/A" } - replaces null or empty text + Null passes through every transform except defaultIfEmpty. Numbers are exact decimals. + + Lookup: { "table": { "N": "New", "S": "Shipped" }, "fallback": "Unknown" } - replaces the value + by comparing it as text; anything not in the table becomes fallback (null when left out). Use it + when the samples show codes being translated. + + ## ListRule: writes an array + + { "over": "order.lines", "target": ["items"], "where": Filter, "fields": [...], "lists": [...] } + + - over: the path of the source list to walk. "" walks the document itself (a CSV, fixed-width or + Excel source is a list of rows, so it is walked with ""). Leaving over out walks nothing, and + the list holds only its fixed entries. + - target: where the array is written. + - fields and lists: applied to each entry, with paths relative to the entry. Lists nest. + - item: one FieldRule (its target unused) that makes each entry a plain value instead of an + object, for ["A1", "B7"] rather than [{ "sku": "A1" }]. + - where: { "field": "qty", "operator": "greaterThan", "value": 0 } skips entries that don't + match. Operators: equal, notEqual (compare as text), greaterThan, greaterThanOrEqual, + lessThan, lessThanOrEqual (compare as numbers). + - fixed and after: entries placed before or after the walked ones, each + { "item": FieldRule } or { "fields": [...], "lists": [...] }, read from the scope the list sits + in. For header and trailer records, or a list built from separate fields. + + A top-level "root" makes the output an array; top-level fields and lists must then be empty. + + ## Formats + + - XML read: the root element's name is the top-level key, attributes are "@name" keys, mixed + text is "#text", repeated elements are lists, and namespaces are dropped. + - XML written: exactly one top-level key, not a list; "@name" targets become attributes; + "@xmlns" or "@xmlns:prefix" targets declare namespaces. + - CSV, fixed-width and Excel read: a list of rows whose values are all text, keyed by header + name, or by position "1", "2", ... without a header. + - CSV, fixed-width and Excel written: a list of rows (use root), or one object for one row. A + nested target becomes a dotted column name. + - Dates: formatDate always reads year-first dates such as 2026-09-10 or 2026-09-10T08:30:00Z. + Set sourceDateOrder to dayFirst or monthFirst when the source writes dates like 10/09/2026. + + # How to choose rules + + - Take every output value from the source when the source has it, directly or after a + transform, lookup or type. Use "fixed" only for values the source can't supply. + - Never write a value that appears in the source sample as fixed text - not as a "fixed" + value, and not as the text of a concat, replace or defaultIfEmpty. It is data that + changes from one document to the next, and freezing it would match this sample only. + When the format can't build an output value from the source (it can't join two fields, + for example), map what you can and leave that value different; it will be finished by + hand. + - The output sample may hold placeholder values. Match its shape and its value types, and + don't copy placeholder values as fixed values when a source field plainly corresponds. + - Write the output keys in the order the output sample has them. + - Keep it minimal: no transform, lookup or type that the samples don't call for. + - When the message has instructions from the person building the mapping, follow them: they + say how the mapping should behave beyond what the samples show, such as codes a lookup + needs that the sample doesn't happen to contain. + """; + + /// At most this many names of each kind go in a message, so a big partner can't crowd out the samples. + public const int NameLimit = 100; + + /// The message for one call: the formats, both samples, and what went wrong last time. + /// The editor's rules, for their formats and format settings. + /// + /// The partner properties and global values the rules may read, by name. Never their values: + /// those stay on the server, and are only filled in when the mapping runs. + /// + /// What the person building the mapping wrote for the model, if anything. + /// Rules to fix rather than start from scratch, with what was wrong with them. + public static string Message(MappingRules formats, string sourceSample, string targetSample, + ValueNames? names = null, string? instructions = null, + (MappingRules Rules, IReadOnlyList Problems)? previous = null) + { + var message = new StringBuilder(); + message.AppendLine("Formats and their settings (copy them into your answer):"); + message.AppendLine(Serialize(FormatsOnly(formats))); + message.AppendLine(); + message.AppendLine(""); + message.AppendLine(sourceSample.Trim()); + message.AppendLine(""); + message.AppendLine(); + message.AppendLine(""); + message.AppendLine(targetSample.Trim()); + message.AppendLine(""); + + if (names is { } known && (known.Partner.Count > 0 || known.Globals.Count > 0)) + { + message.AppendLine(); + message.AppendLine("Values Bitween fills in when the mapping runs, by name only:"); + if (known.Partner.Count > 0) + message.AppendLine("- partner properties: " + string.Join(", ", known.Partner.Take(NameLimit))); + foreach (var (setId, keys) in known.Globals.Take(NameLimit)) + message.AppendLine($"- global value set \"{setId}\", keys: " + string.Join(", ", keys.Take(NameLimit))); + } + + if (!string.IsNullOrWhiteSpace(instructions)) + { + message.AppendLine(); + message.AppendLine(""); + message.AppendLine(instructions.Trim()); + message.AppendLine(""); + } + + if (previous is { } last) + { + message.AppendLine(); + message.AppendLine("These rules were tried and did not produce the output sample:"); + message.AppendLine(Serialize(last.Rules)); + message.AppendLine(); + message.AppendLine("What was wrong:"); + foreach (var problem in last.Problems) message.AppendLine("- " + problem); + message.AppendLine(); + message.AppendLine("Answer with corrected rules. Keep what was right."); + } + + return message.ToString(); + } + + /// + /// The rules in a model's answer, with the editor's formats put back. + /// + /// With a reason a model can act on, when the answer isn't rules. + public static MappingRules Parse(string answer, MappingRules formats) + { + var json = JsonPart(answer) + ?? throw new FormatException("The answer contained no JSON object."); + MappingRules? rules; + try + { + rules = JsonConvert.DeserializeObject(json, Strict); + } + catch (JsonException ex) + { + throw new FormatException($"The answer is not valid rules JSON: {ex.Message}"); + } + if (rules is null) throw new FormatException("The answer was empty."); + if (NullIn(rules) is { } missing) throw new FormatException(missing); + + if (rules.Version != MappingRules.CurrentVersion) + throw new FormatException($"\"version\" must be {MappingRules.CurrentVersion}."); + if (rules.Root is not null && (rules.Fields.Count > 0 || rules.Lists.Count > 0)) + throw new FormatException("With \"root\" set, top-level \"fields\" and \"lists\" must be empty."); + + // Whatever the model wrote, the formats are the editor's: they describe the exchange, and + // the samples were read with them. + rules.SourceFormat = formats.SourceFormat; + rules.TargetFormat = formats.TargetFormat; + rules.SourceCsv = formats.SourceCsv; + rules.TargetCsv = formats.TargetCsv; + rules.SourceFixedWidth = formats.SourceFixedWidth; + rules.TargetFixedWidth = formats.TargetFixedWidth; + rules.SourceExcel = formats.SourceExcel; + rules.TargetExcel = formats.TargetExcel; + return rules; + } + + /// + /// The names of the partner properties and global values a mapping can read - the keys of the + /// context a preview maps against, with every value left behind. + /// + /// A context built with its secrets masked, as the preview's is. + /// + /// Secrets are left out by name as well: a rule reading one would copy a credential into a + /// document, which is never what a sample asks for and is better written by hand if it is meant. + /// + public static ValueNames NamesOf(MappingContext masked) => new( + Plain(masked.Partner), + masked.Globals.OrderBy(set => set.Key, StringComparer.Ordinal) + .Select(set => (set.Key, Keys: Plain(set.Value))) + .Where(set => set.Keys.Count > 0) + .ToDictionary(set => set.Key, set => set.Keys)); + + static IReadOnlyList Plain(IReadOnlyDictionary values) => + values.Where(kv => kv.Value != AdapterSecretProperties.Sentinel) + .Select(kv => kv.Key).Order(StringComparer.Ordinal).ToList(); + + /// + /// Where the rules hold an explicit null for a list or a value source, which the mapper's types + /// start as empty and the code reading them never expects - "fields": null from a model + /// or a hand-written request. Null when there is none. + /// + public static string? NullIn(MappingRules rules) + { + if (rules.Fields is null) return "\"fields\" can't be null; use [] for none."; + if (rules.Lists is null) return "\"lists\" can't be null; use [] for none."; + var lists = new Stack(rules.Lists); + if (rules.Root is not null) lists.Push(rules.Root); + foreach (var field in rules.Fields) + if (FieldNull(field) is { } problem) return problem; + + while (lists.TryPop(out var list)) + { + if (list is null) return "a list can't be null."; + if (list.Target is null || list.Fields is null || list.Lists is null || list.Fixed is null || list.After is null) + return "a list's \"target\", \"fields\", \"lists\", \"fixed\" and \"after\" can't be null; use [] for none."; + if (list.Item is not null && FieldNull(list.Item) is { } itemProblem) return itemProblem; + foreach (var field in list.Fields) + if (FieldNull(field) is { } problem) return problem; + foreach (var entry in list.Fixed.Concat(list.After)) + { + if (entry is null || entry.Fields is null || entry.Lists is null) + return "an entry's \"fields\" and \"lists\" can't be null; use [] for none."; + if (entry.Item is not null && FieldNull(entry.Item) is { } entryProblem) return entryProblem; + foreach (var field in entry.Fields) + if (FieldNull(field) is { } problem) return problem; + foreach (var inner in entry.Lists) lists.Push(inner); + } + foreach (var inner in list.Lists) lists.Push(inner); + } + return null; + } + + static string? FieldNull(FieldRule? field) => + field is null ? "a field rule can't be null." + : field.Target is null || field.From is null ? "a field rule's \"target\" and \"from\" can't be null." + : field.Transform is { Args: null } ? "a transform can't be null." : null; + + /// The rules as the editor stores them: camelCase, enums as camelCase text, nulls left out. + public static string Serialize(MappingRules rules) => JsonConvert.SerializeObject(rules, Wire); + + /// + /// From the first brace to the last: some models wrap JSON in a code fence or a sentence even + /// when asked not to, and that shouldn't cost a second call. + /// + static string? JsonPart(string answer) + { + var start = answer.IndexOf('{'); + var end = answer.LastIndexOf('}'); + return start >= 0 && end > start ? answer[start..(end + 1)] : null; + } + + static MappingRules FormatsOnly(MappingRules rules) => new() + { + SourceFormat = rules.SourceFormat, + TargetFormat = rules.TargetFormat, + SourceDateOrder = rules.SourceDateOrder, + SourceCsv = rules.SourceCsv, + TargetCsv = rules.TargetCsv, + SourceFixedWidth = rules.SourceFixedWidth, + TargetFixedWidth = rules.TargetFixedWidth, + SourceExcel = rules.SourceExcel, + TargetExcel = rules.TargetExcel, + }; + + // An unknown property is an error rather than ignored: it is a rule the model meant and the + // mapper would silently drop. + static readonly JsonSerializerSettings Strict = new() { ContractResolver = new StrictExceptTransforms() }; + + /// + /// Strict for every type but a transform, whose arguments sit beside "fn" as extension data. + /// Strictness set on the serializer would refuse those too: Newtonsoft checks for a missing + /// member before it looks for extension data. + /// + sealed class StrictExceptTransforms : DefaultContractResolver + { + protected override JsonObjectContract CreateObjectContract(Type objectType) + { + var contract = base.CreateObjectContract(objectType); + contract.MissingMemberHandling = objectType == typeof(TransformRule) + ? MissingMemberHandling.Ignore + : MissingMemberHandling.Error; + return contract; + } + } + + static readonly JsonSerializerSettings Wire = new() + { + // Lookup tables are dictionaries whose keys are data, so they keep their case. + ContractResolver = new DefaultContractResolver + { + NamingStrategy = new CamelCaseNamingStrategy { ProcessDictionaryKeys = false }, + }, + Converters = { new StringEnumConverter(new CamelCaseNamingStrategy()) }, + NullValueHandling = NullValueHandling.Ignore, + }; +} + +/// Names a mapping can read values by, without the values. +public sealed record ValueNames( + IReadOnlyList Partner, + IReadOnlyDictionary> Globals); diff --git a/SW.Bitween.Api/Services/Ai/Masking.cs b/SW.Bitween.Api/Services/Ai/Masking.cs new file mode 100644 index 00000000..e3eeb346 --- /dev/null +++ b/SW.Bitween.Api/Services/Ai/Masking.cs @@ -0,0 +1,197 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using SW.Bitween.NativeAdapters.Mapper; +using SW.Bitween.NativeAdapters.Mapper.Formats; + +namespace SW.Bitween.Services.Ai; + +/// +/// Documents and error text with every value taken out, so an AI provider sees how an exchange is +/// shaped and where it broke but none of what it carried. +/// +/// +/// +/// A document keeps its keys, nesting and the type of each value: "name": "Lina" becomes +/// "name": "<text>". That is nearly always enough to explain a failure - a missing field, +/// a number where text was wanted, an empty list - and it is the only thing sent. +/// +/// +/// Error text is free text, so it is masked three ways: a document inside it (an HTTP adapter puts +/// the partner's response body there) is masked as a document; every value the exchange's own +/// documents hold is replaced wherever it appears; and quoted text, long numbers and email +/// addresses are replaced whatever they are. What survives is the error's wording and the code's +/// stack, which is where an explanation comes from. +/// +/// +public sealed class Masking +{ + public const string Text = ""; + public const string Number = ""; + public const string Secret = ""; + public const string Boolean = ""; + + /// A long list shows this many entries, then how many more there were. + const int ListEntries = 3; + + /// How much of one masked document or error is sent. + public const int MaxLength = 12_000; + + // Every value any masked document held, so error text naming one can have it taken out. + readonly HashSet seen = new(StringComparer.OrdinalIgnoreCase); + + /// + /// A document with its values replaced by their types, or null when it can't be read as JSON, XML + /// or CSV - in which case nothing of it is sent. + /// + /// The file's content type, when known; it picks the reader. + public string? Document(string text, string? contentType) + { + foreach (var format in FormatsFor(contentType)) + { + ValueNode tree; + try + { + tree = format.Read(text); + } + catch (DocumentFormatException) + { + continue; + } + + try + { + return Cut(format.Write(Mask(tree))); + } + catch (DocumentFormatException) + { + return null; + } + } + return null; + } + + /// Error text with documents, values and anything that looks like data taken out. + public string Error(string text) + { + // A document inside is masked as a document; the words around it as free text. Free-text + // rules run on the words only, or they would take a masked document's keys as quoted text. + var (before, document, after) = EmbeddedDocument(text); + return Cut(Words(before) + document + Words(after)); + } + + string Words(string text) + { + // Longest first, so "Lina Haddad" goes before "Lina" can leave " Haddad" behind. + foreach (var value in seen.Where(v => v.Length >= 3).OrderByDescending(v => v.Length)) + text = text.Replace(value, Text, StringComparison.OrdinalIgnoreCase); + + // A stack frame's file keeps its name: the folders above it describe the server, not the failure. + text = SourcePath.Replace(text, " in ${file}:line"); + text = Email.Replace(text, ""); + text = Quoted.Replace(text, m => $"{m.Groups["q"].Value}{Text}{m.Groups["q"].Value}"); + text = QueryString.Replace(text, "?"); + // Credentials a partner's error can echo unquoted: "Bearer eyJ...", "api_key=...", or a bare token. + text = Scheme.Replace(text, m => $"{m.Groups["scheme"].Value} {Secret}"); + text = Labelled.Replace(text, m => $"{m.Groups["label"].Value}{m.Groups["sep"].Value}{Secret}"); + text = TokenLike.Replace(text, Secret); + return LongNumber.Replace(text, Number); + } + + ValueNode Mask(ValueNode node) + { + switch (node) + { + case ScalarNode scalar: + switch (scalar.Value) + { + case null: + return scalar; + case string s: + if (s.Trim().Length > 0) seen.Add(s.Trim()); + return ValueNode.Value(s.Length == 0 ? "" : Text); + case bool: + return ValueNode.Value(Boolean); + default: + seen.Add(Convert.ToString(scalar.Value, System.Globalization.CultureInfo.InvariantCulture)!); + return ValueNode.Value(Number); + } + + case ObjectNode obj: + var copy = ValueNode.Object(); + foreach (var (key, child) in obj.Children()) copy.Set(key, Mask(child)); + return copy; + + case ListNode list: + var masked = ValueNode.List(); + foreach (var item in list.Items.Take(ListEntries)) masked.Add(Mask(item)); + // Still walked, so their values are known to be taken out of the error text. + foreach (var item in list.Items.Skip(ListEntries)) Mask(item); + if (list.Items.Count > ListEntries) + masked.Add(ValueNode.Value($"<{list.Items.Count - ListEntries} more entries like these>")); + return masked; + + default: + return node; + } + } + + /// The text before the first JSON or XML document inside it, that document masked, and the text after. + (string Before, string Document, string After) EmbeddedDocument(string text) + { + foreach (var (open, close, format) in new (char, char, IDocumentFormat)[] + { ('{', '}', new JsonFormat()), ('[', ']', new JsonFormat()), ('<', '>', new XmlFormat()) }) + { + var start = text.IndexOf(open); + var end = text.LastIndexOf(close); + if (start < 0 || end <= start) continue; + + var inner = text[start..(end + 1)]; + try + { + var masked = format.Write(Mask(format.Read(inner))); + return (text[..start], masked, text[(end + 1)..]); + } + catch (DocumentFormatException) + { + // Not a document after all - a stack frame's [brackets], say. + } + } + return (text, "", ""); + } + + static IEnumerable FormatsFor(string? contentType) + { + var type = contentType?.ToLowerInvariant() ?? ""; + if (type.Contains("json")) yield return new JsonFormat(); + else if (type.Contains("xml")) yield return new XmlFormat(); + // Read without a header: a header's names would be kept as keys, and a file without one + // would send its first row as them. Every row is masked, a real header row included. + else if (type.Contains("csv")) yield return new CsvFormat(new CsvOptions { HasHeader = false }); + else + { + // Unknown: the two formats that say so of themselves. CSV is never guessed - nearly any + // text reads as CSV, and its values would then be the whole of it. + yield return new JsonFormat(); + yield return new XmlFormat(); + } + } + + static string Cut(string text) => + text.Length <= MaxLength ? text : text[..MaxLength] + $"\n<{text.Length - MaxLength} more characters not sent>"; + + static readonly Regex SourcePath = new(@" in [^\s]*?[\\/](?[^\\/\s]+):line", RegexOptions.Compiled); + static readonly Regex Email = new(@"[\w.+-]+@[\w-]+(\.[\w-]+)+", RegexOptions.Compiled); + static readonly Regex Quoted = new(@"(?['""])(?!<)[^'""\r\n]{1,200}\k", RegexOptions.Compiled); + static readonly Regex QueryString = new(@"\?[^\s'""<>]+", RegexOptions.Compiled); + static readonly Regex LongNumber = new(@"(?bearer|basic)\s+(?!<)[A-Za-z0-9._~+/=-]{8,}", RegexOptions.Compiled | RegexOptions.IgnoreCase); + static readonly Regex Labelled = new( + @"(?