diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c3d365917..767ed26f6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,6 +35,7 @@ jobs: - forest_admin_datasource_snowflake - forest_admin_datasource_mambu_payments - forest_admin_datasource_graphql_hasura + - forest_admin_datasource_pylon steps: - name: Checkout @@ -88,6 +89,7 @@ jobs: - forest_admin_datasource_snowflake - forest_admin_datasource_mambu_payments - forest_admin_datasource_graphql_hasura + - forest_admin_datasource_pylon services: mongodb: image: mongo:latest @@ -166,7 +168,7 @@ jobs: with: verbose: true oidc: true - files: ${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_agent/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_active_record/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_customizer/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_toolkit/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_rails/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_mongoid/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_rpc_agent/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_rpc/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_zendesk/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_snowflake/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_mambu_payments/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_graphql_hasura/coverage.json + files: ${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_agent/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_active_record/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_customizer/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_toolkit/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_rails/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_mongoid/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_rpc_agent/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_rpc/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_zendesk/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_snowflake/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_mambu_payments/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_graphql_hasura/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_pylon/coverage.json deploy: name: Release package diff --git a/.releaserc.js b/.releaserc.js index 76e57c8f0..998aa1a24 100644 --- a/.releaserc.js +++ b/.releaserc.js @@ -31,7 +31,8 @@ module.exports = { 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/version.rb; '+ 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb; '+ 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb; '+ - 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/version.rb; ', + 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/version.rb; '+ + 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/version.rb; ', successCmd: '( cd packages/forest_admin_agent && gem build && gem push forest_admin_agent-*.gem );' + '( cd packages/forest_admin_datasource_active_record && gem build && gem push forest_admin_datasource_active_record-*.gem );' + @@ -45,7 +46,8 @@ module.exports = { '( cd packages/forest_admin_datasource_zendesk && gem build && gem push forest_admin_datasource_zendesk-*.gem );' + '( cd packages/forest_admin_datasource_snowflake && gem build && gem push forest_admin_datasource_snowflake-*.gem );' + '( cd packages/forest_admin_datasource_mambu_payments && gem build && gem push forest_admin_datasource_mambu_payments-*.gem );' + - '( cd packages/forest_admin_datasource_graphql_hasura && gem build && gem push forest_admin_datasource_graphql_hasura-*.gem );' , + '( cd packages/forest_admin_datasource_graphql_hasura && gem build && gem push forest_admin_datasource_graphql_hasura-*.gem );' + + '( cd packages/forest_admin_datasource_pylon && gem build && gem push forest_admin_datasource_pylon-*.gem );' , }, ], [ @@ -68,6 +70,7 @@ module.exports = { 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb', 'packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb', 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/version.rb', + 'packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/version.rb', 'package.json' ], }, diff --git a/.rubocop.yml b/.rubocop.yml index c6985f27d..927a933eb 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -42,6 +42,7 @@ Gemspec/RequireMFA: - 'packages/forest_admin_datasource_snowflake/forest_admin_datasource_snowflake.gemspec' - 'packages/forest_admin_datasource_mambu_payments/forest_admin_datasource_mambu_payments.gemspec' - 'packages/forest_admin_datasource_graphql_hasura/forest_admin_datasource_graphql_hasura.gemspec' + - 'packages/forest_admin_datasource_pylon/forest_admin_datasource_pylon.gemspec' # Offense count: 1 # This cop supports unsafe autocorrection (--autocorrect-all). @@ -133,6 +134,7 @@ Style/MutableConstant: - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/version.rb' - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb' - 'packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb' + - 'packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/version.rb' # Offense count: 38 # This cop supports safe autocorrection (--autocorrect). @@ -217,6 +219,7 @@ Style/StringLiterals: - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/version.rb' - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb' - 'packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb' + - 'packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/version.rb' # Offense count: 1 # This cop supports safe autocorrection (--autocorrect). @@ -262,6 +265,8 @@ Metrics/ParameterLists: - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb' - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/configuration.rb' - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/collections/base_collection.rb' + - 'packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/base_collection.rb' + - 'packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/configuration.rb' - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/datasource.rb' - 'packages/forest_admin_agent/lib/forest_admin_agent/routes/query_handler.rb' - 'packages/forest_admin_agent/lib/forest_admin_agent/services/smart_action_checker.rb' @@ -294,6 +299,7 @@ Metrics/ModuleLength: - 'packages/forest_admin_datasource_customizer/spec/**/*' - 'packages/forest_admin_datasource_zendesk/spec/**/*' - 'packages/forest_admin_datasource_mambu_payments/spec/**/*' + - 'packages/forest_admin_datasource_pylon/spec/**/*' - 'packages/forest_admin_rails/spec/**/*' - 'packages/forest_admin_rpc_agent/spec/**/*' - 'packages/forest_admin_datasource_mongoid/lib/forest_admin_datasource_mongoid/utils/helpers.rb' @@ -369,6 +375,7 @@ Metrics/ClassLength: - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/query/query_builder.rb' - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/query/aggregator.rb' - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/collections/base_collection.rb' + - 'packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/base_collection.rb' - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/collection.rb' - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/datasource.rb' - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/utils/query.rb' diff --git a/packages/forest_admin_datasource_pylon/.gitignore b/packages/forest_admin_datasource_pylon/.gitignore new file mode 100644 index 000000000..06cfcfb83 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/.gitignore @@ -0,0 +1,8 @@ +*.gem +.bundle/ +Gemfile.lock +Gemfile-test.lock +coverage/ +pkg/ +tmp/ +.rspec_status diff --git a/packages/forest_admin_datasource_pylon/.rspec b/packages/forest_admin_datasource_pylon/.rspec new file mode 100644 index 000000000..34c5164d9 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/.rspec @@ -0,0 +1,3 @@ +--format documentation +--color +--require spec_helper diff --git a/packages/forest_admin_datasource_pylon/Gemfile b/packages/forest_admin_datasource_pylon/Gemfile new file mode 100644 index 000000000..c229ff1d5 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/Gemfile @@ -0,0 +1,16 @@ +source 'https://rubygems.org' + +gemspec + +gem 'forest_admin_datasource_customizer' +gem 'forest_admin_datasource_toolkit' +gem 'rake', '~> 13.0' +gem 'rubocop', '1.86.1' +gem 'rubocop-performance', '1.26.1' +gem 'rubocop-rspec', '3.9.0' + +group :development, :test do + gem 'rspec', '~> 3.0' + gem 'simplecov', '~> 0.22', require: false + gem 'webmock', '~> 3.0' +end diff --git a/packages/forest_admin_datasource_pylon/Gemfile-test b/packages/forest_admin_datasource_pylon/Gemfile-test new file mode 100644 index 000000000..58b4383b1 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/Gemfile-test @@ -0,0 +1,19 @@ +source 'https://rubygems.org' + +# Specify your gem's dependencies in forest_admin_datasource_pylon.gemspec +gemspec + +gem 'rake', '~> 13.0' +gem 'rubocop', '1.86.1' +gem 'rubocop-performance', '1.26.1' +gem 'rubocop-rspec', '3.9.0' + +group :development, :test do + gem 'forest_admin_datasource_customizer', path: '../forest_admin_datasource_customizer' + gem 'forest_admin_datasource_toolkit', path: '../forest_admin_datasource_toolkit' + gem 'rspec', '~> 3.0' + gem 'simplecov', '~> 0.22', require: false + gem 'simplecov-html', '~> 0.12.3' + gem 'simplecov_json_formatter', '~> 0.1.4' + gem 'webmock', '~> 3.0' +end diff --git a/packages/forest_admin_datasource_pylon/README.md b/packages/forest_admin_datasource_pylon/README.md new file mode 100644 index 000000000..21e43e805 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/README.md @@ -0,0 +1,159 @@ +# Forest — Pylon datasource + +Surface [Pylon](https://usepylon.com) issues, accounts, contacts, users and teams as Forest +collections, with filters, free-text search, relations, the conversation thread, custom fields, +CRUD writes and two action plugins. + +## Installation + +```ruby +# Gemfile +gem 'forest_admin_datasource_pylon' +``` + +## Usage + +```ruby +# app/lib/forest_admin_rails/create_agent.rb +ForestAdminAgent::Builder::AgentFactory.instance.add_datasource( + ForestAdminDatasourcePylon::Datasource.new(api_key: ENV['PYLON_API_KEY']) +) +``` + +A single Bearer token is the whole configuration. `GET /me` is the health check: it returns the +organization owning the token, which is enough to prove the credentials are usable. + +## Collections + +| Collection | Pylon resource | Read | Write | +| --- | --- | --- | --- | +| `PylonIssue` | `/issues` | `POST /issues/search`, cursor-paginated | create / update / delete | +| `PylonAccount` | `/accounts` | search + list, cursor-paginated | create / update / delete | +| `PylonContact` | `/contacts` | search + list, cursor-paginated | create / update / delete | +| `PylonUser` | `/users` | whole dataset, unpaginated | update | +| `PylonTeam` | `/teams` | whole dataset, unpaginated | create / update | + +`PylonIssue`, `PylonAccount` and `PylonContact` also carry their Pylon custom fields, introspected +at boot from `GET /custom-fields`. + +## What the API cannot do, and what this does about it + +Pylon is a ticketing API, not a database, and several things Forest asks for have no equivalent. +Where that happens this datasource **refuses with a message naming the reason** rather than +answering something that looks right and is not. All of these arrive as a 400 carrying the text. + +- **No aggregate endpoint and no total.** `aggregate` is refused on the cursor-backed collections: + counting the pages a walk collected would answer a fraction of a collection as if it were the + whole of it, so those are not advertised as countable. `PylonUser` and `PylonTeam` are the + exception — their endpoint hands back every record, so a count or a group over it is the figure + a server-side aggregation would have given. +- **No sort parameter on `/issues/search`.** Issues always come back newest first. A requested + order is reported in the log rather than silently swallowed. +- **No `id` filter on the search endpoints.** A primary-key lookup is short-circuited to + `GET /issues/{id}`, one request per id. That fan-out is capped per page (`MAX_ID_LOOKUPS`), and + a selection naming more ids *and* filtering them further is refused, since which records the + page holds could only be known by reading all of them. +- **No joins.** A condition on a relation is answered by reading the foreign collection for its + keys and sending them as an `in`. Past `MAX_RELATION_KEYS` the condition is refused. +- **Search and id lookup are different endpoints.** Combining a free-text search with a filter on + `id` is refused: neither endpoint can do the other's half. +- **Writes are one record per request.** A write reaching more records than one pass covers is + refused up front; one that fails halfway reports exactly which records were written, so a retry + can target the untouched ones rather than performing the write twice. + +## Rate limits + +Pylon meters **per endpoint**, not per token, from 30 to 300 requests a minute depending on the +endpoint. `RateLimits` holds the documented budget of every endpoint this client calls, and +`RateLimiter` spaces requests out so each one is spent rather than exceeded — a sliding window per +endpoint, in front of the 429 retry rather than instead of it. + +The limiter is a smoother, not a guarantee: past `DEFAULT_MAX_WAIT` a request goes out anyway and +the 429 retry takes over, with one log line per endpoint per window saying so. Under real +saturation — several agents or processes on the same token — the retry is the defence. + +To meter on your own side instead, pass `rate_limiter: nil`: + +```ruby +ForestAdminDatasourcePylon::Datasource.new(api_key: ENV['PYLON_API_KEY'], rate_limiter: nil) +``` + +## Boot-time introspection + +Custom fields are read while the datasource is being constructed, one call per object type. That +sits in front of a Rails boot, so it runs on its own connection with short timeouts and a single +quick retry (`boot_open_timeout`, `boot_timeout`, `boot_retry_policy`), and the first failure +stops the remaining object types from being tried. + +An introspection that fails costs the custom columns, not the datasource: the agent boots on the +native schema and says so in the log. + +## Action plugins + +### `CloseIssue` + +Moves the selected issues to a state, single and bulk, one request per issue with a per-id rescue +so one failure does not abort the batch. + +```ruby +@agent.collection :Ticket do |collection| + collection.use(ForestAdminDatasourcePylon::Plugins::CloseIssue, { + datasource: pylon_datasource, # required + issue_id_field: 'pylon_issue_id', # omit when the action sits on PylonIssue itself + state: 'closed', # default + scopes: %i[single bulk] # default + }) +end +``` + +Two things to know before granting it. + +**Which scope bounds it.** The state is written straight through the Pylon client rather than +through `PylonIssue`, because the action is registered on the host collection — but the ids were +read before that write, through the collection the action sits on, and the agent intersects the +operator's scope into the filter that read them. So the scope that bounds this is the scope of the +**host** collection: mounted on `PylonIssue`, a scope or segment on `PylonIssue` bounds exactly what +it closes; mounted on a business collection with `issue_id_field`, what bounds it is the records the +operator may see there, and the issue ids those records carry. In that second form the column is +the authority — an operator who can write it can name any Pylon issue — so treat it as one. + +**The batch is capped at `MAX_TARGETS` (20) issues**, the budget a filter-driven write already gets: +Pylon takes one request per issue, so a wider selection is a long run of sequential writes the +request may time out on, leaving the issues closed up to that point closed and reporting which ones +to nobody. Past the cap the run is refused before its first write, the way `update` and `delete` +refuse a selection costing more than `MAX_WRITE_REQUESTS`. The cap counts the issues named, not the +records selected: a column of issue ids is not a key, so a hundred host records naming ten issues is +a batch of ten. + +### `CreateIssueWithNotification` + +Creates an issue from a form and notifies the requester, optionally writing the new issue id back +onto the host record. + +```ruby +@agent.collection :Customer do |collection| + collection.use(ForestAdminDatasourcePylon::Plugins::CreateIssueWithNotification, { + datasource: pylon_datasource, # required + sender_email: 'support@example.com', # required when the destination is email + issue_id_field: 'pylon_issue_id', # optional id writeback + email_templates: [{ title: 'Outage', content: '

Sorry {{ record.name }}

' }] + }) +end +``` + +`{{ record.field }}` tokens are interpolated from the host record, HTML-escaped in the message +body. Template titles must be unique and cannot be `"No template"`, which names the option that +picks none of them. + +The message body is HTML the operator writes and Pylon delivers to the requester, and the +requester address is a free-text field. Restrict both actions to the roles that should be able to +send mail on your organization's behalf. + +## Development + +```bash +cd packages/forest_admin_datasource_pylon +bundle install +bundle exec rspec +bundle exec rubocop +``` diff --git a/packages/forest_admin_datasource_pylon/Rakefile b/packages/forest_admin_datasource_pylon/Rakefile new file mode 100644 index 000000000..4c774a2bf --- /dev/null +++ b/packages/forest_admin_datasource_pylon/Rakefile @@ -0,0 +1,6 @@ +require 'bundler/gem_tasks' +require 'rspec/core/rake_task' + +RSpec::Core::RakeTask.new(:spec) + +task default: :spec diff --git a/packages/forest_admin_datasource_pylon/forest_admin_datasource_pylon.gemspec b/packages/forest_admin_datasource_pylon/forest_admin_datasource_pylon.gemspec new file mode 100644 index 000000000..ffe0ac579 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/forest_admin_datasource_pylon.gemspec @@ -0,0 +1,36 @@ +lib = File.expand_path('lib', __dir__) +$LOAD_PATH.unshift lib unless $LOAD_PATH.include?(lib) + +require_relative 'lib/forest_admin_datasource_pylon/version' + +Gem::Specification.new do |spec| + spec.name = 'forest_admin_datasource_pylon' + spec.version = ForestAdminDatasourcePylon::VERSION + spec.authors = ['Forest Admin'] + spec.email = ['contact@forestadmin.com'] + spec.homepage = 'https://www.forestadmin.com' + spec.summary = 'Pylon datasource for Forest Admin Ruby agent.' + spec.description = 'Surface Pylon issues, accounts, contacts, users and teams as Forest Admin collections.' + spec.license = 'GPL-3.0' + spec.required_ruby_version = '>= 3.0.0' + + spec.metadata['homepage_uri'] = spec.homepage + spec.metadata['source_code_uri'] = 'https://github.com/ForestAdmin/agent-ruby' + spec.metadata['changelog_uri'] = 'https://github.com/ForestAdmin/agent-ruby/blob/main/CHANGELOG.md' + spec.metadata['rubygems_mfa_required'] = 'false' + + spec.files = Dir.chdir(__dir__) do + `git ls-files -z`.split("\x0").reject do |f| + (File.expand_path(f) == __FILE__) || + f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor Gemfile]) + end + end + spec.bindir = 'exe' + spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) } + spec.require_paths = ['lib'] + + spec.add_dependency 'activesupport', '>= 6.1' + spec.add_dependency 'faraday', '~> 2.0' + spec.add_dependency 'faraday-retry', '~> 2.0' + spec.add_dependency 'zeitwerk', '~> 2.3' +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon.rb new file mode 100644 index 000000000..6dcc910d2 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon.rb @@ -0,0 +1,70 @@ +require_relative 'forest_admin_datasource_pylon/version' +require 'json' +require 'logger' +require 'set' +require 'uri' +require 'zeitwerk' +require 'faraday' +require 'faraday/retry' +require 'forest_admin_datasource_toolkit' + +loader = Zeitwerk::Loader.for_gem +loader.setup + +module ForestAdminDatasourcePylon + class Error < StandardError; end + class ConfigurationError < Error; end + + # A filter Pylon cannot express. It descends from the toolkit's ValidationError + # rather than from the package's own Error so the agent answers 400 carrying + # the message instead of a 500 "Unexpected error": every one of these names a + # condition the operator set and can change, and the message is the only place + # they learn which one. + class UnsupportedOperatorError < ForestAdminDatasourceToolkit::Exceptions::ValidationError; end + + # The three write errors below descend from ValidationError for that same + # reason: each names something the operator did and can undo. + + # A verb Pylon's API has no endpoint for, a field it only accepts in the other + # direction, or a write reaching more records than one pass may cover. + class UnsupportedWriteError < ForestAdminDatasourceToolkit::Exceptions::ValidationError; end + + # A write Pylon performed on some of its records and then failed on: one + # record is one request, so the ones before the failure stay written, and a + # retry of the whole selection would write them a second time. + class PartialWriteError < ForestAdminDatasourceToolkit::Exceptions::ValidationError; end + + # A write Pylon itself refused, carrying the reason it gave — the likeliest + # way a write fails. Only its 4xx travels this way: a 5xx or a dropped + # connection is not the operator's to act on and stays the APIError it was. + class WriteRejectedError < ForestAdminDatasourceToolkit::Exceptions::ValidationError; end + + # Raised when a Pylon API call fails. Carries the HTTP status and the + # (parsed) response body so callers — smart actions in particular — can + # surface Pylon's own validation message instead of a generic string. + class APIError < Error + attr_reader :status, :body + + def initialize(message, status: nil, body: nil) + super(message) + @status = status + @body = body + end + end + + class << self + attr_writer :logger + + def logger + @logger ||= default_logger + end + + private + + def default_logger + return Rails.logger if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger + + Logger.new($stderr).tap { |l| l.progname = 'forest_admin_datasource_pylon' } + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/client.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/client.rb new file mode 100644 index 000000000..517b64299 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/client.rb @@ -0,0 +1,384 @@ +module ForestAdminDatasourcePylon + # Long by line count only: the public surface is one explicit method per Pylon + # endpoint, each delegating to the shared helpers below. + class Client # rubocop:disable Metrics/ClassLength + include Writes + + MAX_SEARCH_LIMIT = 1000 + + # Bounds `collect_pages`, which asks for a whole dataset rather than a + # window: the endpoints it reads answer in one response, so reaching this + # many pages means the API started paginating on its own and the walk is + # spending more of the per-minute budget than the answer is worth. + MAX_COLLECTED_PAGES = 10 + + # `next_cursor` is nil as soon as Pylon stops advertising a next page, so + # callers never have to know how the absence is spelled on the wire. + SearchPage = Struct.new(:records, :next_cursor, keyword_init: true) + + def initialize(configuration) + @configuration = configuration + end + + # Health check: Pylon returns the details of the organization owning the + # token, which is enough to prove the credentials are usable. + def me + must_succeed('me') { extract_data(connection.get('me').body) } + end + + # POST /issues/search accepts an empty body and then returns the most recent + # issues, ordered by `created_at` descending. + def search_issues(limit:, cursor: nil, filter: nil, search_text: nil) + search_resource('issues/search', limit: limit, cursor: cursor, filter: filter, search_text: search_text) + end + + # Accepts either the UUID or the issue number. + def fetch_issue(id) + fetch_resource('issues', id) + end + + # The whole conversation of an issue, oldest message first, or nil when the + # thread could not be read. + # + # `limit` is left out on purpose: Pylon then answers with every message in a + # single response. Asking for a page would hand back the OLDEST messages and + # cut the most recent ones off, which is the half of a conversation nobody + # opens a ticket to read. + # + # The cursor is still followed, defensively: Pylon paginates this endpoint + # when asked to, so a future default page size stays handled rather than + # silently truncating the thread. + def fetch_issue_messages(issue_id) + path = "issues/#{Faraday::Utils.escape(issue_id)}/messages" + + best_effort("fetch_issue_messages(#{issue_id})", default: nil) { must_succeed(path) { collect_pages(path) } } + end + + def search_accounts(limit:, cursor: nil, filter: nil, search_text: nil) + search_resource('accounts/search', limit: limit, cursor: cursor, filter: filter, search_text: search_text) + end + + def list_accounts(limit:, cursor: nil) + list_resource('accounts', limit: limit, cursor: cursor) + end + + # Accepts either the Pylon UUID or the account's external id. + def fetch_account(id) + fetch_resource('accounts', id) + end + + def search_contacts(limit:, cursor: nil, filter: nil, search_text: nil) + search_resource('contacts/search', limit: limit, cursor: cursor, filter: filter, search_text: search_text) + end + + # GET /contacts is paginated exactly like GET /accounts even though the + # OpenAPI spec forgets to document its query parameters. + def list_contacts(limit:, cursor: nil) + list_resource('contacts', limit: limit, cursor: cursor) + end + + def fetch_contact(id) + fetch_resource('contacts', id) + end + + # GET /users is unpaginated. Deactivated agents are included by default so + # that assignees of older issues stay resolvable. + def fetch_users(include_deactivated: true) + fetch_all('users', 'include_deactivated' => include_deactivated) + end + + def fetch_user(id) + fetch_resource('users', id) + end + + # GET /teams is unpaginated and takes no parameter. + def fetch_teams + fetch_all('teams') + end + + def fetch_team(id) + fetch_resource('teams', id) + end + + # The custom-field definitions of one object type. `object_type` is + # mandatory on this endpoint, so a schema spanning several collections costs + # one call per collection rather than one call in total. + # + # Degrades to nil: this is read while the agent boots, and a token missing + # the permission — or a Pylon that happens to be down right then — has to + # cost the operator the custom columns, not the whole datasource. + # + # nil rather than an empty list so the caller can tell a failure from an + # organization that defined no custom field, the two costing very different + # things: the second says nothing about the next object type, the first says + # it will almost certainly fail the same way. + # + # Which is why it goes through `boot_connection`: a call declared best-effort + # has no business holding the boot for the minutes the resilient policy is + # willing to spend waiting a 429 out. The bound is per request, and the walk + # is allowed the same pages as any other — one is what this endpoint answers + # with, having a handful of definitions to return per object type. + def fetch_custom_fields(object_type) + params = { 'object_type' => object_type } + + best_effort("fetch_custom_fields(#{object_type})", default: nil) do + must_succeed('custom-fields') { collect_pages('custom-fields', params, conn: boot_connection) } + end + end + + # The two memoized connections carry the bearer token in their headers, and + # `Faraday::Connection#inspect` prints those in clear: a Client reaching an + # `inspect` by accident leaks the credential whatever `Configuration` does + # about its own. Masked here for that reason, and not only for symmetry. + def inspect + "#<#{self.class.name} base_url=#{@configuration.url.inspect}>" + end + + private + + def search_resource(path, limit:, cursor: nil, filter: nil, search_text: nil) + body = { 'limit' => clamp_limit(limit) } + body['cursor'] = cursor unless blank?(cursor) + body['filter'] = filter unless filter.nil? + body['search_text'] = search_text unless blank?(search_text) + + must_succeed(path) { to_search_page(connection.post(path, body).body, path) } + end + + # `limit` is mandatory on the paginated GET endpoints, unlike their POST + # /search counterparts which default it server-side. + def list_resource(path, limit:, cursor: nil) + params = { 'limit' => clamp_limit(limit) } + params['cursor'] = cursor unless blank?(cursor) + + must_succeed(path) { to_search_page(connection.get(path, params).body, path) } + end + + def fetch_all(path, params = {}) + must_succeed(path) { extract_list(connection.get(path, params).body, path) } + end + + # Every record of a cursor-paginated GET, no window asked for and no limit + # sent. `CursorWalker` answers the other question — the offset/limit window a + # list view asks for — and is not what this needs. + # + # `params` ride along on every page, cursor included: a mandatory parameter + # dropped on the second request answers a different question than the first. + # + # An empty page, a cursor that does not move and a cursor already followed + # all stop the loop: none happens today, but a walk driven by a remote value + # stops on its own terms rather than collecting the same page twice over. + # + # `conn` is what a caller reading on the boot path hands its own connection + # through: every page of the walk is then bounded like the first. + def collect_pages(path, params = {}, conn: connection) + records = [] + cursor = nil + seen = Set.new + pages = 0 + + loop do + query = cursor.nil? ? params : params.merge('cursor' => cursor) + page = to_search_page(conn.get(path, query).body, path) + records.concat(page.records) + pages += 1 + break if page.next_cursor.nil? || page.records.empty? || !seen.add?(page.next_cursor) + + if pages >= MAX_COLLECTED_PAGES + log_pagination_cap(path, pages, records.size) + break + end + + cursor = page.next_cursor + end + + records + end + + def log_pagination_cap(path, pages, collected) + ForestAdminDatasourcePylon.logger.warn( + "[forest_admin_datasource_pylon] Stopped paginating #{path} after #{pages} page(s) / " \ + "#{collected} record(s); the rest is left out." + ) + end + + # The id comes from operator-supplied filter values, so it is escaped before + # being joined to the path. + def fetch_resource(resource, id) + path = "#{resource}/#{Faraday::Utils.escape(id)}" + must_succeed(path) { extract_record(connection.get(path).body, path) } + end + + def clamp_limit(limit) + value = limit.to_i + return 1 if value < 1 + + [value, MAX_SEARCH_LIMIT].min + end + + # Pylon only includes the `pagination` block when a next page exists, so an + # absent block, `has_next_page: false` and an empty cursor all mean "done". + def to_search_page(body, operation) + pagination = body.is_a?(Hash) ? body['pagination'] : nil + cursor = pagination.is_a?(Hash) && pagination['has_next_page'] ? pagination['cursor'] : nil + + SearchPage.new(records: extract_list(body, operation), next_cursor: blank?(cursor) ? nil : cursor) + end + + # A read expects `data` to hold what it asked for, and anything else broke + # the contract. `extract_data` hands an envelope carrying no `data` straight + # back, which `Array()` would then split into `[key, value]` pairs and the + # collection would serialize into rows holding nothing -- a page that looks + # answered and is empty. Refused instead, the way the write path already + # refuses the same shape: see `Client::Writes#extract_written`. + # + # An absent or null `data` stays the empty answer it is: Pylon spells "no + # record" that way, and a search matching nothing is not a broken contract. + def extract_list(body, operation) + data = extract_data(body) + return [] if data.nil? + return data if data.is_a?(Array) + + refuse_body_shape(body, operation, "'data' is not a list") + end + + # The single-record half of the same check. nil travels: it is what a + # caller reads as "no such record". + def extract_record(body, operation) + data = extract_data(body) + return data if data.nil? || data.is_a?(Hash) + + refuse_body_shape(body, operation, "'data' is not a record") + end + + def blank?(value) + value.nil? || value.to_s.empty? + end + + # Pylon wraps payloads in { "data": ..., "pagination": ..., "request_id": ... }. + def extract_data(body) + return nil if body.nil? || body == '' + return body['data'] if body.is_a?(Hash) && body.key?('data') + + body + end + + def must_succeed(operation) + yield + rescue Faraday::Error => e + raise api_error(operation, e) + rescue APIError + # Already mapped, with its status intact; re-wrapping would erase it. + raise + rescue StandardError => e + raise APIError, "Pylon API call failed: #{operation}: #{e.class}: #{e.message}" + end + + # For the calls whose result enriches a page rather than being the page: the + # failure is reported and the default returned, so a degraded thread or a + # missing enrichment costs the operator a column, not the record they opened. + def best_effort(operation, default:) + yield + rescue StandardError => e + ForestAdminDatasourcePylon.logger.warn( + "[forest_admin_datasource_pylon] #{operation} failed; degrading: #{e.class}: #{e.message}" + ) + default + end + + # Builds an APIError preserving the HTTP status and Pylon's own error body so + # smart actions can show the operator the real reason instead of "failed". + def api_error(operation, error) + response = error.respond_to?(:response) ? error.response : nil + status = response.is_a?(Hash) ? response[:status] : nil + body = parse_body(response.is_a?(Hash) ? response[:body] : nil) + detail = error_detail(status, body) || "#{error.class}: #{error.message}" + APIError.new("Pylon API call failed: #{operation}: #{detail}", status: status, body: body) + end + + def error_detail(status, body) + return nil unless status + + "HTTP #{status} #{error_message(body)}".strip + end + + def error_message(parsed) + return parsed.to_s[0, 500] unless parsed.is_a?(Hash) + + nested = parsed['error'] + message = parsed['message'] || (nested.is_a?(Hash) ? nested['message'] : nested) || + join_errors(parsed['errors']) + message = parsed.to_json if message.to_s.empty? + # Truncate before appending: the request_id is what support needs, so it + # must not be the first thing a long error body pushes out. + append_request_id(message.to_s[0, 500], parsed['request_id']) + end + + def append_request_id(message, request_id) + return message unless request_id + + "#{message} (request_id: #{request_id})" + end + + def join_errors(errors) + Array(errors).filter_map { |e| e.is_a?(Hash) ? (e['message'] || e['detail']) : e }.join('; ') + end + + def parse_body(body) + return body unless body.is_a?(String) && !body.empty? + + JSON.parse(body) + rescue JSON::ParserError + body + end + + def connection + @connection ||= build_connection( + retry_policy: @configuration.retry_policy, + timeout: @configuration.timeout, + open_timeout: @configuration.open_timeout + ) + end + + # For what is read while the datasource is being constructed: short timeouts + # and one quick retry, so the introspection cannot turn a Pylon that is down + # into minutes of Rails boot. Memoized separately from `connection`, which + # keeps the patience every later request is entitled to. + def boot_connection + @boot_connection ||= build_connection( + retry_policy: @configuration.boot_retry_policy, + timeout: @configuration.boot_timeout, + open_timeout: @configuration.boot_open_timeout + ) + end + + # Middleware order is deliberate: `raise_error` sits outside the JSON parser + # so it raises with an already-parsed body, and `retry` sits innermost so it + # inspects raw statuses — behind `raise_error` it would never see a 429. + # + # The throttle goes inside `retry`, which is what makes a replay wait for a + # slot like a first attempt: outside it, the middleware would run once for a + # request that reached Pylon three times. + # + # How long a request is allowed to take is the caller's to state, everything + # else being the same on every connection this builds: the limiter included, + # a second connection metering in a window of its own spending the budget of + # the endpoint twice over. + def build_connection(retry_policy:, timeout:, open_timeout:) + Faraday.new(url: @configuration.url) do |f| + f.request :json + f.response :raise_error + f.response :json + f.request :retry, **retry_policy.to_faraday_options + if @configuration.rate_limiter + f.use Throttle, limiter: @configuration.rate_limiter, base_path: @configuration.base_path + end + f.headers['Authorization'] = "Bearer #{@configuration.api_key}" + f.headers['Accept'] = 'application/json' + f.headers['User-Agent'] = "forest_admin_datasource_pylon/#{VERSION}" + f.options.open_timeout = open_timeout + f.options.timeout = timeout + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/client/writes.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/client/writes.rb new file mode 100644 index 000000000..9a50e9cb9 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/client/writes.rb @@ -0,0 +1,88 @@ +module ForestAdminDatasourcePylon + class Client + # The write half of the client: one explicit method per Pylon write + # endpoint, each delegating to the shared helpers below. + # + # Nothing here degrades. `best_effort` exists for the calls whose result + # enriches a page — a thread that could not be read costs a column — where a + # write that silently did nothing would tell the operator their edit landed. + # + # Pylon exposes no POST or DELETE on users, and no DELETE on teams. The + # collections answer those, not the client, which only spells the endpoints + # that exist. + module Writes + # `title` and `body_html` are the two fields POST /issues requires. + def create_issue(attributes) = post_resource('issues', attributes) + def update_issue(id, attributes) = patch_resource('issues', id, attributes) + def delete_issue(id) = delete_resource('issues', id) + + def create_account(attributes) = post_resource('accounts', attributes) + def update_account(id, attributes) = patch_resource('accounts', id, attributes) + def delete_account(id) = delete_resource('accounts', id) + + def create_contact(attributes) = post_resource('contacts', attributes) + def update_contact(id, attributes) = patch_resource('contacts', id, attributes) + def delete_contact(id) = delete_resource('contacts', id) + + def create_team(attributes) = post_resource('teams', attributes) + def update_team(id, attributes) = patch_resource('teams', id, attributes) + + def update_user(id, attributes) = patch_resource('users', id, attributes) + + private + + def post_resource(resource, attributes) + operation = "create(#{resource})" + + must_succeed(operation) { extract_written(connection.post(resource, attributes).body, operation) } + end + + # The id comes from the record the operator acted on, so it is escaped + # before being joined to the path, like every read does. + def patch_resource(resource, id, attributes) + path = "#{resource}/#{Faraday::Utils.escape(id)}" + operation = "update(#{path})" + + must_succeed(operation) { extract_updated(connection.patch(path, attributes).body, operation) } + end + + # Answers true rather than the body: Pylon returns 200 or 204 with nothing + # worth reading, and a caller has no record left to serialize. + def delete_resource(resource, id) + path = "#{resource}/#{Faraday::Utils.escape(id)}" + + must_succeed("delete(#{path})") do + connection.delete(path) + true + end + end + + # Pylon answers a write with the written record under `data`. Anything else + # broke the contract: `extract_data` hands the body back untouched when + # `data` is absent, which is what a read wants and a write must not accept + # — the collection would serialize the envelope into a record with no id. + def extract_written(body, operation) + record = body['data'] if body.is_a?(Hash) + return record if record.is_a?(Hash) + + refuse_body_shape(body, operation, "missing 'data'") + end + + # An update discards its record, so a 204, an empty body or a null `data` + # is the write having landed with nothing to hand back: raising there would + # report a failure on a record Pylon already patched, and abort the records + # a bulk edit had left to write. + def extract_updated(body, operation) + record = body['data'] if body.is_a?(Hash) + return record if record.nil? || record.is_a?(Hash) + + refuse_body_shape(body, operation, "'data' is not a record") + end + + def refuse_body_shape(body, operation, detail) + raise APIError, + "Pylon API #{operation} returned an unexpected body shape (#{detail}): #{body.inspect}" + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/account.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/account.rb new file mode 100644 index 000000000..42bb57aca --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/account.rb @@ -0,0 +1,47 @@ +module ForestAdminDatasourcePylon + module Collections + class Account < CursorCollection + include SchemaDefinition + include Serializer + + # Pylon reads an account's type back as `type` and takes it as + # `account_type`. + RENAMES = { 'type' => 'account_type' }.freeze + + # An account is created enabled; only `PATCH /accounts/{id}` disables one. + UPDATE_ONLY = %w[is_disabled].freeze + + def initialize(datasource, custom_fields: []) + super(datasource, 'PylonAccount', custom_fields: custom_fields, searchable: true) + end + + protected + + def filter_table = ApiFilters + + def create_record(payload) = datasource.client.create_account(payload) + def update_record(id, payload) = datasource.client.update_account(id, payload) + def delete_record(id) = datasource.client.delete_account(id) + + def update_only_fields = UPDATE_ONLY + def payload_renames = RENAMES + + def unsortable_warning + '[forest_admin_datasource_pylon] PylonAccount cannot honour the requested order; neither GET /accounts ' \ + 'nor POST /accounts/search takes a sort parameter, so accounts come back in the order the API imposes.' + end + + def search_page(limit:, cursor:, filter:, search_text:) + datasource.client.search_accounts(limit: limit, cursor: cursor, filter: filter, search_text: search_text) + end + + def list_page(limit:, cursor:) + datasource.client.list_accounts(limit: limit, cursor: cursor) + end + + def fetch_one(id) + datasource.client.fetch_account(id) + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/account/api_filters.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/account/api_filters.rb new file mode 100644 index 000000000..5a866dc08 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/account/api_filters.rb @@ -0,0 +1,43 @@ +module ForestAdminDatasourcePylon + module Collections + class Account < CursorCollection + # The allow-list of `POST /accounts/search`, transcribed from the API + # reference: a field absent from this table cannot be filtered at all, and + # an operator absent from a field's map is rejected by Pylon. + # + # It is the single source of truth for filtering — `define_schema` derives + # every column's `filter_operators` from it, so this collection declares no + # filter the translator would then refuse; the absence family the agent + # derives on top of it is the exception `Query::OperatorMaps::Table` + # describes. + module ApiFilters + Maps = Query::OperatorMaps + + extend Maps::Table + + CUSTOM_FIELD_OPS = Maps::CUSTOM_FIELD_OPS + + # `id` is filtered server-side here, which is what spares this + # collection the primary-key short-circuit Issue needs. + # + # `name` gets SUBSTRING rather than FULL_TEXT: the endpoint accepts + # `string_contains` but no negation of it. `external_ids` is left out + # entirely although the endpoint filters it — the API matches the bare + # external-id strings while the column shows `{external_id, label}` + # objects, so the filter would run on something the operator cannot see. + # The account read endpoint accepts an external id in place of the + # primary key, which is the way to reach a record by one. + # + # No time field is filterable: `created_at`, `updated_at` and + # `latest_customer_activity_time` are absent from the allow-list. + API_FILTERS = { + 'id' => { ops: Maps::EQUALITY }, + 'name' => { ops: Maps::EQUALITY.merge(Maps::SUBSTRING) }, + 'domains' => { ops: Maps::MEMBERSHIP }, + 'tags' => { ops: Maps::MEMBERSHIP }, + 'owner_id' => { ops: Maps::EQUALITY.merge(Maps::PRESENCE) } + }.freeze + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/account/schema_definition.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/account/schema_definition.rb new file mode 100644 index 000000000..bea8623a8 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/account/schema_definition.rb @@ -0,0 +1,91 @@ +module ForestAdminDatasourcePylon + module Collections + class Account < CursorCollection + # A column is writable when `POST /accounts` or `PATCH /accounts/{id}` + # accepts it, in the shape it is read under — the Json columns holding + # objects rather than plain strings are left read-only, see below. No + # column is sortable — neither `GET /accounts` nor `POST /accounts/search` + # exposes a sort parameter, so advertising a sortable column would let the + # UI ask for an order the API cannot honour. + # + # Filter operators are not chosen here: they come from + # `ApiFilters::API_FILTERS`, which mirrors the allow-list of the API. A + # column missing from that table gets no operator, so the UI offers no + # filter of this collection's own that Pylon would refuse — the absence + # family the agent derives above the datasource being the exception + # `Query::OperatorMaps::Table` describes. + module SchemaDefinition + OneToManySchema = BaseCollection::OneToManySchema + + private + + def define_schema + define_identity_fields + define_domain_fields + define_ownership_fields + define_integration_fields + define_time_fields + end + + # The reverse sides of the two ManyToOne relations pointing here. Both + # `/issues/search` and `/contacts/search` filter `account_id` + # server-side, so a related list is one request and no in-memory pass. + # + # `owner_id` stays a plain column: it does point at a PylonUser, and the + # embedder would resolve it like any other key, but nothing in the panel + # asks for the owner of an account yet. + def define_relations + add_field('issues', OneToManySchema.new(foreign_collection: 'PylonIssue', + origin_key: 'account_id', origin_key_target: 'id')) + add_field('contacts', OneToManySchema.new(foreign_collection: 'PylonContact', + origin_key: 'account_id', origin_key_target: 'id')) + end + + def define_identity_fields + add_column('id', 'String', is_primary_key: true) + add_column('name', 'String', writable: true) + # Left as String rather than Enum: Pylon ships customer / partner / + # prospect but lets an organization define its own account types. It + # is written under the name `account_type`, see `Account::RENAMES`. + add_column('type', 'String', writable: true) + # Writable on an update only: an account is created enabled. + add_column('is_disabled', 'Boolean', writable: true) + end + + # `domain` and `primary_domain` carry the same value; both are kept + # because Pylon returns both, and only the `domains` list is filterable. + # Neither is writable: `domains` is the list the API takes, and writing + # one of its two projections would leave the other stale. + def define_domain_fields + add_column('domain', 'String') + add_column('primary_domain', 'String') + add_column('domains', 'Json', writable: true) + add_column('tags', 'Json', writable: true) + end + + def define_ownership_fields + # Flattened from the nested `{ id: ..., email: ... }` object Pylon + # returns; a plain column, see `define_relations` above. + add_column('owner_id', 'String', writable: true) + # Read-only although the endpoint takes it: the column shows + # `{external_id, label}` objects, and the write shape the reference + # documents is not that one — writing one for the other would replace + # the ids of the account with something it cannot read. + add_column('external_ids', 'Json') + end + + # Both belong to the integrations Pylon syncs them from: `crm_settings` + # is absent from every write endpoint, and `channels` — which they do + # take — holds objects, like `external_ids` above. + def define_integration_fields + add_column('channels', 'Json') + add_column('crm_settings', 'Json') + end + + def define_time_fields + %w[created_at updated_at latest_customer_activity_time].each { |field| add_column(field, 'Date') } + end + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/account/serializer.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/account/serializer.rb new file mode 100644 index 000000000..431c53123 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/account/serializer.rb @@ -0,0 +1,21 @@ +module ForestAdminDatasourcePylon + module Collections + class Account < CursorCollection + module Serializer + NATIVE_FIELDS = %w[id name type is_disabled domain primary_domain domains tags external_ids + channels crm_settings created_at updated_at + latest_customer_activity_time].freeze + + private + + def serialize(account) + attrs = account.is_a?(Hash) ? account : {} + record = NATIVE_FIELDS.to_h { |field| [field, attrs[field]] } + record['owner_id'] = nested_id(attrs['owner']) + add_custom_field_values(record, attrs['custom_fields']) + record + end + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/base_collection.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/base_collection.rb new file mode 100644 index 000000000..550fedc4e --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/base_collection.rb @@ -0,0 +1,563 @@ +module ForestAdminDatasourcePylon + module Collections + class BaseCollection < ForestAdminDatasourceToolkit::Collection + include Writes + + ColumnSchema = ForestAdminDatasourceToolkit::Schema::ColumnSchema + ManyToOneSchema = ForestAdminDatasourceToolkit::Schema::Relations::ManyToOneSchema + OneToManySchema = ForestAdminDatasourceToolkit::Schema::Relations::OneToManySchema + Operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators + Branch = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + Leaf = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + ConditionTreeFactory = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::ConditionTreeFactory + Equivalent = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::ConditionTreeEquivalent + SortFactory = ForestAdminDatasourceToolkit::Components::Query::SortUtils::SortFactory + Filter = ForestAdminDatasourceToolkit::Components::Query::Filter + Page = ForestAdminDatasourceToolkit::Components::Query::Page + Projection = ForestAdminDatasourceToolkit::Components::Query::Projection + + # `residual` holds the conditions left over once the primary-key leaf has + # been taken out of the tree, for the caller to apply in memory. + IdLookup = Struct.new(:ids, :residual, keyword_init: true) + + # Mirrors the operators `ConditionTreeLeaf#match` evaluates natively; any + # other operator needs an equivalence for the column's type to be + # evaluable in memory. + IN_MEMORY_OPERATORS = [Operators::IN, Operators::EQUAL, Operators::LESS_THAN, Operators::GREATER_THAN, + Operators::MATCH, Operators::STARTS_WITH, Operators::ENDS_WITH, + Operators::LONGER_THAN, Operators::SHORTER_THAN, Operators::INCLUDES_ALL, + Operators::NOT_IN, Operators::NOT_EQUAL, Operators::NOT_CONTAINS].freeze + + # The operators `ConditionTreeLeaf#match` evaluates by dereferencing the + # column value without a nil guard, unlike the string operators which all + # test `is_a?(String)` first. They need the guard added here. + NIL_UNSAFE_OPERATORS = [Operators::LESS_THAN, Operators::GREATER_THAN, Operators::INCLUDES_ALL].freeze + + # How many foreign keys a resolved relation condition may carry. Past this + # the condition is refused rather than truncated: keeping the first keys + # would answer a narrower question than the one asked, which is the very + # thing this datasource refuses — a result that looks filtered and is not. + MAX_RELATION_KEYS = 500 + + # What a resolved relation condition leaves behind when no foreign record + # matched it. It is not expressible as a filter — `FilterValue` refuses an + # empty `in`, whose Pylon meaning is undocumented and would read as "match + # everything" — so it travels as a marker the read answers with no record. + MATCHES_NOTHING = :pylon_matches_nothing + + attr_reader :custom_fields + + # Template method: subclasses implement `define_schema` and + # `define_relations` as hooks; ordering between them, custom-field + # registration, and the search flag is owned here so collisions are always + # evaluated against the final native schema. + # + # No `countable` counterpart: Pylon exposes no count endpoint and no total, + # and counting the pages a cursor walk collected would answer a fraction of + # a collection as if it were the whole of it — which is why `aggregate` + # refuses rather than approximates. See EXT-7. + def initialize(datasource, name, custom_fields: [], searchable: false, native_driver: nil) + super(datasource, name, native_driver) + define_schema + define_relations + @custom_fields = add_custom_fields(custom_fields) + enable_search if searchable + end + + # How a collection another one points at with a ManyToOne is read in bulk: + # the serialized records of `ids`, indexed by id, missing ids left out. + # + # Public because the caller is the pointing collection, a different object: + # going through the collection rather than through the client is what keeps + # a related record serialized by the collection owning its shape, instead + # of by a second field list kept in the embedder. + def records_indexed_by_id(_ids) + raise NotImplementedError, "#{self.class} did not implement records_indexed_by_id" + end + + # Pylon exposes no aggregate endpoint, and the pages of a cursor walk are + # not the dataset: a count or a group computed over them would look exact + # while answering a fraction. Every column is registered with + # `is_groupable: false` so the UI never offers one, and a chart built + # through the API anyway is refused here rather than through the + # contract's NotImplementedError, which reads as an oversight. + # + # FetchAllCollection, which does hold every record Pylon has, overrides + # this and answers exactly. + def aggregate(_caller, _filter, _aggregation, _limit = nil) + raise UnsupportedOperatorError, + "#{name} cannot be aggregated: Pylon exposes no aggregate endpoint, and counting or grouping the " \ + 'pages the agent walked would answer a fraction of the collection as if it were the whole of it.' + end + + protected + + # Pylon has no `id` filter operator on /issues/search, so collections + # short-circuit primary-key lookups to /resource/{id}. Ids are UUID + # strings — unlike Zendesk, nothing has to be coerced to an integer. + # + # The leaf is also pulled out of a top-level AND, because Forest sends + # `AND(id equal X, )` on a record detail as soon as a scope or a + # segment is set, and `id` is not a field Pylon can filter on. + # + # Every `id` leaf of that AND is taken, not the first: an `and` names the + # records all of its conditions name, so two of them intersect. Left as a + # residual, the second would have the cap applied around the wider set and + # refuse a selection narrower than it — and cost a request per id the + # intersection drops. + def extract_id_lookup(node) + ids = id_values(node) + return IdLookup.new(ids: ids, residual: nil) if ids + return nil unless and_branch?(node) + + named, rest = Array(node.conditions).partition { |condition| id_values(condition) } + return nil if named.empty? + + residual = ConditionTreeFactory.intersect(rest) + ensure_residual_appliable!(residual) + IdLookup.new(ids: intersect_ids(named), residual: guard_nil_comparisons(residual)) + end + + # Disjoint sets name no record, which is the empty lookup — no request — + # rather than a request per id of the first set for a page that is empty + # whatever they answer. + def intersect_ids(conditions) + conditions.map { |condition| id_values(condition) }.reduce(:&) + end + + # Pylon runs the free-text search inside its search endpoint, which the + # primary-key short-circuit does not go through, and neither the fields it + # covers nor its fuzziness can be reproduced in memory. Answering with the + # unsearched record would be the very thing this datasource refuses: a + # result that looks filtered and is not. + def ensure_searchless_lookup!(filter) + search = filter&.search + return if search.nil? || search.to_s.strip.empty? + + raise UnsupportedOperatorError, + "A search cannot be combined with a filter on 'id': Pylon searches through its search endpoint, " \ + 'which cannot filter on id, while an id is read through its own endpoint, which cannot search. ' \ + 'Clear the search or drop the id condition.' + end + + # Yields the filter with every relation condition resolved into a condition + # on this collection's own columns, so the routes below — the search and + # the primary-key lookup, which applies its leftovers in memory — never see + # a `relation:field` leaf. + # + # Answers with no record at all, and no request, when the resolution found + # nothing to match: see `MATCHES_NOTHING`. + def with_resolved_relations(caller, filter) + tree = filter&.condition_tree + return yield(filter) unless tree&.some_leaf { |leaf| leaf.field.to_s.include?(':') } + + resolved = resolve_relation_conditions(caller, tree) + return [] if resolved == MATCHES_NOTHING + + yield(filter.override(condition_tree: resolved)) + end + + # Forest asks for an offset/limit window, Pylon hands out cursor pages: the + # walker bridges the two, `search_page` performs one call, and the records + # it collected are serialized by the collection. + def search_records(caller, filter) + pylon_filter = build_pylon_filter(caller, filter) + search_text = filter&.search + offset, limit = translate_page(filter&.page) + + records = walker.walk(offset: offset, limit: limit) do |batch, cursor| + search_page(limit: batch, cursor: cursor, filter: pylon_filter, search_text: search_text) + end + records.map { |record| serialize(record) } + end + + # One page of the walk, as a Client::SearchPage: the endpoint and its + # parameter names belong to the collection, the walk does not. + def search_page(limit:, cursor:, filter:, search_text:) + raise NotImplementedError, "#{self.class} did not implement search_page" + end + + # Sliced after the lookup, not before, so ids that resolved to nothing + # (404) do not eat into the requested window. + # + # A filter carrying no page — or a page naming no limit — asks for every + # record it matched, and the records are already in hand: there is no + # window to cut. How far the walk that collected them went is a different + # question, answered by `translate_page` and the caps of the walker. + def page_window(records, filter) + page = filter&.page + return records if page.nil? + + offset = page.offset.to_i.clamp(0, nil) + limit = page.limit.to_i + return records.drop(offset) unless limit.positive? + + records[offset, limit] || [] + end + + def build_pylon_filter(caller, filter) + tree = filter&.condition_tree + ensure_no_stray_id!(tree) + Query::ConditionTreeTranslator.call(tree, api_filters: api_filters, timezone: timezone_for(caller)) + end + + # The `ApiFilters` module of the collection, whose table is the single + # source of truth for what its endpoint filters. The empty table is the + # default: a collection read whole and filtered in memory filters nothing + # server-side. + def filter_table = Query::OperatorMaps::EmptyTable + + # What the endpoint filters server-side: the table of the collection, plus + # one entry per custom field — filtered through the very Pylon slug it is + # read by, with the operators the integrator declared on the column. + def api_filters + @api_filters ||= custom_fields.each_with_object(filter_table::API_FILTERS.dup) do |cf, filters| + filters[cf[:column_name]] = filter_table.for_custom_field(cf[:schema]) + end + end + + # A native column: read-only unless the collection declares it `writable`, + # and never groupable, as no Pylon endpoint aggregates. It is not sortable + # either, the ColumnSchema default, because no search endpoint takes a sort + # parameter. Filter operators are not chosen here: they come from + # `filter_table`, which mirrors the allow-list of the API, so a column + # missing from it gets none and the UI offers no filter Pylon would refuse. + def add_column(name, type, is_primary_key: false, writable: false) + add_field(name, ColumnSchema.new(column_type: type, + filter_operators: filter_table.forest_operators(name), + is_primary_key: is_primary_key, + is_groupable: false, + is_read_only: !writable)) + end + + # A record read through the endpoint of an id that is not the primary key + # it answered with. `GET /accounts/{id}` takes an external id and + # `GET /issues/{id}` an issue number, so the record a lookup hands back + # can carry an `id` other than the one the filter asked for: keeping it + # would answer `id equals ` with a row that does not match, where + # the same filter combined with a scope — which goes through the search + # endpoint instead — answers nothing at all. + def matches_id?(record, id) + record['id'].to_s == id.to_s + end + + # An order no endpoint honours is reported rather than silently swallowed: + # the rows come back in whatever order the API imposes. + def warn_unsortable(sort) + return if sort.nil? || sort.empty? || default_pk_sort?(sort) + return if translate_sort(sort, sortable_fields).first + + ForestAdminDatasourcePylon.logger.warn(unsortable_warning) + end + + # Overridden by collections whose endpoint can sort server-side. + def sortable_fields + {} + end + + # Overridden to name the order the endpoint imposes instead, which is what + # tells the operator what they got in place of the order they asked for. + def unsortable_warning + "[forest_admin_datasource_pylon] #{name} cannot honour the requested order." + end + + # An unknown field silently disables sorting: a Pylon endpoint only honours + # the fixed allow-list its collection declares. + def translate_sort(sort, allow_list) + return [nil, nil] if sort.nil? || sort.empty? + + field, ascending = sort_field_and_direction(sort.first) + pylon_field = allow_list[field.to_s] + return [nil, nil] unless pylon_field + + [pylon_field, ascending ? 'asc' : 'desc'] + end + + # The agent injects an ascending primary-key sort whenever the request + # asks for no order, so the default cannot be told apart from a chosen + # order by presence alone. + def default_pk_sort?(sort) + normalized_sort_clauses(sort) == normalized_sort_clauses(SortFactory.by_primary_keys(self)) + end + + # The search box sends an empty string once the operator clears it. + def no_search?(filter) + filter&.search.to_s.strip.empty? + end + + def timezone_for(caller) + return 'UTC' unless caller.respond_to?(:timezone) + + timezone = caller.timezone + timezone.nil? || timezone.to_s.empty? ? 'UTC' : timezone + end + + # A projection naming only relations names no column of this collection, + # and the row it asks for carries none: returning the whole record there + # would serve every native column under a projection that excluded them, + # which is what `Projection#re_project` does not do. + # + # An empty projection is the other case and not that one — it asks for + # nothing rather than for relations alone, and means the record as it is, + # exactly like no projection at all. `ActionContext#get_records` sends one + # on every action whatever fields it was handed, so a row emptied here + # costs the plugins the record they act on: `primary_keys` raises on it, + # and `IssueTargets` reads the raise as "no issue selected". + def project(record, projection) + return record if projection.nil? || Array(projection).empty? + + wanted = Array(projection).map(&:to_s).reject { |p| p.include?(':') } + wanted.to_h { |k| [k, record[k]] } + end + + # A filter carrying no page — or a page naming no limit — asks for every + # record it matched, and travels to the walk as no limit at all rather than + # as `MAX_SEARCH_LIMIT`: a limit standing in for "everything" is one the + # walk cannot tell from a window the caller asked for, so it would stop at + # a thousand records having answered a larger set, and stop silently — the + # truncation warning only fires on a walk that knows it was cut short. + def translate_page(page) + return [0, nil] if page.nil? + + limit = page.limit.to_i + [page.offset.to_i.clamp(0, nil), limit.positive? ? limit : nil] + end + + # Adds custom fields, skipping any whose column name collides with a + # field already declared on the collection, and clamping the declared + # operators to those the API accepts on a custom field — so the schema + # never advertises a filter the translator would then refuse. Returns + # the subset actually added, carrying the clamped schemas, so callers + # can keep their serializer and api_filters in sync with the schema. + def add_custom_fields(custom_fields) + custom_fields.filter_map do |cf| + column_name = cf[:column_name] + if schema[:fields].key?(column_name) + ForestAdminDatasourcePylon.logger.warn( + "[forest_admin_datasource_pylon] Custom field '#{column_name}' on collection " \ + "'#{name}' conflicts with an existing field; skipping." + ) + nil + else + clamped = clamp_custom_field_operators(column_name, cf[:schema]) + add_field(column_name, clamped) + cf.merge(schema: clamped) + end + end + end + + # Operators a custom field may advertise: the ones the endpoint accepts on + # one, read off the same table the native columns come from. Declarations + # outside this list are dropped at registration, so the schema never + # advertises an operator the translator would refuse — and the empty table + # of a collection filtering nothing server-side advertises none. + def allowed_custom_field_operators + filter_table::CUSTOM_FIELD_OPS.keys + end + + private + + def define_schema = raise(NotImplementedError, "#{self.class} did not implement define_schema") + def define_relations = raise(NotImplementedError, "#{self.class} did not implement define_relations") + + def walker + @walker ||= Pagination::CursorWalker.new + end + + # A set of ids, not a list: the same one named twice is one record, so a + # lookup spends one request on it and a delete does not answer 404 the + # second time. The caps count records rather than mentions for the same + # reason. + def id_values(node) + return nil unless node.is_a?(Leaf) && node.field == 'id' + return nil unless [Operators::EQUAL, Operators::IN].include?(node.operator) + + Array(node.value).map(&:to_s).reject(&:empty?).uniq + end + + def and_branch?(node) + node.is_a?(Branch) && node.aggregator.to_s.casecmp('and').zero? + end + + # An `id` the short-circuit could not take out of the tree has no + # translation left: the endpoint filters no id server-side, and an id under + # an OR cannot be narrowed to a lookup because the other side of the union + # would bring in records the lookup never fetched. Worth an error an + # operator can act on rather than the translator's "add it to api_filters", + # because two things they do reach it: the `id equals` filter next to the + # or/and toggle, and an excluding selection — "every record except these" — + # which arrives as `id not_in` and is no filter they wrote. + # + # A collection whose endpoint does filter id declares it in `api_filters` + # and never short-circuits, so the translator handles its ids like any + # other field and there is nothing to refuse. + def ensure_no_stray_id!(node) + return if node.nil? || api_filters.key?('id') + return unless node.some_leaf { |leaf| leaf.field == 'id' } + + raise UnsupportedOperatorError, + "#{name} cannot answer this selection: Pylon cannot filter on id, so the agent reads the records " \ + 'by id and applies the rest in memory, which only an `and` of `id equals` / `id in` conditions ' \ + 'names a set of records to read. An id inside an `or` names none, and neither does an exclusion, ' \ + 'which is what selecting every record except a few sends. Select the records to act on rather ' \ + 'than the ones to leave out, rewrite the filter with `and`, or filter on another field.' + end + + def resolve_relation_conditions(caller, node) + return resolve_relation_branch(caller, node) if node.is_a?(Branch) + return node unless node.field.to_s.include?(':') + + resolve_relation_leaf(caller, node) + end + + # An unmatchable condition empties an `and` and drops out of an `or`, which + # is how it would behave had it been sent as a condition on a column no + # record answers. + def resolve_relation_branch(caller, branch) + resolved = Array(branch.conditions).map { |condition| resolve_relation_conditions(caller, condition) } + return MATCHES_NOTHING if and_branch?(branch) && resolved.include?(MATCHES_NOTHING) + return Branch.new(branch.aggregator, resolved) if and_branch?(branch) + + kept = resolved.reject { |condition| condition == MATCHES_NOTHING } + kept.empty? ? MATCHES_NOTHING : Branch.new(branch.aggregator, kept) + end + + # Pylon has neither a join nor an include parameter, so a `relation:field` + # leaf — which Forest offers as soon as a ManyToOne is declared, and which + # the schema therefore advertises as filterable — has no translation as it + # stands. It is resolved instead, the way `RelationCollectionDecorator` + # resolves one on a relation the customizer added: the foreign collection + # is read for the keys of the records matching the condition, and the leaf + # becomes the `foreign_key in [...]` this collection does filter. + # + # The read is the foreign collection's own, so its endpoint, its operators + # and its refusals apply — and a relation no condition names costs nothing, + # only a filter mentioning it triggers the read. + def resolve_relation_leaf(caller, leaf) + relation = schema[:fields][leaf.field.to_s.split(':').first] + raise_unfilterable_relation(leaf.field) unless resolvable_relation?(relation) + + keys = foreign_keys_matching(caller, relation, leaf) + keys.empty? ? MATCHES_NOTHING : Leaf.new(relation.foreign_key, Operators::IN, keys) + end + + # A relation is resolvable when its foreign key is a column this collection + # filters with `in` — server-side through `api_filters` for the collections + # that search, in memory for the ones read whole. Nothing else is: a + # OneToMany would have to be matched the other way round, which the schema + # never advertises as filterable, and a leaf reaching further than one + # relation is left to the foreign collection, which resolves its own. + def resolvable_relation?(relation) + return false unless relation.is_a?(ManyToOneSchema) + + column = schema[:fields][relation.foreign_key] + column.is_a?(ColumnSchema) && column.filter_operators.include?(Operators::IN) + end + + # One record past the cap is asked for, so an overflow is seen rather than + # guessed from a full page. + def foreign_keys_matching(caller, relation, leaf) + foreign = datasource.get_collection(relation.foreign_collection) + target = relation.foreign_key_target + query = Filter.new(condition_tree: leaf.unnest, + page: Page.new(offset: 0, limit: MAX_RELATION_KEYS + 1)) + + records = foreign.list(caller, query, Projection.new([target])) + raise_too_many_relation_keys(leaf.field, relation) if records.size > MAX_RELATION_KEYS + + records.filter_map { |record| record[target] }.uniq + end + + def raise_unfilterable_relation(field) + relation = schema[:fields][field.to_s.split(':').first] + instead = if relation.respond_to?(:foreign_key) + "Filter on '#{relation.foreign_key}' instead, or set the filter from the " \ + "#{relation.foreign_collection} list." + else + 'Filter on a column of this collection instead.' + end + + raise UnsupportedOperatorError, + "Pylon cannot filter on the related field '#{field}': it has no join, so a condition on a " \ + 'relation is answered by reading the foreign collection for its keys, which this relation ' \ + "does not allow. #{instead}" + end + + def raise_too_many_relation_keys(field, relation) + raise UnsupportedOperatorError, + "The filter on '#{field}' matches more than #{MAX_RELATION_KEYS} #{relation.foreign_collection} " \ + 'records: Pylon has no join, so the condition travels as the list of their keys, and a list this ' \ + 'long is one the endpoint cannot carry. Narrow the condition on the related field, or filter on ' \ + "'#{relation.foreign_key}' directly." + end + + def clamp_custom_field_operators(column_name, schema) + declared = Array(schema.filter_operators) + dropped = declared - allowed_custom_field_operators + return schema if dropped.empty? + + ForestAdminDatasourcePylon.logger.warn( + "[forest_admin_datasource_pylon] Custom field '#{column_name}' on collection '#{name}' declares " \ + "operators the API cannot honour on a custom field (#{dropped.join(", ")}); they are not advertised." + ) + schema.dup.tap { |clamped| clamped.filter_operators = declared - dropped } + end + + # A residual is evaluated by `ConditionTree#apply`, which compares scalar + # values: a Json column holds a list whose Pylon membership semantics + # have no in-memory counterpart, and an operator without an equivalence + # for the column's type has no in-memory evaluation at all. Both would + # silently corrupt the lookup's result, so they are refused instead. + def ensure_residual_appliable!(node) + case node + when Branch then node.conditions.each { |condition| ensure_residual_appliable!(condition) } + when Leaf then raise_unappliable_residual(node) unless residual_leaf_appliable?(node) + end + end + + def residual_leaf_appliable?(leaf) + column = schema[:fields][leaf.field] + return false if column.nil? || column.column_type == 'Json' + + Equivalent.equivalent_tree?(leaf.operator, IN_MEMORY_OPERATORS, column.column_type) + end + + # `ConditionTreeLeaf#match` compares with a bare `<` / `>`, which raises a + # NoMethodError on a column Pylon leaves null -- `resolution_time` on an + # unresolved issue, for one. Pairing the comparison with a presence check + # reproduces what a database does with NULL, excluding the record, and + # rides on the in-memory equivalence PRESENT already has for every type. + def guard_nil_comparisons(node) + return nil if node.nil? + + node.replace_leafs do |leaf| + next leaf unless NIL_UNSAFE_OPERATORS.include?(leaf.operator) + + ConditionTreeFactory.intersect([Leaf.new(leaf.field, Operators::PRESENT), leaf]) + end + end + + def raise_unappliable_residual(leaf) + raise UnsupportedOperatorError, + "Operator '#{leaf.operator}' on field '#{leaf.field}' cannot be combined with a primary-key " \ + 'lookup: Pylon cannot filter on id server-side, so the other conditions run in memory, ' \ + 'which this one does not support.' + end + + def sort_field_and_direction(entry) + return [entry.field, entry.ascending] if entry.respond_to?(:field) + + field = entry.key?(:field) ? entry[:field] : entry['field'] + ascending = entry.key?(:ascending) ? entry[:ascending] : entry['ascending'] + [field, ascending] + end + + def normalized_sort_clauses(sort) + Array(sort).map do |entry| + field, ascending = sort_field_and_direction(entry) + [field.to_s, ascending] + end + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/contact.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/contact.rb new file mode 100644 index 000000000..3799e615e --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/contact.rb @@ -0,0 +1,45 @@ +module ForestAdminDatasourcePylon + module Collections + class Contact < CursorCollection + include SchemaDefinition + include Serializer + + # `POST /contacts` takes the primary address alone; the other ones are set + # on an existing contact, through the list. + CREATE_ONLY = %w[email].freeze + UPDATE_ONLY = %w[emails].freeze + + def initialize(datasource, custom_fields: []) + super(datasource, 'PylonContact', custom_fields: custom_fields, searchable: true) + end + + protected + + def filter_table = ApiFilters + + def create_record(payload) = datasource.client.create_contact(payload) + def update_record(id, payload) = datasource.client.update_contact(id, payload) + def delete_record(id) = datasource.client.delete_contact(id) + + def create_only_fields = CREATE_ONLY + def update_only_fields = UPDATE_ONLY + + def unsortable_warning + '[forest_admin_datasource_pylon] PylonContact cannot honour the requested order; neither GET /contacts ' \ + 'nor POST /contacts/search takes a sort parameter, so contacts come back in the order the API imposes.' + end + + def search_page(limit:, cursor:, filter:, search_text:) + datasource.client.search_contacts(limit: limit, cursor: cursor, filter: filter, search_text: search_text) + end + + def list_page(limit:, cursor:) + datasource.client.list_contacts(limit: limit, cursor: cursor) + end + + def fetch_one(id) + datasource.client.fetch_contact(id) + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/contact/api_filters.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/contact/api_filters.rb new file mode 100644 index 000000000..af2379e8f --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/contact/api_filters.rb @@ -0,0 +1,38 @@ +module ForestAdminDatasourcePylon + module Collections + class Contact < CursorCollection + # The allow-list of `POST /contacts/search`, transcribed from the API + # reference: a field absent from this table cannot be filtered at all, and + # an operator absent from a field's map is rejected by Pylon. + # + # It is the single source of truth for filtering — `define_schema` derives + # every column's `filter_operators` from it, so this collection declares no + # filter the translator would then refuse; the absence family the agent + # derives on top of it is the exception `Query::OperatorMaps::Table` + # describes. + module ApiFilters + Maps = Query::OperatorMaps + + extend Maps::Table + + CUSTOM_FIELD_OPS = Maps::CUSTOM_FIELD_OPS + + # `id` is filtered server-side here, which is what spares this + # collection the primary-key short-circuit Issue needs. + # + # `name` and `email` get SUBSTRING rather than FULL_TEXT: the endpoint + # accepts `string_contains` but no negation of it. `email` filters the + # primary address only, not the `emails` list. + # + # The contacts search offers nothing else: no presence check, no + # filter on the phone numbers, the portal role or the external ids. + API_FILTERS = { + 'id' => { ops: Maps::EQUALITY }, + 'name' => { ops: Maps::EQUALITY.merge(Maps::SUBSTRING) }, + 'email' => { ops: Maps::EQUALITY.merge(Maps::SUBSTRING) }, + 'account_id' => { ops: Maps::EQUALITY } + }.freeze + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/contact/schema_definition.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/contact/schema_definition.rb new file mode 100644 index 000000000..aaa804abd --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/contact/schema_definition.rb @@ -0,0 +1,93 @@ +module ForestAdminDatasourcePylon + module Collections + class Contact < CursorCollection + # A column is writable when `POST /contacts` or `PATCH /contacts/{id}` + # accepts it, in the shape it is read under — the Json columns holding + # objects rather than plain strings are left read-only, see below. No + # column is sortable — neither `GET /contacts` nor `POST /contacts/search` + # exposes a sort parameter, so advertising a sortable column would let the + # UI ask for an order the API cannot honour. + # + # Filter operators are not chosen here: they come from + # `ApiFilters::API_FILTERS`, which mirrors the allow-list of the API. A + # column missing from that table gets no operator, so the UI offers no + # filter of this collection's own that Pylon would refuse — the absence + # family the agent derives above the datasource being the exception + # `Query::OperatorMaps::Table` describes. A contact carries no timestamp at all — + # Pylon returns none. + module SchemaDefinition + ManyToOneSchema = BaseCollection::ManyToOneSchema + OneToManySchema = BaseCollection::OneToManySchema + + private + + def define_schema + define_identity_fields + define_contact_fields + define_portal_fields + end + + # `account_id` is both the key of the relation and a column the contacts + # search filters, which is what lets the account side list its contacts + # server-side and the embedder resolve the account of a page of contacts + # in one request. + # + # `requested_issues` rather than `issues`: a contact is the requester of + # an issue, never its assignee — that side belongs to PylonUser. + def define_relations + add_field('account', ManyToOneSchema.new(foreign_collection: 'PylonAccount', + foreign_key: 'account_id', foreign_key_target: 'id')) + add_field('requested_issues', OneToManySchema.new(foreign_collection: 'PylonIssue', + origin_key: 'requester_id', origin_key_target: 'id')) + end + + def define_identity_fields + add_column('id', 'String', is_primary_key: true) + add_column('name', 'String', writable: true) + # Flattened from the nested `{ id: ..., external_ids: ... }` object + # Pylon returns, and kept as a column next to the `account` relation + # it is the key of: the search endpoint filters it. Writable, which is + # what opens the relation editor — see the party fields of PylonIssue. + add_column('account_id', 'String', writable: true) + # Read-only Json, and deliberately unfilterable although the search + # endpoint does not offer it either: the API matches bare external-id + # strings while the column shows `{external_id, label}` objects, so a + # filter would run on something the operator cannot see. + add_column('external_ids', 'Json') + end + + # `email` and `primary_phone_number` carry the primary value; the lists + # hold every address and number, and neither list is filterable. + # + # `email` is written on a create and `emails` on an update, one direction + # each: `POST /contacts` takes the primary address alone, and the other + # ones are set on an existing contact. Two writable projections of the + # same addresses would otherwise travel in one patch, the list leaving + # out whatever the primary carries. + # + # `phone_numbers` is not writable at all: it holds objects, and the + # shape the endpoint takes them in is not the one the column shows. + def define_contact_fields + add_column('email', 'String', writable: true) + add_column('emails', 'Json', writable: true) + add_column('primary_phone_number', 'String', writable: true) + add_column('phone_numbers', 'Json') + add_column('avatar_url', 'String', writable: true) + end + + def define_portal_fields + # Left as String rather than Enum: Pylon documents no_access / member + # / admin, but an organization can define its own portal roles, which + # is what `portal_role_id` points at. The id is the one written and the + # name is read-only, like `role_id` and `role_name` on PylonUser: + # whichever of two projections Pylon ignored would come back stale. + add_column('portal_role', 'String') + add_column('portal_role_id', 'String', writable: true) + # Owned by the integrations the contact was seen through; no endpoint + # takes it. + add_column('integration_user_ids', 'Json') + end + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/contact/serializer.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/contact/serializer.rb new file mode 100644 index 000000000..fc644edea --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/contact/serializer.rb @@ -0,0 +1,20 @@ +module ForestAdminDatasourcePylon + module Collections + class Contact < CursorCollection + module Serializer + NATIVE_FIELDS = %w[id name email emails primary_phone_number phone_numbers avatar_url + portal_role portal_role_id external_ids integration_user_ids].freeze + + private + + def serialize(contact) + attrs = contact.is_a?(Hash) ? contact : {} + record = NATIVE_FIELDS.to_h { |field| [field, attrs[field]] } + record['account_id'] = nested_id(attrs['account']) + add_custom_field_values(record, attrs['custom_fields']) + record + end + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/cursor_collection.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/cursor_collection.rb new file mode 100644 index 000000000..01aefc63f --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/cursor_collection.rb @@ -0,0 +1,131 @@ +module ForestAdminDatasourcePylon + module Collections + # Base for the collections Pylon exposes through three endpoints: a plain + # cursor-paginated listing (`GET /accounts`, `GET /contacts`: 300 requests + # per minute), a search over the same pages (`POST /accounts/search`: 120) + # and a single record (`GET /accounts/{id}`: 300). + # + # Their search endpoint filters `id` server-side, so — unlike Issue — they + # declare it in `api_filters` and never need the primary-key short-circuit: + # every predicate, `id` included and under an `or` as well, is translated and + # answered by one search request. The routing below is therefore only about + # spending the cheapest budget that answers the question exactly, never about + # what Pylon can express. + # + # None of these endpoints takes a sort parameter, so `sortable_fields` stays + # the empty default of the base and each collection names, through + # `unsortable_warning`, the order it got instead of the one it asked for. + class CursorCollection < BaseCollection + include RecordSerialization + include RelationEmbedder + + # Pylon documents no maximum number of values on an `in` filter; the chunk + # keeps the request body and the page answering it bounded. + ID_CHUNK_SIZE = 100 + + def list(caller, filter, projection) + records = fetch_records(caller, filter) + rows = records.map { |record| project(record, projection) } + embed_relations(records, rows, projection) + rows + end + + # The search endpoint filters `id` server-side, which is what lets a whole + # page of foreign keys be read in one request per chunk. + def records_indexed_by_id(ids) + ids.each_slice(ID_CHUNK_SIZE).with_object({}) do |chunk, indexed| + search_by_ids(chunk).each { |record| indexed[record['id']] = record } + end + end + + protected + + # Every collection read this way has a search endpoint, hence a table of + # its own: the empty default of the base would silently turn each of its + # filters into a refusal. + def filter_table = raise(NotImplementedError, "#{self.class} did not implement filter_table") + + # One page of the listing endpoint, as a Client::SearchPage. + def list_page(limit:, cursor:) = raise(NotImplementedError, "#{self.class} did not implement list_page") + + # One record straight from its own endpoint. + def fetch_one(id) = raise(NotImplementedError, "#{self.class} did not implement fetch_one") + + private + + # The `id` filter goes through the translator rather than being written by + # hand, so the shape on the wire is the one this collection's `api_filters` + # produce — one spelling of an id filter, not two to keep in step. The + # cursor is followed defensively: a chunk is asked for as a single page, + # and Pylon is free to answer it over several. + def search_by_ids(ids) + pylon_filter = Query::ConditionTreeTranslator.call(Leaf.new('id', Operators::IN, ids), + api_filters: api_filters) + records = walker.walk(offset: 0, limit: ids.size) do |batch, cursor| + search_page(limit: batch, cursor: cursor, filter: pylon_filter, search_text: nil) + end + records.map { |record| serialize(record) } + end + + def fetch_records(caller, filter) + warn_unsortable(filter&.sort) + + with_resolved_relations(caller, filter) do |query| + next listed_records(query) if browsing?(query) + + id = single_id_lookup(query) + next page_window(records_by_id(id), query) if id + + search_records(caller, query) + end + end + + # Nothing to filter and nothing to search: the listing endpoint returns + # the same records for a budget two and a half times larger than the + # search one. + def browsing?(filter) + return true if filter.nil? + + filter.condition_tree.nil? && no_search?(filter) + end + + # The walk of `search_records`, over the listing endpoint: it hands out + # cursor pages just the same, it only takes no filter. + def listed_records(filter) + offset, limit = translate_page(filter&.page) + + records = walker.walk(offset: offset, limit: limit) { |batch, cursor| list_page(limit: batch, cursor: cursor) } + records.map { |record| serialize(record) } + end + + # A record detail is `id equals X` alone: reading it through the record + # endpoint keeps the search budget for the pages that need it. + # + # Only a bare leaf takes that path. An `and` also carrying a scope is left + # to the search endpoint, which filters the id and the rest server-side in + # one request — where a lookup would have to apply the leftovers in memory, + # and would refuse the ones it cannot evaluate there. + def single_id_lookup(filter) + tree = filter.condition_tree + return nil unless tree.is_a?(Leaf) && no_search?(filter) + + ids = extract_id_lookup(tree)&.ids + ids&.one? ? ids.first : nil + end + + # A record the operator can no longer reach — deleted, or outside the + # token's scope — reads as "no record" rather than as a failed page. + def records_by_id(id) + record = fetch_one(id) + return [] if record.nil? + + serialized = serialize(record) + matches_id?(serialized, id) ? [serialized] : [] + rescue APIError => e + raise unless e.status == 404 + + [] + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/fetch_all_collection.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/fetch_all_collection.rb new file mode 100644 index 000000000..eb47833cd --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/fetch_all_collection.rb @@ -0,0 +1,192 @@ +module ForestAdminDatasourcePylon + module Collections + # Base for the collections whose Pylon endpoint hands back the whole + # collection of the organization in a single response: `GET /users` and + # `GET /teams` take no cursor, no filter and no sort at all. + # + # Filtering, sorting and paginating that response in memory is exact rather + # than approximate: the records in hand ARE every record Pylon holds, so a + # window cut out of them carries the same rows a server-side query would + # have returned. This is what keeps the in-memory pass out of the trap this + # datasource refuses elsewhere — a result that looks filtered without being + # filtered — which only arises when a single page of a larger dataset is all + # one has. The cost is bandwidth, not correctness. + # + # Each `list` re-reads the endpoint, so what the operator sees is what Pylon + # holds now rather than what it held when the process booted. One request + # per list against the 300 a minute these endpoints grant, spaced out by + # `RateLimiter`, is a budget no list view comes near. + class FetchAllCollection < BaseCollection + # The filters a column may advertise, per column type. Restricted to the + # operators `ConditionTreeLeaf#match` evaluates natively or + # `ConditionTreeEquivalent` rewrites into that native set, because the + # in-memory pass is the only pass there is here: an operator with no + # equivalence makes `match` return nil, which `apply` reads as "no match" + # and would silently empty the page instead of filtering it. + OPERATOR_CANDIDATES = { + 'String' => [Operators::EQUAL, Operators::NOT_EQUAL, Operators::IN, Operators::NOT_IN, + Operators::PRESENT, Operators::BLANK, Operators::CONTAINS, Operators::I_CONTAINS, + Operators::NOT_CONTAINS, Operators::STARTS_WITH, Operators::ENDS_WITH], + 'Boolean' => [Operators::EQUAL, Operators::NOT_EQUAL, Operators::IN, Operators::NOT_IN, + Operators::PRESENT, Operators::BLANK] + }.freeze + + # Candidates are re-checked against the toolkit rather than trusted, so an + # equivalence the toolkit stops providing takes the filter out of the + # schema instead of turning every page using it into an empty one. + def self.operators_for(column_type) + Array(OPERATOR_CANDIDATES[column_type]).select do |operator| + Equivalent.equivalent_tree?(operator, IN_MEMORY_OPERATORS, column_type) + end + end + + # Countable, alone among the Pylon collections: the count answered here is + # taken over every record Pylon holds, not over the pages a cursor walk + # happened to collect, so it is the figure a server-side count would have + # given rather than a fraction of it presented as the whole -- the very + # reason `BaseCollection#aggregate` refuses everywhere else. One request + # per count, against the 300 a minute these endpoints grant. + def initialize(datasource, name, **options) + super + enable_count + end + + def list(caller, filter, projection) + records = sort_in_memory(filtered_records(caller, filter), filter&.sort) + + page_window(records, filter).map { |record| project(record, projection) } + end + + # Exact, like the filter and the sort above it: the records in hand are + # every record Pylon holds, so a count or a group computed over them is + # the one a server-side aggregation would have answered — which is why + # these columns stay groupable where every other Pylon column is not. + # + # The rows are keyed with strings because that is how the agent reads + # them, while `Aggregation#apply` hands them back keyed with symbols. + def aggregate(caller, filter, aggregation, limit = nil) + aggregation.apply(filtered_records(caller, filter), timezone_for(caller), limit) + .map { |row| { 'group' => row[:group], 'value' => row[:value] } } + end + + # One request answers any number of ids: the endpoint hands back the + # complete dataset, so the ids only pick rows out of it. Read again on + # every pass, like `list` — the freshness this collection trades bandwidth + # for is not worth losing to a cache of related records. + # + # Only the wanted entities are serialized: a page of a pointing collection + # asks for a handful of ids, against every record the organization has. + def records_indexed_by_id(ids) + wanted = Array(ids) + + fetch_all.each_with_object({}) do |entity, indexed| + next unless entity.is_a?(Hash) && wanted.include?(entity['id']) + + indexed[entity['id']] = serialize(entity) + end + end + + protected + + # A column is read-only unless the collection declares it `writable`. + # Scalar columns are sortable and groupable because the in-memory sort and + # aggregation honour anything asked of them; a Json column is none of the + # three, as it holds a list whose Pylon semantics have no in-memory + # counterpart — the same reason the primary-key residual guard refuses one. + def add_column(name, type, is_primary_key: false, writable: false) + add_field(name, ColumnSchema.new(column_type: type, + filter_operators: self.class.operators_for(type), + is_primary_key: is_primary_key, + is_sortable: type != 'Json', + is_groupable: type != 'Json', + is_read_only: !writable)) + end + + # Pylon defines custom fields on issues, accounts and contacts only, so + # neither collection read this way has any. Refused rather than ignored: + # `serialize` has no hook here to read a custom-field value with, and the + # in-memory pass no table to clamp the declared operators against, so a + # declaration would register a column reading nil on every row forever. + def add_custom_fields(custom_fields) + return [] if custom_fields.empty? + + raise ConfigurationError, + "#{name} takes no custom field: Pylon defines them on issues, accounts and contacts only." + end + + # The complete collection, straight from its unpaginated endpoint. + def fetch_all = raise(NotImplementedError, "#{self.class} did not implement fetch_all") + + # One Pylon entity flattened into a record matching the schema. + def serialize(_entity) = raise(NotImplementedError, "#{self.class} did not implement serialize") + + private + + # The complete dataset, serialized and narrowed to the rows the filter + # keeps: what `list` pages and what `aggregate` counts are the same rows. + # + # Relation conditions are resolved first, like everywhere else: neither + # collection read this way declares a ManyToOne today, so what this refuses + # is a condition on the reverse side, which `match` would otherwise read as + # a missing column and answer by dropping every row. + def filtered_records(caller, filter) + with_resolved_relations(caller, filter) do |query| + filter_in_memory(fetch_all.map { |entity| serialize(entity) }, caller, query) + end + end + + # The tree is applied over the complete dataset, so the rows it keeps are + # the rows Pylon would have kept. `guard_nil_comparisons` is still worth + # its cost: nothing in the schema advertises a bare comparison, but a + # scope, a segment or a customizer can send one, and it would otherwise + # raise on the nulls Pylon returns for an unset column. + def filter_in_memory(records, caller, filter) + tree = guard_nil_comparisons(filter&.condition_tree) + return records if tree.nil? + + tree.apply(records, self, timezone_for(caller)) + end + + # Every requested order is honoured, including the ascending primary-key + # sort the agent injects when the request asks for none, so there is no + # unsortable order to report. + # + # Neither Ruby's `sort` nor the toolkit's `Sort#apply` can be used as is: + # `sort` is not stable, and `<=>` answers nil on a null, on two booleans + # and on mixed types, which leaves the comparator undefined and the order + # arbitrary. Ties therefore fall back to the position the API returned the + # record in, and values are compared by `compare_values`. + def sort_in_memory(records, sort) + clauses = normalized_sort_clauses(sort) + return records if clauses.empty? + + records.each_with_index.sort do |(left, left_index), (right, right_index)| + compare_clauses(left, right, clauses).nonzero? || (left_index <=> right_index) + end.map(&:first) + end + + def compare_clauses(left, right, clauses) + clauses.each do |field, ascending| + comparison = compare_values(left[field], right[field]) + next if comparison.zero? + + return ascending ? comparison : -comparison + end + + 0 + end + + # Nulls sort last on an ascending order and first on a descending one, the + # way a database orders them; values `<=>` cannot compare — two booleans, + # for one — are compared through their string form rather than left + # undefined, which puts `false` before `true`, again like a database. + def compare_values(left, right) + return 0 if left.nil? && right.nil? + return 1 if left.nil? + return -1 if right.nil? + + (left <=> right) || (left.to_s <=> right.to_s) + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/issue.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/issue.rb new file mode 100644 index 000000000..8367d47ce --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/issue.rb @@ -0,0 +1,128 @@ +module ForestAdminDatasourcePylon + module Collections + class Issue < BaseCollection + include SchemaDefinition + include RecordSerialization + include Serializer + include RelationEmbedder + include MessagesEmbedder + include IdLookupReader + + # `/issues/search` exposes no sort parameter, so the allow-list is empty and + # every requested order is reported instead of being silently swallowed. + # The mechanism stays in place for the collections whose endpoint sorts. + PYLON_SORTABLE = {}.freeze + + # A primary-key lookup spends one `GET /issues/{id}` per id, sequentially. + # The fan-out is bounded like the cursor walk, and for the same reason: + # `RateLimiter` keeps the requests inside the 300 a minute that endpoint + # grants, but nothing makes two hundred round-trips fast. + # + # What it bounds is one page, not the selection behind it: the window is + # taken off the ids first, so a selection wider than this is read a page + # at a time rather than truncated at its first #{MAX_ID_LOOKUPS}. See + # `IdLookupReader`, and the one case that cannot be paged. + MAX_ID_LOOKUPS = 20 + + # The shape of one message inside the `messages` column. Field names follow + # the columns of this collection rather than the payload: Pylon spells them + # `message_html` and `timestamp`, which would put two conventions in the + # same schema for the operator to reconcile. + MESSAGE_THREAD_SCHEMA = { + 'id' => 'String', + 'body_html' => 'String', + 'is_private' => 'Boolean', + 'source' => 'String', + 'thread_id' => 'String', + 'file_urls' => 'Json', + 'created_at' => 'Date', + 'author_name' => 'String', + 'author_email' => 'String', + 'author_avatar_url' => 'String', + 'author_contact_id' => 'String', + 'author_user_id' => 'String' + }.freeze + + # One thread is one `GET /issues/{id}/messages`, and a thread is the whole + # conversation rather than a page of it. A list view asking for more of + # them than this reads the first ones and reports the rest, for the same + # reason MAX_ID_LOOKUPS bounds the primary-key fan-out — sequential + # round-trips, each carrying an unbounded payload. Lower than + # MAX_ID_LOOKUPS because of that payload, not because of the quota: the + # endpoint grants 120 requests a minute. + MAX_MESSAGE_EMBEDS = 10 + + # `body_html` is the first message of the thread, which `POST /issues` + # requires and `PATCH /issues/{id}` does not carry; `author_unverified` + # qualifies that message and travels with it. + CREATE_ONLY = %w[body_html author_unverified].freeze + + # Pylon creates every issue as `new`, of the type it decides, and takes + # both on an update only. + UPDATE_ONLY = %w[state type].freeze + + def initialize(datasource, custom_fields: []) + super(datasource, 'PylonIssue', custom_fields: custom_fields, searchable: true) + end + + def list(caller, filter, projection) + records = fetch_records(caller, filter) + rows = records.map { |record| project(record, projection) } + embed_relations(records, rows, projection) + embed_messages(records, rows) if want_messages?(projection) + rows + end + + protected + + def filter_table = ApiFilters + + def create_record(payload) = datasource.client.create_issue(payload) + def update_record(id, payload) = datasource.client.update_issue(id, payload) + def delete_record(id) = datasource.client.delete_issue(id) + + def create_only_fields = CREATE_ONLY + def update_only_fields = UPDATE_ONLY + + # An issue is read through `GET /issues/{id}`, one request per record: a + # selection resolved or compared that way spends the write budget twice + # over, so it divides the records one pass reaches rather than fitting + # beside them. + def requests_per_record_read = 1 + + # Never past the primary-key fan-out either: a write resolving named ids + # through `list` reads them one request apiece, and a resolution the page + # cap trimmed would write to a subset of the selection while reporting the + # whole of it. The budget is the tighter of the two at today's numbers; + # the clamp keeps that true if either moves. + def max_resolvable_ids(reads: 0) = [super, MAX_ID_LOOKUPS].min + + def sortable_fields + PYLON_SORTABLE + end + + def unsortable_warning + '[forest_admin_datasource_pylon] PylonIssue cannot honour the requested order; ' \ + 'POST /issues/search always returns issues from the most recent to the oldest.' + end + + def search_page(limit:, cursor:, filter:, search_text:) + datasource.client.search_issues(limit: limit, cursor: cursor, filter: filter, search_text: search_text) + end + + private + + def fetch_records(caller, filter) + warn_unsortable(filter&.sort) + + with_resolved_relations(caller, filter) do |query| + lookup = extract_id_lookup(query&.condition_tree) + next search_records(caller, query) unless lookup + + ensure_searchless_lookup!(query) + records_by_id(caller, lookup, query) + end + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/issue/api_filters.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/issue/api_filters.rb new file mode 100644 index 000000000..07b878767 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/issue/api_filters.rb @@ -0,0 +1,41 @@ +module ForestAdminDatasourcePylon + module Collections + class Issue < BaseCollection + # The allow-list of `POST /issues/search`, transcribed from the API + # reference: a field absent from this table cannot be filtered at all, and + # an operator absent from a field's map is rejected by Pylon. + # + # It is the single source of truth for filtering — `define_schema` derives + # every column's `filter_operators` from it, so this collection declares no + # filter the translator would then refuse; the absence family the agent + # derives on top of it is the exception `Query::OperatorMaps::Table` + # describes. + module ApiFilters + Maps = Query::OperatorMaps + + extend Maps::Table + + CUSTOM_FIELD_OPS = Maps::CUSTOM_FIELD_OPS + + # `param` carries the read-to-filter renames: an issue is read with + # `type` / `resolution_time` / `latest_message_time` but filtered on + # `issue_type` / `resolved_at` / `latest_message_activity_at`. + API_FILTERS = { + 'state' => { ops: Maps::EQUALITY }, + 'type' => { param: 'issue_type', ops: Maps::EQUALITY.merge(Maps::PRESENCE) }, + 'account_id' => { ops: Maps::EQUALITY.merge(Maps::PRESENCE) }, + 'requester_id' => { ops: Maps::EQUALITY.merge(Maps::PRESENCE) }, + 'assignee_id' => { ops: Maps::EQUALITY.merge(Maps::PRESENCE) }, + 'team_id' => { ops: Maps::EQUALITY }, + 'title' => { ops: Maps::FULL_TEXT }, + 'body_html' => { ops: Maps::FULL_TEXT }, + 'tags' => { ops: Maps::MEMBERSHIP }, + 'created_at' => { ops: Maps::TIME }, + 'updated_at' => { ops: Maps::TIME }, + 'resolution_time' => { param: 'resolved_at', ops: Maps::TIME }, + 'latest_message_time' => { param: 'latest_message_activity_at', ops: Maps::TIME } + }.freeze + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/issue/id_lookup_reader.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/issue/id_lookup_reader.rb new file mode 100644 index 000000000..723437045 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/issue/id_lookup_reader.rb @@ -0,0 +1,88 @@ +module ForestAdminDatasourcePylon + module Collections + class Issue < BaseCollection + # Reading issues named by their id, which Pylon serves one request at a + # time: `POST /issues/search` cannot filter on id, so the short-circuit + # `BaseCollection` extracts is answered by `GET /issues/{id}` per id. + # + # Split out of the collection for what it costs: everything here exists to + # bound that fan-out, and none of it is about the shape of an issue. + module IdLookupReader + include RecordSerialization + + private + + # The records are already narrowed to the ids the filter asked for, so + # applying the conditions left over by the short-circuit in memory + # cannot return a record the API would have excluded. The reverse — + # dropping a record over a condition memory evaluates differently from + # Pylon — is ruled out by `extract_id_lookup`, which refuses such + # residuals. + # + # Without a residual the ids are the answer, in order, so the window is + # taken off them before any of them is read: one request per record the + # caller asked to see, and none for the records it did not. A residual + # takes that away — which records the window holds is only known once + # they are all read — so past the cap the selection is refused rather + # than answered with a fraction of itself. + def records_by_id(caller, lookup, filter) + return fetch_by_ids(page_of_ids(lookup.ids, filter)) if lookup.residual.nil? + + refuse_wide_lookup(lookup.ids.size) if lookup.ids.size > MAX_ID_LOOKUPS + + page_window(lookup.residual.apply(fetch_by_ids(lookup.ids), self, timezone_for(caller)), filter) + end + + # The ids of the window, capped: `MAX_ID_LOOKUPS` bounds a page rather + # than a selection here, a page being what each of these requests buys. + def page_of_ids(ids, filter) + offset, limit = translate_page(filter&.page) + wanted = limit ? Array(ids[offset, limit]) : ids.drop(offset) + capped = wanted.first(MAX_ID_LOOKUPS) + warn_truncated_lookup(wanted.size) if wanted.size > capped.size + + capped + end + + # `GET /issues/{id}` accepts the issue number as well as the UUID, so a + # record answering with an id other than the one asked for is dropped: + # see `matches_id?`. + def fetch_by_ids(ids) + ids.filter_map do |id| + record = fetch_issue(id) + next if record.nil? + + serialized = serialize(record) + serialized if matches_id?(serialized, id) + end + end + + # A record the operator can no longer reach — deleted, or outside the + # token's scope — reads as "no record" rather than as a failed page. + def fetch_issue(id) + datasource.client.fetch_issue(id) + rescue APIError => e + raise unless e.status == 404 + + nil + end + + def warn_truncated_lookup(asked) + ForestAdminDatasourcePylon.logger.warn( + "[forest_admin_datasource_pylon] Asked for a page of #{asked} issues by id, reading the first " \ + "#{MAX_ID_LOOKUPS}: Pylon answers one issue per request, and the requests are sequential. " \ + 'Ask for a smaller page to reach the records past this point.' + ) + end + + def refuse_wide_lookup(count) + raise UnsupportedOperatorError, + "This selection names #{count} issues by id and filters them further, which PylonIssue answers " \ + "with one request per named issue — more than the #{MAX_ID_LOOKUPS} one page covers. Reading " \ + 'only some of them would drop records the other conditions keep, and silently answer a page ' \ + 'that is missing them. Name fewer issues, or drop the other conditions.' + end + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/issue/messages_embedder.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/issue/messages_embedder.rb new file mode 100644 index 000000000..758e85f9a --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/issue/messages_embedder.rb @@ -0,0 +1,89 @@ +module ForestAdminDatasourcePylon + module Collections + class Issue < BaseCollection + # The conversation of an issue, embedded as a structured array column the + # way the Zendesk datasource embeds a ticket's comments. + # + # Pylon has no way to read the threads of several issues at once, so the + # thread costs one request per row — against an endpoint allowing 20 per + # minute. The fan-out is therefore bounded like the primary-key lookups of + # this collection: truncated with a warning rather than turned into a rate + # limit error halfway through the page. A relation the projection does not + # ask for costs no request at all. + module MessagesEmbedder + include RecordSerialization + + private + + # The thread is read only when the projection names it. A nil projection + # — what a count or an export goes through — asks for the record as + # Pylon returns it, and embeds nothing, exactly like RelationEmbedder: + # spending one request per row on a path that never asked for the + # conversation is the very fan-out MAX_MESSAGE_EMBEDS exists to bound. + def want_messages?(projection) + Array(projection).map(&:to_s).any? { |p| p == 'messages' || p.start_with?('messages:') } + end + + # A row past the cap, and a row whose thread failed to be read, are left + # at nil: "unknown", never the empty list, which would read as "this + # issue has no message" — the kind of answer that looks complete without + # being it. + def embed_messages(records, rows) + embedded = rows.first(MAX_MESSAGE_EMBEDS) + warn_truncated_threads(rows.size) if rows.size > embedded.size + + embedded.each_with_index do |row, index| + messages = datasource.client.fetch_issue_messages(records[index]['id']) + row['messages'] = messages&.map { |message| serialize_message(message) } + end + end + + def serialize_message(message) + attrs = message.is_a?(Hash) ? message : {} + + { + 'id' => attrs['id'], + 'body_html' => attrs['message_html'], + 'is_private' => attrs['is_private'], + 'source' => attrs['source'], + 'thread_id' => attrs['thread_id'], + 'file_urls' => attrs['file_urls'], + 'created_at' => attrs['timestamp'] + }.merge(flatten_author(attrs['author'])) + end + + # Pylon nests the author's contact and user sides side by side, both + # optional and with nothing telling them apart: a message written by an + # agent carries `user`, one written by a customer carries `contact`. Both + # ids are kept, so a message stays traceable to the PylonContact or + # PylonUser record it came from, and the email is taken from whichever + # side is there. + def flatten_author(author) + attrs = author.is_a?(Hash) ? author : {} + contact = attrs['contact'] + user = attrs['user'] + + { + 'author_name' => attrs['name'], + 'author_avatar_url' => attrs['avatar_url'], + 'author_email' => nested_email(contact) || nested_email(user), + 'author_contact_id' => nested_id(contact), + 'author_user_id' => nested_id(user) + } + end + + def nested_email(value) + value['email'] if value.is_a?(Hash) + end + + def warn_truncated_threads(asked) + ForestAdminDatasourcePylon.logger.warn( + "[forest_admin_datasource_pylon] Asked for the message thread of #{asked} issues, reading the first " \ + "#{MAX_MESSAGE_EMBEDS}: one request per issue, each carrying a whole conversation. " \ + 'Narrow the selection, or take the thread out of the projection, to reach the records past this point.' + ) + end + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/issue/schema_definition.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/issue/schema_definition.rb new file mode 100644 index 000000000..dae0fda73 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/issue/schema_definition.rb @@ -0,0 +1,122 @@ +module ForestAdminDatasourcePylon + module Collections + class Issue < BaseCollection + # A column is writable when `POST /issues` or `PATCH /issues/{id}` accepts + # it, the two directions being told apart by `Issue::CREATE_ONLY` and + # `Issue::UPDATE_ONLY`; everything Pylon computes stays read-only. No + # column is sortable, `/issues/search` exposing no sort parameter at all: + # results always come back ordered by `created_at` descending, so + # advertising a sortable column would let the UI ask for an order the API + # cannot honour. + # + # Filter operators are not chosen here: they come from + # `ApiFilters::API_FILTERS`, which mirrors the allow-list of the API. A + # column missing from that table gets no operator, so the UI offers no + # filter of this collection's own that Pylon would refuse — the absence + # family the agent derives above the datasource being the exception + # `Query::OperatorMaps::Table` describes. + module SchemaDefinition + ColumnSchema = BaseCollection::ColumnSchema + ManyToOneSchema = BaseCollection::ManyToOneSchema + Operators = BaseCollection::Operators + + private + + def define_schema + define_identity_fields + define_content_fields + define_party_fields + define_time_fields + end + + # The four parties of an issue, each pointing at the collection owning its + # shape: the flattened `*_id` columns stay, as the keys the relation is + # read through and as the columns the search endpoint filters. + # + # RelationEmbedder resolves them at read time, in bulk. The reverse sides + # are declared by the collections they belong to; `/issues/search` + # filters every one of these keys, so they are answered server-side. + def define_relations + add_field('account', ManyToOneSchema.new(foreign_collection: 'PylonAccount', + foreign_key: 'account_id', foreign_key_target: 'id')) + add_field('requester', ManyToOneSchema.new(foreign_collection: 'PylonContact', + foreign_key: 'requester_id', foreign_key_target: 'id')) + add_field('assignee', ManyToOneSchema.new(foreign_collection: 'PylonUser', + foreign_key: 'assignee_id', foreign_key_target: 'id')) + add_field('team', ManyToOneSchema.new(foreign_collection: 'PylonTeam', + foreign_key: 'team_id', foreign_key_target: 'id')) + end + + def define_identity_fields + # Only equal/in: these are the two the primary-key short-circuit can + # actually serve through GET /issues/{id}. `id` is not part of the + # search allow-list, so it never reaches the translator. + add_field('id', ColumnSchema.new(column_type: 'String', + filter_operators: [Operators::EQUAL, Operators::IN], + is_primary_key: true, is_groupable: false, is_read_only: true)) + add_column('number', 'Number') + add_column('link', 'String') + end + + def define_content_fields + add_column('title', 'String', writable: true) + # Writable on creation only: it is the first message of the thread, + # which `PATCH /issues/{id}` does not carry. + add_column('body_html', 'String', writable: true) + # Left as String rather than Enum: Pylon ships five built-in states + # but organisations define their own on top of them. Writable on an + # update only — every issue is created `new`. + add_column('state', 'String', writable: true) + add_column('type', 'String', writable: true) + # Where the issue came from: Pylon sets it, no endpoint takes it. + add_column('source', 'String') + add_column('tags', 'Json', writable: true) + add_column('customer_portal_visible', 'Boolean', writable: true) + add_column('author_unverified', 'Boolean', writable: true) + add_column('number_of_touches', 'Number') + define_thread_field + end + + # The conversation, embedded at read time by MessagesEmbedder. Declared + # by hand rather than through `add_column`: its type is the shape of one + # message, not a primitive. + # + # Neither filterable nor sortable — `POST /issues/search` covers no + # message field, and the thread is not even part of the payload the + # search endpoint returns — and not groupable either: `ColumnSchema` + # defaults that flag to true, where the `add_column` of the base passes + # false for every Pylon column, a thread being both an array and a value + # the pages of a cursor walk do not carry. + def define_thread_field + add_field('messages', ColumnSchema.new(column_type: [Issue::MESSAGE_THREAD_SCHEMA], + filter_operators: [], is_groupable: false, + is_read_only: true)) + end + + # Flattened from the nested `{id: …}` objects Pylon returns, and kept as + # columns next to the relations they are the keys of: they are what the + # search endpoint filters, on this side and on the reverse one. + # + # Writable, although `GeneratorField` forces a foreign key read-only in + # the emitted schema whatever the datasource says, so the detail view has + # one editor per key rather than two. What the flag opens is that editor: + # the `BelongsTo` reads its own read-only state off the key column, and + # the front sends the choice back as the very column named here. + def define_party_fields + %w[account_id requester_id assignee_id team_id].each do |field| + add_column(field, 'String', writable: true) + end + end + + def define_time_fields + %w[first_response_time resolution_time latest_message_time created_at updated_at].each do |field| + add_column(field, 'Date') + end + %w[time_in_status_seconds business_hours_time_in_status_seconds].each do |field| + add_column(field, 'Json') + end + end + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/issue/serializer.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/issue/serializer.rb new file mode 100644 index 000000000..932df6202 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/issue/serializer.rb @@ -0,0 +1,26 @@ +module ForestAdminDatasourcePylon + module Collections + class Issue < BaseCollection + module Serializer + PARTY_FIELDS = { 'account_id' => 'account', 'requester_id' => 'requester', + 'assignee_id' => 'assignee', 'team_id' => 'team' }.freeze + + NATIVE_FIELDS = %w[id number link title body_html state type source tags + customer_portal_visible author_unverified number_of_touches + first_response_time resolution_time latest_message_time + created_at updated_at time_in_status_seconds + business_hours_time_in_status_seconds].freeze + + private + + def serialize(issue) + attrs = issue.is_a?(Hash) ? issue : {} + record = NATIVE_FIELDS.to_h { |field| [field, attrs[field]] } + PARTY_FIELDS.each { |column, source| record[column] = nested_id(attrs[source]) } + add_custom_field_values(record, attrs['custom_fields']) + record + end + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/record_serialization.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/record_serialization.rb new file mode 100644 index 000000000..3f88fc699 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/record_serialization.rb @@ -0,0 +1,75 @@ +module ForestAdminDatasourcePylon + module Collections + # The two parts of a Pylon payload every collection reads the same way: the + # nested `{ id: ... }` objects it flattens into foreign-key columns, and the + # custom fields the organization defined, which are columns of their own. + module RecordSerialization + private + + def nested_id(value) + value['id'] if value.is_a?(Hash) + end + + def add_custom_field_values(record, values) + custom_fields.each do |cf| + entry = values.is_a?(Hash) ? values[cf[:column_name]] : nil + record[cf[:column_name]] = coerce_custom_field(custom_field_value(entry), cf[:schema].column_type) + end + end + + # Pylon spells a custom field as `slug => {"slug": ..., "value": ...}`, + # with `"values": [...]` instead of `"value"` for multi-value fields. + def custom_field_value(entry) + return entry unless entry.is_a?(Hash) + + entry.key?('value') ? entry['value'] : entry['values'] + end + + # The API reference documents what a custom field *is*, never the form its + # value is read back in, so a Number answering `"42"` is not ruled out. A + # value that is not already of its column's type is therefore converted -- + # the in-memory pass of the primary-key short-circuit would otherwise drop + # a row on `"42" == 42.0`, comparing a read string with the float + # `ConditionTreeParser.cast_to_type` casts the filter to. + # + # A date stays the string it is: the filter carries an ISO8601 string too, + # and comparing two of those is the ordering itself. + def coerce_custom_field(value, column_type) + return nil if value.nil? + + case column_type + when 'Number' then coerce_number(value) + when 'Boolean' then coerce_boolean(value) + else value + end + end + + # A value already numeric is handed back untouched: `ConditionTreeLeaf#match` + # compares with `==` and `Array#include?`, both of which hold across Integer + # and Float, so nothing needs widening -- and widening would make an integer + # field display the `12.0` it does not hold, where `FilterValue#format_float` + # narrows the very same value on the way out. A string is read to the + # tightest form for that reason, the two halves agreeing on what an integer + # looks like. + # + # A number that cannot be read reads as absent rather than as zero. + def coerce_number(value) + return value if value.is_a?(Numeric) + + float = Float(value, exception: false) + return nil if float.nil? + + float == float.to_i ? float.to_i : float + end + + # An empty boolean reads as absent as well: `false` is an answer of its own, + # and Pylon gave none. + def coerce_boolean(value) + return value if [true, false].include?(value) + return nil if value.to_s.strip.empty? + + !%w[false 0 no].include?(value.to_s.strip.downcase) + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/relation_embedder.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/relation_embedder.rb new file mode 100644 index 000000000..64f1af706 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/relation_embedder.rb @@ -0,0 +1,101 @@ +module ForestAdminDatasourcePylon + module Collections + # Forest asks for a ManyToOne relation as `relation:field` entries in the + # projection and expects the related record nested under the relation name on + # every row. Pylon has no join and no include parameter, so the records are + # read from the foreign collection — in bulk, from the foreign keys the + # serialized records already carry, never one request per row. + # + # What to embed comes from the schema rather than from a list kept here: a + # collection embeds whatever ManyToOne relations it declares, and a relation + # the projection does not ask for costs no request at all. + module RelationEmbedder + ManyToOneSchema = BaseCollection::ManyToOneSchema + Projection = BaseCollection::Projection + + private + + # `records` are the serialized records, carrying the foreign keys `project` + # strips off the rows; `rows` are the projected rows, in the same order. + def embed_relations(records, rows, projection) + # Rebuilt rather than read off the argument: `list` is also called with + # a plain array of field names, inside the agent and in the specs, and a + # relation read from one of those would go unprojected -- which is the + # one shape this must not have. + sub_projections = Projection.new(Array(projection).map(&:to_s)).relations + + projected_relations(projection).group_by { |_name, relation| relation.foreign_collection } + .each do |foreign, group| + embed_foreign(foreign, group, records, rows, sub_projections) + end + end + + # Grouped by foreign collection, so two relations pointing at the same one + # are answered by a single read and their ids are deduped together. + # + # A relation the projection asked for is written on every row, whether or + # not it resolved: a null foreign key, and a record the operator can no + # longer reach, both read as "no related record" rather than as a row + # missing the field. + # + # The nested record is cut down to the fields the projection named, the + # way `project` cuts the row it sits on. The foreign collection hands back + # everything it holds -- Pylon has no join, so a related record is read + # through its own endpoint, which takes no field list -- and nesting that + # whole record would answer `account:name` with every column of the + # account, the ones the caller's permissions had the agent take out of the + # projection included. + # + # The primary key is added back rather than assumed: the route's + # projection carries it, but the nested record is what the serializer + # reads an `included` resource's id off, and a caller inside the agent may + # well have projected without it. + def embed_foreign(foreign_collection, relations, records, rows, sub_projections) + ids = foreign_ids(records, relations) + foreign = datasource.get_collection(foreign_collection) + indexed = ids.empty? ? {} : foreign.records_indexed_by_id(ids) + relations.each do |name, relation| + wanted = projected_fields(sub_projections[name], foreign) + rows.each_with_index do |row, index| + related = indexed[records[index][relation.foreign_key]] + row[name] = wanted ? wanted.re_project(related) : related + end + end + end + + # The fields of the relation, plus the primary key of the collection it + # points at. The key is unioned in by hand rather than through + # `Projection#with_pks`, which also walks the relations of the projection + # it is given and dereferences their schema without a nil guard: a + # projection reaching through a relation this collection does not declare + # would raise there, where it used to be ignored. Only the immediate + # foreign key is wanted here, so only that is added. + def projected_fields(sub_projection, foreign) + return nil if sub_projection.nil? + + Projection.new(sub_projection | ForestAdminDatasourceToolkit::Utils::Schema.primary_keys(foreign)) + end + + # A foreign key Pylon left empty asks for nothing — a blank one no more + # than a null one, and it would reach the `in` filter of the read below, + # which refuses a blank inside a list and would fail the whole page over + # one malformed key. The same id is asked for once however many rows point + # at it. + def foreign_ids(records, relations) + keys = relations.map { |_name, relation| relation.foreign_key } + records.flat_map { |record| keys.map { |key| record[key] } } + .reject { |id| id.nil? || id.to_s.empty? } + .uniq + end + + # `account:name` asks for the `account` relation; a projected column, and a + # relation that is not a ManyToOne, name no field to embed. + def projected_relations(projection) + Array(projection).map { |field| field.to_s.split(':').first }.uniq.filter_map do |name| + relation = schema[:fields][name] + [name, relation] if relation.is_a?(ManyToOneSchema) + end + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/team.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/team.rb new file mode 100644 index 000000000..1f621df12 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/team.rb @@ -0,0 +1,49 @@ +module ForestAdminDatasourcePylon + module Collections + class Team < FetchAllCollection + def initialize(datasource) + super(datasource, 'PylonTeam') + end + + protected + + # Pylon exposes no DELETE on a team, so that hook is left to refuse. + def create_record(payload) = datasource.client.create_team(payload) + def update_record(id, payload) = datasource.client.update_team(id, payload) + + def fetch_all + datasource.client.fetch_teams + end + + # `users` is flattened to the ids of the members: the emails Pylon nests + # there belong to PylonUser, which is where they stay up to date. + def serialize(team) + attrs = team.is_a?(Hash) ? team : {} + + { 'id' => attrs['id'], 'name' => attrs['name'], + 'user_ids' => Array(attrs['users']).filter_map { |user| user['id'] if user.is_a?(Hash) } } + end + + private + + # `/issues/search` filters `team_id` server-side, so the issues assigned to + # a team are listed by one request. `user_ids` gets no relation: Pylon + # nests the members here rather than pointing at the team from a user, so + # the membership is a ManyToMany with no join collection to declare it on. + def define_relations + add_field('issues', OneToManySchema.new(foreign_collection: 'PylonIssue', + origin_key: 'team_id', origin_key_target: 'id')) + end + + def define_schema + add_column('id', 'String', is_primary_key: true) + add_column('name', 'String', writable: true) + # A list, so neither filterable nor sortable, and no relation either: + # see `define_relations` above. `POST /teams` and `PATCH /teams/{id}` + # take the members as this very list, and the one sent replaces the + # membership whole. + add_column('user_ids', 'Json', writable: true) + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/user.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/user.rb new file mode 100644 index 000000000..149cb0918 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/user.rb @@ -0,0 +1,69 @@ +module ForestAdminDatasourcePylon + module Collections + class User < FetchAllCollection + NATIVE_FIELDS = %w[id name email emails avatar_url status role_id is_deactivated].freeze + + def initialize(datasource) + super(datasource, 'PylonUser') + end + + protected + + # Pylon exposes no POST and no DELETE on a user: an agent is invited and + # deactivated from Pylon itself, so the other two hooks are left to refuse. + def update_record(id, payload) = datasource.client.update_user(id, payload) + + # `include_deactivated` is left at the client default of true on purpose: + # a deactivated agent stays the assignee and the author of the issues they + # handled, and a record the rest of the panel points at has to stay + # readable. `is_deactivated` is exposed as a column so an operator can + # filter them out when they want to. + def fetch_all + datasource.client.fetch_users + end + + # `role` is flattened to its name only: its id is already carried by + # `role_id`, and the name is what an operator recognises. Its slug is left + # out — Pylon derives it from the name. + def serialize(user) + attrs = user.is_a?(Hash) ? user : {} + role = attrs['role'] + + NATIVE_FIELDS.to_h { |field| [field, attrs[field]] } + .merge('role_name' => role.is_a?(Hash) ? role['name'] : nil) + end + + private + + # `/issues/search` filters `assignee_id` server-side, so the issues of an + # agent are listed by one request. + # + # The teams of a user are left out: Pylon nests the members inside a team + # and exposes no team id on a user, so that side is a ManyToMany with no + # key column to build it on. + def define_relations + add_field('assigned_issues', OneToManySchema.new(foreign_collection: 'PylonIssue', + origin_key: 'assignee_id', origin_key_target: 'id')) + end + + # `PATCH /users/{id}` takes the name, the avatar, the role and the status, + # and nothing else. + def define_schema + add_column('id', 'String', is_primary_key: true) + add_column('name', 'String', writable: true) + add_column('email', 'String') + # The other addresses of the same agent: a list, so it is neither + # filterable nor sortable. `email` carries the primary one. + add_column('emails', 'Json') + add_column('avatar_url', 'String', writable: true) + # Left as String rather than Enum: Pylon documents active / away / + # out_of_office on the update endpoint, but does not promise the read + # side is limited to them. + add_column('status', 'String', writable: true) + add_column('role_id', 'String', writable: true) + add_column('role_name', 'String') + add_column('is_deactivated', 'Boolean') + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/writes.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/writes.rb new file mode 100644 index 000000000..96aa068bc --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/writes.rb @@ -0,0 +1,389 @@ +module ForestAdminDatasourcePylon + module Collections + # The write half of every Pylon collection. Included by `BaseCollection`, so + # a collection only declares the client calls, through the `*_record` hooks, + # and the fields Pylon accepts in one direction only. A hook left alone + # refuses the verb — no POST or DELETE on users, no DELETE on teams — + # instead of the contract's NotImplementedError, read by the agent as a 500. + # + # What may be written is `is_read_only` on the column, the way `api_filters` + # is what may be filtered: no second list to keep in step with the schema. + module Writes # rubocop:disable Metrics/ModuleLength + # Re-declared rather than borrowed from BaseCollection: a method defined + # here resolves a constant against this module and its ancestors, never + # against the class including it. + Filter = ForestAdminDatasourceToolkit::Components::Query::Filter + Page = ForestAdminDatasourceToolkit::Components::Query::Page + Projection = ForestAdminDatasourceToolkit::Components::Query::Projection + Leaf = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + Operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators + + # What one filter-driven update or delete may spend, in requests. Pylon + # writes one record per request, so a selection costing more than this is + # refused rather than written halfway. + # + # The cap is about the round-trips, not the quota: `RateLimiter` spaces the + # requests out inside the documented budget, which no pass this size comes + # near — but no throttling makes a hundred sequential writes a wait the + # operator watches, or a request the agent times out on first. Refusing is + # the honest answer where a partial write is not recoverable. + # + # The budget covers the whole pass, not its writes: where a record is read + # through its own endpoint, resolving the selection costs a request per + # record and reading a stored value costs another, so a cap counting the + # writes alone would let one pass spend three times this. + MAX_WRITE_REQUESTS = 20 + + # The refusal comes before the payload and the ids: everything on the way + # there answers with something else — a count, a field of the wrong + # direction — for a selection that was never the problem. + def create(_caller, data) + refuse_write('created') unless write_endpoint?(:create_record) + + serialize(create_record(build_payload(writable_attributes(data), :create))) + rescue APIError => e + surface_write_rejection(e) + end + + # What the patch may write is settled before the ids are, so a patch naming + # nothing writable is not refused for reaching too many records. + def update(caller, filter, patch) + refuse_write('updated') unless write_endpoint?(:update_record) + + attributes = writable_attributes(patch) + return if attributes.empty? + + ids = ids_for(caller, filter, extra_reads: stored_read?(attributes) ? 1 : 0) + return if ids.empty? + + payload = build_payload(attributes, :update, caller: caller, ids: ids) + return if payload.empty? + + write_each(ids, 'updated') { |id| update_record(id, payload) } + end + + def delete(caller, filter) + refuse_write('deleted') unless write_endpoint?(:delete_record) + + write_each(ids_for(caller, filter), 'deleted') { |id| delete_record(id) } + end + + protected + + # One Pylon write endpoint each, overridden by the collections having one. + def create_record(_payload) = refuse_write('created') + def update_record(_id, _payload) = refuse_write('updated') + def delete_record(_id) = refuse_write('deleted') + + # The Forest schema carries a single read-only flag per column, so both + # directions offer these; the two lists tell them apart at write time. + def create_only_fields = [].freeze + def update_only_fields = [].freeze + + # Columns whose Pylon write name differs from the one they are read under. + def payload_renames = {}.freeze + + # What reading one record costs here. Nothing where the search endpoint + # filters `id`, a whole selection travelling in one request whatever its + # size; one request where an id is read through its own endpoint. + def requests_per_record_read = 0 + + # How many records one write may reach: the budget divided by what each of + # them costs — the write itself, plus the `reads` the path still owes it. + def max_write_targets(reads: 0) + MAX_WRITE_REQUESTS / (1 + (reads * requests_per_record_read)) + end + + # How many ids a filter may name before the resolution is refused rather + # than spent: the same reach, a named id being read before it is written + # to. `nil` is no bound, a read costing nothing per record. + def max_resolvable_ids(reads: 0) + return nil if requests_per_record_read.zero? + + max_write_targets(reads: reads) + end + + # The records a filter-driven write applies to: exact, or refused — the + # caller writes one request per id and reports success for the whole + # selection, so a subset may never be answered quietly. + # + # An `id equals`/`id in` filter alone — what the record detail and the bulk + # selection send — costs no request. Anything else goes through `list`. + def ids_for(caller, filter, extra_reads: 0) + tree = filter&.condition_tree + if (named = id_values(tree)) && no_search?(filter) + cap = max_write_targets(reads: extra_reads) + refuse_too_many_targets(named.size, cap) if named.size > cap + return named + end + + # A selection naming ids is resolved by reading each of them; any other + # one by a single page of the collection's own read, whose cost does not + # grow with the count. + named_ids = filtered_ids(tree) + reads = extra_reads + (named_ids ? 1 : 0) + bound = named_ids && max_resolvable_ids(reads: reads) + refuse_unresolvable_selection(named_ids.size, bound) if bound && named_ids.size > bound + + resolve_ids_by_list(caller, filter, reads: reads) + end + + private + + # The `*_record` hook is the declaration that the collection wired the + # endpoint, read here rather than repeated in a list of supported verbs. + def write_endpoint?(hook) + method(hook).owner != Writes + end + + # One request per record, so a failure on the k-th leaves the k-1 before it + # written. The error names them: raising the API error alone reads as + # "nothing happened", and retrying on that reading would write them twice. + def write_each(ids, verb) + written = [] + + ids.each do |id| + yield id + written << id + rescue APIError => e + # Always raises, so nothing reaches the partial report below: with no + # record written the failure is the whole of what happened. + surface_write_rejection(e) if written.empty? + + refuse_partial_write(verb, written, id, ids.size, e) + end + end + + # A 4xx names something the operator did, and travels as the + # ValidationError whose message the agent surfaces where the APIError it + # arrived as would be answered with 'Unexpected error'. Anything else is + # Pylon or the network failing, which no edit of theirs would change. + # + # Only the write goes through here: a 4xx raised while resolving the + # selection still reaches them as a 500, reporting a read failure as a + # refused write being the worse of the two. + def surface_write_rejection(error) + raise error unless (400..499).cover?(error.status.to_i) + + raise WriteRejectedError, error.message + end + + # One record past the cap is asked for, so an overflow is seen rather than + # guessed from a full page. + def resolve_ids_by_list(caller, filter, reads:) + cap = max_write_targets(reads: reads) + window = Page.new(offset: 0, limit: cap + 1) + query = (filter || Filter.new).override(page: window) + records = list(caller, query, Projection.new(['id'])) + refuse_unbounded_targets(cap) if records.size > cap + + records.filter_map { |record| record['id'] }.uniq + end + + # The ids a filter names, as a leaf of its own or inside a top-level `and`. + # Unlike `extract_id_lookup`, nothing is asserted about the rest of the + # tree — the leftovers travel to `list` — so this counts records *named*, + # never records the write applies to: what it bounds is the resolution. + # + # The `id` leaves of an `and` intersect, as they do there: this is the + # count a refusal is worded around, and the first of two would refuse a + # selection narrower than the number it names. + def filtered_ids(node) + named = id_values(node) + return named if named + return nil unless and_branch?(node) + + named = Array(node.conditions).filter_map { |child| id_values(child) } + return nil if named.empty? + + named.reduce(:&) + end + + # The writable attributes, in the shape the endpoint takes them. + def build_payload(attributes, direction, caller: nil, ids: []) + attrs = honour_write_direction(attributes, direction, caller, ids) + # Pylon fills in what a create leaves out; on an update a nil is the + # operator clearing a value, so it travels. + attrs = attrs.compact if direction == :create + + custom, native = split_custom_fields(attrs) + payload = native.transform_keys { |field| payload_renames.fetch(field, field) } + payload['custom_fields'] = custom unless custom.empty? + payload + end + + # Everything else is dropped rather than refused: the front sends the + # fields of its form, and a read-only one reaching the payload is the + # agent's doing, not a request the operator made. + def writable_attributes(data) + attrs = data.is_a?(Hash) ? data.transform_keys(&:to_s) : {} + + attrs.select { |field, _value| writable_column?(field) } + end + + def writable_column?(field) + column = schema[:fields][field] + + column&.type == 'Column' && !column.is_read_only + end + + # Whether the patch will have `stored_values` read every record it reaches + # before a field of the wrong direction is dropped or refused, which the + # cap has to charge it for: see `ids_for`. + def stored_read?(attributes) = (attributes.keys & create_only_fields).any? + + # A field of the other direction is dropped when it asks for nothing, and + # refused when the operator really changed it: answering an edit with a + # success Pylon did not perform is worse than an error naming the field. + # + # Only a create can tell without reading, Pylon filling it in with exactly + # what a blank value asks for. On an update the stored value is what + # settles it — an unchecked box is nothing over a stored `false`, and a + # real edit over a stored `true`. + def honour_write_direction(attrs, direction, caller, ids) + wrong = attrs.keys & (direction == :create ? update_only_fields : create_only_fields) + return attrs if wrong.empty? + + asked = if direction == :create + wrong.reject { |field| blank_write_value?(attrs[field]) } + else + wrong - unchanged_fields(caller, ids, wrong, attrs) + end + refuse_wrong_direction(asked, direction) unless asked.empty? + + attrs.except(*wrong) + end + + # What a form sends for a field the operator never touched: no value at + # all, an unchecked box, an empty list. A `0` or a string is a value only + # the other endpoint could write. + def blank_write_value?(value) + return true if value.nil? || value == false + return value.empty? if value.respond_to?(:empty?) + + false + end + + # The wrong-direction fields already holding the value the patch asks for. + # One record the read did not hand back is enough to refuse them all: + # nothing here may claim a value is unchanged on a record it never read. + def unchanged_fields(caller, ids, fields, attrs) + return [] if fields.empty? + + stored = stored_values(caller, ids, fields) + return [] if stored.size < ids.size + + fields.select { |field| stored.all? { |record| same_write_value?(record[field], attrs[field]) } } + end + + # Two blanks are the same state: Pylon returns a null where the form sends + # `false` or an empty string for the same untouched field. Strings are + # compared stripped, `body_html` travelling through an editor that may hand + # back the markup it was given re-indented — and refusing an edit nobody + # made is the one error the operator cannot act on. + def same_write_value?(stored, asked) + return true if blank_write_value?(stored) && blank_write_value?(asked) + return stored.to_s.strip == asked.to_s.strip if stored.is_a?(String) || asked.is_a?(String) + + stored == asked + end + + # Read only when the patch names a field of the wrong direction, and only + # for those fields. One request where the endpoint filters `id`, one per + # record where an id is read through its own endpoint — which the cap does + # charge the patch for, `stored_read?` declaring it before the ids are + # resolved. + # + # By id rather than through the caller's filter: that filter was already + # resolved into these ids, so re-running it would spend those requests + # twice and, carrying no page, walk every record it matches. + def stored_values(caller, ids, fields) + query = Filter.new(condition_tree: Leaf.new('id', Operators::IN, ids), + page: Page.new(offset: 0, limit: ids.size)) + + list(caller, query, Projection.new(['id'] + fields)) + end + + # Pylon reads its custom fields back as a map indexed by slug and writes + # them as a list, `values` for a multi-value field and `value` for every + # other — a select by the slug of its option, what the Enum advertises. + def split_custom_fields(attrs) + by_column = custom_fields_by_column + entries = [] + + native = attrs.each_with_object({}) do |(field, value), rest| + custom_field = by_column[field] + custom_field ? entries << custom_field_entry(custom_field, value) : rest[field] = value + end + + [entries, native] + end + + def custom_fields_by_column + @custom_fields_by_column ||= custom_fields.to_h { |field| [field[:column_name], field] } + end + + def custom_field_entry(custom_field, value) + slug = custom_field[:column_name] + return { 'slug' => slug, 'values' => Array(value) } if custom_field[:multi_value] + + { 'slug' => slug, 'value' => value } + end + + def refuse_write(verb) + raise UnsupportedWriteError, + "A #{name} record cannot be #{verb}: the Pylon API exposes no endpoint for it." + end + + # Every offending field at once: refusing them one at a time would have the + # operator undo one, retry, and learn about the next. + def refuse_wrong_direction(fields, direction) + them = fields.one? ? 'it' : 'them' + detail = if direction == :create + "Pylon only accepts #{them} on an existing record: create the record, then edit it." + else + "Pylon only accepts #{them} when the record is created, and exposes no endpoint to change " \ + "#{them} afterwards." + end + + named = fields.map { |field| "'#{field}'" }.join(', ') + raise UnsupportedWriteError, "#{named} cannot be set here on a #{name}: #{detail}" + end + + # The count is exact here, the filter having named the ids. + def refuse_too_many_targets(count, cap) + refuse_write_reach("applies to #{count} #{name} records, more than the #{cap} one pass covers") + end + + # The resolution only knows the selection overflows: reporting the size of + # its window would name 21 records to a selection holding thousands. + def refuse_unbounded_targets(cap) + refuse_write_reach("applies to more than the #{cap} #{name} records one pass covers") + end + + def refuse_write_reach(reach) + raise UnsupportedWriteError, + "This write #{reach}: Pylon writes one record per request, and a write stopping halfway — on a " \ + 'timeout, or on the first record Pylon refuses — would report a success it did not perform. ' \ + 'Narrow the selection to reach the records past this point.' + end + + # How many of the named ids the rest of the filter matches is unknown here, + # so the count is reported as what it is: records named. + def refuse_unresolvable_selection(count, bound) + raise UnsupportedWriteError, + "This write names #{count} #{name} records and filters them further, which #{name} answers with " \ + "one request per named record, on top of the one each write costs: more than the #{bound} one " \ + 'pass covers. Select fewer records, or drop the other conditions to write the ones named.' + end + + def refuse_partial_write(verb, written, failed_id, total, error) + raise PartialWriteError, + "#{written.size} of #{total} #{name} records were #{verb} and then '#{failed_id}' failed: " \ + "#{error.message}. The records already #{verb} are #{written.join(", ")}, and they stay " \ + "#{verb} — the ones after them were left untouched. Retry the write on the untouched records " \ + "alone: retrying the whole selection would perform it twice on the ones already #{verb}." + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/configuration.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/configuration.rb new file mode 100644 index 000000000..859576220 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/configuration.rb @@ -0,0 +1,72 @@ +module ForestAdminDatasourcePylon + class Configuration + DEFAULT_BASE_URL = 'https://api.usepylon.com'.freeze + + attr_reader :api_key, :base_url, :open_timeout, :timeout, :retry_policy, :rate_limiter, + :boot_open_timeout, :boot_timeout, :boot_retry_policy + + # `rate_limiter: nil` takes the throttling out of the stack, leaving the 429 + # retry as the only rate-limit handling. For the deployment metering on its + # own side, or the one that would rather see the 429. + # + # The `boot_` trio governs what the datasource reads while it is being + # constructed, where the wait is a Rails boot the operator sits through + # rather than a request that has already returned a page. + def initialize(api_key:, base_url: nil, open_timeout: 5, timeout: 30, retry_policy: RetryPolicy.new, + rate_limiter: RateLimiter.new, boot_open_timeout: 3, boot_timeout: 10, + boot_retry_policy: RetryPolicy.boot) + @api_key = api_key + @base_url = base_url || DEFAULT_BASE_URL + @open_timeout = open_timeout + @timeout = timeout + @retry_policy = retry_policy + @rate_limiter = rate_limiter + @boot_open_timeout = boot_open_timeout + @boot_timeout = boot_timeout + @boot_retry_policy = boot_retry_policy + validate! + end + + # Pylon exposes unversioned paths (`/issues`, `/me`) directly under the host. + def url + @base_url.chomp('/') + end + + # Whatever precedes the endpoint in the path, for a base url mounted under a + # subpath — an egress proxy, or a mock server. Empty against the API itself. + # `RateLimits` is keyed on the endpoint, so this has to come off a path + # before the table is asked: left on, every anchored rule misses and the + # whole datasource meters in one fallback bucket. + def base_path + @base_path ||= URI.parse(url).path + end + + # `api_key` is a bearer token, and nothing prints a Configuration on + # purpose: what reaches an `inspect` is a Rails error page, or a + # `logger.debug` of something holding one. The default would put the token + # in clear there. + # + # One of three, not the whole of it: the token also rides in the headers of + # the client's Faraday connections, which print them in clear, so `Client` + # and `Datasource` mask their own. Together they cut every path from an + # object this package hands out to the credential. + def inspect + "#<#{self.class.name} base_url=#{@base_url.inspect} api_key=[FILTERED]>" + end + + private + + def validate! + missing = [] + missing << 'api_key' if blank?(@api_key) + return if missing.empty? + + raise ConfigurationError, + "ForestAdminDatasourcePylon missing required config: #{missing.join(", ")}" + end + + def blank?(value) + value.nil? || value.to_s.strip.empty? + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/datasource.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/datasource.rb new file mode 100644 index 000000000..ccd025b2a --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/datasource.rb @@ -0,0 +1,53 @@ +module ForestAdminDatasourcePylon + class Datasource < ForestAdminDatasourceToolkit::Datasource + attr_reader :client, :configuration + + def initialize(api_key:, **options) + super() + @configuration = Configuration.new(api_key: api_key, **options) + @client = Client.new(@configuration) + + register_collections + end + + # The datasource is what a Rails error page or a `logger.debug` is likeliest + # to print, and it holds the client whose connections carry the bearer + # token. Every collection reaches that token the same way, through the + # `@datasource` the toolkit's Collection keeps, so cutting the chain here + # covers them too -- and spares the recursive dump the default `inspect` + # walks into, a datasource and its collections pointing at each other. + def inspect + "#<#{self.class.name} collections=#{collections.keys.inspect}>" + end + + private + + # The five collections are registered together: each one declares relations + # pointing at the others, and a relation whose foreign collection is missing + # is a schema the agent refuses to boot on. + # + # Their custom fields are introspected here, one call per object type, as + # Pylon indexes its definitions by object type and asks for one on every + # call. Each entry stays on the collection it belongs to: a custom field is + # filtered through the very slug it is read by, which the collection's + # `api_filters` already carries, so there is no datasource-wide mapping to + # hold — and two Pylon datasources in the same agent share nothing. + # + # An introspection that fails costs the custom columns, not the datasource: + # `fetch_custom_fields` degrades and the agent boots on the native schema. + # The first failure also stands for the object types after it, each call + # being bounded per request rather than across the three — see + # `CustomFieldsIntrospector`. + def register_collections + custom_fields = Schema::CustomFieldsIntrospector.new(@client) + + add_collection(Collections::Issue.new(self, custom_fields: custom_fields.issue_custom_fields)) + add_collection(Collections::Account.new(self, custom_fields: custom_fields.account_custom_fields)) + add_collection(Collections::Contact.new(self, custom_fields: custom_fields.contact_custom_fields)) + # Pylon carries custom fields on issues, accounts and contacts only: + # neither an agent nor a team has any. + add_collection(Collections::User.new(self)) + add_collection(Collections::Team.new(self)) + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/issue_enums.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/issue_enums.rb new file mode 100644 index 000000000..b463878ad --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/issue_enums.rb @@ -0,0 +1,20 @@ +module ForestAdminDatasourcePylon + # The closed sets `POST /issues` and `PATCH /issues/{id}` document, shared by + # the plugins building forms over them. + module IssueEnums + # Accepted on a create, and absent from every read: Pylon never returns the + # priority of an issue, which is why no column carries it. + PRIORITY = %w[urgent high medium low].freeze + + # Where the first message of a created issue is delivered. `internal` is the + # absence of a delivery, and travels as no `destination_metadata` at all. + DESTINATION = %w[email slack in_app_chat customer_portal sms whatsapp internal].freeze + + INTERNAL_DESTINATION = 'internal'.freeze + + # The one state every organization has: the others Pylon ships, and the + # custom ones defined on top of them, are named by the option that writes + # them rather than listed here. + CLOSED_STATE = 'closed'.freeze + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/pagination/cursor_walker.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/pagination/cursor_walker.rb new file mode 100644 index 000000000..77333f794 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/pagination/cursor_walker.rb @@ -0,0 +1,103 @@ +module ForestAdminDatasourcePylon + module Pagination + # Forest asks for an offset/limit window; Pylon only knows how to hand out + # the next page of a cursor. Bridging the two means walking pages until the + # window is covered, then slicing. Deep offsets therefore cost one request + # per page, which is why the walk is capped: the requests are sequential — + # a cursor is only known once the page before it came back — so an unbounded + # walk is a list view the operator waits on, page after page, well before it + # is a quota `/issues/search` grants 120 requests a minute of. + class CursorWalker + MAX_PAGES = 20 + MAX_RECORDS = 5_000 + + def initialize(max_pages: MAX_PAGES, max_records: MAX_RECORDS) + @max_pages = max_pages + @max_records = max_records + end + + # Yields `(limit, cursor)` and expects a Client::SearchPage back. + # + # A nil limit asks for every record past the offset: the walk then runs + # until Pylon says there is no page left, or until a cap stops it. That + # distinction is the whole point of accepting nil rather than a limit + # standing in for "everything": a walk told to collect a thousand records + # stops at a thousand having covered the window it was given, and reports + # nothing, while a walk told to collect everything and stopped by a cap + # knows it is handing back less than it was asked for, and says so. + def walk(offset:, limit:, &page_source) + offset = offset.to_i.clamp(0, nil) + limit = limit&.to_i + return [] if limit && !limit.positive? + + records = collect(offset, limit, &page_source) + + limit ? (records[offset, limit] || []) : records.drop(offset) + end + + private + + # The walk itself: pages are collected until the window is covered, the + # source says there is nothing left, or a cap stops it. The slicing is + # `walk`'s, this only decides how far to go. + def collect(offset, limit) + needed = limit && (offset + limit) + records = [] + cursor = nil + seen = Set.new + pages = 0 + + loop do + page = yield(batch_size(needed, records.size), cursor) + records.concat(page.records) + pages += 1 + + break if stop?(page, seen) + break if needed && records.size >= needed + + if capped?(pages, records.size) + log_truncation(offset: offset, limit: limit, pages: pages, collected: records.size) + break + end + + cursor = page.next_cursor + end + + records + end + + # An empty page, a cursor that does not move and a cursor already followed + # all stop the walk; Pylon does none of the three today, but a walk driven + # by a remote value stops on its own terms rather than on the caps only. + # + # The whole set is kept rather than the previous cursor alone, like + # `Client#collect_pages`: a cycle wider than one page would otherwise + # collect the same pages over and over until a cap cut it short, and hand + # back the duplicates as records. + def stop?(page, seen) + page.next_cursor.nil? || page.records.empty? || !seen.add?(page.next_cursor) + end + + def capped?(pages, collected) + pages >= @max_pages || collected >= @max_records + end + + # Bounded by the record budget left, and by the window still missing when + # there is one, so the walk never collects past @max_records. + def batch_size(needed, collected) + budget = @max_records - collected + budget = [needed - collected, budget].min if needed + budget.clamp(1, Client::MAX_SEARCH_LIMIT) + end + + def log_truncation(offset:, limit:, pages:, collected:) + window = limit ? "offset=#{offset} limit=#{limit}" : "every record past offset=#{offset}" + ForestAdminDatasourcePylon.logger.warn( + "[forest_admin_datasource_pylon] Stopped paginating after #{pages} page(s) / #{collected} record(s) " \ + "while fetching #{window}; results are truncated. " \ + 'Narrow the filter to reach records past this point.' + ) + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/plugins/close_issue.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/plugins/close_issue.rb new file mode 100644 index 000000000..366147942 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/plugins/close_issue.rb @@ -0,0 +1,116 @@ +module ForestAdminDatasourcePylon + module Plugins + # Moves the selected issues to a state, `closed` unless told otherwise. + # + # One variant per scope, where the Zendesk plugin builds four: Zendesk has + # two terminal statuses to tell apart, Pylon has `closed` and, past it, the + # custom status slugs an organization defines — which the `state` option + # takes, rather than a second dimension of action names nobody would read. + # + # The state is written through the client rather than through the + # collection: the action is registered on the host collection, which is + # rarely PylonIssue, and going through a collection would mean resolving it + # from a datasource the plugin was not given. + class CloseIssue < ForestAdminDatasourceCustomizer::Plugins::Plugin + BaseAction = ForestAdminDatasourceCustomizer::Decorators::Action::BaseAction + ActionScope = ForestAdminDatasourceCustomizer::Decorators::Action::Types::ActionScope + ForestException = ForestAdminDatasourceToolkit::Exceptions::ForestException + + # What one run may write, the budget a filter-driven write already gets: + # this writes one PATCH per issue too, sequentially, and the action runs + # inside a single HTTP request. Past this the batch is refused before the + # first write rather than discovered after the twentieth, where a run cut + # short by a timeout leaves part of the selection moved and reports which + # part to nobody -- `apply_state` only names it once the loop returns. + # + # The collection cannot bound this: the action writes through the client, + # being registered on the host collection rather than on PylonIssue. + MAX_TARGETS = Collections::Writes::MAX_WRITE_REQUESTS + + SCOPE_KEYS = %i[single bulk].freeze + SCOPES = { single: ActionScope::SINGLE, bulk: ActionScope::BULK }.freeze + NAMES = { single: 'Close Pylon issue', bulk: 'Close selected Pylon issues' }.freeze + NAME_OPTIONS = { single: :action_name, bulk: :bulk_action_name }.freeze + + def run(_datasource_customizer, collection_customizer = nil, options = {}) + opts = options.is_a?(Hash) ? options : {} + datasource = opts[:datasource] + raise ForestException, 'CloseIssue plugin requires :datasource' unless datasource + raise ForestException, 'CloseIssue plugin requires a collection' unless collection_customizer + + state = normalize_state(opts[:state]) + + normalize_scopes(opts[:scopes]).each do |scope_key| + collection_customizer.add_action(name_for(scope_key, opts), + build_action(datasource, SCOPES[scope_key], state, opts[:issue_id_field])) + end + end + + private + + # Left unchecked against the states Pylon ships: it takes the slug of a + # custom status just as well, and refusing one would refuse the very + # workflow an organization built. + def normalize_state(value) + state = value.nil? ? IssueEnums::CLOSED_STATE : value.to_s.strip + return state unless state.empty? + + raise ForestException, 'CloseIssue :state cannot be empty.' + end + + # Through `to_s`: a value that is neither a string nor a symbol has no + # `to_sym`, and raising NoMethodError here would hide the unknown-scope + # error that names what was actually passed. + def normalize_scopes(value) + scopes = Array(value).map { |scope| scope.to_s.to_sym }.uniq + scopes = SCOPE_KEYS if scopes.empty? + unknown = scopes - SCOPE_KEYS + return scopes if unknown.empty? + + raise ForestException, + "Unknown CloseIssue scopes: #{unknown.join(", ")}. Allowed: #{SCOPE_KEYS.join(", ")}." + end + + def name_for(scope_key, opts) + opts[NAME_OPTIONS[scope_key]] || NAMES[scope_key] + end + + def build_action(datasource, scope, state, issue_id_field) + BaseAction.new(scope: scope, &executor(datasource, state, issue_id_field)) + end + + def executor(datasource, state, issue_id_field) + lambda do |context, result_builder| + ids = IssueTargets.resolve_issue_ids(context, issue_id_field) + next result_builder.error(message: Messages.no_target(issue_id_field)) if ids.empty? + next result_builder.error(message: Messages.too_many(ids.size, state)) if ids.size > MAX_TARGETS + + succeeded, failed = apply_state(datasource, ids, state) + next result_builder.error(message: Messages.error(failed, state)) if succeeded.empty? + + result_builder.success(message: Messages.success(succeeded, failed, state)) + end + end + + # One rescue per id: a single issue Pylon refuses — deleted, or outside + # the token's scope — must not cost the operator the rest of a selection, + # and what failed is named in the message rather than left to a log. + def apply_state(datasource, ids, state) + succeeded = [] + failed = [] + + ids.each do |id| + datasource.client.update_issue(id, 'state' => state) + succeeded << id + rescue StandardError => e + ForestAdminDatasourcePylon.logger.warn( + "[forest_admin_datasource_pylon] failed to move issue #{id} to '#{state}': #{e.class}: #{e.message}" + ) + failed << [id, "#{e.class}: #{e.message}"] + end + + [succeeded, failed] + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/plugins/close_issue/messages.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/plugins/close_issue/messages.rb new file mode 100644 index 000000000..e75fe686a --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/plugins/close_issue/messages.rb @@ -0,0 +1,62 @@ +module ForestAdminDatasourcePylon + module Plugins + class CloseIssue + # What the operator reads once the batch ran. Every id that failed is + # named: an action reporting a plain success over a batch it only half + # applied is the one thing the panel cannot recover from. + module Messages + module_function + + def success(succeeded, failed, state) + [succeeded_phrase(succeeded, state), failed_phrase(failed)].compact.join(' ') + end + + def error(failed, state) + return "Failed to #{verb(state)} issue #{failed.first.first}: #{failed.first.last}" if failed.size == 1 + + "Failed to #{verb(state)} all #{failed.size} issues. First error: #{failed.first.last}" + end + + # Worded around the selection rather than around the run: what the + # operator can act on is how many issues they picked, and the cap is + # named so the next attempt is a size they can aim for. + def too_many(count, state) + "This selection names #{count} Pylon issues, more than the #{CloseIssue::MAX_TARGETS} one run of " \ + 'this action covers: Pylon takes one request per issue, and a run stopping halfway would leave ' \ + "part of the selection #{past_verb(state)} without naming which part. Select fewer issues, and " \ + 'run the action again on the rest.' + end + + def no_target(field) + return 'No Pylon issue selected.' if field.nil? + + "No Pylon issue id found in '#{field}'." + end + + def succeeded_phrase(succeeded, state) + return nil if succeeded.empty? + + return "Issue #{succeeded.first} #{past_verb(state)}." if succeeded.size == 1 + + "#{succeeded.size} issues #{past_verb(state)}." + end + + def failed_phrase(failed) + return nil if failed.empty? + + "#{failed.size} failed: #{failed.map(&:first).join(", ")}." + end + + # A custom status is named as it is, where the state every organization + # has reads as the verb an operator used to fire the action. + def verb(state) + state == IssueEnums::CLOSED_STATE ? 'close' : "move to #{state}" + end + + def past_verb(state) + state == IssueEnums::CLOSED_STATE ? 'closed' : "moved to #{state}" + end + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/plugins/create_issue_with_notification.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/plugins/create_issue_with_notification.rb new file mode 100644 index 000000000..596bdd78d --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/plugins/create_issue_with_notification.rb @@ -0,0 +1,175 @@ +module ForestAdminDatasourcePylon + module Plugins + # Opens a Pylon issue and delivers its first message to the requester. + # + # Pylon creates the contact on the fly from the form's email, so the action + # can be registered on any host collection — no relation to Pylon needed. + # + # Where Zendesk notifies as a side effect of a public comment, Pylon says it + # outright: `destination_metadata.destination` names the channel the + # issue's `body_html` is delivered through, and no `destination_metadata` at + # all is what leaves the issue internal. The "Send as internal note" + # checkbox is that choice, worded the way the Zendesk plugin words it. + # + # The form is FormBuilder's, the wire payload is Payload's; what is left + # here is the registration, its options, and what the operator reads back. + class CreateIssueWithNotification < ForestAdminDatasourceCustomizer::Plugins::Plugin + BaseAction = ForestAdminDatasourceCustomizer::Decorators::Action::BaseAction + ActionScope = ForestAdminDatasourceCustomizer::Decorators::Action::Types::ActionScope + ForestException = ForestAdminDatasourceToolkit::Exceptions::ForestException + + NAME = 'Create Pylon issue and notify'.freeze + + def run(_datasource_customizer, collection_customizer = nil, options = {}) + options = {} unless options.is_a?(Hash) + datasource = options[:datasource] + raise ForestException, 'CreateIssueWithNotification plugin requires :datasource' unless datasource + raise ForestException, 'CreateIssueWithNotification plugin requires a collection' unless collection_customizer + + opts = options.except(:datasource) + opts[:email_templates] = normalize_templates(opts[:email_templates]) + opts[:destination] = normalize_destination(opts[:destination]) + opts[:priority_override] = normalize_priority(opts[:priority_override]) + require_sender_email!(opts) + + collection_customizer.add_action(opts[:action_name] || NAME, build_action(datasource, opts)) + end + + private + + # The title is what the enum carries and what the content is looked up + # by, so it has to name one template: a duplicate makes the first + # unreachable and would send the other one's content under its name, and + # the sentinel of the "pick nothing" option makes the template it names + # unpickable. Both are configuration, so both are refused at registration + # rather than discovered by whoever sends the wrong message. + def normalize_templates(value) + templates = Array(value).compact + titles = templates.map { |template| template[:title].to_s } + + reserved = titles.include?(FormBuilder::NO_TEMPLATE) + raise ForestException, "An email template cannot be titled #{FormBuilder::NO_TEMPLATE.inspect}." if reserved + + duplicated = titles.tally.select { |_title, count| count > 1 }.keys + raise ForestException, "Duplicate email template titles: #{duplicated.join(", ")}." if duplicated.any? + + templates + end + + def normalize_destination(value) + return Payload::EMAIL_DESTINATION if value.nil? + + normalize(value, IssueEnums::DESTINATION, 'destination') + end + + # `POST /issues` refuses an email delivery that does not name the address it + # is sent from, so the option is mandatory there rather than optional. The + # refusal belongs at registration, where it names the option and the agent + # will not boot without it, rather than at the first execution, where it + # reaches the operator as a Pylon 400 on a form they filled correctly. + def require_sender_email!(opts) + return unless opts[:destination] == Payload::EMAIL_DESTINATION + return if Payload.present?(opts[:sender_email]) + + raise ForestException, + 'CreateIssueWithNotification requires :sender_email when the destination is email. ' \ + 'It must be one of the addresses configured in the Pylon email app.' + end + + def normalize_priority(value) + return nil unless Payload.present?(value) + + normalize(value, IssueEnums::PRIORITY, 'priority') + end + + def normalize(value, allowed, label) + normalized = value.to_s + return normalized if allowed.include?(normalized) + + raise ForestException, + "Unknown CreateIssueWithNotification #{label}: #{normalized}. Allowed: #{allowed.join(", ")}." + end + + def build_action(datasource, opts) + BaseAction.new(scope: ActionScope::SINGLE, form: FormBuilder.build(opts), &executor(datasource, opts)) + end + + def executor(datasource, opts) + lambda do |context, result_builder| + values = submitted_values(context, opts) + email = values['Requester email'] + next result_builder.error(message: 'Requester email is required.') unless Payload.present?(email) + + issue = create_issue(datasource, Payload.build(values, email, opts)) + next result_builder.error(message: issue.last) if issue.is_a?(Array) + + writeback = write_back_issue_id(context, opts[:issue_id_field], issue['id']) + result_builder.success(message: success_message(issue, values, opts, writeback)) + end + end + + # The form only carries the internal-note checkbox when the option asked + # for it, but a form value is not what the form offered: the agent copies + # every key the request sent, matched against a field or not. Left as + # submitted, the flag would create an internal issue on an action + # registered without it — and the requester the plugin exists to notify + # would not be. What the form does not show cannot be sent. + def submitted_values(context, opts) + values = context.form_values + return values if opts[:show_internal_note] + + values.merge(FormBuilder::INTERNAL_NOTE_LABEL => false) + end + + # A 4xx is Pylon naming what the operator filled in, and reaches them as + # the action's own error, message intact: raised, it would leave the agent + # to answer 'Unexpected error' — APIError is none of the classes whose + # message the translator passes through — and to log nothing either, its + # status being under 500. Anything else is Pylon or the network failing, + # which no edit of the form would change, and stays the 500 it is. + def create_issue(datasource, payload) + datasource.client.create_issue(payload) + rescue APIError => e + raise unless (400..499).cover?(e.status.to_i) + + ForestAdminDatasourcePylon.logger.warn( + "[forest_admin_datasource_pylon] Pylon refused the issue creation: #{e.message}" + ) + [:rejected, e.message] + end + + # Best-effort: Pylon has no transaction to roll back, and the issue exists + # whether or not the host record could be stamped with its id. + def write_back_issue_id(context, field, issue_id) + return :skipped if field.nil? + + context.collection.update(context.filter, { field => issue_id }) + :ok + rescue StandardError => e + ForestAdminDatasourcePylon.logger.warn( + "[forest_admin_datasource_pylon] failed to store the issue id in '#{field}': #{e.class}: #{e.message}" + ) + [:failed, "#{e.class}: #{e.message}"] + end + + def success_message(issue, values, opts, writeback) + base = base_success_message(issue, values, opts) + return base unless writeback.is_a?(Array) && writeback.first == :failed + + "#{base} (warning: could not store the issue id on the record: #{writeback.last})" + end + + # The number is what an operator recognises an issue by; the id stands in + # when Pylon answered without one. + def base_success_message(issue, values, opts) + reference = issue['number'] || issue['id'] + destination = Payload.destination_for(values, opts) + if Payload.internal?(destination) + return "Issue ##{reference} created (internal, the requester was not contacted)." + end + + "Issue ##{reference} created and the requester notified by #{destination.tr("_", " ")}." + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/plugins/create_issue_with_notification/form_builder.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/plugins/create_issue_with_notification/form_builder.rb new file mode 100644 index 000000000..e9e32ccca --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/plugins/create_issue_with_notification/form_builder.rb @@ -0,0 +1,173 @@ +require 'cgi' + +module ForestAdminDatasourcePylon + module Plugins + class CreateIssueWithNotification + module FormBuilder + FieldType = ForestAdminDatasourceCustomizer::Decorators::Action::Types::FieldType + + NO_TEMPLATE = 'No template'.freeze + TOKEN_RE = /\{\{\s*record\.([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}/ + + # Shared with `Payload`, which reads the submitted value back, and with + # the executor, which has to neutralise it when the form does not carry + # the field at all. + INTERNAL_NOTE_LABEL = 'Send as internal note'.freeze + + module_function + + # ActionCollectionDecorator rejects forms that mix Page elements with + # non-Page elements, so each mode (flat / wizard) stays homogeneous. + def build(opts) + body = body_fields(opts) + return body if opts[:email_templates].empty? + + [ + { type: 'Layout', component: 'Page', next_button_label: 'Continue', + elements: [template_field(opts[:email_templates])] }, + { type: 'Layout', component: 'Page', previous_button_label: 'Back', + elements: body } + ] + end + + # No Type field, unlike the Zendesk form: `POST /issues` does not take + # one -- Pylon accepts `type` on an update only, which is what + # `Issue::UPDATE_ONLY` already says. + def body_fields(opts) + fields = [requester_field(opts[:requester_email_default]), + subject_field(opts[:default_subject]), + message_field(opts[:default_message], opts[:email_templates])] + fields << priority_field unless present?(opts[:priority_override]) + fields << internal_note_field if opts[:show_internal_note] + fields + end + + def requester_field(default) + { type: FieldType::STRING, label: 'Requester email', is_required: true, + description: 'Email of the Pylon requester; the contact is created on the fly when it is unknown. ' \ + 'Pre-filled from the selected record when available.', + default_value: requester_default(default) } + end + + def template_field(templates) + { type: FieldType::ENUM, label: 'Template', is_required: true, + enum_values: [NO_TEMPLATE] + templates.map { |t| t[:title] }, + default_value: NO_TEMPLATE, + description: 'Pick a template to pre-fill the Message on the next page.' } + end + + def subject_field(default_subject) + { type: FieldType::STRING, label: 'Subject', is_required: true, + default_value: template_default(default_subject, escape_html: false) } + end + + def message_field(default_message, templates) + field = { type: FieldType::STRING, label: 'Message', widget: 'RichText', is_required: true, + description: 'The body of the issue (HTML). Unless it is sent as an internal note, this is ' \ + 'the message Pylon delivers to the requester.' } + default = template_default(default_message, escape_html: true) + return field.merge(default_value: default) if templates.empty? + + # Both keys: `default_value:` fills the first render — drop_default + # runs once, the data key sticking after it — where `value:` is + # re-evaluated by drop_deferred on every fetch, which is what a + # Template change re-fires the message proc through. + field.merge(default_value: default, value: message_value(templates, default)) + end + + # No default: Pylon applies its own when the key is absent, and no + # priority is ever read back — the issue payload does not carry one, so + # nothing in Forest will show the operator what they picked. + def priority_field + { type: FieldType::ENUM, label: 'Priority', enum_values: IssueEnums::PRIORITY, + description: 'Set on creation only; Pylon does not return the priority of an issue, so it is not ' \ + 'shown anywhere in Forest afterwards.' } + end + + def internal_note_field + { type: FieldType::BOOLEAN, label: INTERNAL_NOTE_LABEL, + description: 'When checked, the issue is created without contacting the requester.', + default_value: false } + end + + def requester_default(value) + return nil if value.nil? + return template_default(value, escape_html: false) if value.is_a?(String) + + lambda do |context| + record = fetch_record(context) + record.empty? ? nil : value.call(record) + rescue StandardError => e + ForestAdminDatasourcePylon.logger.warn( + "[forest_admin_datasource_pylon] requester_email_default resolver raised: #{e.class}: #{e.message}" + ) + nil + end + end + + def template_default(template, escape_html:) + return nil unless present?(template) + return template unless template.match?(TOKEN_RE) + + ->(context) { interpolate(template, fetch_record(context), escape_html: escape_html) } + end + + # Returns nil unless Template was just changed, so set_watch_changes + # carries over the user's current Message edits between renders. + # + # Taking a template back restores the configured default rather than + # emptying a required field: it is what the operator was handed before + # they picked one. + def message_value(templates, default) + by_title = templates.to_h { |t| [t[:title], t[:content].to_s] } + lambda do |context| + return nil unless context.field_changed?('Template') + + title = context.get_form_value('Template') + return evaluate_default(default, context) if title == NO_TEMPLATE + + interpolated(by_title[title].to_s, context) + end + end + + # A default carrying tokens is a proc, and one without is the string + # itself; no default at all empties the field, as it always did. + def evaluate_default(default, context) + default.is_a?(Proc) ? default.call(context) : default.to_s + end + + def interpolated(content, context) + return content unless content.match?(TOKEN_RE) + + interpolate(content, fetch_record(context), escape_html: true) + end + + def fetch_record(context) + context.get_record([]) || {} + rescue StandardError => e + ForestAdminDatasourcePylon.logger.warn( + "[forest_admin_datasource_pylon] failed to fetch record for token interpolation: #{e.class}: #{e.message}" + ) + {} + end + + # The message ships as `body_html` and is delivered as such — an + # unescaped `<` or `&` coming from a record value would break the + # outbound message or smuggle markup into it. + def interpolate(template, record, escape_html:) + template.gsub(TOKEN_RE) do + key = ::Regexp.last_match(1) + value = record[key] + next '' if value.nil? + + escape_html ? CGI.escapeHTML(value.to_s) : value.to_s + end + end + + def present?(value) + !value.nil? && value.to_s != '' + end + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/plugins/create_issue_with_notification/payload.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/plugins/create_issue_with_notification/payload.rb new file mode 100644 index 000000000..fcf972450 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/plugins/create_issue_with_notification/payload.rb @@ -0,0 +1,72 @@ +module ForestAdminDatasourcePylon + module Plugins + class CreateIssueWithNotification + # What the filled form becomes on the wire, where FormBuilder owns what + # the operator fills. + module Payload + # Only meaningful on an email delivery: Pylon reads the sending address + # and the copies off the email app they belong to. The address is + # mandatory there — `CreateIssueWithNotification` refuses to register an + # email delivery without one. + EMAIL_DESTINATION = 'email'.freeze + + module_function + + def build(values, email, opts) + payload = { + 'title' => values['Subject'], + 'body_html' => values['Message'], + # Pylon wants a name alongside the address when it creates the + # contact; derive it from the local part. It is ignored when the + # contact already exists. + 'requester_email' => email, + 'requester_name' => derive_requester_name(email) + } + priority = opts[:priority_override] || values['Priority'] + payload['priority'] = priority if present?(priority) + + destination = destination_for(values, opts) + payload['destination_metadata'] = metadata(destination, opts) unless internal?(destination) + payload + end + + # The checkbox wins over the configured destination: it is the + # operator's call, made on the record they are looking at. + def destination_for(values, opts) + truthy?(values[FormBuilder::INTERNAL_NOTE_LABEL]) ? IssueEnums::INTERNAL_DESTINATION : opts[:destination] + end + + # An internal issue travels as no metadata at all rather than as + # `{destination: 'internal'}`: that is the form the API reference names + # for "do not contact the requester", and the one that stays right if + # Pylon ever adds a required companion field to a real destination. + def internal?(destination) + destination == IssueEnums::INTERNAL_DESTINATION + end + + def metadata(destination, opts) + metadata = { 'destination' => destination } + return metadata unless destination == EMAIL_DESTINATION + + metadata['email'] = opts[:sender_email] + metadata['email_ccs'] = Array(opts[:email_ccs]) if Array(opts[:email_ccs]).any? + metadata['email_bccs'] = Array(opts[:email_bccs]) if Array(opts[:email_bccs]).any? + metadata + end + + def derive_requester_name(email) + local = email.to_s.split('@').first.to_s + local.empty? ? email.to_s : local + end + + def truthy?(value) + value == true || value.to_s.casecmp('true').zero? + end + + def present?(value) + !value.nil? && value.to_s != '' + end + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/plugins/issue_targets.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/plugins/issue_targets.rb new file mode 100644 index 000000000..f50a0fc24 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/plugins/issue_targets.rb @@ -0,0 +1,51 @@ +module ForestAdminDatasourcePylon + module Plugins + # Which Pylon issues an action was fired on. + # + # Two shapes, one option: `issue_id_field` names a column of the host + # collection holding a Pylon issue id — a business collection keeping the + # issue it opened — and, left out, the ids are the primary keys of the + # selected records, which is what an action registered on PylonIssue itself + # acts on. + module IssueTargets + module_function + + # A collection whose column was renamed, or a record the scope hides, + # answers "no issue selected" through the action's own message rather than + # through a stack trace in the panel. + # + # A ValidationError is the exception: it is the datasource refusing the + # selection in words written for the operator — PylonIssue naming more + # issues by id than one page of lookups covers, for one — and the agent + # surfaces its message as it is. Swallowed, it would reach them as "no + # issue selected" about a selection they can see they made, which is the + # one thing the message must not say. + def resolve_issue_ids(context, field = nil) + ids = field.nil? ? primary_key_ids(context) : column_ids(context, field) + # Deduplicated: a column of issue ids is not a key, so two selected + # records may name the same issue, which would then be written twice and + # counted twice in what the action reports back. + ids.filter_map do |id| + id.to_s unless id.nil? || id.to_s.empty? + end.uniq + rescue ForestAdminDatasourceToolkit::Exceptions::ValidationError + raise + rescue StandardError => e + source = field ? "from '#{field}'" : 'from the selected records' + ForestAdminDatasourcePylon.logger.warn( + "[forest_admin_datasource_pylon] failed to resolve the issues to act on #{source}: " \ + "#{e.class}: #{e.message}" + ) + [] + end + + def primary_key_ids(context) + Array(context.get_record_ids) + end + + def column_ids(context, field) + context.get_records([field.to_s]).map { |record| record[field.to_s] } + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/query/condition_tree_translator.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/query/condition_tree_translator.rb new file mode 100644 index 000000000..0d448fb6f --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/query/condition_tree_translator.rb @@ -0,0 +1,151 @@ +module ForestAdminDatasourcePylon + module Query + # See https://docs.usepylon.com/pylon-docs/developer/api/api-reference/issues + # + # Translates a Forest condition tree into the structured JSON filter of + # `POST /issues/search`: + # + # leaf -> { 'field' => …, 'operator' => …, 'value' | 'values' => … } + # branch -> { 'operator' => 'and' | 'or', 'subfilters' => [...] } + # + # Pylon nests sub-filters, so — unlike the Zendesk query string — OR is + # translated natively instead of being rejected. + # + # Anything the API cannot express raises UnsupportedOperatorError: a filter + # that is dropped returns unfiltered rows which look filtered. The wire + # format of the values, and the refusals that go with it, live in FilterValue. + class ConditionTreeTranslator + Branch = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + Leaf = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + + # Pylon rejects sub-filters nested deeper than three levels. + MAX_DEPTH = 3 + + LIST_OPERATORS = %w[in not_in].freeze + VALUELESS_OPERATORS = %w[is_set is_unset].freeze + + # The comparisons Pylon reads as a moment in time: what tells FilterValue + # that a bare date is a date, and not a piece of text a field happens to + # hold. Read off the emitted operator rather than off the column type, + # which the translator does not see -- and it is the operator that decides + # the format anyway. + TIME_OPERATORS = %w[time_is_after time_is_before].freeze + + def self.call(condition_tree, api_filters: {}, timezone: nil) + return nil if condition_tree.nil? + + new(api_filters: api_filters, timezone: timezone).translate(condition_tree) + end + + def initialize(api_filters: {}, timezone: nil) + @api_filters = api_filters || {} + @value = FilterValue.new(timezone: timezone) + end + + def translate(node, depth = 1) + case node + when Branch then translate_branch(node, depth) + when Leaf then translate_leaf(node) + else + raise UnsupportedOperatorError, "Unknown condition node: #{node.class}" + end + end + + private + + def translate_branch(branch, depth) + conditions = Array(branch.conditions) + if conditions.empty? + raise UnsupportedOperatorError, "Condition tree aggregator '#{branch.aggregator}' carries no condition." + end + + # Validated before the unwrap below, so a branch is refused on the + # aggregator it carries rather than on how many conditions it holds. + operator = aggregator(branch) + + # A lone condition needs no wrapper, and spending no nesting level on it + # keeps trees the agent builds one branch at a time under Pylon's cap. + return translate(conditions.first, depth) if conditions.size == 1 + + raise_too_deep(depth) if depth > MAX_DEPTH + + { 'operator' => operator, + 'subfilters' => conditions.map { |condition| translate(condition, depth + 1) } } + end + + def aggregator(branch) + value = branch.aggregator.to_s.downcase + return value if %w[and or].include?(value) + + raise UnsupportedOperatorError, + "Unknown condition tree aggregator #{branch.aggregator.inspect}; expected 'And' or 'Or'." + end + + def translate_leaf(leaf) + spec = @api_filters[leaf.field] + raise_unfilterable_field(leaf.field) unless spec + + operator = spec[:ops][leaf.operator] + raise_unsupported_operator(leaf, spec) unless operator + ensure_filterable_absence!(leaf, spec) + + with_value({ 'field' => (spec[:param] || leaf.field).to_s, 'operator' => operator }, operator, leaf) + end + + def with_value(filter, operator, leaf) + return filter if VALUELESS_OPERATORS.include?(operator) + return filter.merge('values' => @value.list(leaf)) if LIST_OPERATORS.include?(operator) + + filter.merge('value' => @value.single(leaf, time: TIME_OPERATORS.include?(operator))) + end + + # `present`, `blank` and `missing` are advertised on every field carrying + # an equality or a membership filter: the agent derives them from those + # above the datasource and rewrites them into a comparison with an empty + # value. Only a field the API reference documents `is_set` / `is_unset` on + # can answer one, and it answers it through those operators, never through + # the rewritten comparison -- which Pylon would match against the empty + # value as if it were a value of its own. + # + # Refused here rather than in FilterValue, which sees the empty value but + # not whether the field has a presence filter to answer it with. + def ensure_filterable_absence!(leaf, spec) + return unless absence_condition?(leaf) + return if spec[:ops].values.any? { |candidate| VALUELESS_OPERATORS.include?(candidate) } + + raise UnsupportedOperatorError, + "Pylon cannot filter '#{leaf.field}' for absence: the field carries no is_set / is_unset filter " \ + 'in the Pylon API reference, so a present, blank or missing condition on it cannot be translated. ' \ + 'Filter for absence on a field that does, or filter on a value instead.' + end + + # The shape the absence operators are rewritten into: a nil value, or a + # list holding nothing but blanks. An empty list is not one of them -- it + # comes from a filter carrying no value at all, which FilterValue reports. + def absence_condition?(leaf) + return true if leaf.value.nil? + return false unless leaf.value.is_a?(Array) && leaf.value.any? + + leaf.value.all? { |value| value.nil? || value.to_s.empty? } + end + + def raise_too_deep(depth) + raise UnsupportedOperatorError, + "Pylon rejects a filter nested deeper than #{MAX_DEPTH} levels (reached #{depth}); " \ + 'flatten the segment or the filter.' + end + + def raise_unfilterable_field(field) + raise UnsupportedOperatorError, + "Pylon cannot filter on '#{field}'; add it to the collection's api_filters " \ + 'after checking it against the Pylon API reference.' + end + + def raise_unsupported_operator(leaf, spec) + raise UnsupportedOperatorError, + "Operator '#{leaf.operator}' is not supported on field '#{leaf.field}'. " \ + "Supported: #{spec[:ops].keys.join(", ")}." + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/query/filter_value.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/query/filter_value.rb new file mode 100644 index 000000000..132452233 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/query/filter_value.rb @@ -0,0 +1,129 @@ +require 'date' +require 'active_support/core_ext/time/zones' + +module ForestAdminDatasourcePylon + module Query + # How a Forest filter value reaches the wire, and every way it can fail to. + # Split from the translator, which knows the shape of the condition tree but + # not the format of what the leaves carry. + class FilterValue + # A date carrying no time of day, which is what the frontend sends for a + # Dateonly column -- a shape only a custom field has, no native column + # being typed that way. + DATE_ONLY = /\A\d{4}-\d{2}-\d{2}\z/ + + def initialize(timezone: nil) + @timezone = timezone.to_s.strip.empty? ? 'UTC' : timezone + end + + # `time` says the operator this value travels with is one of Pylon's time + # comparisons, which is what decides whether a bare date is a date or a + # piece of text: the same string on a text field is a value of its own. + def single(leaf, time: false) + raise_nil_value(leaf.field) if leaf.value.nil? + + format(leaf.value, time: time, field: leaf.field) + end + + # Dropping the blanks would silently answer a different question: `not_in + # [nil, 'open']` was asked to exclude the blank records and would come + # back including them. An empty list is just as bad the other way round, + # translating to a filter matching everything. + # + # No time comparison takes a list, so nothing here is read as a date. + def list(leaf) + values = Array(leaf.value) + raise_empty_list(leaf) if values.empty? + raise_blank_in_list(leaf) if values.any? { |value| value.nil? || value.to_s.empty? } + + values.map { |value| format(value, field: leaf.field) } + end + + private + + # Booleans travel as they are: the filter is JSON, not a query string, so + # only the dates and the numbers the agent widened need a wire format. + def format(value, time: false, field: nil) + case value + when Time, DateTime then value.to_time.utc.iso8601 + when Date then format_date(value) + when Float then format_float(value, field) + when String then time ? format_time_string(value) : value + else value + end + end + + # The agent casts every Number column with `to_f` + # (`ConditionTreeParser.cast_to_type`), so an integer custom field would be + # filtered with `42.0` -- a form none of its values carry. A float with + # nothing after the point travels as the integer it is; a decimal keeps its + # own. + # + # A cast that overflowed to Infinity, or a NaN, is refused rather than + # passed on: `to_i` raises on both, and so does the JSON encoder a step + # later, either way as a 500 naming nothing the operator can act on. + def format_float(value, field) + raise_out_of_range(value, field) unless value.finite? + + value == value.to_i ? value.to_i : value + end + + # `time_is_after` receives a timestamp everywhere else in this datasource, + # a native date column being read and filtered as one: a Dateonly custom + # field cannot be the one field sending the same operator another shape. + # The bound is the one a Ruby `Date` already gets -- midnight in the + # timezone of the caller. + # + # A string this operator cannot read as a date is left to Pylon, which + # names what it refuses better than a guess here would. + def format_time_string(value) + return value unless DATE_ONLY.match?(value) + + format_date(Date.parse(value)) + rescue Date::Error + value + end + + # Only reached by a condition tree built in Ruby -- a segment or a scope + # written as code -- and by the bare date above. Everything else coming + # through HTTP arrives as an ISO8601 timestamp, already expressed in the + # timezone of the caller: the agent casts a date filter with `value.to_s`, + # and the toolkit formats the bounds it derives from Today / Previous* + # itself. + def format_date(value) + Time.use_zone(@timezone) { Time.zone.local(value.year, value.month, value.day).utc.iso8601 } + rescue ArgumentError + ForestAdminDatasourcePylon.logger.warn( + "[forest_admin_datasource_pylon] unknown timezone '#{@timezone}', falling back to UTC" + ) + value.strftime('%Y-%m-%dT00:00:00Z') + end + + # A filter carrying a nil value reads as a presence check on most APIs, + # which is silently the wrong query. + def raise_nil_value(field) + raise UnsupportedOperatorError, + "Filter value on '#{field}' is nil; use the PRESENT or BLANK operator to filter for absence." + end + + def raise_blank_in_list(leaf) + raise UnsupportedOperatorError, + "Operator '#{leaf.operator}' on field '#{leaf.field}' was given a list holding a blank value; " \ + 'Pylon matches an absent value through is_set / is_unset only, so filter for absence with the ' \ + 'PRESENT or BLANK operator rather than listing nil or an empty string.' + end + + def raise_empty_list(leaf) + raise UnsupportedOperatorError, + "Operator '#{leaf.operator}' on field '#{leaf.field}' was given an empty list; " \ + 'pass at least one value.' + end + + def raise_out_of_range(value, field) + raise UnsupportedOperatorError, + "Filter value on '#{field}' is #{value}, which is not a number Pylon can be asked for; " \ + 'filter with a value inside the range of a double.' + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/query/operator_maps.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/query/operator_maps.rb new file mode 100644 index 000000000..3a4230fd8 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/query/operator_maps.rb @@ -0,0 +1,108 @@ +module ForestAdminDatasourcePylon + module Query + # The operator maps the Pylon search endpoints share. A map spells each + # Forest operator as the Pylon operator honouring it; a collection's + # `API_FILTERS` table then assembles, field by field, the maps its endpoint + # accepts according to the API reference. + # + # Sharing the maps is what keeps those tables readable as the allow-lists + # they transcribe, and keeps one wire spelling from being fixed in one + # collection and left wrong in the next. + module OperatorMaps + Operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators + + # `not_equal` is deliberately absent: the toolkit derives it from + # `not_in`, so declaring it would only add a second spelling. + EQUALITY = { Operators::EQUAL => 'equals', + Operators::IN => 'in', + Operators::NOT_IN => 'not_in' }.freeze + + # MISSING is mapped as well although Pylon spells absence one way only: + # the toolkit rewrites it into `equal nil` when it is left out, which the + # translator cannot express, so a field the API does check for absence + # would refuse the very filter it can answer. + PRESENCE = { Operators::PRESENT => 'is_set', + Operators::BLANK => 'is_unset', + Operators::MISSING => 'is_unset' }.freeze + + # Declaring the bare comparisons rather than before/after is what lets + # the toolkit rewrite Today / PreviousWeek / ... into a pair of bounds, + # which is also why `time_range` never has to be emitted. + TIME = { Operators::GREATER_THAN => 'time_is_after', + Operators::LESS_THAN => 'time_is_before' }.freeze + + # Pylon exposes a single substring operator and documents no case + # semantics for it, so both Forest spellings map onto that one operator -- + # or the UI would offer a case-sensitive "contains" and a case-insensitive + # one behaving identically. + SUBSTRING = { Operators::CONTAINS => 'string_contains', + Operators::I_CONTAINS => 'string_contains' }.freeze + + # The same, for the endpoints that also accept the negation. A field + # filtered through SUBSTRING alone must not advertise it: Pylon rejects + # `string_does_not_contain` where it is not documented. + FULL_TEXT = SUBSTRING.merge(Operators::NOT_CONTAINS => 'string_does_not_contain', + Operators::NOT_I_CONTAINS => 'string_does_not_contain').freeze + + # A list-valued field -- `tags`, `domains`: `in` matches it against + # several candidates at once. + # + # Pylon also accepts `contains` / `does_not_contain` on such a field, and + # they are deliberately left out: the columns are typed `Json`, the only + # type the toolkit has for a list, and `Rules` allows a Json column the + # base and array operators alone. A declared `contains` would be refused + # by `ConditionTreeValidator` on the way in -- "the given operator + # 'contains' is not allowed with the columnType schema: 'Json'" -- so the + # UI would offer a filter that errors instead of one Pylon answers. + # Typing the columns `['String']` is not the way out either: no branch of + # `get_allowed_operators_for_column_type` reads an array type, and the + # validator raises a NoMethodError on it. Reaching those two operators + # takes a toolkit change, which is not this datasource's to make here. + MEMBERSHIP = { Operators::IN => 'in', + Operators::NOT_IN => 'not_in' }.freeze + + # A custom field is filtered through its slug, so its operators come from + # the column the integrator declared rather than from a table; every + # search endpoint accepts this same set on one. + CUSTOM_FIELD_OPS = EQUALITY.merge(PRESENCE).merge(TIME).merge(FULL_TEXT).freeze + + # Extended by a collection's `ApiFilters` module, whose `API_FILTERS` is + # the single source of truth for what its endpoint filters: the schema + # derives every column's `filter_operators` from it, so no collection + # declares a filter the translator would then refuse. + # + # One family escapes those tables. The agent derives `present`, `blank` and + # `missing` from an equality or a membership filter, above the datasource, + # and rewrites them into a comparison with an empty value. Only a field + # carrying PRESENCE can answer one -- Pylon matches an absent value through + # `is_set` / `is_unset` alone -- so on every other field the translator + # refuses the rewritten condition and names the filter to change, rather + # than sending a comparison Pylon would answer as if the empty value were + # a value of its own. + module Table + def forest_operators(field) + self::API_FILTERS.dig(field, :ops)&.keys || [] + end + + # Read off the extending module rather than off this one, so the + # `CUSTOM_FIELD_OPS` a collection declares is the single source both + # this spelling and `allowed_custom_field_operators` come from: an + # endpoint accepting less on a custom field narrows one constant. + def for_custom_field(schema) + { ops: self::CUSTOM_FIELD_OPS.slice(*Array(schema&.filter_operators)) } + end + end + + # The table of a collection whose endpoint filters nothing server-side: no + # field, and no operator on a custom field either. It is the default of + # `BaseCollection#filter_table`, so a collection read whole and filtered + # in memory needs no table of its own. + module EmptyTable + extend Table + + API_FILTERS = {}.freeze + CUSTOM_FIELD_OPS = {}.freeze + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/rate_limiter.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/rate_limiter.rb new file mode 100644 index 000000000..7ba45f9ae --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/rate_limiter.rb @@ -0,0 +1,139 @@ +module ForestAdminDatasourcePylon + # Spaces requests out so the budget of an endpoint is spent rather than + # exceeded: a sliding window per endpoint, and a wait until the next slot when + # the window is full. + # + # This sits in front of the 429 retry rather than replacing it. The retry + # answers a 429 Pylon already sent, which costs a full rate-limit window to + # recover from (Retry-After runs up to 60s); waiting half a second for a slot + # is the same throughput at a fraction of the latency. The retry stays as the + # backstop for what this cannot see — another process, or another agent, on + # the same token. + # + # One limiter per Configuration, so per token: Pylon meters the token, and two + # agents holding different ones do not share a budget. + class RateLimiter + WINDOW = 60.0 + + # How long one attempt may be held back before it is let through anyway. The + # limiter exists to avoid a 429, not to guarantee one never happens: past + # this point the window is saturated by more than this agent's own traffic, + # so queueing behind it would trade a retry the client already handles for a + # request the operator watches spin. It goes out, and the warning says why. + # + # Per attempt, not per request: `retry` replays up to `max_retries` times and + # each replay asks for a slot of its own, so a request can spend this bound + # once per attempt, on top of the backoff `retry` waits itself. None of it + # runs under the Faraday timeout, which only covers the adapter. + # + # The region this smooths is narrow, and it is worth being plain about it. + # On a stream over budget the waits accumulate rather than settling, so the + # bound is crossed sooner the further over it sits: measured against a + # 120/min endpoint, a stream 1% over budget throttles its first thousand + # requests and then lets roughly one in twelve through unthrottled, one 5% + # over gives up after two hundred and lets half through, and past ~10% over + # the bound is crossed as soon as the window fills — request 121 — after + # which almost nothing is throttled at all. A burst arriving at once books + # its next slot a whole window out and goes straight through from the first + # request past the budget. Under real saturation the 429 retry is the + # defence, not this; what this buys is the region just over budget, and a + # budget the code knows rather than one it discovers as a 429. + DEFAULT_MAX_WAIT = 5.0 + + attr_reader :max_wait, :window + + def initialize(limits: RateLimits, window: WINDOW, max_wait: DEFAULT_MAX_WAIT, clock: nil, sleeper: nil) + @limits = limits + @window = window.to_f + @max_wait = max_wait.to_f + @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) } + @sleeper = sleeper || ->(seconds) { sleep(seconds) } + @mutex = Mutex.new + @slots = {} + @warned = {} + end + + # Blocks until the endpoint has room, then returns. Called once per attempt, + # retries included: a replayed request spends the budget a first one did. + def acquire(method, path) + rule = @limits.for(method, path) + wait, warn = @mutex.synchronize { reserve(rule) } + + warn_saturated(rule, wait) if warn + return if wait <= 0 || wait > @max_wait + + @sleeper.call(wait) + end + + private + + # The reservation happens under the lock and the waiting outside it: holding + # the mutex across the sleep would serialize every thread behind the slowest + # one, and threads waiting on unrelated endpoints have no reason to queue. + # + # What is recorded is the moment the request will be made, not the moment it + # was asked for, so concurrent callers each take a distinct slot and spread + # out instead of all waking onto the same one. + # + # Returns the wait the caller owes and whether this is the bypass worth a log + # line — both settled here, the second being shared state like the first. + def reserve(rule) + taken = (@slots[rule.name] ||= []) + now = @clock.call + expire(taken, now) + + slot = next_slot(taken, rule.limit, now) + wait = slot - now + bypass = wait > @max_wait + # Past the bound the request goes out now, so the slot it books is now: + # recording the one it declined to wait for would meter a request nobody + # ever made and push the whole window further out. + insert(taken, bypass ? now : slot) + + [wait, bypass && first_warning?(rule, now)] + end + + # The bookings that have left the window are its leading run, the list being + # kept ordered. + def expire(taken, now) + taken.shift(taken.bsearch_index { |at| at > now - @window } || taken.size) + end + + # Now while the window still has room, otherwise a window past the limit-th + # most recent booking, which is the one that has to fall out of it first — + # an index that only reads as such on an ordered list. + def next_slot(taken, limit, now) + return now if taken.size < limit + + taken[taken.size - limit] + @window + end + + # A booking has one position in an ordered list, so it goes there rather than + # onto the end followed by a sort: `now` lands before the slots already + # reserved further out, and the list is what `next_slot` reads an index off. + def insert(taken, booking) + taken.insert(taken.bsearch_index { |at| at >= booking } || taken.size, booking) + end + + # One line per endpoint per window. What the warning reports is a saturation + # that lasts, so a line per request puts one on every request it describes — + # a thousand of them for a couple of minutes over budget, burying the first, + # which is the only one the operator needed. + def first_warning?(rule, now) + last = @warned[rule.name] + return false if last && now - last < @window + + @warned[rule.name] = now + true + end + + def warn_saturated(rule, wait) + ForestAdminDatasourcePylon.logger.warn( + "[forest_admin_datasource_pylon] #{rule.name} is at its budget of #{rule.limit} requests per " \ + "#{@window.round}s; the next slot is #{wait.round(1)}s out, past the #{@max_wait.round(1)}s this waits. " \ + 'Letting the request through — Pylon may answer 429, which the client retries. Further requests over ' \ + "this budget are let through too, and this says so once per #{@window.round}s." + ) + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/rate_limits.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/rate_limits.rb new file mode 100644 index 000000000..0dd780785 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/rate_limits.rb @@ -0,0 +1,96 @@ +module ForestAdminDatasourcePylon + # The per-endpoint budgets Pylon documents, in requests per minute, read off + # the API reference (docs.usepylon.com/pylon-docs/developer/api/api-reference, + # one page per resource — each endpoint states its own "Rate limit:" line). + # + # Pylon meters per endpoint, not per token, so every rule owns its own window: + # two endpoints allowing 300 each are 600 requests, and pooling them would + # throttle at half the budget the API grants. + # + # An endpoint absent from the table falls back to DEFAULT_LIMIT, the lowest + # figure documented anywhere on the API. An undocumented quota is not an + # absent one, and the generous guess is the one an operator discovers as a 429. + # + # Every endpoint the client calls has a rule, writes included: the budget of a + # write is no more uniform than that of a read — `POST /accounts` is granted + # ten times `POST /issues`, and `PATCH /accounts/{id}` two and a half times + # `PATCH /issues/{id}` — so leaving one to the fallback throttles it at a + # fraction of what Pylon grants, which is the failure this table exists to + # avoid rather than a safe default. + class RateLimits + DEFAULT_LIMIT = 30 + + # Ids are escaped before being joined to a path, so a segment never carries + # a slash and this matches exactly one of them. + ID = '[^/]+'.freeze + + Rule = Struct.new(:name, :limit, keyword_init: true) + + # Every rule is anchored, so `/issues`, `/issues/{id}` and + # `/issues/{id}/messages` are three distinct buckets rather than three + # readings of the same prefix, and the order of this list carries no meaning. + # + # `name` is what the window is keyed on and what a log line shows, so it + # spells the endpoint rather than its budget: two endpoints sharing a figure + # must not share a window. + RULES = [ + ['post /issues/search', :post, %r{\A/issues/search\z}, 120], + ['post /accounts/search', :post, %r{\A/accounts/search\z}, 120], + ['post /contacts/search', :post, %r{\A/contacts/search\z}, 120], + ['get /issues/:id/messages', :get, %r{\A/issues/#{ID}/messages\z}, 120], + ['get /issues', :get, %r{\A/issues\z}, 30], + ['post /issues', :post, %r{\A/issues\z}, 30], + ['get /issues/:id', :get, %r{\A/issues/#{ID}\z}, 300], + ['patch /issues/:id', :patch, %r{\A/issues/#{ID}\z}, 120], + ['get /accounts', :get, %r{\A/accounts\z}, 300], + ['get /accounts/:id', :get, %r{\A/accounts/#{ID}\z}, 300], + ['get /contacts', :get, %r{\A/contacts\z}, 300], + ['get /contacts/:id', :get, %r{\A/contacts/#{ID}\z}, 300], + ['get /users', :get, %r{\A/users\z}, 300], + ['get /users/:id', :get, %r{\A/users/#{ID}\z}, 300], + ['get /teams', :get, %r{\A/teams\z}, 300], + ['get /teams/:id', :get, %r{\A/teams/#{ID}\z}, 300], + ['get /custom-fields', :get, %r{\A/custom-fields\z}, 300], + ['get /me', :get, %r{\A/me\z}, 300], + ['post /accounts', :post, %r{\A/accounts\z}, 300], + ['patch /accounts/:id', :patch, %r{\A/accounts/#{ID}\z}, 300], + ['post /contacts', :post, %r{\A/contacts\z}, 300], + ['patch /contacts/:id', :patch, %r{\A/contacts/#{ID}\z}, 300], + ['delete /issues/:id', :delete, %r{\A/issues/#{ID}\z}, 120], + ['patch /teams/:id', :patch, %r{\A/teams/#{ID}\z}, 120], + ['patch /users/:id', :patch, %r{\A/users/#{ID}\z}, 120], + ['delete /accounts/:id', :delete, %r{\A/accounts/#{ID}\z}, 30], + ['delete /contacts/:id', :delete, %r{\A/contacts/#{ID}\z}, 30], + ['post /teams', :post, %r{\A/teams\z}, 30] + ].map { |name, verb, pattern, limit| [verb, pattern, Rule.new(name: name, limit: limit)] }.freeze + + class << self + def for(method, path) + verb = method.to_s.downcase.to_sym + normalized = normalize(path) + + found = RULES.find { |rule_verb, pattern, _rule| rule_verb == verb && pattern.match?(normalized) } + found ? found[2] : fallback(verb, normalized) + end + + private + + # Faraday hands back the path of the resolved URL, which carries the + # leading slash the rules are written against and, on a base url mounted + # under a subpath, whatever precedes it. + def normalize(path) + stripped = path.to_s.chomp('/') + stripped.start_with?('/') ? stripped : "/#{stripped}" + end + + # An unlisted endpoint is bucketed by its first segment rather than by its + # full path: keying on the path would open a window per record id, so a + # fan-out over a hundred records would meter as a hundred endpoints each + # one request in. + def fallback(verb, path) + segment = path.split('/').reject(&:empty?).first + Rule.new(name: "#{verb} /#{segment} (undocumented)", limit: DEFAULT_LIMIT) + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/retry_policy.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/retry_policy.rb new file mode 100644 index 000000000..89c44ec0d --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/retry_policy.rb @@ -0,0 +1,71 @@ +module ForestAdminDatasourcePylon + # Everything governing how the client reacts to a failed request, in one + # place: which statuses and exceptions are worth another attempt, on which + # verbs, and how long to wait. + class RetryPolicy + # Pylon quotas are per-minute, so a 429 routinely carries a Retry-After of up + # to 60s. faraday-retry gives up outright when Retry-After exceeds + # max_interval (`return if retry_after > max_interval`), so the cap has to + # cover a full rate-limit window or the 429 retry never fires when it matters. + DEFAULT_MAX_INTERVAL = 65 + + STATUSES = [429, 502, 503, 504].freeze + + # faraday-retry's defaults plus ConnectionFailed: a dropped connection is + # exactly the transient failure a resilient client should absorb, and it is + # not retried out of the box. + EXCEPTIONS = [ + Errno::ETIMEDOUT, 'Timeout::Error', Faraday::TimeoutError, + Faraday::RetriableResponse, Faraday::ConnectionFailed + ].freeze + + # The verbs that change nothing, so any transient failure is worth another + # attempt. Narrower than faraday-retry's idempotent default: a 502 or a + # dropped connection on the way back from a DELETE Pylon did perform is + # replayed into a 404, which the write path then surfaces as a deletion that + # failed when it landed. + # + # A 429 stays safe to retry on any verb, Pylon having rejected the request + # before processing it, and travels through retry_if rather than through this + # list: faraday-retry ORs the two, so methods can only widen the set, never + # restrict it. + RETRYABLE_METHODS = %i[get head options].freeze + RETRY_IF = ->(env, _exception) { env[:status] == 429 } + + # The cap for a call that must not hold the boot, deliberately below a + # rate-limit window where DEFAULT_MAX_INTERVAL sits above it: a Retry-After + # past the cap makes faraday-retry abandon outright, which is what turns a + # 429 into an immediate give-up rather than a minute of waiting per attempt. + BOOT_MAX_INTERVAL = 2 + + BACKOFF_FACTOR = 2 + + attr_reader :max_retries, :interval, :max_interval + + # One retry rather than none, for what is read once and never revisited: a + # transient failure there costs its result for the whole life of the process, + # and half a second absorbs the hiccup without waiting a 429 out. + def self.boot + new(max_retries: 1, interval: 0.5, max_interval: BOOT_MAX_INTERVAL) + end + + def initialize(max_retries: 3, interval: 0.5, max_interval: DEFAULT_MAX_INTERVAL) + @max_retries = max_retries + @interval = interval + @max_interval = max_interval + end + + def to_faraday_options + { + max: @max_retries, + interval: @interval, + max_interval: @max_interval, + backoff_factor: BACKOFF_FACTOR, + retry_statuses: STATUSES, + exceptions: EXCEPTIONS, + methods: RETRYABLE_METHODS, + retry_if: RETRY_IF + } + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/schema/custom_fields_introspector.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/schema/custom_fields_introspector.rb new file mode 100644 index 000000000..8292c003a --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/schema/custom_fields_introspector.rb @@ -0,0 +1,203 @@ +module ForestAdminDatasourcePylon + module Schema + # Turns the custom fields an organization defined in Pylon into columns, as + # entries shaped `{ column_name:, schema:, multi_value: }` — what + # `add_custom_fields` registers on a collection, and what the payload + # builder writes a value back through. + # + # `column_name` is the Pylon slug verbatim, and there is no second key + # carrying it: the slug is both what a read payload indexes the values by and + # what a search filter sends as `field`, so renaming the column would add a + # mapping to keep in step for nothing. + # + # Pylon defines custom fields per object type, and asks for that type on + # every call: `issue`, `account` and `contact` are the three this datasource + # has a collection for — it also exposes `task`, `project`, `meeting` and + # `opportunity`, while users and teams carry no custom field at all. + class CustomFieldsIntrospector + ColumnSchema = ForestAdminDatasourceToolkit::Schema::ColumnSchema + Maps = Query::OperatorMaps + + # A type absent from this table is skipped rather than guessed at: a column + # whose Forest type does not match what Pylon holds would filter and + # display wrong, which is worse than not being there. + # + # `user` holds a Pylon user id, kept a String rather than turned into a + # relation to PylonUser: a custom field is registered after the relations + # are declared, and a foreign key the operator can read is what this story + # promises. + PYLON_TO_COLUMN_TYPE = { + 'text' => 'String', + 'url' => 'String', + 'user' => 'String', + 'number' => 'Number', + 'decimal' => 'Number', + 'boolean' => 'Boolean', + 'date' => 'Dateonly', + 'datetime' => 'Date', + 'select' => 'Enum', + 'multiselect' => 'Json' + }.freeze + + # The types Pylon writes back through `values` rather than `value`. + MULTI_VALUE_TYPES = %w[multiselect].freeze + + BASE_OPS = (Maps::EQUALITY.keys + Maps::PRESENCE.keys).freeze + + # A date drops the membership operators on the way, `Rules` granting a DATE + # or a DATEONLY column no array operator -- the hazard already documented + # for MEMBERSHIP in `operator_maps.rb`. Native date columns declare the + # comparisons alone for the same reason. + # + # Dropping them is not what keeps `in` out of the UI, though, and nothing + # here can: `OperatorsEquivalenceCollectionDecorator` republishes it from + # the `equal` this set declares, the IN transform depending on EQUAL for + # every column type. A DATEONLY also gets `after_x_hours_ago` / + # `before_x_hours_ago` republished from the comparisons, `Times.compare` + # deriving them for that type where `Rules` refuses them. The validator + # then rejects all three, so the operator is offered a date filter the + # agent answers with a 400. The contradiction is the toolkit's to settle + # and is tracked as PRD-989; the set below is what Pylon accepts, which is + # the only question this table can answer. + TIME_OPS = (BASE_OPS - Maps::MEMBERSHIP.keys + Maps::TIME.keys).freeze + + # Drawn from `CUSTOM_FIELD_OPS`, the set every search endpoint accepts on a + # custom field, so a collection's clamp has nothing to drop. + # + # A Number gets no comparison: Pylon documents `time_is_after` / + # `time_is_before` for the bare comparisons and nothing else, so a numeric + # range would travel as a time filter. A multiselect gets nothing at all — + # its membership operators are not part of what a custom field accepts. + OPERATORS = { + 'String' => (BASE_OPS + Maps::FULL_TEXT.keys).freeze, + 'Enum' => BASE_OPS, + 'Number' => BASE_OPS, + 'Boolean' => BASE_OPS, + 'Date' => TIME_OPS, + 'Dateonly' => TIME_OPS, + 'Json' => [].freeze + }.freeze + + def initialize(client) + @client = client + @unreachable = false + end + + def issue_custom_fields = introspect('issue') + def account_custom_fields = introspect('account') + def contact_custom_fields = introspect('contact') + + private + + # One object type per call, three of them, all of it in front of a Rails + # boot the operator sits through. The first failure stands for the rest: + # a Pylon that is down, or a token missing the permission, fails the two + # that follow the same way, and each is bounded per request rather than + # across the three — so trying them anyway spends the bound three times to + # learn what the first one already said. + def introspect(object_type) + return [] if @unreachable + + definitions = @client.fetch_custom_fields(object_type) + return give_up(object_type) if definitions.nil? + + definitions.filter_map { |raw| build_entry(raw, object_type) } + end + + def give_up(object_type) + @unreachable = true + ForestAdminDatasourcePylon.logger.warn( + "[forest_admin_datasource_pylon] Custom fields could not be read for #{object_type}; " \ + 'the object types after it are left unread rather than held against the same failure. ' \ + 'The datasource boots on the native schema, without the custom columns.' + ) + [] + end + + def build_entry(raw, object_type) + return nil unless raw.is_a?(Hash) + + slug = raw['slug'].to_s + return nil if slug.empty? + + column_type = PYLON_TO_COLUMN_TYPE[raw['type']] + return warn_unknown_type(raw, slug, object_type) if column_type.nil? + + { column_name: slug, schema: build_schema(raw, column_type), + multi_value: MULTI_VALUE_TYPES.include?(raw['type']) } + end + + # A custom field is writable when Pylon says it is: it flags the ones + # synced from an app or an integration, which its own endpoints refuse. + # Nothing is sortable -- no Pylon endpoint takes a sort parameter -- and + # nothing is groupable: one column left groupable turns `supportGroups` on + # for the whole collection, and the group-by the UI then offers errors. + def build_schema(raw, column_type) + opts = { column_type: column_type, + filter_operators: OPERATORS.fetch(column_type, []), + is_read_only: !writable_definition?(raw), + is_sortable: false, + is_groupable: false } + + column_type == 'Enum' ? enum_schema(raw, opts) : ColumnSchema.new(**opts) + end + + # Pylon reads a select back — and filters it — as the slug of the option, + # never as its label, so those are the values the column advertises. + # + # Forest refuses an Enum carrying no value: a select whose options were all + # removed falls back to String, so the column still shows what it holds. + # + # Read-only there whatever Pylon says of the field, unlike every other + # fallback here: a select is written as the slug of one of its options, so + # a free-text editor on one offers the operator no value the endpoint would + # accept. Showing what it holds is the whole of what this can do. + def enum_schema(raw, opts) + values = option_slugs(raw) + return ColumnSchema.new(**opts, enum_values: values) unless values.empty? + + ColumnSchema.new(**opts, column_type: 'String', + filter_operators: OPERATORS.fetch('String'), is_read_only: true) + end + + def option_slugs(raw) + metadata = raw['select_metadata'] + options = metadata.is_a?(Hash) ? metadata['options'] : nil + + Array(options).filter_map do |option| + option['slug'] if option.is_a?(Hash) && !option['slug'].to_s.empty? + end + end + + # Only an explicit `false` opens a custom field to writes. A definition + # carrying no flag at all is left read-only and reported: this datasource + # advertises nothing an endpoint would refuse, and reading the absence as + # "writable" would turn every field synced from an app into an editor whose + # every save Pylon rejects -- where reading it as "read-only" costs the + # capability and says so once per boot. + def writable_definition?(raw) + return true if raw['is_read_only'] == false + return false if raw['is_read_only'] == true + + warn_unflagged_writability(raw) + false + end + + def warn_unflagged_writability(raw) + ForestAdminDatasourcePylon.logger.warn( + "[forest_admin_datasource_pylon] Custom field '#{raw["slug"]}' carries no 'is_read_only' flag; " \ + 'leaving it read-only. Pylon refuses a write on the fields it syncs from an app or an integration, ' \ + 'and nothing here can tell this one apart from those without the flag.' + ) + end + + def warn_unknown_type(raw, slug, object_type) + ForestAdminDatasourcePylon.logger.warn( + "[forest_admin_datasource_pylon] Custom field '#{slug}' on #{object_type} has type " \ + "#{raw["type"].inspect}, which this datasource cannot map to a Forest column; skipping." + ) + nil + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/throttle.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/throttle.rb new file mode 100644 index 000000000..faeeb4e3c --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/throttle.rb @@ -0,0 +1,21 @@ +module ForestAdminDatasourcePylon + # Holds a request until its endpoint has budget. A middleware rather than a + # call in each client method: there is one code path for every request here, + # where the client has twenty, and this one also catches the requests the + # client never issues itself — the replays `retry` performs. + class Throttle < Faraday::Middleware + def initialize(app, limiter:, base_path: '') + super(app) + @limiter = limiter + @base_path = base_path.to_s + end + + # Faraday hands over the path of the resolved url, so a base url mounted + # under a subpath carries that prefix and no rule matches. It comes off + # before the limiter sees the path. + def call(env) + @limiter.acquire(env.method, env.url.path.delete_prefix(@base_path)) + @app.call(env) + end + end +end diff --git a/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/version.rb b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/version.rb new file mode 100644 index 000000000..bc4c6546e --- /dev/null +++ b/packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/version.rb @@ -0,0 +1,3 @@ +module ForestAdminDatasourcePylon + VERSION = "1.39.4" +end diff --git a/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/client/writes_spec.rb b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/client/writes_spec.rb new file mode 100644 index 000000000..4b0e8fe72 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/client/writes_spec.rb @@ -0,0 +1,159 @@ +RSpec.describe ForestAdminDatasourcePylon::Client::Writes do + let(:retry_policy) { ForestAdminDatasourcePylon::RetryPolicy.new(max_retries: 2, interval: 0) } + let(:configuration) { ForestAdminDatasourcePylon::Configuration.new(api_key: 'k', retry_policy: retry_policy) } + let(:client) { ForestAdminDatasourcePylon::Client.new(configuration) } + let(:base) { configuration.url } + + def json(payload, status = 200) + { status: status, body: payload.is_a?(String) ? payload : payload.to_json, + headers: { 'Content-Type' => 'application/json' } } + end + + # One method per endpoint, and the endpoint is the whole of what each one + # knows: the payload is the collection's to build. + describe 'the endpoint each write reaches' do + { + create_issue: [:post, 'issues'], create_account: [:post, 'accounts'], + create_contact: [:post, 'contacts'], create_team: [:post, 'teams'] + }.each do |method, (verb, path)| + it "#{method} posts to /#{path}" do + stub_request(verb, "#{base}/#{path}").to_return(json('data' => { 'id' => 'x' })) + + expect(client.public_send(method, 'name' => 'Acme')).to eq('id' => 'x') + expect(WebMock).to have_requested(verb, "#{base}/#{path}").with(body: { 'name' => 'Acme' }) + end + end + + { + update_issue: 'issues', update_account: 'accounts', update_contact: 'contacts', + update_team: 'teams', update_user: 'users' + }.each do |method, path| + it "#{method} patches /#{path}/{id}" do + stub_request(:patch, "#{base}/#{path}/x1").to_return(json('data' => { 'id' => 'x1' })) + + expect(client.public_send(method, 'x1', 'name' => 'Acme')).to eq('id' => 'x1') + expect(WebMock).to have_requested(:patch, "#{base}/#{path}/x1").with(body: { 'name' => 'Acme' }) + end + end + + { delete_issue: 'issues', delete_account: 'accounts', delete_contact: 'contacts' }.each do |method, path| + it "#{method} deletes /#{path}/{id}" do + stub_request(:delete, "#{base}/#{path}/x1").to_return(status: 204) + + expect(client.public_send(method, 'x1')).to be(true) + expect(WebMock).to have_requested(:delete, "#{base}/#{path}/x1") + end + end + end + + describe 'the record a write answers with' do + it 'unwraps the "data" envelope' do + stub_request(:post, "#{base}/issues") + .to_return(json('data' => { 'id' => 'i1', 'title' => 'Boom' }, 'request_id' => 'req_1')) + + expect(client.create_issue('title' => 'Boom')).to eq('id' => 'i1', 'title' => 'Boom') + end + + # A read hands an unwrapped body back untouched; a write must not, or the + # collection would serialize an envelope into a record carrying no id. + it 'raises when the envelope carries no record' do + stub_request(:post, "#{base}/issues").to_return(json('request_id' => 'req_1')) + + expect { client.create_issue('title' => 'Boom') } + .to raise_error(ForestAdminDatasourcePylon::APIError, /create\(issues\).*unexpected body shape/m) + end + + it 'raises when the record is not an object' do + stub_request(:patch, "#{base}/issues/i1").to_return(json('data' => 'ok')) + + expect { client.update_issue('i1', 'title' => 'Boom') } + .to raise_error(ForestAdminDatasourcePylon::APIError, %r{update\(issues/i1\)}) + end + + # An update's record is discarded by the collection, so an answer carrying + # none is the write having landed with nothing to hand back — raising there + # would report a failure on a record Pylon already patched. + it 'accepts an update answered with no body at all' do + stub_request(:patch, "#{base}/issues/i1").to_return(status: 204) + + expect(client.update_issue('i1', 'title' => 'Boom')).to be_nil + end + + it 'accepts an update answered without a record' do + stub_request(:patch, "#{base}/issues/i1").to_return(json('data' => nil, 'request_id' => 'req_1')) + stub_request(:patch, "#{base}/issues/i2").to_return(json('request_id' => 'req_2')) + + expect(client.update_issue('i1', 'title' => 'Boom')).to be_nil + expect(client.update_issue('i2', 'title' => 'Boom')).to be_nil + end + end + + describe 'a failed write' do + it 'raises an APIError carrying the status and the request id' do + stub_request(:post, "#{base}/issues") + .to_return(json({ 'message' => 'title is required', 'request_id' => 'req_9' }, 422)) + + expect { client.create_issue({}) }.to raise_error(ForestAdminDatasourcePylon::APIError) { |error| + expect(error.status).to eq(422) + expect(error.message).to include('create(issues)', 'HTTP 422', 'title is required', 'req_9') + } + end + + it 'names the deleted record in the operation' do + stub_request(:delete, "#{base}/issues/i1").to_return(json({ 'message' => 'gone' }, 404)) + + expect { client.delete_issue('i1') } + .to raise_error(ForestAdminDatasourcePylon::APIError, %r{delete\(issues/i1\)}) + end + + # Ids reach the client from a filter the operator set, so they are escaped + # rather than joined to the path as they come. + it 'escapes an id that would otherwise alter the request path' do + stub_request(:patch, "#{base}/issues/a%2Fb").to_return(json('data' => { 'id' => 'a/b' })) + + client.update_issue('a/b', 'title' => 'Boom') + + expect(WebMock).to have_requested(:patch, "#{base}/issues/a%2Fb") + end + end + + # A 429 is refused before Pylon processes the request, so replaying it creates + # nothing twice; a 502 may well have created the issue, and is not replayed. + describe 'retrying a write' do + it 'retries a rate-limited create' do + stub_request(:post, "#{base}/issues") + .to_return(json({ 'message' => 'slow down' }, 429)) + .then.to_return(json('data' => { 'id' => 'i1' })) + + expect(client.create_issue('title' => 'Boom')).to eq('id' => 'i1') + expect(WebMock).to have_requested(:post, "#{base}/issues").twice + end + + it 'does not retry a create that failed on a gateway error' do + stub_request(:post, "#{base}/issues").to_return(json({ 'message' => 'bad gateway' }, 502)) + + expect { client.create_issue('title' => 'Boom') }.to raise_error(ForestAdminDatasourcePylon::APIError) + expect(WebMock).to have_requested(:post, "#{base}/issues").once + end + + it 'retries a rate-limited delete' do + stub_request(:delete, "#{base}/issues/i1") + .to_return(json({ 'message' => 'slow down' }, 429)) + .then.to_return(json({}, 204)) + + expect(client.delete_issue('i1')).to be(true) + expect(WebMock).to have_requested(:delete, "#{base}/issues/i1").twice + end + + # A 502 on the way back from a DELETE Pylon did perform would be replayed + # into a 404, which the write path surfaces as a deletion that failed when + # it landed -- a report of something that did not happen. So the gateway + # error stays what it is, on a delete as on a create. + it 'does not retry a delete that failed on a gateway error' do + stub_request(:delete, "#{base}/issues/i1").to_return(json({ 'message' => 'bad gateway' }, 502)) + + expect { client.delete_issue('i1') }.to raise_error(ForestAdminDatasourcePylon::APIError, /502/) + expect(WebMock).to have_requested(:delete, "#{base}/issues/i1").once + end + end +end diff --git a/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/client_spec.rb b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/client_spec.rb new file mode 100644 index 000000000..73ce78548 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/client_spec.rb @@ -0,0 +1,808 @@ +RSpec.describe ForestAdminDatasourcePylon::Client do + let(:retry_policy) { ForestAdminDatasourcePylon::RetryPolicy.new(max_retries: 2, interval: 0) } + let(:configuration) { ForestAdminDatasourcePylon::Configuration.new(api_key: 'k', retry_policy: retry_policy) } + let(:client) { described_class.new(configuration) } + let(:base) { configuration.url } + + def json(payload, status = 200) + { status: status, body: payload.is_a?(String) ? payload : payload.to_json, + headers: { 'Content-Type' => 'application/json' } } + end + + describe 'authentication' do + it 'sends the api key as a Bearer token' do + stub_request(:get, "#{base}/me").to_return(json('data' => {})) + + client.me + expect(WebMock).to have_requested(:get, "#{base}/me") + .with(headers: { 'Authorization' => 'Bearer k', 'Accept' => 'application/json' }) + end + + it 'advertises a versioned user agent' do + stub_request(:get, "#{base}/me").to_return(json('data' => {})) + + client.me + expect(WebMock).to have_requested(:get, "#{base}/me") + .with(headers: { 'User-Agent' => "forest_admin_datasource_pylon/#{ForestAdminDatasourcePylon::VERSION}" }) + end + end + + describe '#me' do + it 'unwraps the "data" envelope' do + stub_request(:get, "#{base}/me").to_return(json('data' => { 'id' => 'org_1', 'name' => 'Acme' }, + 'request_id' => 'req_1')) + + expect(client.me).to eq('id' => 'org_1', 'name' => 'Acme') + end + + it 'returns the body as-is when it is not wrapped' do + stub_request(:get, "#{base}/me").to_return(json('id' => 'org_1')) + + expect(client.me).to eq('id' => 'org_1') + end + + it 'returns nil when the response has an empty body' do + stub_request(:get, "#{base}/me").to_return(status: 200, body: '') + + expect(client.me).to be_nil + end + + it 'wraps an unauthorized response in an APIError carrying status and body' do + stub_request(:get, "#{base}/me").to_return(json({ 'message' => 'invalid token' }, 401)) + + expect { client.me }.to raise_error(ForestAdminDatasourcePylon::APIError) { |error| + expect(error.message).to eq('Pylon API call failed: me: HTTP 401 invalid token') + expect(error.status).to eq(401) + expect(error.body).to eq('message' => 'invalid token') + } + end + + it 'appends the request_id when Pylon returns one' do + stub_request(:get, "#{base}/me").to_return(json({ 'message' => 'boom', 'request_id' => 'req_42' }, 500)) + + expect { client.me } + .to raise_error(ForestAdminDatasourcePylon::APIError, /boom \(request_id: req_42\)/) + end + + it 'keeps the request_id even when the message is truncated' do + body = { 'message' => 'x' * 900, 'request_id' => 'req_42' } + stub_request(:get, "#{base}/me").to_return(json(body, 500)) + + expect { client.me }.to raise_error(ForestAdminDatasourcePylon::APIError, /\(request_id: req_42\)\z/) + end + + it 'reads the message out of a nested error object' do + stub_request(:get, "#{base}/me").to_return(json({ 'error' => { 'message' => 'nested boom' } }, 422)) + + expect { client.me }.to raise_error(ForestAdminDatasourcePylon::APIError, /HTTP 422 nested boom/) + end + + it 'reads the message out of a plain string error' do + stub_request(:get, "#{base}/me").to_return(json({ 'error' => 'flat boom' }, 422)) + + expect { client.me }.to raise_error(ForestAdminDatasourcePylon::APIError, /HTTP 422 flat boom/) + end + + it 'joins an errors array, accepting hashes and bare strings' do + body = { 'errors' => [{ 'message' => 'first' }, { 'detail' => 'second' }, 'third'] } + stub_request(:get, "#{base}/me").to_return(json(body, 422)) + + expect { client.me }.to raise_error(ForestAdminDatasourcePylon::APIError, /first; second; third/) + end + + it 'falls back to the raw body when it is not JSON' do + stub_request(:get, "#{base}/me").to_return(status: 500, body: 'boom') + + expect { client.me }.to raise_error(ForestAdminDatasourcePylon::APIError, /HTTP 500 boom/) + end + + it 'falls back to the serialized payload when no message field is recognised' do + stub_request(:get, "#{base}/me").to_return(json({ 'unexpected' => true }, 400)) + + expect { client.me }.to raise_error(ForestAdminDatasourcePylon::APIError, /HTTP 400 .*unexpected/) + end + + it 'reports a connection failure without an HTTP status' do + stub_request(:get, "#{base}/me").to_timeout + + expect { client.me }.to raise_error(ForestAdminDatasourcePylon::APIError) { |error| + expect(error.status).to be_nil + expect(error.message).to match(/Pylon API call failed: me: Faraday::ConnectionFailed/) + } + end + + it 'wraps a non-Faraday failure in an APIError' do + allow(client).to receive(:extract_data).and_raise(KeyError, 'nope') + stub_request(:get, "#{base}/me").to_return(json('data' => {})) + + expect { client.me }.to raise_error(ForestAdminDatasourcePylon::APIError, /me: KeyError: nope/) + end + + # Callers branch on `status`, so an already-mapped error has to come back + # untouched rather than be re-wrapped by the generic StandardError arm. + it 'lets an already-mapped APIError through with its status intact' do + mapped = ForestAdminDatasourcePylon::APIError.new('boom', status: 404, body: { 'message' => 'boom' }) + allow(client).to receive(:extract_data).and_raise(mapped) + stub_request(:get, "#{base}/me").to_return(json('data' => {})) + + expect { client.me }.to raise_error(ForestAdminDatasourcePylon::APIError) { |error| + expect(error).to be(mapped) + expect(error.status).to eq(404) + } + end + end + + describe '#search_issues' do + it 'posts the limit and returns the records' do + stub_request(:post, "#{base}/issues/search").to_return(json('data' => [{ 'id' => 'i1' }])) + + page = client.search_issues(limit: 2) + + expect(page.records).to eq([{ 'id' => 'i1' }]) + expect(WebMock).to have_requested(:post, "#{base}/issues/search").with(body: { 'limit' => 2 }) + end + + it 'omits cursor, filter and search_text when they are not provided' do + stub_request(:post, "#{base}/issues/search").to_return(json('data' => [])) + + client.search_issues(limit: 5, cursor: nil, filter: nil, search_text: '') + + expect(WebMock).to have_requested(:post, "#{base}/issues/search").with(body: { 'limit' => 5 }) + end + + it 'forwards cursor, filter and search_text when provided' do + stub_request(:post, "#{base}/issues/search").to_return(json('data' => [])) + filter = { 'field' => 'state', 'operator' => 'equals', 'values' => ['new'] } + + client.search_issues(limit: 5, cursor: 'c1', filter: filter, search_text: 'boom') + + expect(WebMock).to have_requested(:post, "#{base}/issues/search") + .with(body: { 'limit' => 5, 'cursor' => 'c1', 'filter' => filter, 'search_text' => 'boom' }) + end + + it 'clamps the limit to the API maximum' do + stub_request(:post, "#{base}/issues/search").to_return(json('data' => [])) + + client.search_issues(limit: 99_999) + + expect(WebMock).to have_requested(:post, "#{base}/issues/search") + .with(body: { 'limit' => described_class::MAX_SEARCH_LIMIT }) + end + + it 'raises the limit to 1 when it is zero or negative' do + stub_request(:post, "#{base}/issues/search").to_return(json('data' => [])) + + client.search_issues(limit: 0) + + expect(WebMock).to have_requested(:post, "#{base}/issues/search").with(body: { 'limit' => 1 }) + end + + it 'exposes the cursor when a next page is advertised' do + stub_request(:post, "#{base}/issues/search") + .to_return(json('data' => [], 'pagination' => { 'cursor' => 'c2', 'has_next_page' => true })) + + expect(client.search_issues(limit: 1).next_cursor).to eq('c2') + end + + # Pylon omits the block entirely on the last page, so absence is the + # common case rather than the edge case. + it 'reports no next cursor when the pagination block is absent' do + stub_request(:post, "#{base}/issues/search").to_return(json('data' => [{ 'id' => 'i1' }])) + + expect(client.search_issues(limit: 1).next_cursor).to be_nil + end + + it 'reports no next cursor when has_next_page is false' do + stub_request(:post, "#{base}/issues/search") + .to_return(json('data' => [], 'pagination' => { 'cursor' => 'c2', 'has_next_page' => false })) + + expect(client.search_issues(limit: 1).next_cursor).to be_nil + end + + it 'reports no next cursor when the advertised cursor is empty' do + stub_request(:post, "#{base}/issues/search") + .to_return(json('data' => [], 'pagination' => { 'cursor' => '', 'has_next_page' => true })) + + expect(client.search_issues(limit: 1).next_cursor).to be_nil + end + + it 'returns no records when the payload carries none' do + stub_request(:post, "#{base}/issues/search").to_return(json('data' => nil)) + + expect(client.search_issues(limit: 1).records).to eq([]) + end + + it 'wraps a failure in an APIError naming the endpoint' do + stub_request(:post, "#{base}/issues/search").to_return(json({ 'message' => 'bad filter' }, 400)) + + expect { client.search_issues(limit: 1) } + .to raise_error(ForestAdminDatasourcePylon::APIError, %r{issues/search: HTTP 400 bad filter}) + end + end + + describe '#fetch_issue' do + it 'unwraps the issue' do + stub_request(:get, "#{base}/issues/i1").to_return(json('data' => { 'id' => 'i1', 'number' => 1 })) + + expect(client.fetch_issue('i1')).to eq('id' => 'i1', 'number' => 1) + end + + it 'accepts an issue number as well as a uuid' do + stub_request(:get, "#{base}/issues/42").to_return(json('data' => { 'id' => 'i1', 'number' => 42 })) + + expect(client.fetch_issue(42)).to include('number' => 42) + end + + it 'escapes an id that would otherwise alter the request path' do + stub_request(:get, "#{base}/issues/..%2Fme").to_return(json('data' => nil)) + + client.fetch_issue('../me') + + expect(WebMock).to have_requested(:get, "#{base}/issues/..%2Fme") + end + + it 'wraps a missing issue in a 404 APIError' do + stub_request(:get, "#{base}/issues/nope").to_return(json({ 'message' => 'not found' }, 404)) + + expect { client.fetch_issue('nope') }.to raise_error(ForestAdminDatasourcePylon::APIError) { |error| + expect(error.status).to eq(404) + } + end + end + + describe '#fetch_issue_messages' do + let(:logger) { instance_double(Logger, warn: nil) } + + def page(records, cursor: nil) + body = { 'data' => records } + body['pagination'] = { 'cursor' => cursor, 'has_next_page' => true } if cursor + json(body) + end + + it 'returns the whole thread of an issue' do + stub_request(:get, "#{base}/issues/i1/messages").to_return(page([{ 'id' => 'm1' }, { 'id' => 'm2' }])) + + expect(client.fetch_issue_messages('i1')).to eq([{ 'id' => 'm1' }, { 'id' => 'm2' }]) + end + + # Omitting `limit` is what makes Pylon answer with every message at once; + # asking for a page would hand back the oldest ones and cut off the rest. + it 'sends no limit, so Pylon answers with every message in one request' do + stub_request(:get, "#{base}/issues/i1/messages").to_return(page([])) + + client.fetch_issue_messages('i1') + + expect(WebMock).to have_requested(:get, "#{base}/issues/i1/messages").with(query: {}).once + end + + it 'escapes an id that would otherwise alter the request path' do + stub_request(:get, "#{base}/issues/..%2Fme/messages").to_return(page([])) + + client.fetch_issue_messages('../me') + + expect(WebMock).to have_requested(:get, "#{base}/issues/..%2Fme/messages") + end + + it 'follows the cursor when Pylon paginates the thread anyway' do + stub_request(:get, "#{base}/issues/i1/messages").with(query: {}) + .to_return(page([{ 'id' => 'm1' }], cursor: 'c1')) + stub_request(:get, "#{base}/issues/i1/messages").with(query: { 'cursor' => 'c1' }) + .to_return(page([{ 'id' => 'm2' }])) + + expect(client.fetch_issue_messages('i1')).to eq([{ 'id' => 'm1' }, { 'id' => 'm2' }]) + end + + it 'stops on a cursor that does not move' do + stub_request(:get, "#{base}/issues/i1/messages").with(query: { 'cursor' => 'c1' }) + .to_return(page([{ 'id' => 'm2' }], cursor: 'c1')) + stub_request(:get, "#{base}/issues/i1/messages").with(query: {}) + .to_return(page([{ 'id' => 'm1' }], cursor: 'c1')) + + expect(client.fetch_issue_messages('i1').size).to eq(2) + end + + it 'stops on a cursor it has already followed' do + stub_request(:get, "#{base}/issues/i1/messages").with(query: {}) + .to_return(page([{ 'id' => 'm1' }], cursor: 'c1')) + stub_request(:get, "#{base}/issues/i1/messages").with(query: { 'cursor' => 'c1' }) + .to_return(page([{ 'id' => 'm2' }], cursor: 'c2')) + stub_request(:get, "#{base}/issues/i1/messages").with(query: { 'cursor' => 'c2' }) + .to_return(page([{ 'id' => 'm3' }], cursor: 'c1')) + + expect(client.fetch_issue_messages('i1')).to eq([{ 'id' => 'm1' }, { 'id' => 'm2' }, { 'id' => 'm3' }]) + expect(WebMock).to have_requested(:get, "#{base}/issues/i1/messages") + .with(query: { 'cursor' => 'c1' }).once + end + + it 'stops on an empty page' do + stub_request(:get, "#{base}/issues/i1/messages").to_return(page([], cursor: 'c1')) + + expect(client.fetch_issue_messages('i1')).to eq([]) + expect(WebMock).to have_requested(:get, "#{base}/issues/i1/messages").once + end + + it 'caps a thread Pylon never stops paginating, and says so' do + allow(ForestAdminDatasourcePylon).to receive(:logger).and_return(logger) + served = 0 + stub_request(:get, %r{/issues/i1/messages}).to_return do + served += 1 + page([{ 'id' => "m#{served}" }], cursor: "c#{served}") + end + + client.fetch_issue_messages('i1') + + expect(served).to eq(described_class::MAX_COLLECTED_PAGES) + expect(logger).to have_received(:warn).with(/Stopped paginating/) + end + + # The thread enriches a page rather than being it: a failure costs the + # operator the column, not the records they opened. + it 'degrades to nil and reports the failure when the thread cannot be read' do + allow(ForestAdminDatasourcePylon).to receive(:logger).and_return(logger) + stub_request(:get, "#{base}/issues/i1/messages").to_return(json({ 'message' => 'boom' }, 500)) + + expect(client.fetch_issue_messages('i1')).to be_nil + expect(logger).to have_received(:warn).with(/fetch_issue_messages\(i1\) failed; degrading.*HTTP 500 boom/) + end + + it 'degrades on a missing issue rather than raising a 404' do + stub_request(:get, "#{base}/issues/nope/messages").to_return(json({ 'message' => 'not found' }, 404)) + + expect(client.fetch_issue_messages('nope')).to be_nil + end + end + + describe '#search_accounts' do + it 'posts the full search envelope and returns the records' do + stub_request(:post, "#{base}/accounts/search").to_return(json('data' => [{ 'id' => 'a1' }])) + filter = { 'field' => 'name', 'operator' => 'equals', 'values' => ['Acme'] } + + page = client.search_accounts(limit: 2, cursor: 'c1', filter: filter, search_text: 'acme') + + expect(page.records).to eq([{ 'id' => 'a1' }]) + expect(WebMock).to have_requested(:post, "#{base}/accounts/search") + .with(body: { 'limit' => 2, 'cursor' => 'c1', 'filter' => filter, 'search_text' => 'acme' }) + end + + it 'omits cursor, filter and search_text when they are not provided' do + stub_request(:post, "#{base}/accounts/search").to_return(json('data' => [])) + + client.search_accounts(limit: 5, cursor: nil, filter: nil, search_text: '') + + expect(WebMock).to have_requested(:post, "#{base}/accounts/search").with(body: { 'limit' => 5 }) + end + + it 'wraps a failure in an APIError naming the endpoint' do + body = { 'message' => 'bad filter', 'request_id' => 'req_7' } + stub_request(:post, "#{base}/accounts/search").to_return(json(body, 400)) + + expect { client.search_accounts(limit: 1) }.to raise_error(ForestAdminDatasourcePylon::APIError) { |error| + expect(error.status).to eq(400) + expect(error.message) + .to eq('Pylon API call failed: accounts/search: HTTP 400 bad filter (request_id: req_7)') + } + end + end + + describe '#search_contacts' do + it 'posts to the contacts endpoint, clamps the limit and exposes the next cursor' do + stub_request(:post, "#{base}/contacts/search") + .to_return(json('data' => [{ 'id' => 'ct1' }], 'pagination' => { 'cursor' => 'c2', 'has_next_page' => true })) + + page = client.search_contacts(limit: 99_999) + + expect(page.records).to eq([{ 'id' => 'ct1' }]) + expect(page.next_cursor).to eq('c2') + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: { 'limit' => described_class::MAX_SEARCH_LIMIT }) + end + end + + describe '#list_accounts' do + # Unlike POST /accounts/search, the paginated GET rejects a request that + # does not carry a limit. + it 'sends the mandatory limit as a query parameter' do + stub_request(:get, "#{base}/accounts").with(query: { 'limit' => '2' }) + .to_return(json('data' => [{ 'id' => 'a1' }])) + + page = client.list_accounts(limit: 2) + + expect(page.records).to eq([{ 'id' => 'a1' }]) + expect(page.next_cursor).to be_nil + end + + it 'forwards the cursor and clamps the limit' do + stub_request(:get, "#{base}/accounts").with(query: { 'limit' => '1000', 'cursor' => 'c1' }) + .to_return(json('data' => [])) + + client.list_accounts(limit: 99_999, cursor: 'c1') + + expect(WebMock).to have_requested(:get, "#{base}/accounts") + .with(query: { 'limit' => '1000', 'cursor' => 'c1' }) + end + + it 'omits an empty cursor' do + stub_request(:get, "#{base}/accounts").with(query: { 'limit' => '5' }).to_return(json('data' => [])) + + client.list_accounts(limit: 5, cursor: '') + + expect(WebMock).to have_requested(:get, "#{base}/accounts").with(query: { 'limit' => '5' }) + end + + it 'exposes the cursor when a next page is advertised' do + stub_request(:get, "#{base}/accounts").with(query: { 'limit' => '1' }) + .to_return(json('data' => [], + 'pagination' => { + 'cursor' => 'c2', 'has_next_page' => true + })) + + expect(client.list_accounts(limit: 1).next_cursor).to eq('c2') + end + + it 'reports no next cursor when has_next_page is false' do + stub_request(:get, "#{base}/accounts").with(query: { 'limit' => '1' }) + .to_return(json('data' => [], + 'pagination' => { + 'cursor' => 'c2', 'has_next_page' => false + })) + + expect(client.list_accounts(limit: 1).next_cursor).to be_nil + end + + it 'wraps a failure in an APIError carrying status, body and request_id' do + body = { 'message' => 'limit is required', 'request_id' => 'req_9' } + stub_request(:get, "#{base}/accounts").with(query: { 'limit' => '1' }).to_return(json(body, 400)) + + expect { client.list_accounts(limit: 1) }.to raise_error(ForestAdminDatasourcePylon::APIError) { |error| + expect(error.status).to eq(400) + expect(error.body).to eq(body) + expect(error.message) + .to eq('Pylon API call failed: accounts: HTTP 400 limit is required (request_id: req_9)') + } + end + end + + describe '#list_contacts' do + # The OpenAPI spec omits the query parameters of GET /contacts, but the + # endpoint paginates exactly like GET /accounts. + it 'paginates like the accounts listing' do + stub_request(:get, "#{base}/contacts").with(query: { 'limit' => '2', 'cursor' => 'c1' }) + .to_return(json('data' => [{ 'id' => 'ct1' }], + 'pagination' => { + 'cursor' => 'c2', 'has_next_page' => true + })) + + page = client.list_contacts(limit: 2, cursor: 'c1') + + expect(page.records).to eq([{ 'id' => 'ct1' }]) + expect(page.next_cursor).to eq('c2') + end + end + + describe '#fetch_account' do + it 'unwraps the account' do + stub_request(:get, "#{base}/accounts/a1").to_return(json('data' => { 'id' => 'a1', 'name' => 'Acme' })) + + expect(client.fetch_account('a1')).to eq('id' => 'a1', 'name' => 'Acme') + end + + it 'accepts an external id as well as a uuid' do + stub_request(:get, "#{base}/accounts/ext%2F42").to_return(json('data' => { 'id' => 'a1' })) + + expect(client.fetch_account('ext/42')).to eq('id' => 'a1') + end + + it 'wraps a missing account in a 404 APIError naming the endpoint' do + stub_request(:get, "#{base}/accounts/nope").to_return(json({ 'message' => 'not found' }, 404)) + + expect { client.fetch_account('nope') }.to raise_error(ForestAdminDatasourcePylon::APIError) { |error| + expect(error.status).to eq(404) + expect(error.message).to match(%r{accounts/nope: HTTP 404 not found}) + } + end + end + + describe '#fetch_contact' do + it 'unwraps the contact' do + stub_request(:get, "#{base}/contacts/ct1").to_return(json('data' => { 'id' => 'ct1', 'email' => 'a@b.c' })) + + expect(client.fetch_contact('ct1')).to eq('id' => 'ct1', 'email' => 'a@b.c') + end + end + + describe '#fetch_users' do + it 'includes deactivated users by default' do + stub_request(:get, "#{base}/users").with(query: { 'include_deactivated' => 'true' }) + .to_return(json('data' => [{ 'id' => 'u1' }, { 'id' => 'u2' }])) + + expect(client.fetch_users).to eq([{ 'id' => 'u1' }, { 'id' => 'u2' }]) + end + + it 'can ask for active users only' do + stub_request(:get, "#{base}/users").with(query: { 'include_deactivated' => 'false' }) + .to_return(json('data' => [])) + + client.fetch_users(include_deactivated: false) + + expect(WebMock).to have_requested(:get, "#{base}/users").with(query: { 'include_deactivated' => 'false' }) + end + + it 'returns an empty array when the payload carries no data' do + stub_request(:get, "#{base}/users").with(query: { 'include_deactivated' => 'true' }) + .to_return(json('data' => nil)) + + expect(client.fetch_users).to eq([]) + end + + it 'wraps a failure in an APIError naming the endpoint' do + stub_request(:get, "#{base}/users").with(query: { 'include_deactivated' => 'true' }) + .to_return(json({ 'message' => 'boom' }, 500)) + + expect { client.fetch_users } + .to raise_error(ForestAdminDatasourcePylon::APIError, /users: HTTP 500 boom/) + end + end + + describe '#fetch_user' do + it 'unwraps the user' do + stub_request(:get, "#{base}/users/u1").to_return(json('data' => { 'id' => 'u1', 'name' => 'Ada' })) + + expect(client.fetch_user('u1')).to eq('id' => 'u1', 'name' => 'Ada') + end + end + + describe '#fetch_teams' do + it 'returns every team without sending any query parameter' do + stub_request(:get, "#{base}/teams").to_return(json('data' => [{ 'id' => 't1' }])) + + expect(client.fetch_teams).to eq([{ 'id' => 't1' }]) + expect(WebMock).to have_requested(:get, "#{base}/teams") + end + end + + describe '#fetch_team' do + it 'unwraps the team' do + stub_request(:get, "#{base}/teams/t1").to_return(json('data' => { 'id' => 't1', 'name' => 'Support' })) + + expect(client.fetch_team('t1')).to eq('id' => 't1', 'name' => 'Support') + end + end + + describe '#fetch_custom_fields' do + let(:logger) { instance_double(Logger, warn: nil) } + + def definitions(records, cursor: nil) + body = { 'data' => records } + body['pagination'] = { 'cursor' => cursor, 'has_next_page' => true } if cursor + json(body) + end + + # `object_type` is mandatory: Pylon answers 400 without it, so the schema of + # each collection is read by its own call. + it 'asks for the definitions of one object type' do + stub_request(:get, "#{base}/custom-fields").with(query: { 'object_type' => 'issue' }) + .to_return(definitions([{ 'slug' => 'severity' }])) + + expect(client.fetch_custom_fields('issue')).to eq([{ 'slug' => 'severity' }]) + end + + # The mandatory parameter has to survive the walk: dropped on the second + # page, it would answer with the fields of another object type or with a 400. + it 'keeps the object type on every page of the walk' do + stub_request(:get, "#{base}/custom-fields").with(query: { 'object_type' => 'issue' }) + .to_return(definitions([{ 'slug' => 'severity' }], cursor: 'c1')) + stub_request(:get, "#{base}/custom-fields").with(query: { 'object_type' => 'issue', 'cursor' => 'c1' }) + .to_return(definitions([{ 'slug' => 'tier' }])) + + expect(client.fetch_custom_fields('issue')).to eq([{ 'slug' => 'severity' }, { 'slug' => 'tier' }]) + end + + it 'returns an empty list when the organization defined no custom field' do + stub_request(:get, "#{base}/custom-fields").with(query: { 'object_type' => 'account' }) + .to_return(definitions(nil)) + + expect(client.fetch_custom_fields('account')).to eq([]) + end + + # Read while the agent boots: the datasource has to come up on its native + # schema rather than fail to come up at all. + it 'degrades to an empty list and reports the failure' do + allow(ForestAdminDatasourcePylon).to receive(:logger).and_return(logger) + stub_request(:get, "#{base}/custom-fields").with(query: { 'object_type' => 'issue' }) + .to_return(json({ 'message' => 'boom' }, 500)) + + expect(client.fetch_custom_fields('issue')).to be_nil + expect(logger).to have_received(:warn).with(/fetch_custom_fields\(issue\) failed; degrading.*HTTP 500 boom/) + end + + it 'degrades when the token is not allowed to read the definitions' do + stub_request(:get, "#{base}/custom-fields").with(query: { 'object_type' => 'issue' }) + .to_return(json({ 'message' => 'forbidden' }, 403)) + + expect(client.fetch_custom_fields('issue')).to be_nil + end + end + + # The introspection runs while the datasource is being constructed, so it is + # read through a connection of its own: what the resilient one is willing to + # wait for is minutes of Rails boot the operator sits through. + describe 'the boot connection' do + let(:fast_boot) { ForestAdminDatasourcePylon::RetryPolicy.new(max_retries: 1, interval: 0, max_interval: 2) } + let(:definitions) { "#{base}/custom-fields" } + + def boot_client(**options) + described_class.new(ForestAdminDatasourcePylon::Configuration.new(api_key: 'k', **options)) + end + + it 'bounds how long one attempt may take' do + conn = client.send(:boot_connection) + + expect([conn.options.open_timeout, conn.options.timeout]).to eq([3, 10]) + end + + it 'honours the configured boot timeouts' do + conn = boot_client(boot_open_timeout: 1, boot_timeout: 2).send(:boot_connection) + + expect([conn.options.open_timeout, conn.options.timeout]).to eq([1, 2]) + end + + # The whole point of the bound: faraday-retry abandons a Retry-After past + # max_interval, so a rate-limited boot degrades at once instead of waiting + # out a window per attempt. + it 'gives up at once on a 429 carrying a whole rate-limit window' do + stub_request(:get, definitions).with(query: { 'object_type' => 'issue' }) + .to_return(status: 429, body: { 'message' => 'slow down' }.to_json, + headers: { 'Content-Type' => 'application/json', + 'Retry-After' => '60' }) + + expect(client.fetch_custom_fields('issue')).to be_nil + expect(WebMock).to have_requested(:get, definitions).with(query: { 'object_type' => 'issue' }).once + end + + # One retry rather than none: the introspection runs once and is never + # revisited, so a hiccup would cost the custom columns for the whole life of + # the process. + it 'retries once a failure Pylon sent without a Retry-After' do + stub_request(:get, definitions).with(query: { 'object_type' => 'issue' }) + .to_return(json({ 'message' => 'slow down' }, 429)) + .then.to_return(json('data' => [{ 'slug' => 'severity' }])) + + expect(boot_client(boot_retry_policy: fast_boot).fetch_custom_fields('issue')) + .to eq([{ 'slug' => 'severity' }]) + expect(WebMock).to have_requested(:get, definitions).with(query: { 'object_type' => 'issue' }).twice + end + + it 'spends no more than that one retry' do + stub_request(:get, definitions).with(query: { 'object_type' => 'issue' }) + .to_return(json({ 'message' => 'slow down' }, 429)) + + expect(boot_client(boot_retry_policy: fast_boot).fetch_custom_fields('issue')).to be_nil + expect(WebMock).to have_requested(:get, definitions).with(query: { 'object_type' => 'issue' }).twice + end + + # Pylon meters the endpoint, so the boot spends the same budget as every + # later call: a window of its own would spend that budget twice over. + it 'meters the introspection on the limiter the rest of the client uses' do + limiter = instance_spy(ForestAdminDatasourcePylon::RateLimiter) + stub_request(:get, definitions).with(query: { 'object_type' => 'issue' }).to_return(json('data' => [])) + + boot_client(rate_limiter: limiter).fetch_custom_fields('issue') + expect(limiter).to have_received(:acquire).with(:get, '/custom-fields') + end + + # The other cursor walk is a request the operator is already waiting on, and + # keeps the patience the resilient policy grants it -- here two retries, + # where the boot connection would have spent one. + it 'leaves every other call on the resilient connection' do + messages = "#{base}/issues/i1/messages" + stub_request(:get, messages).to_return(json({ 'message' => 'slow down' }, 429)) + + expect(client.fetch_issue_messages('i1')).to be_nil + expect(WebMock).to have_requested(:get, messages).times(3) + end + end + + describe 'rate limiting' do + it 'retries a 429 and returns the eventual success' do + stub_request(:get, "#{base}/me") + .to_return(json({ 'message' => 'slow down' }, 429)) + .then.to_return(json('data' => { 'id' => 'org_1' })) + + expect(client.me).to eq('id' => 'org_1') + expect(WebMock).to have_requested(:get, "#{base}/me").twice + end + + it 'retries a 503 and returns the eventual success' do + stub_request(:get, "#{base}/me") + .to_return(json({ 'message' => 'unavailable' }, 503)) + .then.to_return(json('data' => { 'id' => 'org_1' })) + + expect(client.me).to eq('id' => 'org_1') + expect(WebMock).to have_requested(:get, "#{base}/me").twice + end + + it 'gives up after the configured retry budget and raises the last error' do + stub_request(:get, "#{base}/me").to_return(json({ 'message' => 'slow down' }, 429)) + + expect { client.me }.to raise_error(ForestAdminDatasourcePylon::APIError) { |error| + expect(error.status).to eq(429) + } + expect(WebMock).to have_requested(:get, "#{base}/me").times(3) + end + + it 'does not retry a 404' do + stub_request(:get, "#{base}/me").to_return(json({ 'message' => 'nope' }, 404)) + + expect { client.me }.to raise_error(ForestAdminDatasourcePylon::APIError) + expect(WebMock).to have_requested(:get, "#{base}/me").once + end + + it 'retries a dropped connection' do + stub_request(:get, "#{base}/me").to_timeout.then.to_return(json('data' => { 'id' => 'org_1' })) + + expect(client.me).to eq('id' => 'org_1') + expect(WebMock).to have_requested(:get, "#{base}/me").twice + end + + # Regression: max_interval used to be hardcoded to 5s. faraday-retry gives up + # outright when Retry-After exceeds max_interval, so a Pylon 429 carrying a + # per-minute Retry-After silently performed zero retries. + describe 'Retry-After' do + it 'honours a Retry-After that fits within the cap' do + stub_request(:get, "#{base}/me") + .to_return(status: 429, headers: { 'Retry-After' => '0' }) + .then.to_return(json('data' => { 'id' => 'org_1' })) + + expect(client.me).to eq('id' => 'org_1') + expect(WebMock).to have_requested(:get, "#{base}/me").twice + end + + it 'gives up without retrying when Retry-After exceeds the cap' do + impatient = ForestAdminDatasourcePylon::Configuration.new( + api_key: 'k', retry_policy: ForestAdminDatasourcePylon::RetryPolicy.new(interval: 0, max_interval: 1) + ) + stub_request(:get, "#{base}/me").to_return(status: 429, headers: { 'Retry-After' => '5' }) + + expect { described_class.new(impatient).me }.to raise_error(ForestAdminDatasourcePylon::APIError) + expect(WebMock).to have_requested(:get, "#{base}/me").once + end + end + end + + # The write path already refuses a body Pylon answered without `data`; a read + # used to hand the envelope back, and `Array()` split it into `[key, value]` + # pairs the collection serialized into rows holding nothing -- a page that + # looks answered and is empty. + describe 'a response body that broke the contract' do + it 'refuses an envelope carrying no "data" where a list was expected' do + stub_request(:get, "#{base}/teams").to_return(json('teams' => [{ 'id' => 't1' }])) + + expect { client.fetch_teams } + .to raise_error(ForestAdminDatasourcePylon::APIError, /teams.*'data' is not a list/) + end + + it 'refuses a "data" holding something other than a list on a search' do + stub_request(:post, "#{base}/issues/search").to_return(json('data' => { 'id' => 'i1' })) + + expect { client.search_issues(limit: 10) } + .to raise_error(ForestAdminDatasourcePylon::APIError, %r{issues/search.*'data' is not a list}) + end + + it 'refuses a "data" holding something other than a record on a single read' do + stub_request(:get, "#{base}/issues/i1").to_return(json('data' => %w[i1])) + + expect { client.fetch_issue('i1') } + .to raise_error(ForestAdminDatasourcePylon::APIError, %r{issues/i1.*'data' is not a record}) + end + + # Pylon spells "no record" that way, and a search matching nothing is not a + # broken contract. + it 'reads an absent or null "data" as the empty answer it is' do + stub_request(:get, "#{base}/teams").to_return(json('data' => nil)) + stub_request(:get, "#{base}/issues/i1").to_return(json('data' => nil)) + + expect(client.fetch_teams).to eq([]) + expect(client.fetch_issue('i1')).to be_nil + end + end +end diff --git a/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/account_spec.rb b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/account_spec.rb new file mode 100644 index 000000000..49014add9 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/account_spec.rb @@ -0,0 +1,625 @@ +module ForestAdminDatasourcePylon + RSpec.describe Collections::Account do + def filter(condition_tree: nil, search: nil, page: nil, sort: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new( + condition_tree: condition_tree, search: search, page: page, sort: sort + ) + end + + def leaf(field, operator, value = nil) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def branch(aggregator, conditions) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + .new(aggregator, conditions) + end + + def id_leaf(operator, value) + leaf('id', operator, value) + end + + def page(offset, limit) + ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: offset, limit: limit) + end + + def sort_on(field, ascending: true) + ForestAdminDatasourceToolkit::Components::Query::Sort.new([{ field: field, ascending: ascending }]) + end + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + # Trimmed to the shape observed on the API: the owner is a nested object + # carrying an id, and unset values come back as null rather than absent. + def account_payload(id, overrides = {}) + { + 'id' => id, 'name' => 'Acme', 'type' => 'customer', 'is_disabled' => false, + 'domain' => 'acme.com', 'primary_domain' => 'acme.com', 'domains' => %w[acme.com acme.io], + 'tags' => %w[vip], 'owner' => { 'id' => 'usr-1', 'email' => 'ada@acme.com' }, + 'external_ids' => [{ 'external_id' => 'crm-1', 'label' => 'salesforce' }], + 'channels' => [{ 'channel_id' => 'C1', 'source' => 'slack', 'is_primary' => true }], + 'crm_settings' => { 'details' => [{ 'id' => 'crm-1', 'source' => 'salesforce' }] }, + 'custom_fields' => {}, 'created_at' => '2026-08-07T13:06:22Z', 'updated_at' => '2026-08-10T09:00:00Z', + 'latest_customer_activity_time' => nil + }.merge(overrides) + end + + # Relations are fields too; the assertions on the columns select them out. + def columns + collection.fields.select { |_name, field| field.type == 'Column' } + end + + let(:datasource) { ForestAdminDatasourcePylon::Datasource.new(api_key: 'k') } + let(:collection) { described_class.new(datasource) } + let(:base) { datasource.configuration.url } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + + before { stub_custom_fields } + + def stub_list(query, payload) + stub_request(:get, "#{base}/accounts").with(query: query).to_return(json(payload)) + end + + def stub_search(payload = { 'data' => [account_payload('acc-1')] }) + stub_request(:post, "#{base}/accounts/search").to_return(json(payload)) + end + + describe 'schema' do + it 'is named PylonAccount' do + expect(collection.name).to eq('PylonAccount') + end + + it 'declares id as the primary key' do + expect(collection.fields['id'].is_primary_key).to be(true) + end + + # No short-circuit to serve here: /accounts/search filters id itself, so + # the column advertises every operator the endpoint accepts on it. + it 'advertises the id operators the search endpoint filters server-side' do + expect(collection.fields['id'].filter_operators) + .to eq([operators::EQUAL, operators::IN, operators::NOT_IN]) + end + + it 'exposes the native columns observed on the API' do + expect(collection.fields.keys).to include( + 'name', 'type', 'is_disabled', 'domain', 'primary_domain', 'domains', 'tags', + 'owner_id', 'external_ids', 'channels', 'crm_settings', + 'created_at', 'updated_at', 'latest_customer_activity_time' + ) + end + + it 'flattens the owner into a foreign-key column instead of exposing the nested object' do + expect(collection.fields.keys).not_to include('owner') + end + + it 'types the lists as Json and the times as dates' do + expect(collection.fields['domains'].column_type).to eq('Json') + expect(collection.fields['channels'].column_type).to eq('Json') + expect(collection.fields['is_disabled'].column_type).to eq('Boolean') + expect(collection.fields['latest_customer_activity_time'].column_type).to eq('Date') + end + + # Neither endpoint exposes a sort parameter. + it 'declares every column non-sortable' do + expect(columns.values.map(&:is_sortable).uniq).to eq([false]) + end + + # The Json columns holding objects — `external_ids`, `channels` — stay + # read-only although the endpoint takes them: their write shape is not the + # one the column shows. + it 'declares writable exactly the columns an endpoint takes in the shape they are read' do + writable = columns.reject { |_name, column| column.is_read_only }.keys + + expect(writable).to contain_exactly('name', 'type', 'is_disabled', 'domains', 'tags', 'owner_id') + end + + # No Pylon endpoint aggregates, and the pages of a cursor walk are not the + # dataset: a chart grouped by one of these columns would answer a fraction + # as if it were the whole collection. + it 'declares no column groupable' do + expect(columns.values.map(&:is_groupable).uniq).to eq([false]) + end + + # `search_text` is native on /accounts/search, while Pylon exposes neither a + # count endpoint nor a total, so Count stays out until it can be throttled. + it 'enables search and leaves count disabled' do + expect(collection.is_searchable?).to be(true) + expect(collection.is_countable?).to be(false) + end + + it 'advertises only the operators the search allow-list accepts' do + expect(collection.fields['name'].filter_operators) + .to eq([operators::EQUAL, operators::IN, operators::NOT_IN, operators::CONTAINS, operators::I_CONTAINS]) + expect(collection.fields['owner_id'].filter_operators) + .to eq([operators::EQUAL, operators::IN, operators::NOT_IN, + operators::PRESENT, operators::BLANK, operators::MISSING]) + expect(collection.fields['tags'].filter_operators) + .to eq([operators::IN, operators::NOT_IN]) + expect(collection.fields['domains'].filter_operators) + .to eq([operators::IN, operators::NOT_IN]) + end + + # `POST /accounts/search` does filter a list column with `contains`, and + # the schema still must not advertise it: the column is typed Json, the + # only type the toolkit has for a list, and it allows no substring + # operator on one -- the filter would be refused on the way in. + it 'advertises no substring operator on a list column' do + allowed = ForestAdminDatasourceToolkit::Validations::Rules.get_allowed_operators_for_column_type('Json') + + %w[domains tags].each do |field| + expect(collection.fields[field].filter_operators) + .not_to include(operators::CONTAINS, operators::NOT_CONTAINS, operators::I_CONTAINS) + expect(allowed).to include(*collection.fields[field].filter_operators) + end + end + + # /accounts/search accepts `string_contains` on a name but documents no + # negation of it, so the UI must not offer one. + it 'offers no negated substring on name' do + expect(collection.fields['name'].filter_operators) + .not_to include(operators::NOT_CONTAINS, operators::NOT_I_CONTAINS) + end + + # The endpoint does filter external_ids, on the bare id strings, while the + # column shows { external_id, label } objects: the filter would run on + # something the operator cannot see. + it 'advertises no operator on external_ids' do + expect(collection.fields['external_ids'].filter_operators).to eq([]) + end + + # Unlike /issues/search, the accounts search takes no time filter at all. + it 'advertises no operator on the time columns' do + %w[created_at updated_at latest_customer_activity_time].each do |field| + expect(collection.fields[field].filter_operators).to eq([]) + end + end + + it 'advertises no operator on the other columns Pylon cannot filter' do + %w[domain primary_domain type is_disabled channels crm_settings].each do |field| + expect(collection.fields[field].filter_operators).to eq([]) + end + end + end + + describe 'relations' do + let(:one_to_many) { ForestAdminDatasourceToolkit::Schema::Relations::OneToManySchema } + + # The reverse sides of the ManyToOne relations Issue and Contact declare. + # Both endpoints filter `account_id` server-side, so a related list is one + # request and no in-memory pass. + it 'declares the issues and the contacts of an account as OneToMany relations' do + expect(collection.fields.values_at('issues', 'contacts')).to all(be_a(one_to_many)) + expect(collection.fields['issues']) + .to have_attributes(foreign_collection: 'PylonIssue', origin_key: 'account_id', + origin_key_target: 'id') + expect(collection.fields['contacts']) + .to have_attributes(foreign_collection: 'PylonContact', origin_key: 'account_id', + origin_key_target: 'id') + end + + # It points at a PylonUser, but nothing in the panel asks for the owner of + # an account yet. + it 'leaves the owner a plain column' do + expect(collection.fields['owner_id'].type).to eq('Column') + expect(collection.fields.keys).not_to include('owner') + end + end + + # No condition tree and no search: the listing endpoint returns the same + # records for 300 requests per minute where the search endpoint allows 120. + describe '#list without a filter' do + it 'browses the listing endpoint and serializes what it returns' do + stub_list({ 'limit' => '1000' }, 'data' => [account_payload('acc-1')]) + + rows = collection.list(nil, filter, nil) + + expect(rows.size).to eq(1) + expect(rows.first).to include('id' => 'acc-1', 'name' => 'Acme', 'type' => 'customer', + 'domains' => %w[acme.com acme.io], 'tags' => %w[vip], + 'is_disabled' => false, 'latest_customer_activity_time' => nil) + end + + it 'flattens the nested owner into a foreign-key column' do + stub_list({ 'limit' => '1000' }, 'data' => [account_payload('acc-1')]) + + expect(collection.list(nil, filter, nil).first).to include('owner_id' => 'usr-1') + end + + it 'keeps the nested object out of the serialized record' do + stub_list({ 'limit' => '1000' }, 'data' => [account_payload('acc-1')]) + + expect(collection.list(nil, filter, nil).first.keys).not_to include('owner') + end + + it 'reports no owner rather than raising when the account has none' do + stub_list({ 'limit' => '1000' }, 'data' => [account_payload('acc-1', 'owner' => nil)]) + + expect(collection.list(nil, filter, nil).first).to include('owner_id' => nil) + end + + it 'restricts the record to the projection' do + stub_list({ 'limit' => '1000' }, 'data' => [account_payload('acc-1')]) + + expect(collection.list(nil, filter, %w[id name])).to eq([{ 'id' => 'acc-1', 'name' => 'Acme' }]) + end + + it 'never spends the search budget' do + stub_list({ 'limit' => '1000' }, 'data' => [account_payload('acc-1')]) + + collection.list(nil, filter, %w[id]) + + expect(WebMock).not_to have_requested(:post, "#{base}/accounts/search") + end + + # The search box sends an empty string once the operator clears it, which + # is not a search and must not cost a search request. + it 'browses when the search is blank' do + stub_list({ 'limit' => '1000' }, 'data' => [account_payload('acc-1')]) + + expect(collection.list(nil, filter(search: ' '), %w[id])).to eq([{ 'id' => 'acc-1' }]) + end + + it 'browses when Forest sends no filter at all' do + stub_list({ 'limit' => '1000' }, 'data' => [account_payload('acc-1')]) + + expect(collection.list(nil, nil, %w[id])).to eq([{ 'id' => 'acc-1' }]) + end + + it 'forwards the requested page as the listing limit' do + stub_list({ 'limit' => '1' }, 'data' => [account_payload('acc-1')]) + + collection.list(nil, filter(page: page(0, 1)), nil) + + expect(WebMock).to have_requested(:get, "#{base}/accounts").with(query: { 'limit' => '1' }) + end + + it 'walks the cursor until the requested window is covered' do + stub_list({ 'limit' => '3' }, + 'data' => [account_payload('acc-1'), account_payload('acc-2')], + 'pagination' => { 'cursor' => 'c1', 'has_next_page' => true }) + stub_list({ 'limit' => '1', 'cursor' => 'c1' }, 'data' => [account_payload('acc-3')]) + + expect(collection.list(nil, filter(page: page(2, 1)), %w[id])).to eq([{ 'id' => 'acc-3' }]) + end + + # Browsing carries no page when a decorator above asks for the whole + # collection — an emulated sort, an emulated operator — and the walk has to + # answer with the collection rather than with its first page. + it 'walks every page when the read carries no page at all' do + stub_list({ 'limit' => '1000' }, + 'data' => [account_payload('acc-1')], + 'pagination' => { 'cursor' => 'c1', 'has_next_page' => true }) + stub_list({ 'limit' => '1000', 'cursor' => 'c1' }, 'data' => [account_payload('acc-2')]) + + expect(collection.list(nil, filter, %w[id])).to eq([{ 'id' => 'acc-1' }, { 'id' => 'acc-2' }]) + end + + it 'returns an empty list when the organization has no account' do + stub_list({ 'limit' => '1000' }, 'data' => []) + + expect(collection.list(nil, filter, nil)).to eq([]) + end + end + + describe '#list with a filter' do + before { stub_search } + + it 'sends the translated condition tree to the search endpoint' do + collection.list(nil, filter(condition_tree: leaf('name', operators::I_CONTAINS, 'acm')), %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/accounts/search").with( + body: { 'limit' => Client::MAX_SEARCH_LIMIT, + 'filter' => { 'field' => 'name', 'operator' => 'string_contains', 'value' => 'acm' } } + ) + end + + it 'translates a membership filter on a list column' do + collection.list(nil, filter(condition_tree: leaf('tags', operators::IN, %w[vip])), %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/accounts/search").with( + body: hash_including('filter' => { 'field' => 'tags', 'operator' => 'in', 'values' => %w[vip] }) + ) + end + + it 'translates a presence filter with no value' do + collection.list(nil, filter(condition_tree: leaf('owner_id', operators::BLANK)), %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/accounts/search").with( + body: hash_including('filter' => { 'field' => 'owner_id', 'operator' => 'is_unset' }) + ) + end + + it 'sends a free-text search as search_text, intersected with the filter' do + query = filter(condition_tree: leaf('tags', operators::IN, %w[vip]), search: 'acme') + + collection.list(nil, query, %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/accounts/search").with( + body: { 'limit' => Client::MAX_SEARCH_LIMIT, 'search_text' => 'acme', + 'filter' => { 'field' => 'tags', 'operator' => 'in', 'values' => %w[vip] } } + ) + end + + # The listing endpoint cannot search, so a search alone is worth the + # search endpoint even with nothing to filter. + it 'searches on a free-text search alone' do + collection.list(nil, filter(search: 'acme'), %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/accounts/search") + .with(body: { 'limit' => Client::MAX_SEARCH_LIMIT, 'search_text' => 'acme' }) + expect(WebMock).not_to have_requested(:get, "#{base}/accounts") + end + + # `id` is a filter field of this endpoint, so it needs no short-circuit and + # an `or` cannot widen anything: it is translated like any other field. + it 'translates an id filter server-side, even under an or' do + tree = branch('Or', [id_leaf(operators::EQUAL, 'acc-1'), leaf('name', operators::EQUAL, 'Acme')]) + + collection.list(nil, filter(condition_tree: tree), %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/accounts/search").with( + body: hash_including( + 'filter' => { 'operator' => 'or', + 'subfilters' => [{ 'field' => 'id', 'operator' => 'equals', 'value' => 'acc-1' }, + { 'field' => 'name', 'operator' => 'equals', 'value' => 'Acme' }] } + ) + ) + end + + # The whole point of translating rather than dropping: a predicate Pylon + # cannot express fails loudly instead of returning unfiltered rows. + it 'raises rather than returning unfiltered rows for a field Pylon cannot filter' do + query = filter(condition_tree: leaf('created_at', operators::GREATER_THAN, '2026-01-01T00:00:00Z')) + + expect { collection.list(nil, query, %w[id]) } + .to raise_error(UnsupportedOperatorError, /cannot filter on 'created_at'/) + expect(WebMock).not_to have_requested(:post, "#{base}/accounts/search") + end + + it 'raises rather than searching for an operator the endpoint refuses on a field' do + expect { collection.list(nil, filter(condition_tree: leaf('name', operators::NOT_CONTAINS, 'a')), %w[id]) } + .to raise_error(UnsupportedOperatorError, /not supported on field 'name'/) + end + + it 'keeps the same filter across every page of the walk' do + stub_request(:post, "#{base}/accounts/search") + .with(body: hash_including('limit' => 3)) + .to_return(json('data' => [account_payload('acc-1'), account_payload('acc-2')], + 'pagination' => { 'cursor' => 'c1', 'has_next_page' => true })) + stub_request(:post, "#{base}/accounts/search") + .with(body: hash_including('cursor' => 'c1')) + .to_return(json('data' => [account_payload('acc-3')])) + query = filter(condition_tree: leaf('tags', operators::IN, %w[vip]), page: page(2, 1)) + + expect(collection.list(nil, query, %w[id])).to eq([{ 'id' => 'acc-3' }]) + expect(WebMock).to have_requested(:post, "#{base}/accounts/search") + .with(body: hash_including('filter' => { 'field' => 'tags', 'operator' => 'in', + 'values' => %w[vip] })).twice + end + end + + # A record detail is `id equals X` alone: reading it through GET + # /accounts/{id} spends the 300 requests/minute budget instead of the 120 of + # the search endpoint. + describe '#list on a single-id filter' do + it 'reads the account through its own endpoint instead of searching' do + stub_request(:get, "#{base}/accounts/acc-1").to_return(json('data' => account_payload('acc-1'))) + + rows = collection.list(nil, filter(condition_tree: id_leaf(operators::EQUAL, 'acc-1')), %w[id name]) + + expect(rows).to eq([{ 'id' => 'acc-1', 'name' => 'Acme' }]) + expect(WebMock).not_to have_requested(:post, "#{base}/accounts/search") + end + + it 'still reads by id when the search is empty' do + stub_request(:get, "#{base}/accounts/acc-1").to_return(json('data' => account_payload('acc-1'))) + query = filter(condition_tree: id_leaf(operators::EQUAL, 'acc-1'), search: '') + + expect(collection.list(nil, query, %w[id])).to eq([{ 'id' => 'acc-1' }]) + end + + it 'reports no record when the account no longer exists' do + stub_request(:get, "#{base}/accounts/gone").to_return(json({ 'message' => 'not found' }, 404)) + + expect(collection.list(nil, filter(condition_tree: id_leaf(operators::EQUAL, 'gone')), %w[id])).to eq([]) + end + + # A blank body would otherwise serialize into a record whose every column, + # id included, is null: a row the panel shows and cannot open. + it 'reports no record rather than a blank one when Pylon answers with no data' do + stub_request(:get, "#{base}/accounts/acc-1").to_return(status: 200, body: '') + + expect(collection.list(nil, filter(condition_tree: id_leaf(operators::EQUAL, 'acc-1')), %w[id])).to eq([]) + end + + # `GET /accounts/{id}` accepts an external id in place of the primary key + # and answers with the account carrying its own UUID. The row would not + # match the filter that asked for it, and the same filter combined with a + # scope — which goes through the search endpoint — answers nothing. + it 'reports no record when the endpoint answered an alias of the primary key' do + stub_request(:get, "#{base}/accounts/crm-1").to_return(json('data' => account_payload('acc-1'))) + query = filter(condition_tree: id_leaf(operators::EQUAL, 'crm-1')) + + expect(collection.list(nil, query, %w[id name])).to eq([]) + end + + it 'propagates a failure that is not a missing record' do + stub_request(:get, "#{base}/accounts/acc-1").to_return(json({ 'message' => 'boom' }, 500)) + + expect { collection.list(nil, filter(condition_tree: id_leaf(operators::EQUAL, 'acc-1')), %w[id]) } + .to raise_error(APIError) + end + + it 'applies the requested page to the record it read' do + stub_request(:get, "#{base}/accounts/acc-1").to_return(json('data' => account_payload('acc-1'))) + query = filter(condition_tree: id_leaf(operators::EQUAL, 'acc-1'), page: page(1, 1)) + + expect(collection.list(nil, query, %w[id])).to eq([]) + end + + # One search request answers several ids exactly, where one GET per id + # would burn the budget of the whole agent. + it 'searches instead when several ids are asked for' do + stub_search('data' => [account_payload('acc-1'), account_payload('acc-2')]) + + rows = collection.list(nil, filter(condition_tree: id_leaf(operators::IN, %w[acc-1 acc-2])), %w[id]) + + expect(rows).to eq([{ 'id' => 'acc-1' }, { 'id' => 'acc-2' }]) + expect(WebMock).to have_requested(:post, "#{base}/accounts/search").with( + body: hash_including('filter' => { 'field' => 'id', 'operator' => 'in', + 'values' => %w[acc-1 acc-2] }) + ) + end + + # Forest sends `AND(id equal X, )` on a record detail as soon as a + # scope or a segment is set. The search endpoint filters both server-side, + # so nothing has to be applied in memory -- and a scope on a list column, + # which no in-memory pass could evaluate, is answered rather than refused. + it 'searches instead when the filter carries more conditions' do + stub_search + tree = branch('And', [id_leaf(operators::EQUAL, 'acc-1'), leaf('tags', operators::IN, %w[vip])]) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id])).to eq([{ 'id' => 'acc-1' }]) + expect(WebMock).to have_requested(:post, "#{base}/accounts/search").with( + body: hash_including( + 'filter' => { 'operator' => 'and', + 'subfilters' => [{ 'field' => 'id', 'operator' => 'equals', 'value' => 'acc-1' }, + { 'field' => 'tags', 'operator' => 'in', 'values' => %w[vip] }] } + ) + ) + end + + # Both are honoured at once here, unlike on the collections whose endpoint + # cannot filter an id: the search intersects the filter server-side. + it 'searches instead when a search is combined with the id' do + stub_search + query = filter(condition_tree: id_leaf(operators::EQUAL, 'acc-1'), search: 'acme') + + expect(collection.list(nil, query, %w[id])).to eq([{ 'id' => 'acc-1' }]) + expect(WebMock).to have_requested(:post, "#{base}/accounts/search").with( + body: { 'limit' => Client::MAX_SEARCH_LIMIT, 'search_text' => 'acme', + 'filter' => { 'field' => 'id', 'operator' => 'equals', 'value' => 'acc-1' } } + ) + end + + it 'searches instead when the id is filtered out rather than in' do + stub_search + collection.list(nil, filter(condition_tree: id_leaf(operators::NOT_IN, %w[acc-9])), %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/accounts/search").with( + body: hash_including('filter' => { 'field' => 'id', 'operator' => 'not_in', 'values' => %w[acc-9] }) + ) + end + end + + describe 'custom fields' do + let(:column) do + ForestAdminDatasourceToolkit::Schema::ColumnSchema.new(column_type: 'String', + filter_operators: [operators::EQUAL]) + end + let(:collection) do + described_class.new(datasource, custom_fields: [{ column_name: 'tier', schema: column }, + { column_name: 'zones', schema: column }]) + end + + it 'serializes single- and multi-value custom fields' do + fields = { 'tier' => { 'slug' => 'tier', 'value' => 'gold' }, + 'zones' => { 'slug' => 'zones', 'values' => %w[eu us] } } + stub_list({ 'limit' => '1000' }, 'data' => [account_payload('acc-1', 'custom_fields' => fields)]) + + expect(collection.list(nil, filter, nil).first).to include('tier' => 'gold', 'zones' => %w[eu us]) + end + + it 'yields nil for a custom field the account does not carry' do + stub_list({ 'limit' => '1000' }, 'data' => [account_payload('acc-1', 'custom_fields' => nil)]) + + expect(collection.list(nil, filter, nil).first).to include('tier' => nil, 'zones' => nil) + end + + # Pylon accepts a custom-field slug as a filter field, with the operators + # the integrator declared on the column. + it 'filters a custom field through its slug' do + stub_search('data' => []) + + collection.list(nil, filter(condition_tree: leaf('tier', operators::EQUAL, 'gold')), %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/accounts/search") + .with(body: hash_including('filter' => { 'field' => 'tier', 'operator' => 'equals', + 'value' => 'gold' })) + end + + it 'refuses an operator the custom field does not declare' do + expect { collection.list(nil, filter(condition_tree: leaf('tier', operators::CONTAINS, 'go')), %w[id]) } + .to raise_error(UnsupportedOperatorError, /not supported on field 'tier'/) + end + + # Clamped at registration, so the schema never advertises an operator the + # translator would refuse at query time. + it 'drops a declared operator Pylon cannot honour on a custom field and warns' do + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + declared = ForestAdminDatasourceToolkit::Schema::ColumnSchema + .new(column_type: 'String', filter_operators: [operators::EQUAL, operators::STARTS_WITH]) + + clamped = described_class.new(datasource, custom_fields: [{ column_name: 'tier', schema: declared }]) + + expect(clamped.fields['tier'].filter_operators).to eq([operators::EQUAL]) + expect(ForestAdminDatasourcePylon.logger) + .to have_received(:warn).with(/cannot honour on a custom field \(starts_with\)/) + end + end + + describe '#list with a sort' do + before { allow(ForestAdminDatasourcePylon.logger).to receive(:warn) } + + # No Pylon endpoint of this collection takes a sort parameter, so the + # order is reported instead of being silently swallowed. + it 'warns that the requested order cannot be honoured, naming what happens instead' do + stub_list({ 'limit' => '1000' }, 'data' => [account_payload('acc-1')]) + + collection.list(nil, filter(sort: sort_on('name')), %w[id]) + + expect(ForestAdminDatasourcePylon.logger) + .to have_received(:warn).with(/PylonAccount cannot honour the requested order.+order the API imposes/) + end + + it 'stays quiet when Forest asks for no order' do + stub_list({ 'limit' => '1000' }, 'data' => [account_payload('acc-1')]) + + collection.list(nil, filter, %w[id]) + + expect(ForestAdminDatasourcePylon.logger).not_to have_received(:warn) + end + + # The agent injects an ascending primary-key sort whenever the request asks + # for no order; only an order someone actually chose is reported. + it 'stays quiet on the default primary-key sort the agent injects' do + stub_list({ 'limit' => '1000' }, 'data' => [account_payload('acc-1')]) + + collection.list(nil, filter(sort: sort_on('id')), %w[id]) + + expect(ForestAdminDatasourcePylon.logger).not_to have_received(:warn) + end + + it 'warns on the search path too' do + stub_search + + collection.list(nil, filter(condition_tree: leaf('name', operators::EQUAL, 'Acme'), + sort: sort_on('id', ascending: false)), %w[id]) + + expect(ForestAdminDatasourcePylon.logger).to have_received(:warn).with(/cannot honour the requested order/) + end + + it 'warns on the record path too' do + stub_request(:get, "#{base}/accounts/acc-1").to_return(json('data' => account_payload('acc-1'))) + + collection.list(nil, filter(condition_tree: id_leaf(operators::EQUAL, 'acc-1'), sort: sort_on('name')), + %w[id]) + + expect(ForestAdminDatasourcePylon.logger).to have_received(:warn).with(/cannot honour the requested order/) + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/base_collection_spec.rb b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/base_collection_spec.rb new file mode 100644 index 000000000..d949818f9 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/base_collection_spec.rb @@ -0,0 +1,724 @@ +module ForestAdminDatasourcePylon + RSpec.describe Collections::BaseCollection do + def leaf(field, operator, value) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def branch(aggregator, conditions) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + .new(aggregator, conditions) + end + + def filter(condition_tree: nil, search: nil, page: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new( + condition_tree: condition_tree, search: search, page: page + ) + end + + def page(offset, limit) + ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: offset, limit: limit) + end + + def sort(field, ascending: true) + ForestAdminDatasourceToolkit::Components::Query::Sort.new([{ field: field, ascending: ascending }]) + end + + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + + let(:datasource) do + instance_double(ForestAdminDatasourcePylon::Datasource, + client: instance_double(ForestAdminDatasourcePylon::Client)) + end + + let(:subclass) do + Class.new(described_class) do + # Minimal schema: the residual guard looks the columns up, and the + # default-sort check needs a primary key to compare against. + def define_schema + column = Collections::BaseCollection::ColumnSchema + add_field('id', column.new(column_type: 'String', is_primary_key: true)) + add_field('state', column.new(column_type: 'String')) + add_field('type', column.new(column_type: 'String')) + add_field('tags', column.new(column_type: 'Json')) + add_field('resolved_at', column.new(column_type: 'Date')) + add_field('account_id', column.new(column_type: 'String', + filter_operators: [Collections::BaseCollection::Operators::IN])) + add_field('owner_id', column.new(column_type: 'String')) + end + + # Two ManyToOne, so the resolution can be observed on a relation whose + # foreign key this collection filters and on one whose foreign key it + # does not -- alongside a prefix naming no relation at all. + def define_relations + add_field('account', Collections::BaseCollection::ManyToOneSchema.new( + foreign_collection: 'PylonAccount', foreign_key: 'account_id', + foreign_key_target: 'id' + )) + add_field('owner', Collections::BaseCollection::ManyToOneSchema.new( + foreign_collection: 'PylonUser', foreign_key: 'owner_id', + foreign_key_target: 'id' + )) + end + + public :extract_id_lookup, :project, :translate_page, :add_custom_fields, + :translate_sort, :timezone_for, :build_pylon_filter, :api_filters, :default_pk_sort?, + :ensure_searchless_lookup!, :search_records, :page_window, :warn_unsortable, + :with_resolved_relations + end + end + + # Implements the two hooks the read pipeline leaves to the collection: one + # page of the cursor walk, and the serialization of what it collected. + let(:searching_subclass) do + Class.new(subclass) do + attr_accessor :pages + attr_reader :calls + + # A field the endpoint filters, so the translated filter handed to + # `search_page` can be observed. + def api_filters + operators = Collections::BaseCollection::Operators + { 'state' => { ops: { operators::EQUAL => 'equals' } } } + end + + protected + + def search_page(limit:, cursor:, filter:, search_text:) + @calls ||= [] + @calls << { limit: limit, cursor: cursor, filter: filter, search_text: search_text } + @pages.shift || Client::SearchPage.new(records: [], next_cursor: nil) + end + + private + + def serialize(record) = record.merge('serialized' => true) + end + end + + let(:collection) { subclass.new(datasource, 'X') } + + def searching(*pages) + searching_subclass.new(datasource, 'X').tap { |collection| collection.pages = pages } + end + + def search_page(records, next_cursor = nil) + Client::SearchPage.new(records: records, next_cursor: next_cursor) + end + + describe 'subclass contract' do + it 'raises NotImplementedError naming define_schema when the hook is missing' do + expect { Class.new(described_class).new(datasource, 'X') } + .to raise_error(NotImplementedError, /define_schema/) + end + + it 'raises NotImplementedError naming define_relations when only define_schema is implemented' do + incomplete = Class.new(described_class) { def define_schema; end } + + expect { incomplete.new(datasource, 'X') }.to raise_error(NotImplementedError, /define_relations/) + end + + it 'raises NotImplementedError naming search_page when the walk reaches the endpoint hook' do + expect { collection.search_records(nil, filter) }.to raise_error(NotImplementedError, /search_page/) + end + + # Reached only by a collection declaring a ManyToOne to this one: an + # unresolvable relation names the missing hook rather than embedding nil. + it 'raises NotImplementedError naming records_indexed_by_id when a relation points here' do + expect { collection.records_indexed_by_id(%w[uuid-1]) } + .to raise_error(NotImplementedError, /records_indexed_by_id/) + end + end + + describe 'search/count flags' do + # Inverted from the Zendesk template: only the endpoints that really carry + # `search_text` opt in, and Count has no Pylon equivalent at all. + it 'leaves search and count disabled by default' do + expect(collection.is_searchable?).to be(false) + expect(collection.is_countable?).to be(false) + end + + # Count has no Pylon equivalent at all, so nothing opts into it: the + # collections walking a cursor refuse `aggregate` outright, and the two + # reading their whole dataset are not advertised as countable either. + it 'honours searchable: true from super, count having no opt-in' do + opted_in = subclass.new(datasource, 'X', searchable: true) + + expect(opted_in.is_searchable?).to be(true) + expect(opted_in.is_countable?).to be(false) + end + end + + describe 'refusals' do + # The agent answers 400 carrying the message for a ValidationError, and 500 + # "Unexpected error" for anything else. Every refusal of this datasource + # names a filter the operator set and can change, and the message is the + # only place they learn which one. + it 'refuses through an error the agent answers 400 for' do + expect(UnsupportedOperatorError.new('nope')) + .to be_a(ForestAdminDatasourceToolkit::Exceptions::ValidationError) + end + + # No Pylon endpoint aggregates, and a count over the pages the agent walked + # would answer a fraction of the collection as if it were all of it. The + # contract's NotImplementedError would read as an oversight instead. + it 'refuses to aggregate, naming the collection' do + expect { collection.aggregate(nil, nil, nil) } + .to raise_error(UnsupportedOperatorError, /X cannot be aggregated/) + end + + # A relation whose foreign key this collection does not filter cannot be + # resolved: the keys read from the foreign collection would have nothing + # to be matched against. + it 'refuses a filter on a relation whose foreign key it cannot filter' do + query = filter(condition_tree: leaf('owner:name', operators::EQUAL, 'Bob')) + + expect { collection.with_resolved_relations(nil, query) { |q| q } } + .to raise_error(UnsupportedOperatorError, + /related field 'owner:name'.*Filter on 'owner_id' instead.*PylonUser list/m) + end + + it 'refuses one whose prefix names no relation of the collection' do + query = filter(condition_tree: leaf('nope:name', operators::EQUAL, 'Acme')) + + expect { collection.with_resolved_relations(nil, query) { |q| q } } + .to raise_error(UnsupportedOperatorError, /Filter on a column of this collection instead/) + end + end + + # Pylon has no join, so a condition on a related field is answered by reading + # the foreign collection for the keys matching it. + describe '#with_resolved_relations' do + let(:foreign) { instance_double(Collections::Account) } + + before { allow(datasource).to receive(:get_collection).with('PylonAccount').and_return(foreign) } + + def resolved(tree) + collection.with_resolved_relations(nil, filter(condition_tree: tree), &:condition_tree) + end + + def returning(*ids) + allow(foreign).to receive(:list) { ids.map { |id| { 'id' => id } } } + end + + it 'leaves a filter naming no relation untouched, without reading anything' do + tree = leaf('state', operators::EQUAL, 'new') + + expect(resolved(tree)).to be(tree) + expect(datasource).not_to have_received(:get_collection) + end + + it 'rewrites the leaf into the foreign keys of the matching records' do + returning('acc-1', 'acc-2') + + expect(resolved(leaf('account:name', operators::EQUAL, 'Acme')).to_h) + .to eq(field: 'account_id', operator: operators::IN, value: %w[acc-1 acc-2]) + end + + # The condition reaches the foreign collection unnested, asking only for + # the key it is matched against, and bounded so an overflow is seen. + it 'reads the foreign collection for the target key alone' do + returning('acc-1') + resolved(leaf('account:name', operators::EQUAL, 'Acme')) + + expect(foreign).to have_received(:list) do |_caller, query, projection| + expect(query.condition_tree.to_h).to eq(field: 'name', operator: operators::EQUAL, value: 'Acme') + expect(query.page.to_h).to eq(offset: 0, limit: Collections::BaseCollection::MAX_RELATION_KEYS + 1) + expect(projection).to eq(['id']) + end + end + + it 'resolves a relation nested inside a branch, leaving the other conditions in place' do + returning('acc-1') + tree = branch('And', [leaf('state', operators::EQUAL, 'new'), leaf('account:name', operators::EQUAL, 'Acme')]) + + expect(resolved(tree).to_h[:conditions].last) + .to eq(field: 'account_id', operator: operators::IN, value: %w[acc-1]) + end + + # No foreign record matched, so no record of this collection can: answered + # without a request, rather than with an empty `in` reading as "everything". + it 'answers with no record when nothing matched, without running the read' do + returning + ran = false + query = filter(condition_tree: leaf('account:name', operators::EQUAL, 'Acme')) + + expect(collection.with_resolved_relations(nil, query) { ran = true }).to eq([]) + expect(ran).to be(false) + end + + it 'empties an and whose relation matched nothing' do + returning + tree = branch('And', [leaf('state', operators::EQUAL, 'new'), leaf('account:name', operators::EQUAL, 'Acme')]) + + expect(collection.with_resolved_relations(nil, filter(condition_tree: tree)) { |q| q }).to eq([]) + end + + # The other side of the union still selects records, so the unmatchable + # branch drops out instead of emptying the read. + it 'drops an unmatchable branch out of an or' do + returning + tree = branch('Or', [leaf('state', operators::EQUAL, 'new'), leaf('account:name', operators::EQUAL, 'Acme')]) + + expect(resolved(tree).to_h) + .to eq(aggregator: 'Or', conditions: [{ field: 'state', operator: operators::EQUAL, value: 'new' }]) + end + + it 'answers with no record when every branch of an or matched nothing' do + returning + tree = branch('Or', [leaf('account:name', operators::EQUAL, 'A'), leaf('account:name', operators::EQUAL, 'B')]) + + expect(collection.with_resolved_relations(nil, filter(condition_tree: tree)) { |q| q }).to eq([]) + end + + # Truncating would answer a narrower question than the one asked, without + # saying so. + it 'refuses to truncate a relation matching more records than the cap' do + returning(*Array.new(Collections::BaseCollection::MAX_RELATION_KEYS + 1) { |i| "acc-#{i}" }) + + expect { resolved(leaf('account:name', operators::EQUAL, 'Acme')) } + .to raise_error(UnsupportedOperatorError, /matches more than 500 PylonAccount records/) + end + end + + describe '#extract_id_lookup' do + it 'extracts a single id from an equality leaf' do + lookup = collection.extract_id_lookup(leaf('id', operators::EQUAL, 'uuid-1')) + + expect(lookup.ids).to eq(['uuid-1']) + expect(lookup.residual).to be_nil + end + + it 'extracts every id from an in leaf' do + node = leaf('id', operators::IN, %w[uuid-1 uuid-2]) + + expect(collection.extract_id_lookup(node).ids).to eq(%w[uuid-1 uuid-2]) + end + + # Pylon ids are uuids: unlike Zendesk's integer ids, nothing is coerced. + it 'keeps the id as an opaque string' do + expect(collection.extract_id_lookup(leaf('id', operators::EQUAL, 42)).ids).to eq(['42']) + end + + it 'drops empty values' do + expect(collection.extract_id_lookup(leaf('id', operators::IN, ['uuid-1', ''])).ids).to eq(['uuid-1']) + end + + it 'ignores a leaf on another field' do + expect(collection.extract_id_lookup(leaf('state', operators::EQUAL, 'new'))).to be_nil + end + + it 'ignores an operator the short-circuit cannot serve' do + expect(collection.extract_id_lookup(leaf('id', operators::NOT_EQUAL, 'uuid-1'))).to be_nil + end + + it 'ignores a nil condition tree' do + expect(collection.extract_id_lookup(nil)).to be_nil + end + + # Forest sends `AND(id equal X, )` on a record detail as soon as a + # scope or a segment is set, and `id` is not a Pylon filter field. + it 'pulls the id out of a top-level and, returning the other conditions' do + scope = leaf('state', operators::EQUAL, 'new') + lookup = collection.extract_id_lookup(branch('And', [leaf('id', operators::EQUAL, 'uuid-1'), scope])) + + expect(lookup.ids).to eq(['uuid-1']) + expect(lookup.residual.to_h).to eq(scope.to_h) + end + + it 'keeps the remaining conditions grouped when more than one is left' do + conditions = [leaf('id', operators::IN, %w[uuid-1]), leaf('state', operators::EQUAL, 'new'), + leaf('type', operators::EQUAL, 'ticket')] + + residual = collection.extract_id_lookup(branch('And', conditions)).residual + + expect(residual.to_h).to eq(aggregator: 'And', conditions: conditions.drop(1).map(&:to_h)) + end + + it 'reports no residual when the and carries the id alone' do + lookup = collection.extract_id_lookup(branch('And', [leaf('id', operators::EQUAL, 'uuid-1')])) + + expect(lookup.residual).to be_nil + end + + # An OR cannot be narrowed to the ids: the other side of the union would + # bring in records the short-circuit never fetched. + it 'ignores an id nested in an or' do + node = branch('Or', [leaf('id', operators::EQUAL, 'uuid-1'), leaf('state', operators::EQUAL, 'new')]) + + expect(collection.extract_id_lookup(node)).to be_nil + end + + it 'ignores an and carrying no id condition' do + node = branch('And', [leaf('state', operators::EQUAL, 'new'), leaf('type', operators::EQUAL, 'ticket')]) + + expect(collection.extract_id_lookup(node)).to be_nil + end + + # A Json column holds a list whose Pylon membership semantics have no + # in-memory counterpart: the lookup is refused rather than mis-filtered. + it 'refuses a lookup whose residual it cannot evaluate in memory' do + node = branch('And', [leaf('id', operators::EQUAL, 'uuid-1'), leaf('tags', operators::CONTAINS, 'vip')]) + + expect { collection.extract_id_lookup(node) } + .to raise_error(UnsupportedOperatorError, /cannot be combined with a primary-key lookup/) + end + + it 'refuses a residual on a field the schema does not declare' do + node = branch('And', [leaf('id', operators::EQUAL, 'uuid-1'), leaf('ghost', operators::EQUAL, 'x')]) + + expect { collection.extract_id_lookup(node) } + .to raise_error(UnsupportedOperatorError, /field 'ghost'/) + end + + # `ConditionTreeLeaf#match` compares with a bare `>`, which raises a + # NoMethodError on the nil Pylon returns for an unresolved issue. The + # guard pairs the comparison with a presence check, the way a database + # excludes a NULL row from a comparison. + describe 'guarding a comparison against a null column' do + let(:comparison) { leaf('resolved_at', operators::GREATER_THAN, '2026-01-01T00:00:00Z') } + let(:residual) do + collection.extract_id_lookup(branch('And', [leaf('id', operators::EQUAL, 'uuid-1'), comparison])).residual + end + + it 'pairs the comparison with a presence check' do + expect(residual.to_h).to eq( + aggregator: 'And', + conditions: [{ field: 'resolved_at', operator: operators::PRESENT, value: nil }, comparison.to_h] + ) + end + + it 'excludes the record instead of raising when the column is null' do + expect { residual.apply([{ 'id' => 'uuid-1', 'resolved_at' => nil }], collection, 'UTC') } + .not_to raise_error + expect(residual.apply([{ 'id' => 'uuid-1', 'resolved_at' => nil }], collection, 'UTC')).to eq([]) + end + + it 'still keeps a record the comparison matches' do + record = { 'id' => 'uuid-1', 'resolved_at' => '2026-08-07T13:06:22Z' } + + expect(residual.apply([record], collection, 'UTC')).to eq([record]) + end + + it 'leaves an operator that needs no guard untouched' do + equality = leaf('state', operators::EQUAL, 'new') + node = branch('And', [leaf('id', operators::EQUAL, 'uuid-1'), equality]) + + expect(collection.extract_id_lookup(node).residual.to_h).to eq(equality.to_h) + end + end + end + + describe '#ensure_searchless_lookup!' do + # Pylon searches through its search endpoint, which cannot filter on id, + # and reads an id through its own, which cannot search: honouring both at + # once is impossible, and honouring one silently is what this refuses. + it 'refuses a lookup carrying a free-text search' do + expect { collection.ensure_searchless_lookup!(filter(search: 'boom')) } + .to raise_error(UnsupportedOperatorError, /search cannot be combined with a filter on 'id'/) + end + + it 'accepts a lookup carrying no search' do + expect { collection.ensure_searchless_lookup!(filter) }.not_to raise_error + expect { collection.ensure_searchless_lookup!(nil) }.not_to raise_error + expect { collection.ensure_searchless_lookup!(filter(search: ' ')) }.not_to raise_error + end + end + + describe '#default_pk_sort?' do + # The agent injects this exact sort whenever the request asks for no order. + it 'recognises the ascending primary-key sort the agent injects' do + expect(collection.default_pk_sort?(sort('id'))).to be(true) + end + + it 'does not mistake a chosen order for the default' do + expect(collection.default_pk_sort?(sort('id', ascending: false))).to be(false) + expect(collection.default_pk_sort?(sort('state'))).to be(false) + end + end + + describe '#translate_sort' do + let(:allow_list) { { 'created_at' => 'created_at' } } + + it 'maps an allowed field to its api name and direction' do + expect(collection.translate_sort(sort('created_at'), allow_list)).to eq(%w[created_at asc]) + expect(collection.translate_sort(sort('created_at', ascending: false), allow_list)) + .to eq(%w[created_at desc]) + end + + it 'reports no sort for a field outside the allow-list' do + expect(collection.translate_sort(sort('title'), allow_list)).to eq([nil, nil]) + end + + it 'reports no sort when Forest asks for none' do + expect(collection.translate_sort(nil, allow_list)).to eq([nil, nil]) + expect(collection.translate_sort([], allow_list)).to eq([nil, nil]) + end + + it 'reads a plain hash clause as well as a Sort entry' do + expect(collection.translate_sort([{ field: 'created_at', ascending: false }], allow_list)) + .to eq(%w[created_at desc]) + expect(collection.translate_sort([{ 'field' => 'created_at', 'ascending' => true }], allow_list)) + .to eq(%w[created_at asc]) + end + end + + describe '#timezone_for' do + it 'falls back to UTC when the caller carries no timezone' do + expect(collection.timezone_for(nil)).to eq('UTC') + expect(collection.timezone_for(instance_double(ForestAdminDatasourceToolkit::Components::Caller, + timezone: nil))).to eq('UTC') + expect(collection.timezone_for(instance_double(ForestAdminDatasourceToolkit::Components::Caller, + timezone: ''))).to eq('UTC') + end + + it 'uses the timezone of the caller' do + caller = instance_double(ForestAdminDatasourceToolkit::Components::Caller, timezone: 'Europe/Paris') + + expect(collection.timezone_for(caller)).to eq('Europe/Paris') + end + end + + describe '#build_pylon_filter' do + # A collection that declares nothing filterable is the safe default: every + # predicate is refused rather than silently dropped. + it 'declares no server-side filter by default' do + expect(collection.api_filters).to eq({}) + expect { collection.build_pylon_filter(nil, filter(condition_tree: leaf('state', operators::EQUAL, 'new'))) } + .to raise_error(ForestAdminDatasourcePylon::UnsupportedOperatorError, /cannot filter on 'state'/) + end + + it 'returns no filter when there is no condition tree' do + expect(collection.build_pylon_filter(nil, filter)).to be_nil + expect(collection.build_pylon_filter(nil, nil)).to be_nil + end + + # The UI offers both an `id equals` filter and the and/or toggle, so this + # is reachable by an operator and deserves better than the translator's + # "add it to the collection's api_filters". + it 'reports an id the short-circuit could not reach with an actionable error' do + node = branch('Or', [leaf('id', operators::EQUAL, 'uuid-1'), leaf('state', operators::EQUAL, 'new')]) + + expect { collection.build_pylon_filter(nil, filter(condition_tree: node)) } + .to raise_error(UnsupportedOperatorError, /An id inside an `or` names none/) + end + + # A collection whose endpoint filters id server-side never short-circuits, + # so there is nothing an `or` could widen: id is translated like any other + # field, including under an aggregator. + it 'translates an id the collection declares in api_filters, even inside an or' do + filtering = Class.new(subclass) do + def api_filters + operators = Collections::BaseCollection::Operators + { 'id' => { ops: { operators::EQUAL => 'equals' } }, + 'state' => { ops: { operators::EQUAL => 'equals' } } } + end + end.new(datasource, 'X') + node = branch('Or', [leaf('id', operators::EQUAL, 'uuid-1'), leaf('state', operators::EQUAL, 'new')]) + + expect(filtering.build_pylon_filter(nil, filter(condition_tree: node))).to eq( + 'operator' => 'or', + 'subfilters' => [{ 'field' => 'id', 'operator' => 'equals', 'value' => 'uuid-1' }, + { 'field' => 'state', 'operator' => 'equals', 'value' => 'new' }] + ) + end + end + + describe '#search_records' do + it 'walks a single page and serializes what it collected' do + collection = searching(search_page([{ 'id' => 'a' }, { 'id' => 'b' }])) + + expect(collection.search_records(nil, filter)) + .to eq([{ 'id' => 'a', 'serialized' => true }, { 'id' => 'b', 'serialized' => true }]) + expect(collection.calls) + .to eq([{ limit: Client::MAX_SEARCH_LIMIT, cursor: nil, filter: nil, search_text: nil }]) + end + + # The regression guarded here: a page-less read used to travel to the walk + # as `limit: MAX_SEARCH_LIMIT`, which the walk could not tell from a window + # the caller asked for — so it stopped at the first full page and answered a + # larger set with its first thousand records, and did so without a warning. + it 'follows the cursor past the first page when the filter carries no page' do + collection = searching(search_page([{ 'id' => 'a' }], 'c1'), + search_page([{ 'id' => 'b' }], 'c2'), + search_page([{ 'id' => 'c' }])) + + expect(collection.search_records(nil, filter).map { |record| record['id'] }).to eq(%w[a b c]) + expect(collection.calls.map { |call| call[:cursor] }).to eq([nil, 'c1', 'c2']) + end + + # The walker asks for the window still missing and hands back the cursor of + # the previous page; the filter and the search stay the same throughout. + it 'follows the cursor until the requested window is covered' do + collection = searching(search_page([{ 'id' => 'a' }, { 'id' => 'b' }], 'c1'), + search_page([{ 'id' => 'c' }])) + query = filter(condition_tree: leaf('state', operators::EQUAL, 'new'), search: 'boom', page: page(2, 1)) + + expect(collection.search_records(nil, query)).to eq([{ 'id' => 'c', 'serialized' => true }]) + expect(collection.calls).to eq( + [{ limit: 3, cursor: nil, filter: { 'field' => 'state', 'operator' => 'equals', 'value' => 'new' }, + search_text: 'boom' }, + { limit: 1, cursor: 'c1', filter: { 'field' => 'state', 'operator' => 'equals', 'value' => 'new' }, + search_text: 'boom' }] + ) + end + + it 'refuses a predicate the endpoint cannot express instead of searching unfiltered' do + collection = searching(search_page([{ 'id' => 'a' }])) + + expect { collection.search_records(nil, filter(condition_tree: leaf('type', operators::EQUAL, 'x'))) } + .to raise_error(UnsupportedOperatorError, /cannot filter on 'type'/) + expect(collection.calls).to be_nil + end + end + + describe '#page_window' do + let(:records) { [{ 'id' => 'a' }, { 'id' => 'b' }, { 'id' => 'c' }] } + + it 'slices the requested window out of the records' do + expect(collection.page_window(records, filter(page: page(1, 1)))).to eq([{ 'id' => 'b' }]) + end + + it 'returns every record when Forest asks for no page' do + expect(collection.page_window(records, nil)).to eq(records) + end + + it 'reports an empty window rather than nil past the last record' do + expect(collection.page_window(records, filter(page: page(10, 5)))).to eq([]) + end + end + + describe '#warn_unsortable' do + before { allow(ForestAdminDatasourcePylon.logger).to receive(:warn) } + + # The empty default matches an endpoint exposing no sort parameter: the + # order is reported instead of being silently swallowed. + it 'reports a chosen order the collection cannot honour, naming it' do + collection.warn_unsortable(sort('state')) + + expect(ForestAdminDatasourcePylon.logger) + .to have_received(:warn).with('[forest_admin_datasource_pylon] X cannot honour the requested order.') + end + + it 'stays quiet on no order and on the default primary-key sort the agent injects' do + collection.warn_unsortable(nil) + collection.warn_unsortable([]) + collection.warn_unsortable(sort('id')) + + expect(ForestAdminDatasourcePylon.logger).not_to have_received(:warn) + end + + it 'stays quiet on an order the endpoint does sort by' do + sorting = Class.new(subclass) { def sortable_fields = { 'state' => 'state' } }.new(datasource, 'X') + + sorting.warn_unsortable(sort('state')) + + expect(ForestAdminDatasourcePylon.logger).not_to have_received(:warn) + end + end + + describe '#project' do + let(:record) { { 'id' => 'uuid-1', 'title' => 'Boom', 'state' => 'new' } } + + it 'returns the record untouched when there is no projection' do + expect(collection.project(record, nil)).to eq(record) + end + + # What `ActionContext#get_records` sends on every action, whatever fields + # it was handed. Emptied here, the row costs a plugin the record it acts + # on rather than a column: `primary_keys` raises on it. + it 'returns the record untouched for an empty projection' do + expect(collection.project(record, [])).to eq(record) + expect( + collection.project(record, ForestAdminDatasourceToolkit::Components::Query::Projection.new) + ).to eq(record) + end + + it 'keeps only the projected fields' do + expect(collection.project(record, %w[id title])).to eq('id' => 'uuid-1', 'title' => 'Boom') + end + + it 'yields nil for a projected field the record does not carry' do + expect(collection.project(record, %w[id missing])).to eq('id' => 'uuid-1', 'missing' => nil) + end + + # The relation is embedded onto the row afterwards; what `project` owns is + # the columns, and a projection naming none asks for none. Serving the + # whole record here would put every native column under a projection that + # excluded them. + it 'keeps no column when only relation paths are asked for' do + expect(collection.project(record, ['account:name'])).to eq({}) + end + + it 'keeps the columns of a projection mixing the two' do + expect(collection.project(record, ['id', 'account:name'])).to eq('id' => 'uuid-1') + end + end + + describe '#translate_page' do + it 'asks for every record when Forest sends no page' do + expect(collection.translate_page(nil)).to eq([0, nil]) + end + + it 'passes the offset and limit through' do + expect(collection.translate_page(page(10, 25))).to eq([10, 25]) + end + + it 'asks for every record when the page carries no limit' do + expect(collection.translate_page(page(0, nil))).to eq([0, nil]) + end + + it 'clamps a negative offset to zero' do + expect(collection.translate_page(page(-5, 10))).to eq([0, 10]) + end + end + + describe '#add_custom_fields' do + let(:schema) { ForestAdminDatasourceToolkit::Schema::ColumnSchema.new(column_type: 'String') } + + it 'adds a field and reports it as added' do + added = collection.add_custom_fields([{ column_name: 'severity', schema: schema }]) + + expect(added.map { |cf| cf[:column_name] }).to eq(['severity']) + expect(collection.fields).to have_key('severity') + end + + it 'skips a field colliding with an existing one and warns' do + collection.add_field('severity', schema) + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + + added = collection.add_custom_fields([{ column_name: 'severity', schema: schema }]) + + expect(added).to be_empty + expect(ForestAdminDatasourcePylon.logger).to have_received(:warn).with(/conflicts with an existing field/) + end + + # The safe default matches the empty api_filters: a collection that + # filters nothing server-side must not advertise custom-field filters + # either, or the translator would refuse at query time what the schema + # offered. + it 'clamps the declared operators to the collection allow-list and warns' do + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + declared = ForestAdminDatasourceToolkit::Schema::ColumnSchema + .new(column_type: 'String', filter_operators: [operators::EQUAL]) + + added = collection.add_custom_fields([{ column_name: 'severity', schema: declared }]) + + expect(collection.fields['severity'].filter_operators).to eq([]) + expect(added.first[:schema].filter_operators).to eq([]) + expect(ForestAdminDatasourcePylon.logger) + .to have_received(:warn).with(/cannot honour on a custom field \(equal\)/) + end + + it 'clamps a copy, leaving the integrator schema object untouched' do + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + declared = ForestAdminDatasourceToolkit::Schema::ColumnSchema + .new(column_type: 'String', filter_operators: [operators::EQUAL]) + + collection.add_custom_fields([{ column_name: 'severity', schema: declared }]) + + expect(declared.filter_operators).to eq([operators::EQUAL]) + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/contact_spec.rb b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/contact_spec.rb new file mode 100644 index 000000000..7dfba3851 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/contact_spec.rb @@ -0,0 +1,494 @@ +module ForestAdminDatasourcePylon + RSpec.describe Collections::Contact do + def filter(condition_tree: nil, search: nil, page: nil, sort: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new( + condition_tree: condition_tree, search: search, page: page, sort: sort + ) + end + + def leaf(field, operator, value = nil) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def branch(aggregator, conditions) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + .new(aggregator, conditions) + end + + def id_leaf(operator, value) + leaf('id', operator, value) + end + + def page(offset, limit) + ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: offset, limit: limit) + end + + def sort_on(field, ascending: true) + ForestAdminDatasourceToolkit::Components::Query::Sort.new([{ field: field, ascending: ascending }]) + end + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + # Trimmed to the shape observed on the API: the account is a nested object + # carrying an id, and unset values come back as null rather than absent. + def contact_payload(id, overrides = {}) + { + 'id' => id, 'name' => 'Ada Lovelace', 'email' => 'ada@acme.com', + 'emails' => %w[ada@acme.com ada@acme.io], 'account' => { 'id' => 'acc-1', 'external_ids' => nil }, + 'avatar_url' => 'https://usepylon.com/ada.png', 'portal_role' => 'admin', 'portal_role_id' => 'role-1', + 'primary_phone_number' => '+33100000000', 'phone_numbers' => %w[+33100000000], + 'external_ids' => [{ 'external_id' => 'crm-9', 'label' => 'hubspot' }], + 'integration_user_ids' => [{ 'id' => 'U1', 'source' => 'slack' }], 'custom_fields' => {} + }.merge(overrides) + end + + # Relations are fields too; the assertions on the columns select them out. + def columns + collection.fields.select { |_name, field| field.type == 'Column' } + end + + let(:datasource) { ForestAdminDatasourcePylon::Datasource.new(api_key: 'k') } + let(:collection) { described_class.new(datasource) } + let(:base) { datasource.configuration.url } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + + before { stub_custom_fields } + + def stub_list(query, payload) + stub_request(:get, "#{base}/contacts").with(query: query).to_return(json(payload)) + end + + def stub_search(payload = { 'data' => [contact_payload('con-1')] }) + stub_request(:post, "#{base}/contacts/search").to_return(json(payload)) + end + + describe 'schema' do + it 'is named PylonContact' do + expect(collection.name).to eq('PylonContact') + end + + it 'declares id as the primary key' do + expect(collection.fields['id'].is_primary_key).to be(true) + end + + # No short-circuit to serve here: /contacts/search filters id itself, so + # the column advertises every operator the endpoint accepts on it. + it 'advertises the id operators the search endpoint filters server-side' do + expect(collection.fields['id'].filter_operators) + .to eq([operators::EQUAL, operators::IN, operators::NOT_IN]) + end + + it 'exposes the native columns observed on the API' do + expect(collection.fields.keys).to include( + 'name', 'email', 'emails', 'account_id', 'avatar_url', 'portal_role', 'portal_role_id', + 'primary_phone_number', 'phone_numbers', 'external_ids', 'integration_user_ids' + ) + end + + # The nested object Pylon returns becomes the key column; the `account` + # field of the schema is the relation read through that key. + it 'flattens the account into a foreign-key column instead of exposing the nested object' do + expect(columns.keys).to include('account_id') + expect(columns.keys).not_to include('account') + end + + # Pylon returns no timestamp at all on a contact. + it 'declares no time column' do + expect(columns.values.map(&:column_type)).not_to include('Date') + end + + it 'types the lists as Json' do + expect(collection.fields['emails'].column_type).to eq('Json') + expect(collection.fields['phone_numbers'].column_type).to eq('Json') + expect(collection.fields['integration_user_ids'].column_type).to eq('Json') + end + + # Neither endpoint exposes a sort parameter. + it 'declares every column non-sortable' do + expect(columns.values.map(&:is_sortable).uniq).to eq([false]) + end + + # `phone_numbers` and `external_ids` stay read-only although the endpoint + # takes them: they hold objects, in a shape the write side does not + # document as the one the column shows. `portal_role` stays read-only next + # to the `portal_role_id` it is the name of, so one patch never carries two + # projections of the same role. + it 'declares writable exactly the columns an endpoint takes in the shape they are read' do + writable = columns.reject { |_name, column| column.is_read_only }.keys + + expect(writable).to contain_exactly('name', 'account_id', 'email', 'emails', 'avatar_url', + 'primary_phone_number', 'portal_role_id') + end + + # No Pylon endpoint aggregates, and the pages of a cursor walk are not the + # dataset: a chart grouped by one of these columns would answer a fraction + # as if it were the whole collection. + it 'declares no column groupable' do + expect(columns.values.map(&:is_groupable).uniq).to eq([false]) + end + + it 'enables search and leaves count disabled' do + expect(collection.is_searchable?).to be(true) + expect(collection.is_countable?).to be(false) + end + + it 'advertises only the operators the search allow-list accepts' do + expect(collection.fields['name'].filter_operators) + .to eq([operators::EQUAL, operators::IN, operators::NOT_IN, operators::CONTAINS, operators::I_CONTAINS]) + expect(collection.fields['email'].filter_operators) + .to eq([operators::EQUAL, operators::IN, operators::NOT_IN, operators::CONTAINS, operators::I_CONTAINS]) + expect(collection.fields['account_id'].filter_operators) + .to eq([operators::EQUAL, operators::IN, operators::NOT_IN]) + end + + # /contacts/search accepts `string_contains` but documents no negation of + # it, so the UI must not offer one. + it 'offers no negated substring on name and email' do + expect(collection.fields['name'].filter_operators) + .not_to include(operators::NOT_CONTAINS, operators::NOT_I_CONTAINS) + expect(collection.fields['email'].filter_operators) + .not_to include(operators::NOT_CONTAINS, operators::NOT_I_CONTAINS) + end + + # The endpoint filters no external id on a contact, and the column shows + # { external_id, label } objects the filter would not match anyway. + it 'advertises no operator on external_ids' do + expect(collection.fields['external_ids'].filter_operators).to eq([]) + end + + # `email` filters the primary address only: the lists, the phone numbers + # and the portal role are absent from the allow-list. + it 'advertises no operator on the columns Pylon cannot filter' do + %w[emails phone_numbers primary_phone_number avatar_url portal_role portal_role_id + integration_user_ids].each do |field| + expect(collection.fields[field].filter_operators).to eq([]) + end + end + end + + describe 'relations' do + it 'points at the account of a contact through the flattened foreign key' do + expect(collection.fields['account']) + .to be_a(ForestAdminDatasourceToolkit::Schema::Relations::ManyToOneSchema) + .and have_attributes(foreign_collection: 'PylonAccount', foreign_key: 'account_id', + foreign_key_target: 'id') + end + + # `requested_issues` rather than `issues`: a contact is the requester of an + # issue, never its assignee -- that side belongs to PylonUser. + it 'declares the issues a contact requested, read through requester_id' do + expect(collection.fields['requested_issues']) + .to be_a(ForestAdminDatasourceToolkit::Schema::Relations::OneToManySchema) + .and have_attributes(foreign_collection: 'PylonIssue', origin_key: 'requester_id', + origin_key_target: 'id') + end + end + + # No condition tree and no search: the listing endpoint returns the same + # records for 300 requests per minute where the search endpoint allows 120. + describe '#list without a filter' do + it 'browses the listing endpoint and serializes what it returns' do + stub_list({ 'limit' => '1000' }, 'data' => [contact_payload('con-1')]) + + rows = collection.list(nil, filter, nil) + + expect(rows.size).to eq(1) + expect(rows.first).to include('id' => 'con-1', 'name' => 'Ada Lovelace', 'email' => 'ada@acme.com', + 'emails' => %w[ada@acme.com ada@acme.io], 'portal_role' => 'admin', + 'primary_phone_number' => '+33100000000') + end + + it 'flattens the nested account into a foreign-key column' do + stub_list({ 'limit' => '1000' }, 'data' => [contact_payload('con-1')]) + + expect(collection.list(nil, filter, nil).first).to include('account_id' => 'acc-1') + end + + it 'keeps the nested object out of the serialized record' do + stub_list({ 'limit' => '1000' }, 'data' => [contact_payload('con-1')]) + + expect(collection.list(nil, filter, nil).first.keys).not_to include('account') + end + + it 'reports no account rather than raising when the contact has none' do + stub_list({ 'limit' => '1000' }, 'data' => [contact_payload('con-1', 'account' => nil)]) + + expect(collection.list(nil, filter, nil).first).to include('account_id' => nil) + end + + it 'restricts the record to the projection' do + stub_list({ 'limit' => '1000' }, 'data' => [contact_payload('con-1')]) + + expect(collection.list(nil, filter, %w[id email])) + .to eq([{ 'id' => 'con-1', 'email' => 'ada@acme.com' }]) + end + + it 'never spends the search budget' do + stub_list({ 'limit' => '1000' }, 'data' => [contact_payload('con-1')]) + + collection.list(nil, filter, %w[id]) + + expect(WebMock).not_to have_requested(:post, "#{base}/contacts/search") + end + + it 'walks the cursor until the requested window is covered' do + stub_list({ 'limit' => '3' }, + 'data' => [contact_payload('con-1'), contact_payload('con-2')], + 'pagination' => { 'cursor' => 'c1', 'has_next_page' => true }) + stub_list({ 'limit' => '1', 'cursor' => 'c1' }, 'data' => [contact_payload('con-3')]) + + expect(collection.list(nil, filter(page: page(2, 1)), %w[id])).to eq([{ 'id' => 'con-3' }]) + end + + it 'returns an empty list when the organization has no contact' do + stub_list({ 'limit' => '1000' }, 'data' => []) + + expect(collection.list(nil, filter, nil)).to eq([]) + end + end + + describe '#list with a filter' do + before { stub_search } + + it 'sends the translated condition tree to the search endpoint' do + collection.list(nil, filter(condition_tree: leaf('email', operators::I_CONTAINS, '@acme')), %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search").with( + body: { 'limit' => Client::MAX_SEARCH_LIMIT, + 'filter' => { 'field' => 'email', 'operator' => 'string_contains', 'value' => '@acme' } } + ) + end + + it 'translates a filter on the account the contact belongs to' do + collection.list(nil, filter(condition_tree: leaf('account_id', operators::IN, %w[acc-1 acc-2])), %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search").with( + body: hash_including('filter' => { 'field' => 'account_id', 'operator' => 'in', + 'values' => %w[acc-1 acc-2] }) + ) + end + + it 'sends a free-text search as search_text, intersected with the filter' do + query = filter(condition_tree: leaf('name', operators::EQUAL, 'Ada Lovelace'), search: 'ada') + + collection.list(nil, query, %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search").with( + body: { 'limit' => Client::MAX_SEARCH_LIMIT, 'search_text' => 'ada', + 'filter' => { 'field' => 'name', 'operator' => 'equals', 'value' => 'Ada Lovelace' } } + ) + end + + # The listing endpoint cannot search, so a search alone is worth the + # search endpoint even with nothing to filter. + it 'searches on a free-text search alone' do + collection.list(nil, filter(search: 'ada'), %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: { 'limit' => Client::MAX_SEARCH_LIMIT, 'search_text' => 'ada' }) + expect(WebMock).not_to have_requested(:get, "#{base}/contacts") + end + + # `id` is a filter field of this endpoint, so it needs no short-circuit and + # an `or` cannot widen anything: it is translated like any other field. + it 'translates an id filter server-side, even under an or' do + tree = branch('Or', [id_leaf(operators::EQUAL, 'con-1'), leaf('email', operators::EQUAL, 'ada@acme.com')]) + + collection.list(nil, filter(condition_tree: tree), %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search").with( + body: hash_including( + 'filter' => { 'operator' => 'or', + 'subfilters' => [{ 'field' => 'id', 'operator' => 'equals', 'value' => 'con-1' }, + { 'field' => 'email', 'operator' => 'equals', + 'value' => 'ada@acme.com' }] } + ) + ) + end + + # The whole point of translating rather than dropping: a predicate Pylon + # cannot express fails loudly instead of returning unfiltered rows. + it 'raises rather than returning unfiltered rows for a field Pylon cannot filter' do + query = filter(condition_tree: leaf('portal_role', operators::EQUAL, 'admin')) + + expect { collection.list(nil, query, %w[id]) } + .to raise_error(UnsupportedOperatorError, /cannot filter on 'portal_role'/) + expect(WebMock).not_to have_requested(:post, "#{base}/contacts/search") + end + + it 'raises rather than searching for an operator the endpoint refuses on a field' do + query = filter(condition_tree: leaf('email', operators::NOT_I_CONTAINS, '@acme')) + + expect { collection.list(nil, query, %w[id]) } + .to raise_error(UnsupportedOperatorError, /not supported on field 'email'/) + end + end + + # A record detail is `id equals X` alone: reading it through GET + # /contacts/{id} spends the 300 requests/minute budget instead of the 120 of + # the search endpoint. + describe '#list on a single-id filter' do + it 'reads the contact through its own endpoint instead of searching' do + stub_request(:get, "#{base}/contacts/con-1").to_return(json('data' => contact_payload('con-1'))) + + rows = collection.list(nil, filter(condition_tree: id_leaf(operators::EQUAL, 'con-1')), %w[id name]) + + expect(rows).to eq([{ 'id' => 'con-1', 'name' => 'Ada Lovelace' }]) + expect(WebMock).not_to have_requested(:post, "#{base}/contacts/search") + end + + it 'reports no record when the contact no longer exists' do + stub_request(:get, "#{base}/contacts/gone").to_return(json({ 'message' => 'not found' }, 404)) + + expect(collection.list(nil, filter(condition_tree: id_leaf(operators::EQUAL, 'gone')), %w[id])).to eq([]) + end + + it 'propagates a failure that is not a missing record' do + stub_request(:get, "#{base}/contacts/con-1").to_return(json({ 'message' => 'boom' }, 500)) + + expect { collection.list(nil, filter(condition_tree: id_leaf(operators::EQUAL, 'con-1')), %w[id]) } + .to raise_error(APIError) + end + + it 'applies the requested page to the record it read' do + stub_request(:get, "#{base}/contacts/con-1").to_return(json('data' => contact_payload('con-1'))) + query = filter(condition_tree: id_leaf(operators::EQUAL, 'con-1'), page: page(1, 1)) + + expect(collection.list(nil, query, %w[id])).to eq([]) + end + + # One search request answers several ids exactly, where one GET per id + # would burn the budget of the whole agent. + it 'searches instead when several ids are asked for' do + stub_search('data' => [contact_payload('con-1'), contact_payload('con-2')]) + + rows = collection.list(nil, filter(condition_tree: id_leaf(operators::IN, %w[con-1 con-2])), %w[id]) + + expect(rows).to eq([{ 'id' => 'con-1' }, { 'id' => 'con-2' }]) + expect(WebMock).to have_requested(:post, "#{base}/contacts/search").with( + body: hash_including('filter' => { 'field' => 'id', 'operator' => 'in', + 'values' => %w[con-1 con-2] }) + ) + end + + # Forest sends `AND(id equal X, )` on a record detail as soon as a + # scope or a segment is set; the search endpoint filters both server-side. + it 'searches instead when the filter carries more conditions' do + stub_search + tree = branch('And', [id_leaf(operators::EQUAL, 'con-1'), leaf('account_id', operators::EQUAL, 'acc-1')]) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id])).to eq([{ 'id' => 'con-1' }]) + expect(WebMock).to have_requested(:post, "#{base}/contacts/search").with( + body: hash_including( + 'filter' => { 'operator' => 'and', + 'subfilters' => [{ 'field' => 'id', 'operator' => 'equals', 'value' => 'con-1' }, + { 'field' => 'account_id', 'operator' => 'equals', + 'value' => 'acc-1' }] } + ) + ) + end + + it 'searches instead when a search is combined with the id' do + stub_search + query = filter(condition_tree: id_leaf(operators::EQUAL, 'con-1'), search: 'ada') + + expect(collection.list(nil, query, %w[id])).to eq([{ 'id' => 'con-1' }]) + expect(WebMock).to have_requested(:post, "#{base}/contacts/search").with( + body: { 'limit' => Client::MAX_SEARCH_LIMIT, 'search_text' => 'ada', + 'filter' => { 'field' => 'id', 'operator' => 'equals', 'value' => 'con-1' } } + ) + end + end + + describe 'custom fields' do + let(:column) do + ForestAdminDatasourceToolkit::Schema::ColumnSchema.new(column_type: 'String', + filter_operators: [operators::EQUAL]) + end + let(:collection) do + described_class.new(datasource, custom_fields: [{ column_name: 'seniority', schema: column }, + { column_name: 'products', schema: column }]) + end + + it 'serializes single- and multi-value custom fields' do + fields = { 'seniority' => { 'slug' => 'seniority', 'value' => 'champion' }, + 'products' => { 'slug' => 'products', 'values' => %w[api portal] } } + stub_list({ 'limit' => '1000' }, 'data' => [contact_payload('con-1', 'custom_fields' => fields)]) + + expect(collection.list(nil, filter, nil).first) + .to include('seniority' => 'champion', 'products' => %w[api portal]) + end + + it 'yields nil for a custom field the contact does not carry' do + stub_list({ 'limit' => '1000' }, 'data' => [contact_payload('con-1', 'custom_fields' => nil)]) + + expect(collection.list(nil, filter, nil).first).to include('seniority' => nil, 'products' => nil) + end + + it 'filters a custom field through its slug' do + stub_search('data' => []) + + collection.list(nil, filter(condition_tree: leaf('seniority', operators::EQUAL, 'champion')), %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('filter' => { 'field' => 'seniority', 'operator' => 'equals', + 'value' => 'champion' })) + end + + it 'refuses an operator the custom field does not declare' do + query = filter(condition_tree: leaf('seniority', operators::CONTAINS, 'cham')) + + expect { collection.list(nil, query, %w[id]) } + .to raise_error(UnsupportedOperatorError, /not supported on field 'seniority'/) + end + + # Clamped at registration, so the schema never advertises an operator the + # translator would refuse at query time. + it 'drops a declared operator Pylon cannot honour on a custom field and warns' do + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + declared = ForestAdminDatasourceToolkit::Schema::ColumnSchema + .new(column_type: 'String', filter_operators: [operators::EQUAL, operators::STARTS_WITH]) + + clamped = described_class.new(datasource, custom_fields: [{ column_name: 'seniority', schema: declared }]) + + expect(clamped.fields['seniority'].filter_operators).to eq([operators::EQUAL]) + expect(ForestAdminDatasourcePylon.logger) + .to have_received(:warn).with(/cannot honour on a custom field \(starts_with\)/) + end + end + + describe '#list with a sort' do + before { allow(ForestAdminDatasourcePylon.logger).to receive(:warn) } + + it 'warns that the requested order cannot be honoured, naming what happens instead' do + stub_list({ 'limit' => '1000' }, 'data' => [contact_payload('con-1')]) + + collection.list(nil, filter(sort: sort_on('email')), %w[id]) + + expect(ForestAdminDatasourcePylon.logger) + .to have_received(:warn).with(/PylonContact cannot honour the requested order.+order the API imposes/) + end + + it 'stays quiet when Forest asks for no order and on the default primary-key sort' do + stub_list({ 'limit' => '1000' }, 'data' => [contact_payload('con-1')]) + + collection.list(nil, filter, %w[id]) + collection.list(nil, filter(sort: sort_on('id')), %w[id]) + + expect(ForestAdminDatasourcePylon.logger).not_to have_received(:warn) + end + + it 'warns on the search path too' do + stub_search + + collection.list(nil, filter(condition_tree: leaf('email', operators::EQUAL, 'ada@acme.com'), + sort: sort_on('name')), %w[id]) + + expect(ForestAdminDatasourcePylon.logger).to have_received(:warn).with(/cannot honour the requested order/) + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/fetch_all_collection_spec.rb b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/fetch_all_collection_spec.rb new file mode 100644 index 000000000..1252638ae --- /dev/null +++ b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/fetch_all_collection_spec.rb @@ -0,0 +1,376 @@ +module ForestAdminDatasourcePylon + RSpec.describe Collections::FetchAllCollection do + def leaf(field, operator, value = nil) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def filter(condition_tree: nil, page: nil, sort: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new( + condition_tree: condition_tree, page: page, sort: sort + ) + end + + def page(offset, limit) + ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: offset, limit: limit) + end + + def aggregation(operation, field: nil, groups: []) + ForestAdminDatasourceToolkit::Components::Query::Aggregation.new( + operation: operation, field: field, groups: groups + ) + end + + def sort(*clauses) + ForestAdminDatasourceToolkit::Components::Query::Sort.new(clauses) + end + + def by(field, ascending: true) + { field: field, ascending: ascending } + end + + def ids(records) + records.map { |record| record['id'] } + end + + # Mirrors `residual_leaf_appliable?`: an operator is evaluable in memory when + # `ConditionTreeLeaf#match` handles it natively or the toolkit can rewrite it + # into operators that it does. + def appliable?(operator, column_type) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::ConditionTreeEquivalent + .equivalent_tree?(operator, Collections::BaseCollection::IN_MEMORY_OPERATORS, column_type) + end + + def advertised(collection) + collection.fields.flat_map do |name, column| + column.filter_operators.map { |operator| [name, operator, column.column_type] } + end + end + + # A filter value of the shape the operator expects, so every advertised + # operator can be run for real over the dataset. + def value_for(operator, column_type) + return nil if [operators::PRESENT, operators::BLANK].include?(operator) + + sample = column_type == 'Boolean' ? true : 'Bob' + [operators::IN, operators::NOT_IN].include?(operator) ? [sample] : sample + end + + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + let(:datasource) { instance_double(ForestAdminDatasourcePylon::Datasource) } + + # A collection over an in-memory dataset: the endpoint hook hands back what + # the example set, so the shared flow can be observed without the HTTP layer. + let(:subclass) do + Class.new(described_class) do + attr_accessor :entities + + def define_schema + add_column('id', 'String', is_primary_key: true) + add_column('name', 'String') + add_column('flag', 'Boolean') + add_column('resolved_at', 'Date') + add_column('list', 'Json') + end + + def define_relations; end + + protected + + def fetch_all + @entities + end + + def serialize(entity) + entity + end + end + end + + # Deliberately out of order, with a record whose every value is null: the + # API imposes no order, and Pylon spells an unset value as null. + let(:entities) do + [{ 'id' => 'u2', 'name' => 'Bob', 'flag' => false, 'resolved_at' => nil, 'list' => %w[a] }, + { 'id' => 'u1', 'name' => 'alice', 'flag' => true, 'resolved_at' => '2026-01-02T00:00:00Z', 'list' => [] }, + { 'id' => 'u3', 'name' => nil, 'flag' => nil, 'resolved_at' => nil, 'list' => nil }] + end + + let(:collection) { subclass.new(datasource, 'X').tap { |instance| instance.entities = entities } } + + describe 'subclass contract' do + let(:schema_only) do + Class.new(described_class) do + def define_schema; end + def define_relations; end + end + end + + it 'names fetch_all when the endpoint hook is missing' do + expect { schema_only.new(datasource, 'X').list(nil, nil, nil) } + .to raise_error(NotImplementedError, /did not implement fetch_all/) + end + + it 'names serialize when the serialization hook is missing' do + fetching = Class.new(schema_only) do + protected + + def fetch_all + [{ 'id' => 'u1' }] + end + end + + expect { fetching.new(datasource, 'X').list(nil, nil, nil) } + .to raise_error(NotImplementedError, /did not implement serialize/) + end + + # Neither endpoint carries a search parameter. Count is the one capability + # a collection holding the whole dataset can answer exactly, where every + # cursor-backed one would be counting the pages it happened to walk. + it 'leaves search disabled and enables count' do + expect(collection.is_searchable?).to be(false) + expect(collection.is_countable?).to be(true) + end + + # A group is computed over the complete dataset here, unlike every other + # Pylon collection: a chart grouped by a scalar column is exact. + it 'declares the scalar columns groupable and the json one not' do + expect(collection.fields['name'].is_groupable).to be(true) + expect(collection.fields['list'].is_groupable).to be(false) + end + + # Pylon defines custom fields on issues, accounts and contacts only, and + # nothing here reads a custom-field value nor clamps its operators: the + # column would read nil on every row forever. + it 'refuses a custom field rather than registering a column nothing fills' do + declared = { column_name: 'tier', + schema: ForestAdminDatasourceToolkit::Schema::ColumnSchema.new(column_type: 'String') } + + expect { subclass.new(datasource, 'X', custom_fields: [declared]) } + .to raise_error(ConfigurationError, /takes no custom field/) + end + end + + # How a collection pointing here with a ManyToOne resolves its foreign keys. + # The endpoint hands back the complete dataset, so the ids only pick rows out + # of it and any number of them costs the same single read. + describe '#records_indexed_by_id' do + it 'indexes the wanted records by id' do + expect(collection.records_indexed_by_id(%w[u3 u1])) + .to eq('u1' => entities[1], 'u3' => entities[2]) + end + + it 'leaves out an id the endpoint no longer returns rather than indexing a blank record' do + expect(collection.records_indexed_by_id(%w[u1 gone]).keys).to eq(%w[u1]) + end + + # A page of the pointing collection asks for a handful of ids against every + # record the organization has: serializing the whole dataset to slice it + # afterwards would pay for all of them, on every page. + it 'serializes the wanted records only' do + counting = Class.new(subclass) do + attr_reader :serialized + + protected + + def serialize(entity) + (@serialized ||= []) << entity['id'] + entity + end + end.new(datasource, 'X') + counting.entities = entities + + counting.records_indexed_by_id(%w[u1]) + + expect(counting.serialized).to eq(%w[u1]) + end + end + + # Exact where every other Pylon collection has to refuse: the records in + # hand are every record Pylon holds, so a count over them is the count a + # server-side aggregation would have answered. + describe '#aggregate' do + it 'counts the records the filter keeps' do + expect(collection.aggregate(nil, filter, aggregation('Count'))) + .to eq([{ 'group' => {}, 'value' => 3 }]) + expect(collection.aggregate(nil, filter(condition_tree: leaf('name', operators::PRESENT)), + aggregation('Count'))) + .to eq([{ 'group' => {}, 'value' => 2 }]) + end + + it 'groups by a column' do + rows = collection.aggregate(nil, filter, aggregation('Count', groups: [{ field: 'flag' }])) + + expect(rows).to contain_exactly({ 'group' => { 'flag' => false }, 'value' => 1 }, + { 'group' => { 'flag' => true }, 'value' => 1 }, + { 'group' => { 'flag' => nil }, 'value' => 1 }) + end + + it 'honours the limit the chart asks for' do + rows = collection.aggregate(nil, filter, aggregation('Count', groups: [{ field: 'name' }]), 1) + + expect(rows.size).to eq(1) + end + end + + describe '.operators_for' do + it 'advertises the string filters the in-memory pass can evaluate' do + expect(described_class.operators_for('String')) + .to eq([operators::EQUAL, operators::NOT_EQUAL, operators::IN, operators::NOT_IN, + operators::PRESENT, operators::BLANK, operators::CONTAINS, operators::I_CONTAINS, + operators::NOT_CONTAINS, operators::STARTS_WITH, operators::ENDS_WITH]) + end + + it 'advertises the boolean filters the in-memory pass can evaluate' do + expect(described_class.operators_for('Boolean')) + .to eq([operators::EQUAL, operators::NOT_EQUAL, operators::IN, operators::NOT_IN, + operators::PRESENT, operators::BLANK]) + end + + # A Json column holds a list whose Pylon semantics have no in-memory + # counterpart, and a type with no candidate is not guessed at: both + # advertise nothing rather than a filter that could answer wrongly. + it 'advertises no filter on a json column nor on a type it has no candidate for' do + expect(described_class.operators_for('Json')).to eq([]) + expect(described_class.operators_for('Date')).to eq([]) + end + + # The schema is the contract the UI builds its filters from: an operator + # in it that memory cannot evaluate would silently empty the page. + it 'advertises only operators the in-memory pass can evaluate' do + unappliable = advertised(collection).reject { |_field, operator, type| appliable?(operator, type) } + + expect(unappliable).to be_empty + end + + it 'evaluates every advertised operator over the dataset, nulls included' do + failures = advertised(collection).filter_map do |field, operator, type| + collection.list(nil, filter(condition_tree: leaf(field, operator, value_for(operator, type))), nil) + nil + rescue StandardError => e + "#{field} #{operator}: #{e.class}: #{e.message}" + end + + expect(failures).to be_empty + end + end + + describe '#list' do + it 'serializes every record the endpoint returned when nothing is filtered' do + expect(ids(collection.list(nil, filter, nil))).to eq(%w[u2 u1 u3]) + expect(collection.list(nil, nil, nil).size).to eq(3) + end + + it 'restricts the records to the projection' do + expect(collection.list(nil, filter(sort: sort(by('id'))), %w[id name])) + .to eq([{ 'id' => 'u1', 'name' => 'alice' }, { 'id' => 'u2', 'name' => 'Bob' }, + { 'id' => 'u3', 'name' => nil }]) + end + + # Cut out of the complete, ordered dataset, so the window holds the rows a + # server-side query would have returned for it. + it 'slices the requested page out of the ordered records' do + expect(ids(collection.list(nil, filter(sort: sort(by('id')), page: page(1, 1)), nil))).to eq(%w[u2]) + end + + # A read asking for no page asks for the whole dataset, which is the read + # `SortCollectionDecorator` performs to build its reference order: capping + # it at the search limit would drop every record past the thousandth and + # leave the decorator without a position for them. + it 'answers a page-less read with every record, past the search limit' do + collection.entities = Array.new(Client::MAX_SEARCH_LIMIT + 5) { |i| { 'id' => "u#{i}" } } + + expect(collection.list(nil, filter, nil).size).to eq(Client::MAX_SEARCH_LIMIT + 5) + expect(collection.list(nil, nil, nil).size).to eq(Client::MAX_SEARCH_LIMIT + 5) + end + + # The same read with an offset and no limit -- which `Page#apply` reads as + # "to the end of the records" too: the offset is honoured, the tail is not + # cut. + it 'honours an offset carrying no limit without capping the tail' do + collection.entities = Array.new(Client::MAX_SEARCH_LIMIT + 5) { |i| { 'id' => "u#{i}" } } + query = filter(page: page(2, nil)) + + expect(collection.list(nil, query, nil).size).to eq(Client::MAX_SEARCH_LIMIT + 3) + end + end + + describe '#list with a filter' do + def filtered(field, operator, value = nil) + ids(collection.list(nil, filter(condition_tree: leaf(field, operator, value)), nil)) + end + + it 'keeps the records the condition tree matches, in the order the API returned them' do + expect(filtered('id', operators::IN, %w[u1 u2])).to eq(%w[u2 u1]) + expect(filtered('name', operators::CONTAINS, 'li')).to eq(%w[u1]) + expect(filtered('flag', operators::EQUAL, true)).to eq(%w[u1]) + end + + # The default search the agent builds for a non-searchable collection is a + # case-insensitive contains, which is why the operator is advertised. + it 'matches a contains regardless of case through i_contains' do + expect(filtered('name', operators::I_CONTAINS, 'ALI')).to eq(%w[u1]) + expect(filtered('name', operators::CONTAINS, 'ALI')).to eq([]) + end + + it 'reads a null column as blank rather than as a value' do + expect(filtered('name', operators::BLANK)).to eq(%w[u3]) + expect(filtered('name', operators::PRESENT)).to eq(%w[u2 u1]) + expect(filtered('flag', operators::BLANK)).to eq(%w[u3]) + end + + # `ConditionTreeLeaf#match` compares with a bare `>`, which raises on the + # null Pylon returns for an unset column. Nothing in the schema advertises + # the comparison, but a scope or a customizer can still send one. + it 'excludes a null column from a comparison instead of raising' do + expect(filtered('resolved_at', operators::GREATER_THAN, '2026-01-01T00:00:00Z')).to eq(%w[u1]) + end + end + + describe '#list with a sort' do + def sorted(*clauses) + ids(collection.list(nil, filter(sort: sort(*clauses)), nil)) + end + + it 'honours an ascending and a descending order' do + expect(sorted(by('name'))).to eq(%w[u2 u1 u3]) + expect(sorted(by('name', ascending: false))).to eq(%w[u3 u1 u2]) + end + + # The agent injects this exact order whenever the request asks for none, + # and unlike the endpoints Pylon orders itself, it is honoured here. + it 'honours the ascending primary-key sort the agent injects' do + expect(sorted(by('id'))).to eq(%w[u1 u2 u3]) + end + + # `false <=> true` is nil in Ruby: the order would be arbitrary without a + # comparator of its own. + it 'orders a boolean column, false first, the way a database does' do + expect(sorted(by('flag'))).to eq(%w[u2 u1 u3]) + expect(sorted(by('flag', ascending: false))).to eq(%w[u3 u1 u2]) + end + + it 'moves on to the next clause for the records the first cannot tell apart' do + collection.entities = [{ 'id' => 'u2', 'flag' => true, 'name' => 'Bob' }, + { 'id' => 'u1', 'flag' => true, 'name' => 'alice' }, + { 'id' => 'u3', 'flag' => false, 'name' => 'Zoe' }] + + expect(sorted(by('flag'), by('name'))).to eq(%w[u3 u2 u1]) + end + + # Ruby's `sort` is not stable, so the position the API returned the record + # in breaks the ties: the same request cannot answer in a different order. + it 'keeps the order the API returned for records the sort cannot tell apart' do + collection.entities = [{ 'id' => 'u3', 'name' => 'same' }, { 'id' => 'u1', 'name' => 'same' }, + { 'id' => 'u2', 'name' => 'same' }] + + expect(sorted(by('name'))).to eq(%w[u3 u1 u2]) + end + + it 'keeps two null columns in place rather than comparing them' do + collection.entities = [{ 'id' => 'u2', 'name' => nil }, { 'id' => 'u1', 'name' => nil }] + + expect(sorted(by('name'))).to eq(%w[u2 u1]) + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/issue/messages_embedder_spec.rb b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/issue/messages_embedder_spec.rb new file mode 100644 index 000000000..46ad3dddd --- /dev/null +++ b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/issue/messages_embedder_spec.rb @@ -0,0 +1,242 @@ +module ForestAdminDatasourcePylon + # Observed through PylonIssue, the only collection carrying a conversation. + RSpec.describe Collections::Issue::MessagesEmbedder do + def filter(condition_tree: nil, search: nil, page: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new( + condition_tree: condition_tree, search: search, page: page + ) + end + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def issue_payload(id, overrides = {}) + { 'id' => id, 'title' => 'Boom' }.merge(overrides) + end + + # Trimmed to the shape observed on the API: the author carries a contact or + # a user side, and unset values come back as null rather than absent. + def message_payload(id, overrides = {}) + { + 'id' => id, 'message_html' => '

hello

', 'is_private' => false, 'source' => 'email', + 'thread_id' => nil, 'file_urls' => [], 'timestamp' => '2026-08-07T13:06:22Z', + 'author' => { 'name' => 'Ada', 'avatar_url' => 'https://cdn/ada.png', + 'contact' => { 'id' => 'con-1', 'email' => 'ada@acme.com' }, 'user' => nil } + }.merge(overrides) + end + + def stub_issues(*payloads) + stub_request(:post, "#{base}/issues/search").to_return(json('data' => payloads)) + end + + def stub_messages(issue_id, *payloads) + stub_request(:get, "#{base}/issues/#{issue_id}/messages").to_return(json('data' => payloads)) + end + + let(:datasource) { ForestAdminDatasourcePylon::Datasource.new(api_key: 'k') } + let(:issues) { datasource.get_collection('PylonIssue') } + let(:base) { datasource.configuration.url } + let(:logger) { instance_double(Logger, warn: nil) } + + before { stub_custom_fields } + + describe 'the schema of the column' do + it 'declares messages as an array of message shapes' do + expect(issues.fields['messages'].column_type).to eq([Collections::Issue::MESSAGE_THREAD_SCHEMA]) + end + + it 'names the fields after the columns of the collection, not after the payload' do + expect(Collections::Issue::MESSAGE_THREAD_SCHEMA.keys).to include('body_html', 'created_at') + expect(Collections::Issue::MESSAGE_THREAD_SCHEMA.keys).not_to include('message_html', 'timestamp') + end + + it 'advertises no filter and no sort on a thread the search endpoint does not cover' do + expect(issues.fields['messages'].filter_operators).to eq([]) + expect(issues.fields['messages'].is_sortable).to be(false) + end + + it 'is read-only, like every other column of this story' do + expect(issues.fields['messages'].is_read_only).to be(true) + end + end + + describe 'what the projection asks for' do + before { stub_issues(issue_payload('i1')) } + + it 'reads the thread when the projection names it' do + stub_messages('i1', message_payload('msg-1')) + + rows = issues.list(nil, filter, %w[id messages]) + + expect(rows.first['messages'].map { |m| m['id'] }).to eq(%w[msg-1]) + expect(WebMock).to have_requested(:get, "#{base}/issues/i1/messages").once + end + + it 'reads the thread when the projection reaches inside it' do + stub_messages('i1', message_payload('msg-1')) + + expect(issues.list(nil, filter, %w[id messages:body_html]).first['messages']).not_to be_nil + end + + it 'reads no thread when the projection does not name it' do + rows = issues.list(nil, filter, %w[id title]) + + expect(rows.first).not_to have_key('messages') + expect(WebMock).not_to have_requested(:get, "#{base}/issues/i1/messages") + end + + # A count and an export both go through a nil projection: neither asked + # for the conversation, and both would pay one request per row for it. + it 'reads no thread when the projection is nil' do + expect(issues.list(nil, filter, nil).first).not_to have_key('messages') + expect(WebMock).not_to have_requested(:get, "#{base}/issues/i1/messages") + end + end + + describe 'the shape of an embedded message' do + before { stub_issues(issue_payload('i1')) } + + it 'renames message_html and timestamp after the columns of the collection' do + stub_messages('i1', message_payload('msg-1')) + + message = issues.list(nil, filter, %w[id messages]).first['messages'].first + + expect(message).to include('id' => 'msg-1', 'body_html' => '

hello

', 'is_private' => false, + 'source' => 'email', 'created_at' => '2026-08-07T13:06:22Z') + end + + it 'fills every field the column schema declares' do + stub_messages('i1', message_payload('msg-1')) + + message = issues.list(nil, filter, %w[id messages]).first['messages'].first + + expect(message.keys).to match_array(Collections::Issue::MESSAGE_THREAD_SCHEMA.keys) + end + + it 'keeps the thread in the order Pylon returns it, oldest first' do + stub_messages('i1', message_payload('msg-1'), message_payload('msg-2'), message_payload('msg-3')) + + rows = issues.list(nil, filter, %w[id messages]) + + expect(rows.first['messages'].map { |m| m['id'] }).to eq(%w[msg-1 msg-2 msg-3]) + end + + it 'embeds an empty thread as an empty list' do + stub_messages('i1') + + expect(issues.list(nil, filter, %w[id messages]).first['messages']).to eq([]) + end + end + + describe 'the author of a message' do + before { stub_issues(issue_payload('i1')) } + + def author_of(payload) + stub_messages('i1', payload) + issues.list(nil, filter, %w[id messages]).first['messages'].first + end + + it 'flattens the contact side of a message written by a customer' do + expect(author_of(message_payload('msg-1'))).to include( + 'author_name' => 'Ada', 'author_avatar_url' => 'https://cdn/ada.png', + 'author_email' => 'ada@acme.com', 'author_contact_id' => 'con-1', 'author_user_id' => nil + ) + end + + it 'flattens the user side of a message written by an agent' do + author = { 'name' => 'Grace', 'avatar_url' => nil, 'contact' => nil, + 'user' => { 'id' => 'usr-1', 'email' => 'grace@support.io' } } + + expect(author_of(message_payload('msg-1', 'author' => author))).to include( + 'author_name' => 'Grace', 'author_email' => 'grace@support.io', + 'author_contact_id' => nil, 'author_user_id' => 'usr-1' + ) + end + + # Pylon puts both sides side by side with nothing telling them apart; the + # contact is the one the customer-facing thread is about. + it 'prefers the contact email when both sides are there' do + author = { 'name' => 'Ada', 'contact' => { 'id' => 'con-1', 'email' => 'ada@acme.com' }, + 'user' => { 'id' => 'usr-1', 'email' => 'ada@support.io' } } + + expect(author_of(message_payload('msg-1', 'author' => author))).to include( + 'author_email' => 'ada@acme.com', 'author_contact_id' => 'con-1', 'author_user_id' => 'usr-1' + ) + end + + it 'leaves every author field null when the message carries no author' do + expect(author_of(message_payload('msg-1', 'author' => nil))).to include( + 'author_name' => nil, 'author_email' => nil, 'author_avatar_url' => nil, + 'author_contact_id' => nil, 'author_user_id' => nil + ) + end + end + + describe 'the fan-out it allows' do + let(:cap) { Collections::Issue::MAX_MESSAGE_EMBEDS } + + before do + stub_issues(*(1..(cap + 2)).map { |i| issue_payload("i#{i}") }) + (1..(cap + 2)).each { |i| stub_messages("i#{i}", message_payload("msg-#{i}")) } + end + + it 'reads one thread per row up to the cap' do + issues.list(nil, filter, %w[id messages]) + + expect(WebMock).to have_requested(:get, "#{base}/issues/i#{cap}/messages").once + expect(WebMock).not_to have_requested(:get, "#{base}/issues/i#{cap + 1}/messages") + end + + # Never the empty list: "no thread read" is not "this issue has no + # message", and the operator has to be able to tell them apart. + it 'leaves the rows past the cap at nil rather than at an empty thread' do + rows = issues.list(nil, filter, %w[id messages]) + + expect(rows.first(cap).map { |row| row['messages'] }).to all(be_an(Array)) + expect(rows.drop(cap).map { |row| row['messages'] }).to eq([nil, nil]) + end + + it 'reports the rows it left out' do + allow(ForestAdminDatasourcePylon).to receive(:logger).and_return(logger) + + issues.list(nil, filter, %w[id messages]) + + expect(logger).to have_received(:warn).with(/Asked for the message thread of #{cap + 2} issues/) + end + + it 'reports nothing when the page fits under the cap' do + allow(ForestAdminDatasourcePylon).to receive(:logger).and_return(logger) + stub_issues(issue_payload('i1')) + + issues.list(nil, filter, %w[id messages]) + + expect(logger).not_to have_received(:warn) + end + end + + describe 'when the thread cannot be read' do + before { stub_issues(issue_payload('i1'), issue_payload('i2')) } + + it 'serves the page with the column left at nil' do + stub_request(:get, "#{base}/issues/i1/messages").to_return(json({ 'message' => 'boom' }, 500)) + stub_messages('i2', message_payload('msg-2')) + + rows = issues.list(nil, filter, %w[id messages]) + + expect(rows.map { |row| row['id'] }).to eq(%w[i1 i2]) + expect(rows.first['messages']).to be_nil + expect(rows.last['messages'].map { |m| m['id'] }).to eq(%w[msg-2]) + end + + it 'reports the degradation' do + allow(ForestAdminDatasourcePylon).to receive(:logger).and_return(logger) + stub_request(:get, %r{/issues/i\d/messages}).to_return(json({ 'message' => 'boom' }, 500)) + + issues.list(nil, filter, %w[id messages]) + + expect(logger).to have_received(:warn).with(/fetch_issue_messages\(i1\) failed; degrading/) + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/issue_spec.rb b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/issue_spec.rb new file mode 100644 index 000000000..bf0950442 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/issue_spec.rb @@ -0,0 +1,752 @@ +module ForestAdminDatasourcePylon + RSpec.describe Collections::Issue do + def filter(condition_tree: nil, search: nil, page: nil, sort: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new( + condition_tree: condition_tree, search: search, page: page, sort: sort + ) + end + + def leaf(field, operator, value) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def branch(aggregator, conditions) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + .new(aggregator, conditions) + end + + def id_leaf(operator, value) + leaf('id', operator, value) + end + + def page(offset, limit) + ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: offset, limit: limit) + end + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + # Trimmed to the shape observed on the API: parties are nested objects + # carrying an id, and unset values come back as null rather than absent. + def issue_payload(id, overrides = {}) + { + 'id' => id, 'number' => 12, 'title' => 'Boom', 'link' => "https://app.usepylon.com/issues?issueNumber=#{id}", + 'body_html' => '

boom

', 'state' => 'new', 'type' => 'ticket', 'source' => 'manual', + 'account' => { 'id' => 'acc-1', 'external_ids' => nil }, 'requester' => { 'id' => 'req-1' }, + 'assignee' => nil, 'team' => nil, 'tags' => %w[urgent], 'custom_fields' => {}, + 'first_response_time' => nil, 'resolution_time' => nil, 'latest_message_time' => '2026-08-07T13:06:22Z', + 'created_at' => '2026-08-07T13:06:22Z', 'updated_at' => '2026-08-07T13:06:22Z', + 'customer_portal_visible' => false, 'number_of_touches' => 0, 'author_unverified' => false, + 'time_in_status_seconds' => { 'open' => 701_211 }, + 'business_hours_time_in_status_seconds' => { 'open' => 172_799 } + }.merge(overrides) + end + + # Relations are fields too; the assertions on the columns select them out. + def columns + collection.fields.select { |_name, field| field.type == 'Column' } + end + + let(:datasource) { ForestAdminDatasourcePylon::Datasource.new(api_key: 'k') } + let(:collection) { datasource.get_collection('PylonIssue') } + let(:base) { datasource.configuration.url } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + + before { stub_custom_fields } + + describe 'schema' do + it 'is named PylonIssue' do + expect(collection.name).to eq('PylonIssue') + end + + it 'declares id as the primary key' do + expect(collection.fields['id'].is_primary_key).to be(true) + end + + it 'only advertises the id operators the short-circuit can serve' do + expect(collection.fields['id'].filter_operators).to eq([operators::EQUAL, operators::IN]) + end + + it 'exposes the native columns observed on the API' do + expect(collection.fields.keys).to include( + 'number', 'title', 'body_html', 'state', 'type', 'source', 'link', 'tags', + 'account_id', 'requester_id', 'assignee_id', 'team_id', + 'first_response_time', 'resolution_time', 'latest_message_time', 'created_at', 'updated_at', + 'customer_portal_visible', 'author_unverified', 'number_of_touches', + 'time_in_status_seconds', 'business_hours_time_in_status_seconds' + ) + end + + it 'types the response-time columns as dates, not durations' do + expect(collection.fields['first_response_time'].column_type).to eq('Date') + expect(collection.fields['resolution_time'].column_type).to eq('Date') + end + + # /issues/search exposes no sort parameter. + it 'declares every column non-sortable' do + expect(columns.values.map(&:is_sortable).uniq).to eq([false]) + end + + # Writable is what POST /issues or PATCH /issues/{id} accepts; everything + # Pylon computes itself stays read-only. + it 'declares writable exactly the columns an endpoint takes' do + writable = columns.reject { |_name, column| column.is_read_only }.keys + + expect(writable).to contain_exactly('title', 'body_html', 'state', 'type', 'tags', + 'customer_portal_visible', 'author_unverified', + 'account_id', 'requester_id', 'assignee_id', 'team_id') + end + + # No Pylon endpoint aggregates, and the pages of a cursor walk are not the + # dataset: a chart grouped by one of these columns would answer a fraction + # as if it were the whole collection. + it 'declares no column groupable' do + expect(columns.values.map(&:is_groupable).uniq).to eq([false]) + end + + # `search_text` is native on /issues/search, while Pylon exposes neither a + # count endpoint nor a total, so Count stays out until it can be throttled. + it 'enables search and leaves count disabled' do + expect(collection.is_searchable?).to be(true) + expect(collection.is_countable?).to be(false) + end + + it 'advertises only the operators the search allow-list accepts' do + expect(collection.fields['state'].filter_operators).to eq([operators::EQUAL, operators::IN, + operators::NOT_IN]) + expect(collection.fields['assignee_id'].filter_operators) + .to eq([operators::EQUAL, operators::IN, operators::NOT_IN, + operators::PRESENT, operators::BLANK, operators::MISSING]) + expect(collection.fields['title'].filter_operators) + .to eq([operators::CONTAINS, operators::I_CONTAINS, operators::NOT_CONTAINS, operators::NOT_I_CONTAINS]) + end + + # Declaring the bare comparisons is what lets the toolkit rewrite Today / + # PreviousWeek / ... into a pair of bounds Pylon understands. + it 'advertises the date columns with the two bounds Pylon accepts' do + expect(collection.fields['created_at'].filter_operators) + .to eq([operators::GREATER_THAN, operators::LESS_THAN]) + end + + # /issues/search cannot filter on them, so offering the filter would only + # produce an error once the operator used it. + it 'advertises no operator on the columns Pylon cannot filter' do + %w[number source number_of_touches first_response_time link + customer_portal_visible time_in_status_seconds].each do |field| + expect(collection.fields[field].filter_operators).to eq([]) + end + end + end + + describe 'relations' do + let(:many_to_one) { ForestAdminDatasourceToolkit::Schema::Relations::ManyToOneSchema } + + it 'declares a ManyToOne for each of the four parties of an issue' do + expect(collection.fields.values_at('account', 'requester', 'assignee', 'team')) + .to all(be_a(many_to_one)) + end + + it 'points each one at the collection owning its shape, through the flattened foreign key' do + expect(collection.fields['account']) + .to have_attributes(foreign_collection: 'PylonAccount', foreign_key: 'account_id', + foreign_key_target: 'id') + expect(collection.fields['requester']) + .to have_attributes(foreign_collection: 'PylonContact', foreign_key: 'requester_id', + foreign_key_target: 'id') + expect(collection.fields['assignee']) + .to have_attributes(foreign_collection: 'PylonUser', foreign_key: 'assignee_id', + foreign_key_target: 'id') + expect(collection.fields['team']) + .to have_attributes(foreign_collection: 'PylonTeam', foreign_key: 'team_id', + foreign_key_target: 'id') + end + + # The key is what `/issues/search` filters, on this side and on the reverse + # one, so it stays a column of its own next to the relation. + it 'keeps the foreign keys as columns' do + expect(columns.keys).to include('account_id', 'requester_id', 'assignee_id', 'team_id') + end + end + + describe '#list' do + it 'searches for the most recent issues and serializes them' do + stub_request(:post, "#{base}/issues/search").to_return(json('data' => [issue_payload('i1')])) + + rows = collection.list(nil, filter, nil) + + expect(rows.size).to eq(1) + expect(rows.first).to include('id' => 'i1', 'title' => 'Boom', 'state' => 'new', + 'tags' => ['urgent'], 'number_of_touches' => 0) + end + + it 'flattens the nested parties into foreign-key columns' do + stub_request(:post, "#{base}/issues/search").to_return(json('data' => [issue_payload('i1')])) + + expect(collection.list(nil, filter, nil).first) + .to include('account_id' => 'acc-1', 'requester_id' => 'req-1', + 'assignee_id' => nil, 'team_id' => nil) + end + + it 'keeps the nested objects out of the serialized record' do + stub_request(:post, "#{base}/issues/search").to_return(json('data' => [issue_payload('i1')])) + + expect(collection.list(nil, filter, nil).first.keys) + .not_to include('account', 'requester', 'assignee', 'team') + end + + it 'restricts the record to the projection' do + stub_request(:post, "#{base}/issues/search").to_return(json('data' => [issue_payload('i1')])) + + expect(collection.list(nil, filter, %w[id title])).to eq([{ 'id' => 'i1', 'title' => 'Boom' }]) + end + + it 'forwards the requested page as the search limit' do + stub_request(:post, "#{base}/issues/search").to_return(json('data' => [issue_payload('i1')])) + + collection.list(nil, filter(page: page(0, 1)), nil) + + expect(WebMock).to have_requested(:post, "#{base}/issues/search").with(body: { 'limit' => 1 }) + end + + it 'returns an empty list when the search returns nothing' do + stub_request(:post, "#{base}/issues/search").to_return(json('data' => [])) + + expect(collection.list(nil, filter, nil)).to eq([]) + end + end + + describe '#list across cursor pages' do + it 'walks the cursor until the requested window is covered' do + stub_request(:post, "#{base}/issues/search") + .with(body: { 'limit' => 3 }) + .to_return(json('data' => [issue_payload('i1'), issue_payload('i2')], + 'pagination' => { 'cursor' => 'c1', 'has_next_page' => true })) + stub_request(:post, "#{base}/issues/search") + .with(body: { 'limit' => 1, 'cursor' => 'c1' }) + .to_return(json('data' => [issue_payload('i3')])) + + rows = collection.list(nil, filter(page: page(2, 1)), %w[id]) + + expect(rows).to eq([{ 'id' => 'i3' }]) + end + + it 'stops on the last page even when the window is not filled' do + stub_request(:post, "#{base}/issues/search").to_return(json('data' => [issue_payload('i1')])) + + rows = collection.list(nil, filter(page: page(0, 50)), %w[id]) + + expect(rows).to eq([{ 'id' => 'i1' }]) + expect(WebMock).to have_requested(:post, "#{base}/issues/search").once + end + end + + describe '#list on a primary-key lookup' do + it 'reads the issue directly instead of searching' do + stub_request(:get, "#{base}/issues/i1").to_return(json('data' => issue_payload('i1'))) + + rows = collection.list(nil, filter(condition_tree: id_leaf(operators::EQUAL, 'i1')), %w[id title]) + + expect(rows).to eq([{ 'id' => 'i1', 'title' => 'Boom' }]) + expect(WebMock).not_to have_requested(:post, "#{base}/issues/search") + end + + it 'reads every id of an in filter, preserving their order' do + stub_request(:get, "#{base}/issues/i1").to_return(json('data' => issue_payload('i1'))) + stub_request(:get, "#{base}/issues/i2").to_return(json('data' => issue_payload('i2'))) + + rows = collection.list(nil, filter(condition_tree: id_leaf(operators::IN, %w[i1 i2])), %w[id]) + + expect(rows).to eq([{ 'id' => 'i1' }, { 'id' => 'i2' }]) + end + + it 'applies the requested page to an in lookup' do + %w[i1 i2 i3].each do |id| + stub_request(:get, "#{base}/issues/#{id}").to_return(json('data' => issue_payload(id))) + end + query = filter(condition_tree: id_leaf(operators::IN, %w[i1 i2 i3]), page: page(1, 1)) + + expect(collection.list(nil, query, %w[id])).to eq([{ 'id' => 'i2' }]) + end + + # The window is taken off the ids, so it is the ids of the page that are + # read and no others: one that no longer resolves leaves a gap rather + # than pulling the next id in behind it, which would cost a request per + # record outside the page to find out. + it 'reads only the ids of the page, gap included, when one no longer exists' do + stub_request(:get, "#{base}/issues/gone").to_return(json({ 'message' => 'not found' }, 404)) + %w[i2 i3].each do |id| + stub_request(:get, "#{base}/issues/#{id}").to_return(json('data' => issue_payload(id))) + end + query = filter(condition_tree: id_leaf(operators::IN, %w[gone i2 i3]), page: page(0, 2)) + + expect(collection.list(nil, query, %w[id])).to eq([{ 'id' => 'i2' }]) + expect(WebMock).not_to have_requested(:get, "#{base}/issues/i3") + end + + # The cap bounds a page, not a selection: an offset past it used to read + # the first MAX_ID_LOOKUPS ids and answer an empty page from them. + it 'answers a page past the lookup cap instead of an empty one' do + ids = Array.new(30) { |i| "i#{i + 1}" } + %w[i21 i22].each do |id| + stub_request(:get, "#{base}/issues/#{id}").to_return(json('data' => issue_payload(id))) + end + query = filter(condition_tree: id_leaf(operators::IN, ids), page: page(20, 2)) + + expect(collection.list(nil, query, %w[id])).to eq([{ 'id' => 'i21' }, { 'id' => 'i22' }]) + expect(WebMock).not_to have_requested(:get, "#{base}/issues/i1") + end + + # An `and` names the records all of its conditions name, so two `id` + # leaves intersect: keeping only the first would apply the cap to the + # wider set and refuse a selection narrower than it. + it 'intersects two id conditions of an and rather than refusing over the wider one' do + wide = Array.new(Collections::Issue::MAX_ID_LOOKUPS + 5) { |i| "i#{i}" } + %w[i2 i4].each do |id| + stub_request(:get, "#{base}/issues/#{id}").to_return(json('data' => issue_payload(id))) + end + tree = branch('And', [id_leaf(operators::IN, wide), id_leaf(operators::IN, %w[i2 i4])]) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id])) + .to eq([{ 'id' => 'i2' }, { 'id' => 'i4' }]) + expect(WebMock).not_to have_requested(:get, "#{base}/issues/i0") + end + + it 'reads nothing when the two id conditions of an and are disjoint' do + tree = branch('And', [id_leaf(operators::IN, %w[i1 i2]), id_leaf(operators::EQUAL, 'i9')]) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id])).to eq([]) + expect(WebMock).not_to have_requested(:get, "#{base}/issues/i1") + end + + # Which records the window holds is only known once they are all read, so + # a wide selection carrying other conditions cannot be answered a page at + # a time, and is refused rather than answered with a fraction of itself. + it 'refuses a selection past the cap that filters the named issues further' do + ids = Array.new(Collections::Issue::MAX_ID_LOOKUPS + 5) { |i| "i#{i}" } + tree = branch('And', [id_leaf(operators::IN, ids), leaf('state', operators::EQUAL, 'new')]) + + expect { collection.list(nil, filter(condition_tree: tree), %w[id]) } + .to raise_error(UnsupportedOperatorError, /names #{ids.size} issues by id/) + end + + it 'skips an issue that no longer exists' do + stub_request(:get, "#{base}/issues/i1").to_return(json('data' => issue_payload('i1'))) + stub_request(:get, "#{base}/issues/gone").to_return(json({ 'message' => 'not found' }, 404)) + + rows = collection.list(nil, filter(condition_tree: id_leaf(operators::IN, %w[i1 gone])), %w[id]) + + expect(rows).to eq([{ 'id' => 'i1' }]) + end + + it 'propagates a failure that is not a missing record' do + stub_request(:get, "#{base}/issues/i1").to_return(json({ 'message' => 'boom' }, 500)) + + expect { collection.list(nil, filter(condition_tree: id_leaf(operators::EQUAL, 'i1')), %w[id]) } + .to raise_error(APIError) + end + + # `GET /issues/{id}` accepts the issue number as well as the UUID and + # answers with the issue carrying its own id: keeping it would answer + # `id equals 42` with a row whose id is not 42. + it 'reports no record when the endpoint answered an alias of the primary key' do + stub_request(:get, "#{base}/issues/42").to_return(json('data' => issue_payload('i1'))) + + expect(collection.list(nil, filter(condition_tree: id_leaf(operators::EQUAL, '42')), %w[id])).to eq([]) + end + end + + describe 'custom fields' do + let(:column) do + ForestAdminDatasourceToolkit::Schema::ColumnSchema.new(column_type: 'String', + filter_operators: [operators::EQUAL]) + end + let(:collection) do + described_class.new(datasource, custom_fields: [{ column_name: 'severity', schema: column }, + { column_name: 'zones', schema: column }]) + end + + it 'serializes single- and multi-value custom fields' do + fields = { 'severity' => { 'slug' => 'severity', 'value' => 'high' }, + 'zones' => { 'slug' => 'zones', 'values' => %w[eu us] } } + stub_request(:post, "#{base}/issues/search") + .to_return(json('data' => [issue_payload('i1', 'custom_fields' => fields)])) + + expect(collection.list(nil, filter, nil).first).to include('severity' => 'high', 'zones' => %w[eu us]) + end + + it 'yields nil for a custom field the issue does not carry' do + stub_request(:post, "#{base}/issues/search").to_return(json('data' => [issue_payload('i1')])) + + expect(collection.list(nil, filter, nil).first).to include('severity' => nil, 'zones' => nil) + end + + # Pylon accepts a custom-field slug as a filter field, with the operators + # the integrator declared on the column. + it 'filters a custom field through its slug' do + stub_request(:post, "#{base}/issues/search").to_return(json('data' => [])) + + collection.list(nil, filter(condition_tree: leaf('severity', operators::EQUAL, 'high')), %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/issues/search") + .with(body: hash_including('filter' => { 'field' => 'severity', 'operator' => 'equals', + 'value' => 'high' })) + end + + it 'refuses an operator the custom field does not declare' do + expect { collection.list(nil, filter(condition_tree: leaf('severity', operators::CONTAINS, 'hi')), %w[id]) } + .to raise_error(UnsupportedOperatorError, /not supported on field 'severity'/) + end + + # Clamped at registration, so the schema never advertises an operator + # the translator would refuse at query time. + it 'drops a declared operator Pylon cannot honour on a custom field and warns' do + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + declared = ForestAdminDatasourceToolkit::Schema::ColumnSchema + .new(column_type: 'String', filter_operators: [operators::EQUAL, operators::STARTS_WITH]) + + clamped = described_class.new(datasource, custom_fields: [{ column_name: 'severity', schema: declared }]) + + expect(clamped.fields['severity'].filter_operators).to eq([operators::EQUAL]) + expect(ForestAdminDatasourcePylon.logger) + .to have_received(:warn).with(/cannot honour on a custom field \(starts_with\)/) + end + + # The API reference documents what a custom field is, never the form its + # value is read back in. The column holds the form the agent gives a filter + # value of the same type, so the two stay comparable whichever one Pylon + # answered with. + describe 'the form a value is read in' do + def typed(type) + ForestAdminDatasourceToolkit::Schema::ColumnSchema + .new(column_type: type, filter_operators: [operators::EQUAL]) + end + + def entry(slug, value) + { slug => { 'slug' => slug, 'value' => value } } + end + + def read(fields) + stub_request(:post, "#{base}/issues/search") + .to_return(json('data' => [issue_payload('i1', 'custom_fields' => fields)])) + + collection.list(nil, filter, nil).first + end + + let(:collection) do + described_class.new(datasource, custom_fields: [{ column_name: 'nps', schema: typed('Number') }, + { column_name: 'vip', schema: typed('Boolean') }, + { column_name: 'renewal', schema: typed('Dateonly') }]) + end + + # A value already numeric keeps its own form: widening `42` into `42.0` + # would display a decimal the field does not hold, where the wire half + # narrows the same value back. A string is read to the tightest form, so + # both halves agree on what an integer looks like. + [[42, 42], ['42', 42], [' 42 ', 42], [42.0, 42.0], [42.5, 42.5], ['42.5', 42.5], + ['42.0', 42]].each do |raw, expected| + it "reads a number answered as #{raw.inspect} as #{expected.inspect}" do + expect(read(entry('nps', raw))['nps']).to eql(expected) + end + end + + it 'reads a number it cannot make sense of as absent rather than as zero' do + expect(read(entry('nps', 'n/a'))['nps']).to be_nil + end + + [[true, true], ['true', true], [false, false], ['false', false], ['0', false]].each do |raw, expected| + it "reads a boolean answered as #{raw.inspect} as #{expected}" do + expect(read(entry('vip', raw))['vip']).to be(expected) + end + end + + it 'reads an empty boolean as absent, false being an answer of its own' do + expect(read(entry('vip', ''))['vip']).to be_nil + end + + it 'leaves a date the ISO string the filter is compared with' do + expect(read(entry('renewal', '2026-08-01'))['renewal']).to eq('2026-08-01') + end + + it 'leaves a field the issue does not carry absent' do + expect(read({})).to include('nps' => nil, 'vip' => nil, 'renewal' => nil) + end + + # The one place the form decides the result: an id filter is answered by + # `GET /issues/{id}` and the rest is applied in memory, so a number left + # as `"42"` would be compared with the `42.0` the agent casts the filter + # to, and the row would be dropped without a word. Reading it as the + # Integer `42` is enough -- `match` compares with `==`. + it 'keeps a row matched on a number combined with an id lookup' do + stub_request(:get, "#{base}/issues/i1") + .to_return(json('data' => issue_payload('i1', 'custom_fields' => entry('nps', '42')))) + tree = branch('And', [id_leaf(operators::EQUAL, 'i1'), leaf('nps', operators::EQUAL, 42.0)]) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id nps])) + .to eq([{ 'id' => 'i1', 'nps' => 42 }]) + end + + # The same row, matched through a membership filter: `Array#include?` + # compares with `==` too, so the Integer read answers the float list. + it 'keeps a row matched on a number list combined with an id lookup' do + stub_request(:get, "#{base}/issues/i1") + .to_return(json('data' => issue_payload('i1', 'custom_fields' => entry('nps', '42')))) + tree = branch('And', [id_leaf(operators::EQUAL, 'i1'), leaf('nps', operators::IN, [42.0, 7.0])]) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id nps])) + .to eq([{ 'id' => 'i1', 'nps' => 42 }]) + end + end + end + + describe '#list with a filter' do + before { stub_request(:post, "#{base}/issues/search").to_return(json('data' => [issue_payload('i1')])) } + + it 'sends the translated condition tree to the search endpoint' do + collection.list(nil, filter(condition_tree: leaf('state', operators::EQUAL, 'closed')), %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/issues/search").with( + body: { 'limit' => Client::MAX_SEARCH_LIMIT, + 'filter' => { 'field' => 'state', 'operator' => 'equals', 'value' => 'closed' } } + ) + end + + it 'sends a free-text search as search_text, intersected with the filter' do + query = filter(condition_tree: leaf('team_id', operators::EQUAL, 'team-1'), search: 'boom') + + collection.list(nil, query, %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/issues/search").with( + body: { 'limit' => Client::MAX_SEARCH_LIMIT, 'search_text' => 'boom', + 'filter' => { 'field' => 'team_id', 'operator' => 'equals', 'value' => 'team-1' } } + ) + end + + # The whole point of translating rather than dropping: a predicate Pylon + # cannot express fails loudly instead of returning unfiltered rows. + it 'raises rather than returning unfiltered rows for a predicate Pylon refuses' do + expect { collection.list(nil, filter(condition_tree: leaf('number', operators::EQUAL, 12)), %w[id]) } + .to raise_error(UnsupportedOperatorError, /cannot filter on 'number'/) + expect(WebMock).not_to have_requested(:post, "#{base}/issues/search") + end + + # Declaring the four ManyToOne relations is what makes a filter on a + # related field reachable, and the schema advertises it as filterable. + # Pylon has no join, so it is answered by reading the accounts matching + # the condition and sending their ids as the `account_id` filter the + # issues endpoint does take. + it 'answers a filter on a related field with the keys of the matching records' do + stub_request(:post, "#{base}/accounts/search") + .to_return(json('data' => [{ 'id' => 'acc-1' }, { 'id' => 'acc-2' }])) + stub_request(:post, "#{base}/issues/search").to_return(json('data' => [])) + + collection.list(nil, filter(condition_tree: leaf('account:name', operators::CONTAINS, 'Acme')), %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/accounts/search").with( + body: hash_including('filter' => { 'field' => 'name', 'operator' => 'string_contains', + 'value' => 'Acme' }) + ) + expect(WebMock).to have_requested(:post, "#{base}/issues/search").with( + body: hash_including('filter' => { 'field' => 'account_id', 'operator' => 'in', + 'values' => %w[acc-1 acc-2] }) + ) + end + + # No account matched, so no issue can: answered without asking the issues + # endpoint, where an empty `in` would have read as no filter at all. + it 'answers with no issue when no related record matched, without searching' do + stub_request(:post, "#{base}/accounts/search").to_return(json('data' => [])) + + expect(collection.list(nil, filter(condition_tree: leaf('account:name', operators::CONTAINS, 'Acme')), + %w[id])).to eq([]) + expect(WebMock).not_to have_requested(:post, "#{base}/issues/search") + end + + # Resolved before the short-circuit reads the tree, so the relation + # becomes an `account_id in [...]` the id lookup can apply in memory. + it 'resolves a filter on a related field combined with an id' do + stub_request(:post, "#{base}/accounts/search").to_return(json('data' => [{ 'id' => 'acc-1' }])) + stub_request(:get, "#{base}/issues/i1").to_return(json('data' => issue_payload('i1'))) + conditions = [id_leaf(operators::EQUAL, 'i1'), leaf('account:name', operators::CONTAINS, 'Acme')] + + expect(collection.list(nil, filter(condition_tree: branch('And', conditions)), %w[id])) + .to eq([{ 'id' => 'i1' }]) + end + + # A relation the filter never names costs nothing: no foreign read is + # triggered by declaring it. + it 'reads no foreign collection for a filter naming none' do + stub_request(:post, "#{base}/issues/search").to_return(json('data' => [])) + + collection.list(nil, filter(condition_tree: leaf('state', operators::EQUAL, 'new')), %w[id]) + + expect(WebMock).not_to have_requested(:post, "#{base}/accounts/search") + end + + it 'keeps the same filter across every page of the walk' do + stub_request(:post, "#{base}/issues/search") + .with(body: hash_including('limit' => 3)) + .to_return(json('data' => [issue_payload('i1'), issue_payload('i2')], + 'pagination' => { 'cursor' => 'c1', 'has_next_page' => true })) + stub_request(:post, "#{base}/issues/search") + .with(body: hash_including('cursor' => 'c1')) + .to_return(json('data' => [issue_payload('i3')])) + query = filter(condition_tree: leaf('state', operators::EQUAL, 'new'), page: page(2, 1)) + + expect(collection.list(nil, query, %w[id])).to eq([{ 'id' => 'i3' }]) + expect(WebMock).to have_requested(:post, "#{base}/issues/search") + .with(body: hash_including('filter' => { 'field' => 'state', 'operator' => 'equals', + 'value' => 'new' })).twice + end + end + + describe '#list on a primary-key lookup carrying more conditions' do + # Forest sends `AND(id equal X, )` on a record detail as soon as a + # scope or a segment is set; `id` is not a Pylon filter field, so the + # lookup has to survive the extra conditions. + it 'reads the issue by id and applies the leftover conditions in memory' do + stub_request(:get, "#{base}/issues/i1").to_return(json('data' => issue_payload('i1'))) + tree = branch('And', [id_leaf(operators::EQUAL, 'i1'), leaf('state', operators::EQUAL, 'new')]) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id])).to eq([{ 'id' => 'i1' }]) + expect(WebMock).not_to have_requested(:post, "#{base}/issues/search") + end + + it 'drops the record when a leftover condition does not match' do + stub_request(:get, "#{base}/issues/i1").to_return(json('data' => issue_payload('i1'))) + tree = branch('And', [id_leaf(operators::EQUAL, 'i1'), leaf('state', operators::EQUAL, 'closed')]) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id])).to eq([]) + end + + # `tags` holds a list whose Pylon membership semantics have no in-memory + # counterpart: the lookup is refused rather than silently mis-filtered. + it 'refuses a leftover condition it cannot evaluate in memory' do + tree = branch('And', [id_leaf(operators::EQUAL, 'i1'), leaf('tags', operators::CONTAINS, 'urgent')]) + + expect { collection.list(nil, filter(condition_tree: tree), %w[id]) } + .to raise_error(UnsupportedOperatorError, /cannot be combined with a primary-key lookup/) + expect(WebMock).not_to have_requested(:get, "#{base}/issues/i1") + end + + # An unresolved issue carries `resolution_time: nil`, and a bare `nil > + # value` raises: a scope or a segment dated on that column used to turn + # every record detail it did not match into a 500. + it 'excludes the record instead of raising when a dated condition meets a null column' do + stub_request(:get, "#{base}/issues/i1").to_return(json('data' => issue_payload('i1'))) + tree = branch('And', [id_leaf(operators::EQUAL, 'i1'), + leaf('resolution_time', operators::GREATER_THAN, '2026-01-01T00:00:00Z')]) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id])).to eq([]) + end + + it 'keeps the record when the dated condition matches' do + stub_request(:get, "#{base}/issues/i1") + .to_return(json('data' => issue_payload('i1', 'resolution_time' => '2026-08-07T13:06:22Z'))) + tree = branch('And', [id_leaf(operators::EQUAL, 'i1'), + leaf('resolution_time', operators::GREATER_THAN, '2026-01-01T00:00:00Z')]) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id])).to eq([{ 'id' => 'i1' }]) + end + end + + describe '#list on a primary-key lookup carrying a search' do + # The short-circuit never reaches /issues/search, so honouring the search + # is impossible: returning the record unsearched would be a result that + # looks searched and is not. + it 'refuses to answer rather than dropping the search' do + query = filter(condition_tree: id_leaf(operators::EQUAL, 'i1'), search: 'boom') + + expect { collection.list(nil, query, %w[id]) } + .to raise_error(UnsupportedOperatorError, /search cannot be combined with a filter on 'id'/) + expect(WebMock).not_to have_requested(:get, "#{base}/issues/i1") + end + + it 'still serves the lookup when the search is empty' do + stub_request(:get, "#{base}/issues/i1").to_return(json('data' => issue_payload('i1'))) + query = filter(condition_tree: id_leaf(operators::EQUAL, 'i1'), search: '') + + expect(collection.list(nil, query, %w[id])).to eq([{ 'id' => 'i1' }]) + end + end + + describe '#list on a primary-key lookup of many ids' do + # One GET per id against the same 20 req/min budget the cursor walk is + # capped for, so the fan-out is bounded the same way. + it 'reads at most the capped number of ids and warns' do + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + ids = Array.new(Collections::Issue::MAX_ID_LOOKUPS + 5) { |i| "i#{i}" } + ids.each { |id| stub_request(:get, "#{base}/issues/#{id}").to_return(json('data' => issue_payload(id))) } + + rows = collection.list(nil, filter(condition_tree: id_leaf(operators::IN, ids)), %w[id]) + + expect(rows.size).to eq(Collections::Issue::MAX_ID_LOOKUPS) + expect(ForestAdminDatasourcePylon.logger).to have_received(:warn).with(/reading the first 20/) + end + + it 'stays quiet and reads every id under the cap' do + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + %w[i1 i2].each { |id| stub_request(:get, "#{base}/issues/#{id}").to_return(json('data' => issue_payload(id))) } + + collection.list(nil, filter(condition_tree: id_leaf(operators::IN, %w[i1 i2])), %w[id]) + + expect(ForestAdminDatasourcePylon.logger).not_to have_received(:warn) + end + end + + describe '#list with a sort' do + before do + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + stub_request(:post, "#{base}/issues/search").to_return(json('data' => [issue_payload('i1')])) + end + + # /issues/search has no sort parameter, so the order is reported instead of + # being silently swallowed. + it 'warns that the requested order cannot be honoured' do + sort = ForestAdminDatasourceToolkit::Components::Query::Sort + .new([{ field: 'created_at', ascending: true }]) + + collection.list(nil, filter(sort: sort), %w[id]) + + expect(ForestAdminDatasourcePylon.logger).to have_received(:warn).with(/cannot honour the requested order/) + end + + it 'stays quiet when Forest asks for no order' do + collection.list(nil, filter, %w[id]) + + expect(ForestAdminDatasourcePylon.logger).not_to have_received(:warn) + end + + # The agent injects an ascending primary-key sort whenever the request + # asks for no order; only an order someone actually chose is reported. + it 'stays quiet on the default primary-key sort the agent injects' do + sort = ForestAdminDatasourceToolkit::Components::Query::Sort.new([{ field: 'id', ascending: true }]) + + collection.list(nil, filter(sort: sort), %w[id]) + + expect(ForestAdminDatasourcePylon.logger).not_to have_received(:warn) + end + + it 'warns on a chosen order, even one on the primary key' do + sort = ForestAdminDatasourceToolkit::Components::Query::Sort.new([{ field: 'id', ascending: false }]) + + collection.list(nil, filter(sort: sort), %w[id]) + + expect(ForestAdminDatasourcePylon.logger).to have_received(:warn).with(/cannot honour the requested order/) + end + + # The lookup returns the records in the order of the ids, so an order the + # operator chose is just as unhonoured there as on the search path. + it 'warns on the primary-key path too' do + stub_request(:get, "#{base}/issues/i1").to_return(json('data' => issue_payload('i1'))) + sort = ForestAdminDatasourceToolkit::Components::Query::Sort + .new([{ field: 'created_at', ascending: true }]) + + collection.list(nil, filter(condition_tree: id_leaf(operators::EQUAL, 'i1'), sort: sort), %w[id]) + + expect(ForestAdminDatasourcePylon.logger).to have_received(:warn).with(/cannot honour the requested order/) + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/relation_embedder_spec.rb b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/relation_embedder_spec.rb new file mode 100644 index 000000000..9cf076959 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/relation_embedder_spec.rb @@ -0,0 +1,368 @@ +module ForestAdminDatasourcePylon + # Observed through the two list paths that embed: PylonIssue, whose four + # ManyToOne relations reach both kinds of foreign collection, and PylonContact, + # which embeds through the cursor-paginated pipeline. + RSpec.describe Collections::RelationEmbedder do + def filter(condition_tree: nil, search: nil, page: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new( + condition_tree: condition_tree, search: search, page: page + ) + end + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + # Trimmed to what the embedder reads: the nested parties. The columns left + # out serialize to nil, which is what Pylon returns for them anyway. + def issue_payload(id, overrides = {}) + { 'id' => id, 'title' => 'Boom', 'account' => { 'id' => 'acc-1' }, + 'requester' => { 'id' => 'con-1' }, 'assignee' => { 'id' => 'usr-1' }, + 'team' => { 'id' => 'team-1' } }.merge(overrides) + end + + def account_payload(id, overrides = {}) + { 'id' => id, 'name' => 'Acme', 'domains' => %w[acme.com], + 'owner' => { 'id' => 'usr-9', 'email' => 'ada@acme.com' } }.merge(overrides) + end + + def contact_payload(id, overrides = {}) + { 'id' => id, 'name' => 'Ada', 'email' => 'ada@acme.com', + 'account' => { 'id' => 'acc-1' } }.merge(overrides) + end + + def user_payload(id, overrides = {}) + { 'id' => id, 'name' => 'Alice', 'role' => { 'id' => 'role-1', 'name' => 'Admin' } }.merge(overrides) + end + + def team_payload(id, overrides = {}) + { 'id' => id, 'name' => 'Support', 'users' => [{ 'id' => 'usr-1' }] }.merge(overrides) + end + + def id_filter(values) + { 'field' => 'id', 'operator' => 'in', 'values' => values } + end + + def stub_issues(*payloads) + stub_request(:post, "#{base}/issues/search").to_return(json('data' => payloads)) + end + + def stub_accounts(*payloads) + stub_request(:post, "#{base}/accounts/search").to_return(json('data' => payloads)) + end + + def stub_contact_search(*payloads) + stub_request(:post, "#{base}/contacts/search").to_return(json('data' => payloads)) + end + + def stub_contact_list(*payloads) + stub_request(:get, "#{base}/contacts").with(query: { 'limit' => '1000' }) + .to_return(json('data' => payloads)) + end + + def stub_users(*payloads) + stub_request(:get, "#{base}/users").with(query: { 'include_deactivated' => 'true' }) + .to_return(json('data' => payloads)) + end + + def stub_teams(*payloads) + stub_request(:get, "#{base}/teams").to_return(json('data' => payloads)) + end + + # The columns the foreign collection declares, which is what its serializer + # is expected to fill in on the embedded record. + def columns_of(name) + datasource.get_collection(name).fields.select { |_field, schema| schema.type == 'Column' }.keys + end + + let(:datasource) { ForestAdminDatasourcePylon::Datasource.new(api_key: 'k') } + let(:issues) { datasource.get_collection('PylonIssue') } + let(:contacts) { datasource.get_collection('PylonContact') } + let(:base) { datasource.configuration.url } + + before { stub_custom_fields } + + describe 'what the projection asks for' do + before { stub_issues(issue_payload('i1')) } + + it 'resolves only the relations the projection names' do + stub_accounts(account_payload('acc-1')) + + row = issues.list(nil, filter, %w[id account:name]).first + + expect(row['account']).to include('id' => 'acc-1', 'name' => 'Acme') + expect(WebMock).to have_requested(:post, "#{base}/accounts/search") + expect(WebMock).not_to have_requested(:post, "#{base}/contacts/search") + expect(WebMock).not_to have_requested(:get, "#{base}/users") + expect(WebMock).not_to have_requested(:get, "#{base}/teams") + end + + it 'reads no foreign collection when the projection holds columns only' do + rows = issues.list(nil, filter, %w[id title account_id]) + + expect(rows).to eq([{ 'id' => 'i1', 'title' => 'Boom', 'account_id' => 'acc-1' }]) + expect(WebMock).not_to have_requested(:post, "#{base}/accounts/search") + end + + # A nil projection returns every column and no relation: `list` is also + # called that way by a count or an export. + it 'reads no foreign collection when the projection is nil' do + expect(issues.list(nil, filter, nil).first).not_to have_key('account') + expect(WebMock).not_to have_requested(:post, "#{base}/accounts/search") + end + + # `account_id` is a column of the projection, not a relation prefix. + it 'embeds nothing for a projected column bearing a relation name' do + expect(issues.list(nil, filter, %w[account_id]).first.keys).to eq(%w[account_id]) + end + end + + describe 'the ids it asks for' do + it 'asks for an id once however many rows point at it' do + stub_issues(issue_payload('i1'), issue_payload('i2'), issue_payload('i3', 'account' => { 'id' => 'acc-2' })) + stub_accounts(account_payload('acc-1'), account_payload('acc-2')) + + rows = issues.list(nil, filter, %w[id account:name]) + + expect(rows.map { |row| row['account']['id'] }).to eq(%w[acc-1 acc-1 acc-2]) + expect(WebMock).to have_requested(:post, "#{base}/accounts/search") + .with(body: hash_including('filter' => id_filter(%w[acc-1 acc-2]))).once + end + + # Pylon documents no maximum on an `in` filter; the chunk keeps the request + # body and the page answering it bounded. + it 'chunks the ids, and asks for no more records than the chunk holds' do + chunk = Collections::CursorCollection::ID_CHUNK_SIZE + ids = (1..(chunk + 50)).map { |index| "acc-#{index}" } + stub_issues(*ids.map { |id| issue_payload("i-#{id}", 'account' => { 'id' => id }) }) + stub_accounts + + issues.list(nil, filter, %w[id account:name]) + + expect(WebMock).to have_requested(:post, "#{base}/accounts/search") + .with(body: { 'limit' => chunk, 'filter' => id_filter(ids.first(chunk)) }) + expect(WebMock).to have_requested(:post, "#{base}/accounts/search") + .with(body: { 'limit' => 50, 'filter' => id_filter(ids.last(50)) }) + expect(WebMock).to have_requested(:post, "#{base}/accounts/search").twice + end + + it 'sends no request at all when every foreign key of the page is null' do + stub_issues(issue_payload('i1', 'team' => nil)) + + expect(issues.list(nil, filter, %w[id team:name]).first).to eq('id' => 'i1', 'team' => nil) + expect(WebMock).not_to have_requested(:get, "#{base}/teams") + end + + it 'leaves the null key out of the ids it asks for' do + stub_issues(issue_payload('i1'), issue_payload('i2', 'account' => nil)) + stub_accounts(account_payload('acc-1')) + + rows = issues.list(nil, filter, %w[id account:name]) + + expect(rows.map { |row| row['account'] }).to eq([rows.first['account'], nil]) + expect(WebMock).to have_requested(:post, "#{base}/accounts/search") + .with(body: hash_including('filter' => id_filter(%w[acc-1]))) + end + + # A blank key would otherwise reach the `in` filter of the read, which + # refuses a blank inside a list: one malformed key would fail the page. + it 'leaves a blank key out of the ids it asks for, like a null one' do + stub_issues(issue_payload('i1'), issue_payload('i2', 'account' => { 'id' => '' })) + stub_accounts(account_payload('acc-1')) + + rows = issues.list(nil, filter, %w[id account:name]) + + expect(rows.map { |row| row['account'] }).to eq([rows.first['account'], nil]) + expect(WebMock).to have_requested(:post, "#{base}/accounts/search") + .with(body: hash_including('filter' => id_filter(%w[acc-1]))) + end + + it 'sends no request at all when every foreign key of the page is blank' do + stub_issues(issue_payload('i1', 'team' => { 'id' => '' })) + + expect(issues.list(nil, filter, %w[id team:name]).first).to eq('id' => 'i1', 'team' => nil) + expect(WebMock).not_to have_requested(:get, "#{base}/teams") + end + + # Deleted, merged, or outside the scope of the token: the row says so + # rather than carrying a blank record the panel would offer to open. + it 'embeds no record for a foreign key the endpoint no longer answers' do + stub_issues(issue_payload('i1')) + stub_accounts + + expect(issues.list(nil, filter, %w[id account:name]).first).to eq('id' => 'i1', 'account' => nil) + end + end + + describe 'how it reads the foreign collections' do + before { stub_issues(issue_payload('i1'), issue_payload('i2', 'assignee' => { 'id' => 'usr-2' })) } + + # `GET /users` and `GET /teams` hand back the complete dataset, so the ids + # only pick rows out of one response. + it 'reads an unpaginated collection once and indexes it by id' do + stub_users(user_payload('usr-1'), user_payload('usr-2', 'name' => 'Bob')) + + rows = issues.list(nil, filter, %w[id assignee:name]) + + expect(rows.map { |row| row['assignee']['name'] }).to eq(%w[Alice Bob]) + expect(WebMock).to have_requested(:get, "#{base}/users") + .with(query: { 'include_deactivated' => 'true' }).once + end + + # A chunk is asked for as a single page, and Pylon is free to answer it over + # several: the ids left out of the first page are read from the next one + # rather than reported as records that no longer exist. + it 'follows the cursor when Pylon answers a chunk over several pages' do + stub_issues(issue_payload('i1'), issue_payload('i2', 'account' => { 'id' => 'acc-2' })) + stub_request(:post, "#{base}/accounts/search").with(body: hash_including('limit' => 2)) + .to_return(json('data' => [account_payload('acc-1')], + 'pagination' => { 'cursor' => 'c1', + 'has_next_page' => true })) + stub_request(:post, "#{base}/accounts/search").with(body: hash_including('cursor' => 'c1')) + .to_return(json('data' => [account_payload('acc-2')])) + + rows = issues.list(nil, filter, %w[id account:name]) + + expect(rows.map { |row| row['account']['id'] }).to eq(%w[acc-1 acc-2]) + expect(WebMock).to have_requested(:post, "#{base}/accounts/search").twice + end + + # One read per foreign collection, and one collection per relation here: + # nothing is shared, so the four are read side by side. + it 'reads each foreign collection the projection reaches' do + stub_accounts(account_payload('acc-1')) + stub_contact_search(contact_payload('con-1')) + stub_users(user_payload('usr-1'), user_payload('usr-2')) + stub_teams(team_payload('team-1')) + + row = issues.list(nil, filter, %w[id account:name requester:name assignee:name team:name]).first + + expect(row.keys).to eq(%w[id account requester assignee team]) + expect(row['requester']).to include('id' => 'con-1', 'name' => 'Ada') + expect(row['team']).to include('id' => 'team-1', 'name' => 'Support') + end + end + + describe 'the shape of the embedded record' do + before { stub_issues(issue_payload('i1')) } + + # The foreign endpoint takes no field list, so the collection reads the + # whole record and the projection is applied to it here. Serving all of it + # would answer `team:name` with columns the projection excluded -- and the + # agent redacts that projection per collection, so the excluded ones are + # also the ones the caller may not read. + it 'carries the projected fields of the foreign collection, and nothing else' do + stub_teams(team_payload('team-1')) + + expect(issues.list(nil, filter, %w[id team:name]).first['team']) + .to eq('id' => 'team-1', 'name' => 'Support') + end + + it 'leaves out the columns of the foreign collection the projection did not name' do + stub_accounts(account_payload('acc-1')) + + embedded = issues.list(nil, filter, %w[id account:name]).first['account'] + + expect(embedded.keys).to match_array(%w[id name]) + expect(columns_of('PylonAccount') - embedded.keys).to include('crm_settings', 'external_ids') + end + + # `with_pks` puts it in the projection the route sends, but a caller inside + # the agent may project without it, and the id is what the serializer + # builds the nested resource around. + it 'keeps the primary key of the foreign record even when the projection leaves it out' do + stub_teams(team_payload('team-1')) + + expect(issues.list(nil, filter, %w[id team:user_ids]).first['team']) + .to eq('id' => 'team-1', 'user_ids' => %w[usr-1]) + end + + it 'flattens the nested objects of the foreign record the way its own list does' do + stub_accounts(account_payload('acc-1')) + stub_users(user_payload('usr-1')) + + row = issues.list(nil, filter, %w[id account:owner_id assignee:role_name]).first + + expect(row['account']).to eq('id' => 'acc-1', 'owner_id' => 'usr-9') + expect(row['assignee']).to eq('id' => 'usr-1', 'role_name' => 'Admin') + end + end + + describe 'the order of the rows' do + it 'writes each related record on its own row' do + stub_issues(issue_payload('i1', 'account' => { 'id' => 'acc-2' }), + issue_payload('i2', 'account' => nil), + issue_payload('i3')) + stub_accounts(account_payload('acc-1', 'name' => 'First'), account_payload('acc-2', 'name' => 'Second')) + + rows = issues.list(nil, filter, %w[id account:name]) + + expect(rows.map { |row| row['id'] }).to eq(%w[i1 i2 i3]) + expect(rows.map { |row| row['account']&.fetch('name') }).to eq(['Second', nil, 'First']) + end + + # The window is cut out before the relations are read, so the ids asked for + # are those of the rows the operator sees. + it 'reads the relations of the requested page only' do + stub_issues(issue_payload('i1'), issue_payload('i2', 'account' => { 'id' => 'acc-2' })) + stub_accounts(account_payload('acc-2')) + page = ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: 1, limit: 1) + + rows = issues.list(nil, filter(page: page), %w[id account:name]) + + expect(rows.map { |row| row['id'] }).to eq(%w[i2]) + expect(WebMock).to have_requested(:post, "#{base}/accounts/search") + .with(body: hash_including('filter' => id_filter(%w[acc-2]))) + end + end + + # The cursor-paginated pipeline embeds the same way, on all three of its + # paths: the listing endpoint, the search endpoint and the record endpoint. + describe 'PylonContact#list' do + it 'embeds the account of a browsed page' do + stub_contact_list(contact_payload('con-1')) + stub_accounts(account_payload('acc-1')) + + row = contacts.list(nil, filter, %w[id account:name]).first + + expect(row).to eq('id' => 'con-1', 'account' => row['account']) + expect(row['account']).to include('id' => 'acc-1', 'name' => 'Acme') + end + + it 'embeds the account of a searched page' do + stub_contact_search(contact_payload('con-1')) + stub_accounts(account_payload('acc-1')) + + row = contacts.list(nil, filter(search: 'ada'), %w[id account:name]).first + + expect(row['account']).to include('id' => 'acc-1') + end + + it 'embeds the account of a record read through its own endpoint' do + stub_request(:get, "#{base}/contacts/con-1").to_return(json('data' => contact_payload('con-1'))) + stub_accounts(account_payload('acc-1')) + tree = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new('id', ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators::EQUAL, 'con-1') + + row = contacts.list(nil, filter(condition_tree: tree), %w[id account:name]).first + + expect(row['account']).to include('id' => 'acc-1') + end + end + + # A OneToMany is not embedded: the agent lists the far side with a query of + # its own, filtered on the origin key -- which every reverse side here is + # filterable on server-side. PylonUser and PylonTeam declare nothing else, so + # they embed nothing at all. + describe 'a collection declaring no ManyToOne' do + it 'embeds nothing, and reads nothing besides its own endpoint' do + stub_users(user_payload('usr-1')) + + rows = datasource.get_collection('PylonUser').list(nil, filter, %w[id assigned_issues:id]) + + expect(rows).to eq([{ 'id' => 'usr-1' }]) + expect(WebMock).not_to have_requested(:post, "#{base}/issues/search") + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/team_spec.rb b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/team_spec.rb new file mode 100644 index 000000000..b0b6c502e --- /dev/null +++ b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/team_spec.rb @@ -0,0 +1,225 @@ +module ForestAdminDatasourcePylon + RSpec.describe Collections::Team do + def filter(condition_tree: nil, page: nil, sort: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new( + condition_tree: condition_tree, page: page, sort: sort + ) + end + + def leaf(field, operator, value = nil) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def page(offset, limit) + ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: offset, limit: limit) + end + + def sort(field, ascending: true) + ForestAdminDatasourceToolkit::Components::Query::Sort.new([{ field: field, ascending: ascending }]) + end + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + # Pylon nests the members as `{id, email}` objects, and a team with none + # comes back with a null rather than an empty list. + def team_payload(id, overrides = {}) + { 'id' => id, 'name' => 'Support', + 'users' => [{ 'id' => 'u1', 'email' => 'alice@acme.io' }, + { 'id' => 'u2', 'email' => 'bob@acme.io' }] }.merge(overrides) + end + + def stub_teams(*payloads) + stub_request(:get, "#{base}/teams").to_return(json('data' => payloads)) + end + + def ids(records) + records.map { |record| record['id'] } + end + + # Mirrors `residual_leaf_appliable?`: an operator is evaluable in memory when + # `ConditionTreeLeaf#match` handles it natively or the toolkit can rewrite it + # into operators that it does. + def appliable?(operator, column_type) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::ConditionTreeEquivalent + .equivalent_tree?(operator, Collections::BaseCollection::IN_MEMORY_OPERATORS, column_type) + end + + # Relations are fields too; the assertions on the columns select them out. + def columns + collection.fields.select { |_name, field| field.type == 'Column' } + end + + let(:datasource) { ForestAdminDatasourcePylon::Datasource.new(api_key: 'k') } + let(:collection) { described_class.new(datasource) } + let(:base) { datasource.configuration.url } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + + before { stub_custom_fields } + + describe 'schema' do + it 'is named PylonTeam' do + expect(collection.name).to eq('PylonTeam') + end + + it 'exposes the columns observed on the API, with the members flattened to their ids' do + expect(columns.keys).to eq(%w[id name user_ids]) + expect(collection.fields['user_ids'].column_type).to eq('Json') + expect(collection.fields['id'].column_type).to eq('String') + end + + it 'declares id as the primary key' do + expect(collection.fields['id'].is_primary_key).to be(true) + end + + # The order is honoured in memory over the complete dataset, so both + # scalar columns can be sorted on. + it 'declares both scalar columns sortable' do + expect(columns.except('user_ids').values.map(&:is_sortable).uniq).to eq([true]) + expect(collection.fields['user_ids'].is_sortable).to be(false) + end + + # POST /teams and PATCH /teams/{id} take the name and the members, and + # Pylon names the id itself. + it 'declares writable exactly the columns an endpoint takes' do + writable = columns.reject { |_name, column| column.is_read_only }.keys + + expect(writable).to contain_exactly('name', 'user_ids') + end + + # GET /teams carries neither a search nor a filter parameter. It does hand + # back every team, so the count is exact rather than a fraction of one. + it 'leaves search disabled and enables count' do + expect(collection.is_searchable?).to be(false) + expect(collection.is_countable?).to be(true) + end + + it 'advertises the string filters on the string columns' do + expect(collection.fields['name'].filter_operators) + .to eq([operators::EQUAL, operators::NOT_EQUAL, operators::IN, operators::NOT_IN, + operators::PRESENT, operators::BLANK, operators::CONTAINS, operators::I_CONTAINS, + operators::NOT_CONTAINS, operators::STARTS_WITH, operators::ENDS_WITH]) + expect(collection.fields['id'].filter_operators).to eq(collection.fields['name'].filter_operators) + end + + # The membership holds a list, whose Pylon semantics have no in-memory + # counterpart -- and no relation either, see the relations below. + it 'advertises no filter on the membership' do + expect(collection.fields['user_ids'].filter_operators).to eq([]) + end + + # Every filter of the schema is answered in memory: one the in-memory pass + # cannot evaluate would silently empty the page instead of filtering it. + it 'advertises only operators the in-memory pass can evaluate' do + unappliable = columns.flat_map do |name, column| + column.filter_operators.reject { |operator| appliable?(operator, column.column_type) } + .map { |operator| "#{name}: #{operator}" } + end + + expect(unappliable).to be_empty + end + end + + describe 'relations' do + # `/issues/search` filters `team_id` server-side, so the issues assigned to + # a team are listed by one request. + it 'declares the issues assigned to a team, read through team_id' do + expect(collection.fields['issues']) + .to be_a(ForestAdminDatasourceToolkit::Schema::Relations::OneToManySchema) + .and have_attributes(foreign_collection: 'PylonIssue', origin_key: 'team_id', + origin_key_target: 'id') + end + + # Pylon nests the members here rather than pointing at the team from a + # user, so the membership is a ManyToMany with no join collection to + # declare it on: `user_ids` stays a plain column. + it 'declares no relation for the membership' do + expect(collection.fields.keys).to eq(%w[id name user_ids issues]) + end + end + + describe '#list' do + it 'reads every team of the organization and serializes them' do + stub_teams(team_payload('t1'), team_payload('t2', 'name' => 'Billing')) + + rows = collection.list(nil, filter, nil) + + expect(ids(rows)).to eq(%w[t1 t2]) + expect(rows.first).to eq('id' => 't1', 'name' => 'Support', 'user_ids' => %w[u1 u2]) + end + + it 'flattens the members to their ids and keeps the nested objects out' do + stub_teams(team_payload('t1'), team_payload('t2', 'users' => nil)) + + rows = collection.list(nil, filter(sort: sort('id')), nil) + + expect(rows.map { |row| row['user_ids'] }).to eq([%w[u1 u2], []]) + expect(rows.first.keys).not_to include('users') + end + + it 'restricts the record to the projection' do + stub_teams(team_payload('t1')) + + expect(collection.list(nil, filter, %w[id name])).to eq([{ 'id' => 't1', 'name' => 'Support' }]) + end + + it 'returns an empty list when the organization has no team' do + stub_teams + + expect(collection.list(nil, filter, nil)).to eq([]) + end + + # Freshness over rate-limit thrift: one call per list, and no record kept + # from the previous one. + it 'reads the endpoint once per list, and again on the next one' do + stub_teams(team_payload('t1')) + + 2.times { collection.list(nil, filter, nil) } + + expect(WebMock).to have_requested(:get, "#{base}/teams").twice + end + + it 'propagates the API error rather than answering with no team' do + stub_request(:get, "#{base}/teams").to_return(json({ 'message' => 'boom' }, 500)) + + expect { collection.list(nil, filter, %w[id]) }.to raise_error(APIError) + end + end + + describe '#list with a filter, a sort and a page' do + before do + stub_teams(team_payload('t2', 'name' => 'Billing'), team_payload('t1'), + team_payload('t3', 'name' => nil, 'users' => [])) + end + + def filtered(field, operator, value = nil) + ids(collection.list(nil, filter(condition_tree: leaf(field, operator, value)), nil)) + end + + # The single response holds every team, so the filter running in memory + # answers exactly what a server-side filter would have. + it 'filters the complete dataset in memory' do + expect(filtered('name', operators::CONTAINS, 'ill')).to eq(%w[t2]) + expect(filtered('id', operators::IN, %w[t1 t3])).to eq(%w[t1 t3]) + expect(filtered('name', operators::BLANK)).to eq(%w[t3]) + end + + it 'honours the requested order in memory, ascending and descending' do + expect(ids(collection.list(nil, filter(sort: sort('name')), nil))).to eq(%w[t2 t1 t3]) + expect(ids(collection.list(nil, filter(sort: sort('name', ascending: false)), nil))).to eq(%w[t3 t1 t2]) + end + + it 'honours the ascending primary-key sort the agent injects when nothing is asked for' do + expect(ids(collection.list(nil, filter(sort: sort('id')), nil))).to eq(%w[t1 t2 t3]) + end + + it 'slices the requested window out of the ordered records' do + query = filter(sort: sort('id'), page: page(2, 5)) + + expect(ids(collection.list(nil, query, %w[id]))).to eq(%w[t3]) + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/user_spec.rb b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/user_spec.rb new file mode 100644 index 000000000..7f720687c --- /dev/null +++ b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/user_spec.rb @@ -0,0 +1,275 @@ +module ForestAdminDatasourcePylon + RSpec.describe Collections::User do + def filter(condition_tree: nil, page: nil, sort: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new( + condition_tree: condition_tree, page: page, sort: sort + ) + end + + def leaf(field, operator, value = nil) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def page(offset, limit) + ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: offset, limit: limit) + end + + def sort(field, ascending: true) + ForestAdminDatasourceToolkit::Components::Query::Sort.new([{ field: field, ascending: ascending }]) + end + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + # Trimmed to the shape observed on the API: `role` is a nested object and an + # unset value comes back as null rather than absent. + def user_payload(id, overrides = {}) + { + 'id' => id, 'name' => 'Alice', 'email' => 'alice@acme.io', 'emails' => %w[alice@acme.io a@acme.io], + 'avatar_url' => 'https://cdn.usepylon.com/alice.png', 'status' => 'active', 'role_id' => 'role-1', + 'role' => { 'id' => 'role-1', 'name' => 'Admin', 'slug' => 'admin' }, 'is_deactivated' => false + }.merge(overrides) + end + + def stub_users(*payloads) + stub_request(:get, "#{base}/users").with(query: { 'include_deactivated' => 'true' }) + .to_return(json('data' => payloads)) + end + + def ids(records) + records.map { |record| record['id'] } + end + + # Mirrors `residual_leaf_appliable?`: an operator is evaluable in memory when + # `ConditionTreeLeaf#match` handles it natively or the toolkit can rewrite it + # into operators that it does. + def appliable?(operator, column_type) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::ConditionTreeEquivalent + .equivalent_tree?(operator, Collections::BaseCollection::IN_MEMORY_OPERATORS, column_type) + end + + # Relations are fields too; the assertions on the columns select them out. + def columns + collection.fields.select { |_name, field| field.type == 'Column' } + end + + let(:datasource) { ForestAdminDatasourcePylon::Datasource.new(api_key: 'k') } + let(:collection) { described_class.new(datasource) } + let(:base) { datasource.configuration.url } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + + before { stub_custom_fields } + + describe 'schema' do + it 'is named PylonUser' do + expect(collection.name).to eq('PylonUser') + end + + it 'exposes the columns observed on the API, with the role flattened to its name' do + expect(columns.keys) + .to eq(%w[id name email emails avatar_url status role_id role_name is_deactivated]) + end + + it 'declares id as the primary key' do + expect(collection.fields['id'].is_primary_key).to be(true) + end + + it 'types the list of addresses as json and the deactivation flag as a boolean' do + expect(collection.fields['emails'].column_type).to eq('Json') + expect(collection.fields['is_deactivated'].column_type).to eq('Boolean') + expect(columns.except('emails', 'is_deactivated').values.map(&:column_type).uniq) + .to eq(['String']) + end + + # The order is honoured in memory over the complete dataset, so every + # scalar column can be sorted on. + it 'declares every scalar column sortable' do + expect(columns.except('emails').values.map(&:is_sortable).uniq).to eq([true]) + expect(collection.fields['emails'].is_sortable).to be(false) + end + + # PATCH /users/{id} takes these four and nothing else: an address is + # proven by the agent signing in, and the deactivation happens in Pylon. + it 'declares writable exactly the columns the update endpoint takes' do + writable = columns.reject { |_name, column| column.is_read_only }.keys + + expect(writable).to contain_exactly('name', 'avatar_url', 'status', 'role_id') + end + + # GET /users carries no search parameter. It does hand back every agent, so + # the count is exact rather than a fraction of one. + it 'leaves search disabled and enables count' do + expect(collection.is_searchable?).to be(false) + expect(collection.is_countable?).to be(true) + end + + it 'advertises the string filters on the string columns' do + expect(collection.fields['name'].filter_operators) + .to eq([operators::EQUAL, operators::NOT_EQUAL, operators::IN, operators::NOT_IN, + operators::PRESENT, operators::BLANK, operators::CONTAINS, operators::I_CONTAINS, + operators::NOT_CONTAINS, operators::STARTS_WITH, operators::ENDS_WITH]) + expect(collection.fields['id'].filter_operators).to eq(collection.fields['name'].filter_operators) + end + + it 'advertises the boolean filters on the deactivation flag' do + expect(collection.fields['is_deactivated'].filter_operators) + .to eq([operators::EQUAL, operators::NOT_EQUAL, operators::IN, operators::NOT_IN, + operators::PRESENT, operators::BLANK]) + end + + # The list of addresses has no in-memory filter that would mean what an + # operator expects, so it advertises none. + it 'advertises no filter on the list of addresses' do + expect(collection.fields['emails'].filter_operators).to eq([]) + end + + # Every filter of the schema is answered in memory: one the in-memory pass + # cannot evaluate would silently empty the page instead of filtering it. + it 'advertises only operators the in-memory pass can evaluate' do + unappliable = columns.flat_map do |name, column| + column.filter_operators.reject { |operator| appliable?(operator, column.column_type) } + .map { |operator| "#{name}: #{operator}" } + end + + expect(unappliable).to be_empty + end + end + + describe 'relations' do + # `/issues/search` filters `assignee_id` server-side, so the issues of an + # agent are listed by one request. + it 'declares the issues assigned to a user, read through assignee_id' do + expect(collection.fields['assigned_issues']) + .to be_a(ForestAdminDatasourceToolkit::Schema::Relations::OneToManySchema) + .and have_attributes(foreign_collection: 'PylonIssue', origin_key: 'assignee_id', + origin_key_target: 'id') + end + + # Pylon nests the members inside a team and exposes no team id on a user, + # so that side is a ManyToMany with no key column to build it on. + it 'declares no relation to the teams a user belongs to' do + expect(collection.fields.keys - columns.keys).to eq(%w[assigned_issues]) + end + end + + describe '#list' do + it 'reads every user of the organization and serializes them' do + stub_users(user_payload('u1'), user_payload('u2', 'name' => 'Bob')) + + rows = collection.list(nil, filter, nil) + + expect(ids(rows)).to eq(%w[u1 u2]) + expect(rows.first).to eq('id' => 'u1', 'name' => 'Alice', 'email' => 'alice@acme.io', + 'emails' => %w[alice@acme.io a@acme.io], 'status' => 'active', + 'avatar_url' => 'https://cdn.usepylon.com/alice.png', + 'role_id' => 'role-1', 'role_name' => 'Admin', 'is_deactivated' => false) + end + + # A deactivated agent stays the assignee of the issues they handled, so the + # record the rest of the panel points at has to stay readable. + it 'asks for the deactivated users too' do + stub_users(user_payload('u1')) + + collection.list(nil, filter, nil) + + expect(WebMock).to have_requested(:get, "#{base}/users") + .with(query: { 'include_deactivated' => 'true' }) + end + + it 'flattens the nested role and keeps the object out of the record' do + stub_users(user_payload('u1'), user_payload('u2', 'role' => nil)) + + rows = collection.list(nil, filter(sort: sort('id')), nil) + + expect(rows.map { |row| row['role_name'] }).to eq(['Admin', nil]) + expect(rows.first.keys).not_to include('role') + end + + it 'restricts the record to the projection' do + stub_users(user_payload('u1')) + + expect(collection.list(nil, filter, %w[id name])).to eq([{ 'id' => 'u1', 'name' => 'Alice' }]) + end + + it 'returns an empty list when the organization has no user' do + stub_users + + expect(collection.list(nil, filter, nil)).to eq([]) + end + + # Freshness over rate-limit thrift: one call per list, and no record kept + # from the previous one. + it 'reads the endpoint once per list, and again on the next one' do + stub_users(user_payload('u1')) + + 2.times { collection.list(nil, filter, nil) } + + expect(WebMock).to have_requested(:get, "#{base}/users") + .with(query: { 'include_deactivated' => 'true' }).twice + end + end + + describe '#list with a filter' do + before do + stub_users(user_payload('u1'), user_payload('u2', 'name' => 'Bob', 'is_deactivated' => true), + user_payload('u3', 'name' => nil, 'role' => nil, 'avatar_url' => nil)) + end + + def filtered(field, operator, value = nil) + ids(collection.list(nil, filter(condition_tree: leaf(field, operator, value)), nil)) + end + + # The single response holds every user, so the filter running in memory + # answers exactly what a server-side filter would have. + it 'filters the complete dataset in memory' do + expect(filtered('name', operators::CONTAINS, 'li')).to eq(%w[u1]) + expect(filtered('is_deactivated', operators::EQUAL, true)).to eq(%w[u2]) + expect(filtered('id', operators::IN, %w[u1 u3])).to eq(%w[u1 u3]) + end + + it 'reads the columns Pylon leaves null as blank instead of crashing' do + expect(filtered('name', operators::BLANK)).to eq(%w[u3]) + expect(filtered('name', operators::CONTAINS, 'li')).to eq(%w[u1]) + expect(filtered('role_name', operators::PRESENT)).to eq(%w[u1 u2]) + expect(filtered('avatar_url', operators::NOT_CONTAINS, 'alice')).to eq(%w[u3]) + end + end + + describe '#list with a sort and a page' do + before do + stub_users(user_payload('u2', 'name' => 'Zoe'), user_payload('u1'), + user_payload('u3', 'name' => 'Carol', 'is_deactivated' => true)) + end + + it 'honours the requested order in memory, ascending and descending' do + expect(ids(collection.list(nil, filter(sort: sort('name')), nil))).to eq(%w[u1 u3 u2]) + expect(ids(collection.list(nil, filter(sort: sort('name', ascending: false)), nil))).to eq(%w[u2 u3 u1]) + end + + it 'honours the ascending primary-key sort the agent injects when nothing is asked for' do + expect(ids(collection.list(nil, filter(sort: sort('id')), nil))).to eq(%w[u1 u2 u3]) + end + + it 'orders the deactivation flag, false first, the way a database does' do + expect(ids(collection.list(nil, filter(sort: sort('is_deactivated')), nil))).to eq(%w[u2 u1 u3]) + end + + it 'slices the requested window out of the ordered records' do + query = filter(sort: sort('id'), page: page(1, 2)) + + expect(ids(collection.list(nil, query, %w[id]))).to eq(%w[u2 u3]) + end + end + + describe '#list when the endpoint fails' do + it 'propagates the API error rather than answering with no user' do + stub_request(:get, "#{base}/users").with(query: { 'include_deactivated' => 'true' }) + .to_return(json({ 'message' => 'boom' }, 500)) + + expect { collection.list(nil, filter, %w[id]) }.to raise_error(APIError) + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/writes_spec.rb b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/writes_spec.rb new file mode 100644 index 000000000..7595e609d --- /dev/null +++ b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/collections/writes_spec.rb @@ -0,0 +1,692 @@ +module ForestAdminDatasourcePylon + RSpec.describe Collections::Writes do + def filter(condition_tree: nil, search: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition_tree, search: search) + end + + def leaf(field, operator, value) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def branch(aggregator, conditions) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + .new(aggregator, conditions) + end + + def id_filter(operator, value) + filter(condition_tree: leaf('id', operator, value)) + end + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def custom_field(type, slug, extra = {}) + { 'id' => "cf_#{slug}", 'slug' => slug, 'label' => slug, 'type' => type, + 'object_type' => 'issue', 'is_read_only' => false }.merge(extra) + end + + def options(*slugs) + { 'select_metadata' => { 'options' => slugs.map { |slug| { 'label' => slug.upcase, 'slug' => slug } } } } + end + + def issue_payload(id, overrides = {}) + { 'id' => id, 'number' => 12, 'title' => 'Boom', 'body_html' => '

boom

', 'state' => 'new', + 'type' => 'ticket', 'source' => 'manual', 'account' => { 'id' => 'acc-1' }, 'tags' => %w[urgent], + 'custom_fields' => {}, 'created_at' => '2026-08-07T13:06:22Z' }.merge(overrides) + end + + # A text field, a select read and written by the slug of its option, a + # multiselect Pylon takes through `values`, and one it syncs from an app and + # refuses to be written. + let(:issue_custom_fields) do + [custom_field('text', 'severity'), + custom_field('select', 'priority', options('p1', 'p2')), + custom_field('multiselect', 'regions', options('us', 'emea')), + custom_field('text', 'synced_id', 'is_read_only' => true)] + end + + let(:datasource) { Datasource.new(api_key: 'k') } + let(:base) { datasource.configuration.url } + let(:issues) { datasource.get_collection('PylonIssue') } + let(:accounts) { datasource.get_collection('PylonAccount') } + let(:contacts) { datasource.get_collection('PylonContact') } + let(:teams) { datasource.get_collection('PylonTeam') } + let(:users) { datasource.get_collection('PylonUser') } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + + before { stub_custom_fields(issue: issue_custom_fields) } + + describe '#create' do + it 'posts the writable columns and answers with the serialized record' do + stub_request(:post, "#{base}/issues").to_return(json('data' => issue_payload('i1'))) + + record = issues.create(nil, 'title' => 'Boom', 'body_html' => '

boom

', 'tags' => %w[urgent], + 'account_id' => 'acc-1') + + expect(record).to include('id' => 'i1', 'title' => 'Boom', 'account_id' => 'acc-1') + expect(WebMock).to have_requested(:post, "#{base}/issues") + .with(body: { 'title' => 'Boom', 'body_html' => '

boom

', + 'tags' => %w[urgent], 'account_id' => 'acc-1' }) + end + + # The schema is the single source of truth for what may be written: a + # read-only column reaching the payload is the agent's doing, not a request + # the operator made, so it is dropped rather than refused. + it 'drops the read-only columns and the keys the schema does not know' do + stub_request(:post, "#{base}/issues").to_return(json('data' => issue_payload('i1'))) + + issues.create(nil, 'title' => 'Boom', 'id' => 'i9', 'number' => 3, 'link' => 'http://x', + 'created_at' => '2026-01-01', 'source' => 'manual', 'messages' => [], + 'number_of_touches' => 4, 'not_a_column' => 'x') + + expect(WebMock).to have_requested(:post, "#{base}/issues").with(body: { 'title' => 'Boom' }) + end + + # Pylon fills in what a create leaves out, so a form field the operator + # never touched travels as nothing at all rather than as an explicit null. + it 'drops the columns left empty' do + stub_request(:post, "#{base}/issues").to_return(json('data' => issue_payload('i1'))) + + issues.create(nil, 'title' => 'Boom', 'team_id' => nil, 'tags' => nil) + + expect(WebMock).to have_requested(:post, "#{base}/issues").with(body: { 'title' => 'Boom' }) + end + end + + describe '#create with custom fields' do + it 'writes them as a list, through `value` or `values`, and leaves the synced one out' do + stub_request(:post, "#{base}/issues").to_return(json('data' => issue_payload('i1'))) + + issues.create(nil, 'title' => 'Boom', 'severity' => 'high', 'priority' => 'p2', + 'regions' => %w[us emea], 'synced_id' => 'zzz') + + expect(WebMock).to have_requested(:post, "#{base}/issues").with( + body: { 'title' => 'Boom', + 'custom_fields' => [{ 'slug' => 'severity', 'value' => 'high' }, + { 'slug' => 'priority', 'value' => 'p2' }, + { 'slug' => 'regions', 'values' => %w[us emea] }] } + ) + end + + it 'sends no custom_fields key when none was set' do + stub_request(:post, "#{base}/issues").to_return(json('data' => issue_payload('i1'))) + + issues.create(nil, 'title' => 'Boom') + + expect(WebMock).to(have_requested(:post, "#{base}/issues").with { |req| !req.body.include?('custom_fields') }) + end + end + + # Pylon takes `state` and `type` on an update only, and Forest has one + # read-only flag per column to say so with. + describe '#create naming a field Pylon only takes on an update' do + it 'refuses the create, naming the field' do + expect { issues.create(nil, 'title' => 'Boom', 'state' => 'closed') } + .to raise_error(UnsupportedWriteError, /'state' cannot be set here on a PylonIssue/) + end + + it 'asks for nothing when the field carries no value' do + stub_request(:post, "#{base}/issues").to_return(json('data' => issue_payload('i1'))) + + issues.create(nil, 'title' => 'Boom', 'state' => nil, 'type' => '') + + expect(WebMock).to have_requested(:post, "#{base}/issues").with(body: { 'title' => 'Boom' }) + end + end + + describe '#update' do + it 'patches the record the filter names, without reading it back first' do + stub_request(:patch, "#{base}/issues/i1").to_return(json('data' => issue_payload('i1'))) + + issues.update(nil, id_filter(operators::EQUAL, 'i1'), 'title' => 'Louder') + + expect(WebMock).to have_requested(:patch, "#{base}/issues/i1").with(body: { 'title' => 'Louder' }) + expect(WebMock).not_to have_requested(:get, "#{base}/issues/i1") + end + + it 'patches every record an `in` filter names' do + stub_request(:patch, "#{base}/issues/i1").to_return(json('data' => issue_payload('i1'))) + stub_request(:patch, "#{base}/issues/i2").to_return(json('data' => issue_payload('i2'))) + + issues.update(nil, id_filter(operators::IN, %w[i1 i2]), 'state' => 'closed') + + expect(WebMock).to have_requested(:patch, "#{base}/issues/i1").with(body: { 'state' => 'closed' }) + expect(WebMock).to have_requested(:patch, "#{base}/issues/i2").with(body: { 'state' => 'closed' }) + end + + it 'sends nothing when every key of the patch is read-only' do + issues.update(nil, id_filter(operators::EQUAL, 'i1'), 'number' => 9, 'link' => 'http://x') + + expect(WebMock).not_to have_requested(:patch, "#{base}/issues/i1") + end + + # The cap bounds a write, and a patch naming nothing writable is not one: + # it is settled before the ids are, so the selection is never resolved and + # never refused for its width. + it 'sends nothing, and refuses nothing, when the patch is read-only over a wide selection' do + ids = Array.new(21) { |index| "i#{index}" } + + expect { issues.update(nil, id_filter(operators::IN, ids), 'number' => 9) }.not_to raise_error + expect(WebMock).not_to have_requested(:patch, %r{/issues/}) + end + + # The scope the operator's role carries rides along as an `and`, so the ids + # are resolved through the collection's own read and a record the scope + # excludes is never written to. + it 'resolves the ids through a read when the filter carries more than an id' do + stub_request(:get, "#{base}/issues/i1").to_return(json('data' => issue_payload('i1', 'state' => 'closed'))) + + issues.update(nil, filter(condition_tree: branch('And', [leaf('id', operators::EQUAL, 'i1'), + leaf('state', operators::EQUAL, 'new')])), + 'title' => 'Louder') + + expect(WebMock).to have_requested(:get, "#{base}/issues/i1") + expect(WebMock).not_to have_requested(:patch, "#{base}/issues/i1") + end + end + + # The record a write answers with is discarded here, so a patch Pylon + # answers with no body at all wrote the record just the same — and the rest + # of the selection is written rather than aborted on it. + describe '#update answered with no record' do + it 'writes every record of the selection' do + stub_request(:patch, "#{base}/issues/i1").to_return(status: 204) + stub_request(:patch, "#{base}/issues/i2").to_return(json('data' => nil)) + + issues.update(nil, id_filter(operators::IN, %w[i1 i2]), 'state' => 'closed') + + expect(WebMock).to have_requested(:patch, "#{base}/issues/i1") + expect(WebMock).to have_requested(:patch, "#{base}/issues/i2") + end + end + + # One record is one request, so a failure on the k-th leaves the k-1 before + # it written and written for good: the error names them, where the API error + # alone would read as "the write failed, nothing happened" and a retry of the + # whole selection would write them twice. + describe 'a write failing partway through the selection' do + it 'names the records already written' do + stub_request(:patch, "#{base}/issues/i1").to_return(json('data' => issue_payload('i1'))) + stub_request(:patch, "#{base}/issues/i2").to_return(json({ 'message' => 'state is invalid' }, 422)) + + expect { issues.update(nil, id_filter(operators::IN, %w[i1 i2]), 'state' => 'closed') } + .to raise_error(PartialWriteError, /1 of 2 PylonIssue records were updated and then 'i2' failed/) + expect(WebMock).to have_requested(:patch, "#{base}/issues/i1") + end + + it 'names the records already deleted' do + stub_request(:delete, "#{base}/issues/i1").to_return(status: 204) + stub_request(:delete, "#{base}/issues/i2").to_return(json({ 'message' => 'gone' }, 404)) + + expect { issues.delete(nil, id_filter(operators::IN, %w[i1 i2])) } + .to raise_error(PartialWriteError, /records already deleted are i1/) + end + + # Nothing was written, so the failure is the whole of what happened and + # travels as the reason Pylon gave — as a ValidationError, the agent + # answering an APIError with 'Unexpected error' whatever it carries. + it 'surfaces the refusal itself when the first record failed' do + stub_request(:delete, "#{base}/issues/i1").to_return(json({ 'message' => 'gone' }, 404)) + + expect { issues.delete(nil, id_filter(operators::IN, %w[i1 i2])) } + .to raise_error(WriteRejectedError, %r{delete\(issues/i1\).*gone}m) + end + end + + # Pylon's own refusal is the likeliest way a write fails, and `APIError` + # descends from the package's Error, which the agent's translator answers + # with 'Unexpected error': a 4xx is re-raised as a ValidationError so the + # reason reaches the operator, and nothing else is. + describe 'a write Pylon refused' do + it 'surfaces the reason a rejected create was given' do + stub_request(:post, "#{base}/issues") + .to_return(json({ 'message' => 'title is required' }, 422)) + + expect { issues.create(nil, 'title' => '', 'body_html' => '

b

') } + .to raise_error(WriteRejectedError, /title is required/) + end + + it 'surfaces the reason a rejected update was given' do + stub_request(:patch, "#{base}/issues/i1").to_return(json({ 'message' => 'unknown state' }, 422)) + + expect { issues.update(nil, id_filter(operators::EQUAL, 'i1'), 'state' => 'nope') } + .to raise_error(WriteRejectedError, /unknown state/) + end + + # Not the operator's to fix, and not theirs to be told to fix: a gateway + # error stays the APIError it was, carrying its status for the agent to + # answer with. + it 'leaves a Pylon-side failure as it was' do + stub_request(:post, "#{base}/issues").to_return(json({ 'message' => 'boom' }, 500)) + + expect { issues.create(nil, 'title' => 'Boom', 'body_html' => '

b

') } + .to raise_error(APIError) { |error| expect(error.status).to eq(500) } + end + end + + # `id in` names the records to act on, and the same one named twice is one + # record: writing it twice would answer 404 on the second delete and report + # a partial failure of a delete that fully succeeded. + describe 'an id named twice in the same selection' do + it 'writes the record once' do + stub_request(:patch, "#{base}/issues/i1").to_return(json('data' => issue_payload('i1'))) + + issues.update(nil, id_filter(operators::IN, %w[i1 i1]), 'state' => 'closed') + + expect(WebMock).to have_requested(:patch, "#{base}/issues/i1").once + end + + # The caps bound records, not mentions: a selection naming the same id + # over and over reaches one record and is not refused for reaching many. + it 'counts it once against the cap' do + stub_request(:patch, "#{base}/issues/i1").to_return(json('data' => issue_payload('i1'))) + + issues.update(nil, id_filter(operators::IN, ['i1'] * 25), 'state' => 'closed') + + expect(WebMock).to have_requested(:patch, "#{base}/issues/i1").once + end + end + + describe '#update naming a field Pylon only takes on a create' do + it 'refuses the update when the operator changed it' do + stub_request(:get, "#{base}/issues/i1").to_return(json('data' => issue_payload('i1'))) + + expect { issues.update(nil, id_filter(operators::EQUAL, 'i1'), 'body_html' => '

louder

') } + .to raise_error(UnsupportedWriteError, /'body_html' cannot be set here on a PylonIssue/) + end + + # A form resending an untouched field asks for nothing, so the rest of the + # edit goes through rather than erroring on a value nobody changed. + it 'drops it, and writes the rest, when it holds the value already stored' do + stub_request(:get, "#{base}/issues/i1").to_return(json('data' => issue_payload('i1'))) + stub_request(:patch, "#{base}/issues/i1").to_return(json('data' => issue_payload('i1'))) + + issues.update(nil, id_filter(operators::EQUAL, 'i1'), + 'body_html' => '

boom

', 'title' => 'Louder') + + expect(WebMock).to have_requested(:patch, "#{base}/issues/i1").with(body: { 'title' => 'Louder' }) + end + + # An unchecked box over a record holding nothing is not an edit: Pylon + # returns a null where the form sends `false`, and refusing that pair + # would fail every edit whose form carries one. + it 'drops a boolean left false over a record holding nothing' do + stub_request(:get, "#{base}/issues/i1").to_return(json('data' => issue_payload('i1'))) + stub_request(:patch, "#{base}/issues/i1").to_return(json('data' => issue_payload('i1'))) + + issues.update(nil, id_filter(operators::EQUAL, 'i1'), 'author_unverified' => false, 'title' => 'Louder') + + expect(WebMock).to have_requested(:patch, "#{base}/issues/i1").with(body: { 'title' => 'Louder' }) + end + + # Over a record holding `true` the same `false` is the operator unchecking + # the box: Pylon cannot write it, and dropping it would report an edit it + # never performed. + it 'refuses a boolean the operator unchecked' do + stub_request(:get, "#{base}/issues/i1") + .to_return(json('data' => issue_payload('i1', 'author_unverified' => true))) + + expect { issues.update(nil, id_filter(operators::EQUAL, 'i1'), 'author_unverified' => false) } + .to raise_error(UnsupportedWriteError, /'author_unverified' cannot be set here on a PylonIssue/) + end + + # Same story for a value cleared rather than unchecked: an empty body over + # a stored one is an edit, where an empty body over an empty one is not. + it 'refuses a string the operator cleared' do + stub_request(:get, "#{base}/issues/i1").to_return(json('data' => issue_payload('i1'))) + + expect { issues.update(nil, id_filter(operators::EQUAL, 'i1'), 'body_html' => '') } + .to raise_error(UnsupportedWriteError, /'body_html' cannot be set here on a PylonIssue/) + end + + # The markup an editor hands back may be the markup it was given, + # re-indented. Refusing that would name a field the operator never touched. + it 'drops a string the editor only re-indented' do + stub_request(:get, "#{base}/issues/i1").to_return(json('data' => issue_payload('i1'))) + stub_request(:patch, "#{base}/issues/i1").to_return(json('data' => issue_payload('i1'))) + + issues.update(nil, id_filter(operators::EQUAL, 'i1'), + 'body_html' => "

boom

\n", 'title' => 'Louder') + + expect(WebMock).to have_requested(:patch, "#{base}/issues/i1").with(body: { 'title' => 'Louder' }) + end + + # Nothing here may claim a value is unchanged on a record it never read, + # and one record short of the selection is enough: dropping the field would + # write the rest of the patch to a record whose stored value is unknown and + # report the whole edit as performed. + it 'refuses it when one of the named records could not be read' do + stub_request(:get, "#{base}/issues/i1").to_return(json('data' => issue_payload('i1'))) + stub_request(:get, "#{base}/issues/i2").to_return(json({ 'message' => 'gone' }, 404)) + + expect do + issues.update(nil, id_filter(operators::IN, %w[i1 i2]), + 'body_html' => '

boom

', 'title' => 'Louder') + end.to raise_error(UnsupportedWriteError, /'body_html' cannot be set here on a PylonIssue/) + expect(WebMock).not_to have_requested(:patch, %r{/issues/}) + end + + # Both offending fields at once: refusing them one at a time would have the + # operator undo one, retry, and learn about the next. + it 'names every field of the wrong direction in one message' do + stub_request(:get, "#{base}/issues/i1") + .to_return(json('data' => issue_payload('i1', 'author_unverified' => true))) + + expect do + issues.update(nil, id_filter(operators::EQUAL, 'i1'), + 'body_html' => '

louder

', 'author_unverified' => false) + end.to raise_error(UnsupportedWriteError, /'body_html', 'author_unverified' cannot be set here/) + end + + # The filter was already resolved into ids, so reading it again would spend + # the same requests twice and, carrying no page of its own, walk every + # record it matches instead of the one about to be written. + it 'reads the stored value by id rather than running the filter a second time' do + contact = { 'id' => 'c1', 'name' => 'Ada', 'email' => 'ada@acme.test' } + stub_request(:post, "#{base}/contacts/search").to_return(json('data' => [contact])) + stub_request(:get, "#{base}/contacts/c1").to_return(json('data' => contact)) + stub_request(:patch, "#{base}/contacts/c1").to_return(json('data' => contact)) + + contacts.update(nil, filter(condition_tree: leaf('name', operators::EQUAL, 'Ada')), + 'email' => 'ada@acme.test', 'avatar_url' => 'http://x') + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search").once + expect(WebMock).to have_requested(:get, "#{base}/contacts/c1") + expect(WebMock).to have_requested(:patch, "#{base}/contacts/c1").with(body: { 'avatar_url' => 'http://x' }) + end + end + + describe '#delete' do + it 'deletes every record the filter names' do + stub_request(:delete, "#{base}/issues/i1").to_return(status: 204) + stub_request(:delete, "#{base}/issues/i2").to_return(status: 204) + + issues.delete(nil, id_filter(operators::IN, %w[i1 i2])) + + expect(WebMock).to have_requested(:delete, "#{base}/issues/i1") + expect(WebMock).to have_requested(:delete, "#{base}/issues/i2") + end + + it 'deletes nothing when the filter matches no record' do + stub_request(:post, "#{base}/accounts/search").to_return(json('data' => [])) + + accounts.delete(nil, filter(condition_tree: leaf('name', operators::EQUAL, 'Nope'))) + + expect(WebMock).not_to have_requested(:delete, %r{/accounts/}) + end + end + + # One request per record, sequentially, and no way to undo the ones already + # written: past the cap the write is refused rather than applied to the first + # records and reported as done for the whole selection. + describe 'a write reaching more records than one pass covers' do + it 'refuses it before spending a single request' do + ids = Array.new(21) { |index| "i#{index}" } + + expect { issues.delete(nil, id_filter(operators::IN, ids)) } + .to raise_error(UnsupportedWriteError, /applies to 21 PylonIssue records/) + expect(WebMock).not_to have_requested(:delete, %r{/issues/}) + end + + # The resolution asks for one record past the cap, so the overflow is seen + # rather than counted: reporting the window as a count would name 21 to an + # operator whose selection holds thousands. + it 'refuses it when the overflow only shows once the filter is resolved, without naming a count' do + stub_request(:post, "#{base}/accounts/search") + .to_return(json('data' => Array.new(21) { |index| { 'id' => "a#{index}", 'name' => 'Acme' } })) + + expect { accounts.delete(nil, filter(condition_tree: leaf('name', operators::EQUAL, 'Acme'))) } + .to raise_error(UnsupportedWriteError, /applies to more than the 20 PylonAccount records one pass covers/) + expect(WebMock).not_to have_requested(:delete, %r{/accounts/}) + end + + # The ids a filter names are not the records the write applies to when the + # filter narrows them further: the collections filtering `id` server-side + # learn the real count in one request, and write it. + it 'writes the records a narrowed selection really matches' do + stub_request(:post, "#{base}/accounts/search") + .to_return(json('data' => [{ 'id' => 'a1', 'name' => 'Acme' }])) + stub_request(:patch, "#{base}/accounts/a1").to_return(json('data' => { 'id' => 'a1' })) + + ids = Array.new(25) { |index| "a#{index}" } + accounts.update(nil, filter(condition_tree: branch('And', [leaf('id', operators::IN, ids), + leaf('name', operators::EQUAL, 'Acme')])), + 'name' => 'Acme Inc') + + expect(WebMock).to have_requested(:patch, "#{base}/accounts/a1").with(body: { 'name' => 'Acme Inc' }) + end + + # An issue is read one request per named id, so past the fan-out the + # resolution would stop short and the write would cover part of the + # selection. Refused — as the ids it names, never as records it was found + # to apply to. + it 'refuses more named ids than the collection can resolve, without claiming they all match' do + ids = Array.new(25) { |index| "i#{index}" } + + expect do + issues.update(nil, filter(condition_tree: branch('And', [leaf('id', operators::IN, ids), + leaf('state', operators::EQUAL, 'new')])), + 'title' => 'Louder') + end.to raise_error(UnsupportedWriteError, /names 25 PylonIssue records and filters them further/) + expect(WebMock).not_to have_requested(:get, %r{/issues/}) + end + + # The count a refusal is worded around is what the `and` names, and an + # `and` of two `id` conditions names their intersection: the wider of the + # two would refuse a selection of three. + it 'counts the intersection of two id conditions, not the wider of them' do + wide = Array.new(25) { |index| "i#{index}" } + %w[i2 i4].each do |id| + stub_request(:get, "#{base}/issues/#{id}") + .to_return(json('data' => { 'id' => id, 'state' => 'new' })) + stub_request(:patch, "#{base}/issues/#{id}").to_return(json('data' => { 'id' => id })) + end + tree = branch('And', [leaf('id', operators::IN, wide), leaf('id', operators::IN, %w[i2 i4])]) + + issues.update(nil, filter(condition_tree: tree), 'title' => 'Louder') + + %w[i2 i4].each { |id| expect(WebMock).to have_requested(:patch, "#{base}/issues/#{id}") } + expect(WebMock).not_to have_requested(:get, "#{base}/issues/i0") + end + + # The cap is a budget of requests, not of writes: where a record is read + # through its own endpoint, every read the path owes it comes out of the + # same twenty, so the records one pass reaches halve for each of them. + it 'halves the reach when the patch has every record read before it is written' do + ids = Array.new(11) { |index| "i#{index}" } + + expect { issues.update(nil, id_filter(operators::IN, ids), 'body_html' => '

x

', 'title' => 'Louder') } + .to raise_error(UnsupportedWriteError, /applies to 11 PylonIssue records, more than the 10 one pass/) + expect(WebMock).not_to have_requested(:get, %r{/issues/}) + expect(WebMock).not_to have_requested(:patch, %r{/issues/}) + end + + # The same eleven records, with nothing to read before writing to them. + it 'keeps the full reach when the patch owes the records no read' do + stub_request(:patch, %r{/issues/i\d+}).to_return(json('data' => issue_payload('i1'))) + + issues.update(nil, id_filter(operators::IN, Array.new(11) { |index| "i#{index}" }), 'title' => 'Louder') + + expect(WebMock).to have_requested(:patch, %r{/issues/i\d+}).times(11) + end + + # A selection naming ids is resolved by reading each of them, so it is + # bounded by the same halved reach — and refused before the first of those + # reads rather than after twenty of them. + it 'refuses a resolution costing a request per record past the halved reach' do + ids = Array.new(12) { |index| "i#{index}" } + + expect do + issues.update(nil, filter(condition_tree: branch('And', [leaf('id', operators::IN, ids), + leaf('state', operators::EQUAL, 'new')])), + 'title' => 'Louder') + end.to raise_error(UnsupportedWriteError, /names 12 PylonIssue records .* more than the 10 one pass covers/m) + expect(WebMock).not_to have_requested(:get, %r{/issues/}) + end + + # A selection naming no id is resolved by one page of the search endpoint, + # whose cost does not grow with the count: nothing to charge per record, so + # the full reach stands. + it 'keeps the full reach when the resolution costs one request whatever the count' do + stub_request(:post, "#{base}/issues/search") + .to_return(json('data' => Array.new(21) { |index| issue_payload("i#{index}") })) + + expect { issues.delete(nil, filter(condition_tree: leaf('state', operators::EQUAL, 'new'))) } + .to raise_error(UnsupportedWriteError, /more than the 20 PylonIssue records one pass covers/) + expect(WebMock).to have_requested(:post, "#{base}/issues/search").once + end + + # And nothing is charged per record where a record costs no request of its + # own: the search endpoint filters `id`, so reading the stored value of a + # whole selection is one request, whatever the reach. + it 'keeps the full reach on a collection whose read does not fan out' do + ids = Array.new(11) { |index| "c#{index}" } + stored = ids.map { |id| { 'id' => id, 'name' => 'Ada', 'email' => 'ada@acme.test' } } + stub_request(:post, "#{base}/contacts/search").to_return(json('data' => stored)) + stub_request(:patch, %r{/contacts/c\d+}).to_return(json('data' => stored.first)) + + contacts.update(nil, id_filter(operators::IN, ids), 'email' => 'ada@acme.test', 'name' => 'Ada Lovelace') + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search").once + expect(WebMock).to have_requested(:patch, %r{/contacts/c\d+}).times(11) + end + + # "Select all except these" reaches PylonIssue as `id not_in`, which its + # endpoint cannot filter: the read refuses it, and so does the delete. The + # message names that selection rather than the `and`/`or` of a filter the + # operator never wrote. + it 'refuses an excluding selection on the collection that cannot filter an id' do + expect { issues.delete(nil, id_filter(operators::NOT_IN, %w[i1])) } + .to raise_error(UnsupportedOperatorError, + /Select the records to act on rather than the ones to leave out/) + end + end + + describe 'a verb Pylon exposes no endpoint for' do + it 'refuses to create a user' do + expect { users.create(nil, 'name' => 'Ada') } + .to raise_error(UnsupportedWriteError, /A PylonUser record cannot be created/) + end + + it 'refuses to delete a user' do + expect { users.delete(nil, id_filter(operators::EQUAL, 'u1')) } + .to raise_error(UnsupportedWriteError, /A PylonUser record cannot be deleted/) + end + + it 'refuses to delete a team' do + expect { teams.delete(nil, id_filter(operators::EQUAL, 't1')) } + .to raise_error(UnsupportedWriteError, /A PylonTeam record cannot be deleted/) + end + + # The refusal holds whatever the selection reaches, so it comes before the + # ids are resolved: answering with the cap would send the operator to + # narrow a selection that was never the problem, and answering a selection + # matching nothing with a silent success would report a delete on a + # collection that cannot perform one. + it 'refuses a selection wider than the cap without naming the cap' do + expect { teams.delete(nil, id_filter(operators::IN, (1..25).map { |i| "t#{i}" })) } + .to raise_error(UnsupportedWriteError, /A PylonTeam record cannot be deleted/) + end + + it 'refuses a selection matching nothing rather than answering it' do + expect { teams.delete(nil, filter(condition_tree: leaf('name', operators::EQUAL, 'nope'))) } + .to raise_error(UnsupportedWriteError, /A PylonTeam record cannot be deleted/) + end + + it 'refuses before spending a request to resolve the selection' do + expect { teams.delete(nil, filter(condition_tree: leaf('name', operators::EQUAL, 'x'))) } + .to raise_error(UnsupportedWriteError) + expect(WebMock).not_to have_requested(:get, "#{base}/teams") + end + end + + describe 'the collections read through their own endpoints' do + it 'writes an account type under the name Pylon takes it as' do + stub_request(:post, "#{base}/accounts").to_return(json('data' => { 'id' => 'a1', 'name' => 'Acme' })) + + accounts.create(nil, 'name' => 'Acme', 'type' => 'customer', 'domains' => %w[acme.test]) + + expect(WebMock).to have_requested(:post, "#{base}/accounts") + .with(body: { 'name' => 'Acme', 'account_type' => 'customer', 'domains' => %w[acme.test] }) + end + + it 'refuses to disable an account that does not exist yet' do + expect { accounts.create(nil, 'name' => 'Acme', 'is_disabled' => true) } + .to raise_error(UnsupportedWriteError, /'is_disabled' cannot be set here on a PylonAccount/) + end + + # An account is created enabled, which is what the form asks for when the + # box is left unchecked: the create it produces is the one requested. + it 'creates an account whose update-only boolean is left false' do + stub_request(:post, "#{base}/accounts").to_return(json('data' => { 'id' => 'a1', 'name' => 'Acme' })) + + accounts.create(nil, 'name' => 'Acme', 'is_disabled' => false) + + expect(WebMock).to have_requested(:post, "#{base}/accounts").with(body: { 'name' => 'Acme' }) + end + + # `POST /contacts` takes the primary address and `PATCH /contacts/{id}` the + # list, so one payload never carries both projections of the addresses. + it 'creates a contact with its primary address' do + stub_request(:post, "#{base}/contacts").to_return(json('data' => { 'id' => 'c1', 'name' => 'Ada' })) + + contacts.create(nil, 'name' => 'Ada', 'email' => 'ada@acme.test', 'emails' => []) + + expect(WebMock).to have_requested(:post, "#{base}/contacts") + .with(body: { 'name' => 'Ada', 'email' => 'ada@acme.test' }) + end + + it 'refuses to change the primary address of an existing contact' do + stub_request(:get, "#{base}/contacts/c1") + .to_return(json('data' => { 'id' => 'c1', 'name' => 'Ada', 'email' => 'ada@acme.test' })) + + expect { contacts.update(nil, id_filter(operators::EQUAL, 'c1'), 'email' => 'new@acme.test') } + .to raise_error(UnsupportedWriteError, /'email' cannot be set here on a PylonContact/) + end + + it 'patches a contact' do + stub_request(:patch, "#{base}/contacts/c1").to_return(json('data' => { 'id' => 'c1', 'name' => 'Ada' })) + + contacts.update(nil, id_filter(operators::EQUAL, 'c1'), 'name' => 'Ada', 'account_id' => 'a1') + + expect(WebMock).to have_requested(:patch, "#{base}/contacts/c1") + .with(body: { 'name' => 'Ada', 'account_id' => 'a1' }) + end + + # The only create whose record is serialized by a collection read in whole: + # `POST /teams` answers with the members nested where the column carries + # their ids, so the flattening the read side does has to run here too. + it 'creates a team and flattens the members of the record it answers with' do + stub_request(:post, "#{base}/teams") + .to_return(json('data' => { 'id' => 't1', 'name' => 'Support', + 'users' => [{ 'id' => 'u1', 'email' => 'ada@acme.test' }, + { 'id' => 'u2' }] })) + + record = teams.create(nil, 'name' => 'Support', 'user_ids' => %w[u1 u2]) + + expect(record).to eq('id' => 't1', 'name' => 'Support', 'user_ids' => %w[u1 u2]) + expect(WebMock).to have_requested(:post, "#{base}/teams") + .with(body: { 'name' => 'Support', 'user_ids' => %w[u1 u2] }) + end + + it 'replaces the members of a team' do + stub_request(:patch, "#{base}/teams/t1").to_return(json('data' => { 'id' => 't1', 'name' => 'Support' })) + + teams.update(nil, id_filter(operators::EQUAL, 't1'), 'name' => 'Support', 'user_ids' => %w[u1 u2]) + + expect(WebMock).to have_requested(:patch, "#{base}/teams/t1") + .with(body: { 'name' => 'Support', 'user_ids' => %w[u1 u2] }) + end + + it 'patches the status of a user' do + stub_request(:patch, "#{base}/users/u1").to_return(json('data' => { 'id' => 'u1', 'name' => 'Ada' })) + + users.update(nil, id_filter(operators::EQUAL, 'u1'), 'status' => 'away', 'email' => 'ada@acme.test') + + expect(WebMock).to have_requested(:patch, "#{base}/users/u1").with(body: { 'status' => 'away' }) + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/configuration_spec.rb b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/configuration_spec.rb new file mode 100644 index 000000000..1054aa049 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/configuration_spec.rb @@ -0,0 +1,157 @@ +RSpec.describe ForestAdminDatasourcePylon::Configuration do + let(:valid_args) { { api_key: 'pk_test_xyz' } } + + describe '#initialize' do + it 'accepts a valid api_key' do + expect(described_class.new(**valid_args).api_key).to eq('pk_test_xyz') + end + + it 'raises a ConfigurationError when api_key is nil' do + expect { described_class.new(api_key: nil) } + .to raise_error(ForestAdminDatasourcePylon::ConfigurationError, /api_key/) + end + + it 'raises a ConfigurationError when api_key is blank' do + expect { described_class.new(api_key: ' ') } + .to raise_error(ForestAdminDatasourcePylon::ConfigurationError, /api_key/) + end + + it 'defaults to the public Pylon base URL' do + expect(described_class.new(**valid_args).base_url).to eq('https://api.usepylon.com') + end + + it 'honours an explicit base_url override' do + config = described_class.new(**valid_args, base_url: 'https://example.test') + expect(config.base_url).to eq('https://example.test') + end + + it 'defaults the timeouts' do + config = described_class.new(**valid_args) + expect([config.open_timeout, config.timeout]).to eq([5, 30]) + end + + it 'keeps configurable timeouts' do + config = described_class.new(**valid_args, open_timeout: 1, timeout: 2) + expect([config.open_timeout, config.timeout]).to eq([1, 2]) + end + + # What is read while the datasource is being constructed is waited on by a + # Rails boot, not by a request that has already returned a page. + it 'defaults the boot timeouts under the ones every other call gets' do + config = described_class.new(**valid_args) + expect([config.boot_open_timeout, config.boot_timeout]).to eq([3, 10]) + end + + it 'keeps configurable boot timeouts' do + config = described_class.new(**valid_args, boot_open_timeout: 1, boot_timeout: 2) + expect([config.boot_open_timeout, config.boot_timeout]).to eq([1, 2]) + end + + it 'defaults the boot retry policy to one bounded retry' do + policy = described_class.new(**valid_args).boot_retry_policy + expect([policy.max_retries, policy.max_interval]).to eq([1, 2]) + end + + it 'accepts an injected boot retry policy' do + policy = ForestAdminDatasourcePylon::RetryPolicy.new(max_retries: 0) + expect(described_class.new(**valid_args, boot_retry_policy: policy).boot_retry_policy).to be(policy) + end + + it 'defaults to a standard retry policy' do + expect(described_class.new(**valid_args).retry_policy) + .to be_a(ForestAdminDatasourcePylon::RetryPolicy) + end + + it 'accepts an injected retry policy' do + policy = ForestAdminDatasourcePylon::RetryPolicy.new(max_retries: 9) + expect(described_class.new(**valid_args, retry_policy: policy).retry_policy).to be(policy) + end + + it 'throttles by default' do + expect(described_class.new(**valid_args).rate_limiter) + .to be_a(ForestAdminDatasourcePylon::RateLimiter) + end + + it 'accepts an injected limiter' do + limiter = ForestAdminDatasourcePylon::RateLimiter.new(max_wait: 1) + expect(described_class.new(**valid_args, rate_limiter: limiter).rate_limiter).to be(limiter) + end + + # One limiter per configuration, so per token: Pylon meters the token, and + # two agents holding different ones do not share a budget. + it 'gives each configuration its own window' do + first = described_class.new(**valid_args) + expect(described_class.new(**valid_args).rate_limiter).not_to be(first.rate_limiter) + end + + it 'takes the throttling out of the stack when handed nil' do + expect(described_class.new(**valid_args, rate_limiter: nil).rate_limiter).to be_nil + end + end + + describe '#url' do + it 'returns the base URL unversioned' do + expect(described_class.new(**valid_args).url).to eq('https://api.usepylon.com') + end + + it 'trims a trailing slash' do + config = described_class.new(**valid_args, base_url: 'https://example.test/') + expect(config.url).to eq('https://example.test') + end + end + + describe '#base_path' do + it 'is empty against the API itself, which mounts its endpoints on the host' do + expect(described_class.new(**valid_args).base_path).to eq('') + end + + # `RateLimits` is keyed on the endpoint, so a base url mounted under a + # subpath — an egress proxy, a mock server — has to have that prefix taken + # off a path before the table is asked. + it 'is the prefix a base url mounted under a subpath puts in front of every path' do + config = described_class.new(**valid_args, base_url: 'https://proxy.test/pylon/v1') + expect(config.base_path).to eq('/pylon/v1') + end + + it 'carries no trailing slash, `url` having trimmed it' do + config = described_class.new(**valid_args, base_url: 'https://proxy.test/pylon/') + expect(config.base_path).to eq('/pylon') + end + end + + # Nothing prints a Configuration on purpose: what reaches an `inspect` is a + # Rails error page, or a `logger.debug` of the datasource. The default would + # print the bearer token in clear in both. + describe '#inspect' do + it 'masks the api key, and keeps the base url a reader needs to place it' do + output = described_class.new(**valid_args, base_url: 'https://proxy.test/pylon').inspect + + expect(output).to include('[FILTERED]', 'https://proxy.test/pylon') + expect(output).not_to include('pk_test_xyz') + end + + # Masking the Configuration alone is not enough: the token also rides in the + # headers of the client's Faraday connections, which `Faraday::Connection` + # prints in clear, and every collection reaches those through the + # datasource. The whole graph is asserted so a new holder cannot reopen a + # path quietly. + it 'is masked on every object this package hands out, not on the configuration alone' do + stub_custom_fields + datasource = ForestAdminDatasourcePylon::Datasource.new(api_key: 'pk_test_xyz') + datasource.client.send(:connection) + + [datasource, datasource.client, datasource.configuration, + datasource.get_collection('PylonIssue'), datasource.get_collection('PylonUser')].each do |object| + expect(object.inspect).not_to include('pk_test_xyz') + end + end + + it 'keeps each of them worth printing' do + stub_custom_fields + datasource = ForestAdminDatasourcePylon::Datasource.new(api_key: 'pk_test_xyz') + + expect(datasource.inspect).to include('PylonIssue', 'PylonTeam') + expect(datasource.client.inspect).to include('https://api.usepylon.com') + end + end +end diff --git a/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/datasource_spec.rb b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/datasource_spec.rb new file mode 100644 index 000000000..e67058018 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/datasource_spec.rb @@ -0,0 +1,184 @@ +RSpec.describe ForestAdminDatasourcePylon::Datasource do + let(:datasource) { described_class.new(api_key: 'k') } + let(:base) { ForestAdminDatasourcePylon::Configuration::DEFAULT_BASE_URL } + + before { stub_custom_fields } + + def definition(slug, type: 'text', **extra) + { 'id' => "cf_#{slug}", 'slug' => slug, 'label' => slug.capitalize, 'type' => type }.merge(extra) + end + + def select_definition(slug, type: 'select', options: %w[p1 p2]) + definition(slug, type: type, + 'select_metadata' => { 'options' => options.map { |o| { 'label' => o.upcase, 'slug' => o } } }) + end + + def leaf(field, operator, value) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def filter(condition_tree) + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition_tree) + end + + def stub_issues(custom_fields) + stub_request(:post, "#{base}/issues/search") + .to_return(status: 200, headers: { 'Content-Type' => 'application/json' }, + body: { 'data' => [{ 'id' => 'i1', 'custom_fields' => custom_fields }] }.to_json) + end + + it 'builds a configuration from the api key' do + expect(datasource.configuration.api_key).to eq('k') + expect(datasource.configuration.url).to eq(ForestAdminDatasourcePylon::Configuration::DEFAULT_BASE_URL) + end + + it 'forwards the remaining options to the configuration' do + stub_custom_fields(base: 'https://pylon.test') + custom = described_class.new(api_key: 'k', base_url: 'https://pylon.test/', timeout: 3) + + expect(custom.configuration.url).to eq('https://pylon.test') + expect(custom.configuration.timeout).to eq(3) + end + + it 'exposes a client built on that configuration' do + expect(datasource.client).to be_a(ForestAdminDatasourcePylon::Client) + end + + it 'registers the five collections Pylon exposes' do + expect(datasource.collections.keys) + .to eq(%w[PylonIssue PylonAccount PylonContact PylonUser PylonTeam]) + expect(datasource.get_collection('PylonIssue')).to be_a(ForestAdminDatasourcePylon::Collections::Issue) + expect(datasource.get_collection('PylonAccount')).to be_a(ForestAdminDatasourcePylon::Collections::Account) + expect(datasource.get_collection('PylonContact')).to be_a(ForestAdminDatasourcePylon::Collections::Contact) + expect(datasource.get_collection('PylonUser')).to be_a(ForestAdminDatasourcePylon::Collections::User) + expect(datasource.get_collection('PylonTeam')).to be_a(ForestAdminDatasourcePylon::Collections::Team) + end + + # Every relation declared by one of them points at another: a foreign + # collection left unregistered is a schema the agent refuses to boot on. + it 'registers a collection for every foreign collection its relations point at' do + relations = datasource.collections.values.flat_map do |collection| + collection.fields.values.reject { |field| field.type == 'Column' } + end + + expect(relations).not_to be_empty + expect(relations.map(&:foreign_collection).uniq - datasource.collections.keys).to be_empty + end + + it 'refuses to build without an api key' do + expect { described_class.new(api_key: nil) }.to raise_error(ForestAdminDatasourcePylon::ConfigurationError) + end + + describe 'custom fields' do + # Pylon indexes its definitions by object type and asks for one on every + # call, so the three collections carrying custom fields cost one call each. + it 'reads the definitions of the three object types Pylon carries them on' do + datasource + + %w[issue account contact].each do |object_type| + expect(WebMock).to have_requested(:get, "#{base}/custom-fields") + .with(query: { 'object_type' => object_type }).once + end + end + + # A definition is a column of the collection it was declared on, and of no + # other: Pylon scopes a custom field to one object type. + it 'adds each definition to the collection its object type registers' do + stub_custom_fields(issue: [definition('severity')], account: [definition('tier')], + contact: [definition('nps', type: 'number')]) + + expect(datasource.get_collection('PylonIssue').fields).to have_key('severity') + expect(datasource.get_collection('PylonAccount').fields).to have_key('tier') + expect(datasource.get_collection('PylonContact').fields['nps'].column_type).to eq('Number') + expect(datasource.get_collection('PylonIssue').fields).not_to have_key('tier') + end + + # Users and teams have no custom field at all in Pylon, so nothing is asked + # for them -- an absent call rather than one answering an empty list. + it 'asks for no definition on users and teams' do + datasource + + %w[user team].each do |object_type| + expect(WebMock).not_to have_requested(:get, "#{base}/custom-fields") + .with(query: { 'object_type' => object_type }) + end + end + + # The definitions are read while the agent boots: a Pylon that is down, or a + # token without the permission, has to cost the operator the custom columns + # rather than the agent. + it 'boots on the native schema when the definitions cannot be read' do + stub_request(:get, "#{base}/custom-fields").to_return(status: 500, body: '{}', + headers: { 'Content-Type' => 'application/json' }) + + expect { datasource }.not_to raise_error + expect(datasource.collections.keys).to include('PylonIssue') + expect(datasource.get_collection('PylonIssue').custom_fields).to eq([]) + end + + # A custom field is filtered through the very slug it is read by, so nothing + # is held datasource-wide -- and two Pylon organizations in the same agent + # cannot end up advertising each other's columns. + it 'keeps the custom fields of one datasource out of another' do + stub_custom_fields(issue: [definition('severity')]) + first = described_class.new(api_key: 'k') + + stub_custom_fields(issue: [definition('tier')]) + second = described_class.new(api_key: 'k') + + expect(first.get_collection('PylonIssue').fields).to have_key('severity') + expect(first.get_collection('PylonIssue').fields).not_to have_key('tier') + expect(second.get_collection('PylonIssue').fields).to have_key('tier') + expect(second.get_collection('PylonIssue').fields).not_to have_key('severity') + end + + # The one place the introspected shape meets the read pipeline: a definition + # becomes a column, the payload's value is read through it, and a filter on + # it travels as the slug Pylon indexes it by -- the three are held together + # by the slug alone, so a spec on each in isolation would not catch a rename. + describe 'the round trip of an introspected field' do + it 'reads a select back and filters it by the option slug' do + stub_custom_fields(issue: [select_definition('priority_level')]) + stub_issues('priority_level' => { 'slug' => 'priority_level', 'value' => 'p1' }) + collection = datasource.get_collection('PylonIssue') + + rows = collection.list(nil, filter(leaf('priority_level', 'equal', 'p1')), nil) + + expect(collection.fields['priority_level'].enum_values).to eq(%w[p1 p2]) + expect(rows.first).to include('priority_level' => 'p1') + expect(WebMock).to have_requested(:post, "#{base}/issues/search") + .with(body: hash_including('filter' => { 'field' => 'priority_level', + 'operator' => 'equals', 'value' => 'p1' })) + end + + # A multiselect is read out of `values` rather than `value`, and Pylon + # accepts no filter on one -- the column carries the list and nothing else. + it 'reads a multiselect as the list of its option slugs, unfilterable' do + stub_custom_fields(issue: [select_definition('zones', type: 'multiselect', options: %w[eu us])]) + stub_issues('zones' => { 'slug' => 'zones', 'values' => %w[eu us] }) + collection = datasource.get_collection('PylonIssue') + + rows = collection.list(nil, nil, nil) + + expect(collection.fields['zones'].column_type).to eq('Json') + expect(collection.fields['zones'].filter_operators).to eq([]) + expect(rows.first).to include('zones' => %w[eu us]) + expect { collection.list(nil, filter(leaf('zones', 'contains', 'eu')), nil) } + .to raise_error(ForestAdminDatasourcePylon::UnsupportedOperatorError, /not supported on field 'zones'/) + end + end + + # Registration evaluates the collision against the final native schema, so a + # slug shadowing a column the collection already declares is left out. + it 'skips a definition colliding with a native column' do + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + stub_custom_fields(issue: [definition('title')]) + + expect(datasource.get_collection('PylonIssue').custom_fields).to eq([]) + expect(datasource.get_collection('PylonIssue').fields['title'].filter_operators).not_to be_empty + expect(ForestAdminDatasourcePylon.logger) + .to have_received(:warn).with(/'title' on collection 'PylonIssue' conflicts/) + end + end +end diff --git a/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/pagination/cursor_walker_spec.rb b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/pagination/cursor_walker_spec.rb new file mode 100644 index 000000000..dd83f35ac --- /dev/null +++ b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/pagination/cursor_walker_spec.rb @@ -0,0 +1,165 @@ +RSpec.describe ForestAdminDatasourcePylon::Pagination::CursorWalker do + let(:calls) { [] } + + def search_page(ids, next_cursor) + ForestAdminDatasourcePylon::Client::SearchPage.new( + records: ids.map { |id| { 'id' => id } }, next_cursor: next_cursor + ) + end + + # Serves `pages` in order and records the (limit, cursor) each call was made + # with, so a spec can assert on how the walk was driven. + def source(pages) + proc do |limit, cursor| + calls << { limit: limit, cursor: cursor } + pages.fetch(calls.size - 1, search_page([], nil)) + end + end + + def walk(pages, offset:, limit:, walker: described_class.new) + walker.walk(offset: offset, limit: limit, &source(pages)) + end + + it 'returns the records of a single page when it covers the window' do + records = walk([search_page(%w[a b c], nil)], offset: 0, limit: 3) + + expect(records).to eq([{ 'id' => 'a' }, { 'id' => 'b' }, { 'id' => 'c' }]) + expect(calls).to eq([{ limit: 3, cursor: nil }]) + end + + it 'walks pages until the window is covered, then slices off the offset' do + pages = [search_page(%w[a b], 'c1'), search_page(%w[c d], 'c2'), search_page(%w[e f], nil)] + + expect(walk(pages, offset: 4, limit: 2)).to eq([{ 'id' => 'e' }, { 'id' => 'f' }]) + expect(calls.map { |call| call[:cursor] }).to eq([nil, 'c1', 'c2']) + end + + it 'asks only for what is still missing on each page' do + pages = [search_page(%w[a b], 'c1'), search_page(%w[c], 'c2'), search_page(%w[d], nil)] + + walk(pages, offset: 0, limit: 4) + + expect(calls.map { |call| call[:limit] }).to eq([4, 2, 1]) + end + + it 'never requests more than the API maximum in one call' do + walk([search_page(%w[a], nil)], offset: 0, limit: 50_000) + + expect(calls.first[:limit]).to eq(ForestAdminDatasourcePylon::Client::MAX_SEARCH_LIMIT) + end + + it 'stops at the last page and returns fewer records than asked' do + expect(walk([search_page(%w[a b], nil)], offset: 0, limit: 10).size).to eq(2) + expect(calls.size).to eq(1) + end + + it 'returns an empty window when the offset is past the end' do + expect(walk([search_page(%w[a b], nil)], offset: 50, limit: 10)).to eq([]) + end + + it 'fetches nothing when the limit is not positive' do + expect(walk([search_page(%w[a], nil)], offset: 0, limit: 0)).to eq([]) + expect(calls).to be_empty + end + + describe 'a nil limit' do + it 'walks every page the API hands out instead of stopping at one window' do + pages = [search_page(%w[a b], 'c1'), search_page(%w[c d], 'c2'), search_page(%w[e], nil)] + + expect(walk(pages, offset: 0, limit: nil).size).to eq(5) + expect(calls.size).to eq(3) + end + + it 'asks for the whole record budget on each page' do + walk([search_page(%w[a], nil)], offset: 0, limit: nil) + + expect(calls.first[:limit]).to eq(ForestAdminDatasourcePylon::Client::MAX_SEARCH_LIMIT) + end + + it 'still drops the offset' do + pages = [search_page(%w[a b c], 'c1'), search_page(%w[d], nil)] + + expect(walk(pages, offset: 2, limit: nil)).to eq([{ 'id' => 'c' }, { 'id' => 'd' }]) + end + + it 'costs a single request when the first page is the last' do + expect(walk([search_page(%w[a b], nil)], offset: 0, limit: nil).size).to eq(2) + expect(calls.size).to eq(1) + end + end + + describe 'truncation' do + before { allow(ForestAdminDatasourcePylon.logger).to receive(:warn) } + + it 'stops at the page cap and logs a warning' do + pages = Array.new(5) { |i| search_page(["r#{i}"], "c#{i}") } + + expect(walk(pages, offset: 0, limit: 100, walker: described_class.new(max_pages: 3)).size).to eq(3) + expect(calls.size).to eq(3) + expect(ForestAdminDatasourcePylon.logger).to have_received(:warn).with(/Stopped paginating after 3 page/) + end + + it 'stops at the record cap and logs a warning' do + pages = Array.new(5) { |i| search_page(%W[a#{i} b#{i}], "c#{i}") } + + expect(walk(pages, offset: 0, limit: 100, walker: described_class.new(max_records: 4)).size).to eq(4) + expect(ForestAdminDatasourcePylon.logger).to have_received(:warn).with(/4 record\(s\)/) + end + + it 'never asks for more than the remaining record budget' do + pages = [search_page(%w[a b], 'c1'), search_page(%w[c], 'c2')] + + walk(pages, offset: 0, limit: 100, walker: described_class.new(max_records: 3)) + + expect(calls.map { |call| call[:limit] }).to eq([3, 1]) + end + + it 'does not warn when the walk ends naturally' do + walk([search_page(%w[a b], nil)], offset: 0, limit: 100) + + expect(ForestAdminDatasourcePylon.logger).not_to have_received(:warn) + end + + it 'warns when a cap cuts a walk that asked for every record' do + pages = Array.new(5) { |i| search_page(%W[a#{i} b#{i}], "c#{i}") } + + expect(walk(pages, offset: 0, limit: nil, walker: described_class.new(max_records: 4)).size).to eq(4) + expect(ForestAdminDatasourcePylon.logger) + .to have_received(:warn).with(/every record past offset=0; results are truncated/) + end + + it 'does not warn when a window the caller asked for is covered exactly' do + pages = [search_page(%w[a b], 'c1'), search_page(%w[c d], 'c2')] + + expect(walk(pages, offset: 0, limit: 2).size).to eq(2) + expect(ForestAdminDatasourcePylon.logger).not_to have_received(:warn) + end + end + + describe 'defensive stops' do + it 'stops when a page comes back empty despite advertising a next page' do + pages = [search_page([], 'c1'), search_page(%w[a], 'c2')] + + expect(walk(pages, offset: 0, limit: 10)).to eq([]) + expect(calls.size).to eq(1) + end + + it 'stops when the cursor does not advance' do + pages = [search_page(%w[a], 'same'), search_page(%w[b], 'same'), search_page(%w[c], 'same')] + + expect(walk(pages, offset: 0, limit: 10).size).to eq(2) + expect(calls.size).to eq(2) + end + + # Every cursor followed is remembered, not the last one alone, the way + # `Client#collect_pages` walks: a cycle wider than one page would otherwise + # run to a cap and hand the same records back twice over. + it 'stops on a cursor it has already followed, however wide the cycle' do + pages = [search_page(%w[a], 'c1'), search_page(%w[b], 'c2'), search_page(%w[c], 'c1'), + search_page(%w[d], 'c2')] + + expect(walk(pages, offset: 0, limit: 10)).to eq([{ 'id' => 'a' }, { 'id' => 'b' }, { 'id' => 'c' }]) + expect(calls.map { |call| call[:cursor] }).to eq([nil, 'c1', 'c2']) + end + end +end diff --git a/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/plugins/close_issue_spec.rb b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/plugins/close_issue_spec.rb new file mode 100644 index 000000000..ea25bd6e1 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/plugins/close_issue_spec.rb @@ -0,0 +1,281 @@ +module ForestAdminDatasourcePylon + # Stand-in for an action context. The executor only asks it which records were + # selected -- through a column of the host collection, or through their + # primary keys -- so the full ActionContext is not needed here. + class FakeCloseContext + def initialize(records: [], record_ids: []) + @records = records + @record_ids = record_ids + end + + def get_records(_fields = []) = @records + + # Named after the ActionContext method it stands in for. + def get_record_ids = @record_ids # rubocop:disable Naming/AccessorMethodName + end + + RSpec.describe Plugins::CloseIssue do + let(:client) { instance_double(ForestAdminDatasourcePylon::Client) } + let(:datasource) { instance_double(ForestAdminDatasourcePylon::Datasource, client: client) } + let(:result_builder) { ForestAdminDatasourceCustomizer::Decorators::Action::ResultBuilder.new } + let(:action_scope) { ForestAdminDatasourceCustomizer::Decorators::Action::Types::ActionScope } + let(:forest_exception) { ForestAdminDatasourceToolkit::Exceptions::ForestException } + let(:issue_id_field) { 'pylon_issue_id' } + let(:collection_customizer) do + Class.new do + attr_reader :registered + + def initialize = @registered = {} + def add_action(name, action) = @registered[name] = action + end.new + end + + def register(opts = {}) + described_class.new.run(nil, collection_customizer, { datasource: datasource }.merge(opts)) + collection_customizer.registered + end + + describe '#run' do + it 'registers one variant per scope' do + register + + expect(collection_customizer.registered.keys) + .to contain_exactly('Close Pylon issue', 'Close selected Pylon issues') + end + + it 'binds the matching ActionScope to each variant' do + registered = register + + expect(registered['Close Pylon issue'].scope).to eq(action_scope::SINGLE) + expect(registered['Close selected Pylon issues'].scope).to eq(action_scope::BULK) + end + + it 'honors :scopes to keep only the requested variant' do + register(scopes: %i[single]) + + expect(collection_customizer.registered.keys).to contain_exactly('Close Pylon issue') + end + + it 'accepts string scopes as well as symbols' do + register(scopes: %w[bulk]) + + expect(collection_customizer.registered.keys).to contain_exactly('Close selected Pylon issues') + end + + it 'honors the custom names of both variants' do + register(action_name: 'Resolve', bulk_action_name: 'Resolve all') + + expect(collection_customizer.registered.keys).to contain_exactly('Resolve', 'Resolve all') + end + + it 'raises a ForestException on an unknown scope' do + expect { register(scopes: %i[single weird]) }.to raise_error(forest_exception, /Unknown.*weird/) + end + + # Through `to_s`, so a value carrying no `to_sym` reaches the error that + # names it rather than a NoMethodError naming nothing. + it 'names a scope that is neither a symbol nor a string' do + expect { register(scopes: [1]) }.to raise_error(forest_exception, /Unknown.*1/) + end + + it 'raises a ForestException on an empty state' do + expect { register(state: ' ') }.to raise_error(forest_exception, /state cannot be empty/) + end + + it 'raises a ForestException without :datasource' do + expect { described_class.new.run(nil, collection_customizer, {}) } + .to raise_error(forest_exception, /datasource/) + end + + it 'raises a ForestException without a collection' do + expect { described_class.new.run(nil, nil, datasource: datasource) } + .to raise_error(forest_exception, /collection/) + end + end + + describe 'the issues an execution acts on' do + let(:single) { register(scopes: %i[single], issue_id_field: issue_id_field)['Close Pylon issue'] } + let(:on_primary_key) { register(scopes: %i[single])['Close Pylon issue'] } + + it 'reads the id from the configured column of the host record' do + allow(client).to receive(:update_issue) + + result = single.execute.call(FakeCloseContext.new(records: [{ issue_id_field => 'i1' }]), result_builder) + + expect(client).to have_received(:update_issue).with('i1', 'state' => 'closed') + expect(result[:type]).to eq('Success') + expect(result[:message]).to include('Issue i1 closed.') + end + + # What an action registered on PylonIssue itself acts on: no column to + # name, the record's own primary key is the issue. + it 'reads the primary keys when no column is configured' do + allow(client).to receive(:update_issue) + + result = on_primary_key.execute.call(FakeCloseContext.new(record_ids: %w[i7]), result_builder) + + expect(client).to have_received(:update_issue).with('i7', 'state' => 'closed') + expect(result[:message]).to include('Issue i7 closed.') + end + + # A column of issue ids is not a key, so two host records may name the + # same issue: writing it twice and counting it twice is one bug apiece. + it 'writes an issue named by several of the selected records once' do + allow(client).to receive(:update_issue) + records = [{ issue_id_field => 'i1' }, { issue_id_field => 'i1' }, { issue_id_field => 'i2' }] + + result = single.execute.call(FakeCloseContext.new(records: records), result_builder) + + expect(client).to have_received(:update_issue).with('i1', 'state' => 'closed').once + expect(client).to have_received(:update_issue).with('i2', 'state' => 'closed').once + expect(result[:message]).to include('2 issues') + end + + it 'returns an error naming the column when no host record carries an id' do + allow(client).to receive(:update_issue) + + result = single.execute.call(FakeCloseContext.new(records: [{ issue_id_field => nil }]), result_builder) + + expect(client).not_to have_received(:update_issue) + expect(result[:type]).to eq('Error') + expect(result[:message]).to include(issue_id_field) + end + + it 'returns an error when nothing was selected' do + allow(client).to receive(:update_issue) + + result = on_primary_key.execute.call(FakeCloseContext.new(record_ids: []), result_builder) + + expect(client).not_to have_received(:update_issue) + expect(result[:message]).to eq('No Pylon issue selected.') + end + + it 'refuses a selection wider than one run covers, before writing anything' do + ids = Array.new(described_class::MAX_TARGETS + 1) { |i| "i#{i}" } + allow(client).to receive(:update_issue) + + result = on_primary_key.execute.call(FakeCloseContext.new(record_ids: ids), result_builder) + + expect(client).not_to have_received(:update_issue) + expect(result[:type]).to eq('Error') + expect(result[:message]).to include(ids.size.to_s, described_class::MAX_TARGETS.to_s) + end + + it 'writes a selection of exactly what one run covers' do + ids = Array.new(described_class::MAX_TARGETS) { |i| "i#{i}" } + allow(client).to receive(:update_issue) + + result = on_primary_key.execute.call(FakeCloseContext.new(record_ids: ids), result_builder) + + expect(client).to have_received(:update_issue).exactly(ids.size).times + expect(result[:type]).to eq('Success') + end + + # The cap counts issues, not selected records: a column of issue ids is + # not a key, so a wide selection naming few issues is written rather than + # refused on a count the operator cannot see. + it 'counts the issues named rather than the records selected' do + records = Array.new(described_class::MAX_TARGETS + 5) { { issue_id_field => 'i1' } } + allow(client).to receive(:update_issue) + + result = single.execute.call(FakeCloseContext.new(records: records), result_builder) + + expect(client).to have_received(:update_issue).once + expect(result[:type]).to eq('Success') + end + + it 'logs and answers an error when the records cannot be read at all' do + context = instance_double(ForestAdminDatasourceCustomizer::Decorators::Action::Context::ActionContextSingle) + allow(context).to receive(:get_records).and_raise(StandardError, 'boom') + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + allow(client).to receive(:update_issue) + + result = single.execute.call(context, result_builder) + + expect(client).not_to have_received(:update_issue) + expect(result[:type]).to eq('Error') + expect(ForestAdminDatasourcePylon.logger).to have_received(:warn) + .with(a_string_including(issue_id_field, 'boom')) + end + + # A ValidationError is the collection refusing the selection in words + # written for the operator -- PylonIssue naming more issues by id than one + # page of lookups covers, for one. Degraded to an empty list, it would be + # reported as "no issue selected" about a selection they can see they made. + it 'lets a refusal of the selection through rather than reporting nothing selected' do + context = instance_double(ForestAdminDatasourceCustomizer::Decorators::Action::Context::ActionContext) + allow(context).to receive(:get_record_ids).and_raise(UnsupportedOperatorError, 'names 25 issues by id') + allow(client).to receive(:update_issue) + + expect { on_primary_key.execute.call(context, result_builder) } + .to raise_error(UnsupportedOperatorError, /names 25 issues by id/) + expect(client).not_to have_received(:update_issue) + end + end + + describe 'the state an execution writes' do + let(:bulk) { register(scopes: %i[bulk])['Close selected Pylon issues'] } + + it 'patches every selected issue' do + allow(client).to receive(:update_issue) + + result = bulk.execute.call(FakeCloseContext.new(record_ids: %w[i1 i2 i3]), result_builder) + + %w[i1 i2 i3].each { |id| expect(client).to have_received(:update_issue).with(id, 'state' => 'closed') } + expect(result[:message]).to include('3 issues closed.') + end + + # An organization defining its own statuses names the slug it wants; Pylon + # takes it exactly like a standard one. + it 'writes the configured state, custom slug included, and says so' do + allow(client).to receive(:update_issue) + action = register(scopes: %i[single], state: 'on_hold')['Close Pylon issue'] + + result = action.execute.call(FakeCloseContext.new(record_ids: %w[i1]), result_builder) + + expect(client).to have_received(:update_issue).with('i1', 'state' => 'on_hold') + expect(result[:message]).to include('Issue i1 moved to on_hold.') + end + end + + describe 'an execution Pylon refuses' do + let(:bulk) { register(scopes: %i[bulk])['Close selected Pylon issues'] } + let(:context) { FakeCloseContext.new(record_ids: %w[i1 i2 i3]) } + + # The point of the per-id rescue: one refusal costs one issue, and the + # operator is told which, rather than reading a success over a batch that + # was only half applied. + it 'keeps going and names what failed' do + allow(client).to receive(:update_issue) + allow(client).to receive(:update_issue).with('i2', anything).and_raise(APIError, 'HTTP 404 not found') + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + + result = bulk.execute.call(context, result_builder) + + expect(result[:type]).to eq('Success') + expect(result[:message]).to include('2 issues closed.', '1 failed: i2.') + expect(ForestAdminDatasourcePylon.logger).to have_received(:warn).with(a_string_including('i2', 'not found')) + end + + it 'answers an error when every issue failed' do + allow(client).to receive(:update_issue).and_raise(APIError, 'HTTP 403 forbidden') + allow(ForestAdminDatasourcePylon.logger).to receive(:warn).exactly(3).times + + result = bulk.execute.call(context, result_builder) + + expect(result[:type]).to eq('Error') + expect(result[:message]).to include('Failed to close all 3 issues', 'forbidden') + end + + it 'names the single issue that failed' do + allow(client).to receive(:update_issue).and_raise(APIError, 'HTTP 403 forbidden') + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + action = register(scopes: %i[single])['Close Pylon issue'] + + result = action.execute.call(FakeCloseContext.new(record_ids: %w[i1]), result_builder) + + expect(result[:message]).to include('Failed to close issue i1', 'forbidden') + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/plugins/create_issue_with_notification_spec.rb b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/plugins/create_issue_with_notification_spec.rb new file mode 100644 index 000000000..4dbd5ee5e --- /dev/null +++ b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/plugins/create_issue_with_notification_spec.rb @@ -0,0 +1,451 @@ +module ForestAdminDatasourcePylon + # Stand-in for an ActionContextSingle. Not a Struct: `Struct` mixes in + # Enumerable, which already defines `#filter`, so a `:filter` member is both + # a cop violation and genuinely ambiguous. + class FakeCreateContext + attr_reader :form_values, :collection, :filter + + def initialize(form_values: {}, collection: nil, filter: nil, record: {}) + @form_values = form_values + @collection = collection + @filter = filter + @record = record + end + + def get_record(_fields = []) = @record + end + + RSpec.describe Plugins::CreateIssueWithNotification do + let(:client) { instance_double(ForestAdminDatasourcePylon::Client) } + let(:datasource) { instance_double(ForestAdminDatasourcePylon::Datasource, client: client) } + let(:result_builder) { ForestAdminDatasourceCustomizer::Decorators::Action::ResultBuilder.new } + let(:action_scope) { ForestAdminDatasourceCustomizer::Decorators::Action::Types::ActionScope } + let(:forest_exception) { ForestAdminDatasourceToolkit::Exceptions::ForestException } + let(:collection_customizer) do + Class.new do + attr_reader :registered + + def initialize = @registered = {} + def add_action(name, action) = @registered[name] = action + end.new + end + + # `sender_email` rides along by default: Pylon refuses an email delivery + # that does not name the sending address, so the plugin refuses to register + # one, and an email destination is what the options default to. + def register(opts = {}) + options = { datasource: datasource, sender_email: 'support@acme.test' }.merge(opts) + described_class.new.run(nil, collection_customizer, options) + collection_customizer.registered[opts[:action_name] || described_class::NAME] + end + + def form_values(overrides = {}) + { 'Requester email' => 'ada@acme.test', 'Subject' => 'Boom', + 'Message' => '

it broke

' }.merge(overrides) + end + + def run_action(action, values, context_options = {}) + action.execute.call(FakeCreateContext.new(form_values: values, **context_options), result_builder) + end + + describe '#run' do + it 'registers a SINGLE-scoped action under the default name' do + action = register + + expect(collection_customizer.registered.keys).to contain_exactly(described_class::NAME) + expect(action.scope).to eq(action_scope::SINGLE) + end + + # No Type field, unlike the Zendesk form: POST /issues does not take one. + it 'builds the form Pylon accepts on a create' do + expect(register.form.map { |field| field[:label] }) + .to eq(['Requester email', 'Subject', 'Message', 'Priority']) + end + + it 'drops the Priority field when the priority is imposed' do + expect(register(priority_override: 'urgent').form.map { |field| field[:label] }) + .to eq(['Requester email', 'Subject', 'Message']) + end + + it 'adds the internal-note checkbox when asked for' do + expect(register(show_internal_note: true).form.map { |field| field[:label] }) + .to eq(['Requester email', 'Subject', 'Message', 'Priority', 'Send as internal note']) + end + + # The title is the enum value and the key the content is looked up by, so + # it has to name one template. + it 'refuses two templates sharing a title, which would hide the first' do + expect { register(email_templates: [{ title: 'Outage', content: 'a' }, { title: 'Outage', content: 'b' }]) } + .to raise_error(forest_exception, /Duplicate email template titles: Outage/) + end + + it 'refuses a template titled like the option that picks none of them' do + expect { register(email_templates: [{ title: 'No template', content: 'a' }]) } + .to raise_error(forest_exception, /cannot be titled/) + end + + it 'splits the form in two pages when templates are configured' do + form = register(email_templates: [{ title: 'Outage', content: 'Sorry' }]).form + + expect(form.map { |element| element[:component] }).to eq(%w[Page Page]) + expect(form.first[:elements].map { |field| field[:label] }).to eq(['Template']) + end + + it 'sends the message as RichText and requires the requester' do + form = register.form + + expect(form.find { |field| field[:label] == 'Message' }[:widget]).to eq('RichText') + expect(form.first[:is_required]).to be(true) + end + + it 'offers the priorities Pylon documents, with no default' do + priority = register.form.find { |field| field[:label] == 'Priority' } + + expect(priority[:enum_values]).to eq(IssueEnums::PRIORITY) + expect(priority[:default_value]).to be_nil + end + + # The plain hashes above only become form elements once the agent builds + # them, and a form the factory refuses -- Page elements mixed with plain + # ones, two fields sharing a label -- fails in the panel, not here. + it 'builds through the real form factory, flat and paged alike' do + flat = register(show_internal_note: true) + paged = register(action_name: 'paged', email_templates: [{ title: 'Outage', content: 'Sorry' }]) + + expect { flat.build_elements.validate_fields_ids }.not_to raise_error + expect { paged.build_elements.validate_fields_ids }.not_to raise_error + expect(flat.static_form).to be(true) + expect(paged.static_form).to be(false) + end + + it 'honors :action_name' do + register + register(action_name: 'Open a ticket') + + expect(collection_customizer.registered.keys).to contain_exactly(described_class::NAME, 'Open a ticket') + end + + it 'raises a ForestException on an unknown destination' do + expect { register(destination: 'pigeon') }.to raise_error(forest_exception, /Unknown.*pigeon/) + end + + it 'refuses an email delivery that names no sending address' do + expect { described_class.new.run(nil, collection_customizer, datasource: datasource) } + .to raise_error(forest_exception, /:sender_email when the destination is email/) + end + + # The address belongs to the email app; the other channels never carry it. + it 'asks for no sending address on another channel' do + expect { described_class.new.run(nil, collection_customizer, datasource: datasource, destination: 'slack') } + .not_to raise_error + end + + it 'raises a ForestException on an unknown priority' do + expect { register(priority_override: 'critical') }.to raise_error(forest_exception, /Unknown.*critical/) + end + + it 'raises a ForestException without :datasource' do + expect { described_class.new.run(nil, collection_customizer, {}) } + .to raise_error(forest_exception, /datasource/) + end + + it 'raises a ForestException without a collection' do + expect { described_class.new.run(nil, nil, datasource: datasource) } + .to raise_error(forest_exception, /collection/) + end + end + + describe 'the issue an execution creates' do + it 'posts the form as an issue delivered to the requester by email' do + allow(client).to receive(:create_issue).and_return({ 'id' => 'i1', 'number' => 12 }) + + result = run_action(register, form_values('Priority' => 'high')) + + expect(client).to have_received(:create_issue).with( + 'title' => 'Boom', 'body_html' => '

it broke

', + 'requester_email' => 'ada@acme.test', 'requester_name' => 'ada', + 'priority' => 'high', + 'destination_metadata' => { 'destination' => 'email', 'email' => 'support@acme.test' } + ) + expect(result[:type]).to eq('Success') + expect(result[:message]).to include('Issue #12 created and the requester notified by email.') + end + + it 'carries the sending address and the copies of an email delivery' do + allow(client).to receive(:create_issue).and_return({ 'id' => 'i1' }) + + run_action(register(email_ccs: ['lead@acme.test'], email_bccs: ['audit@acme.test']), form_values) + + expect(client).to have_received(:create_issue).with( + hash_including('destination_metadata' => { 'destination' => 'email', 'email' => 'support@acme.test', + 'email_ccs' => ['lead@acme.test'], + 'email_bccs' => ['audit@acme.test'] }) + ) + end + + # Those three belong to the email app they are configured on; another + # channel would carry them for nothing. + it 'leaves the email settings out of another channel' do + allow(client).to receive(:create_issue).and_return({ 'id' => 'i1' }) + + run_action(register(destination: 'slack'), form_values) + + expect(client).to have_received(:create_issue) + .with(hash_including('destination_metadata' => { 'destination' => 'slack' })) + end + + # "Do not contact the requester" is the absence of the key, which is the + # form the API reference names for it. + it 'sends no destination at all when the operator asks for an internal issue' do + allow(client).to receive(:create_issue).and_return({ 'id' => 'i1', 'number' => 12 }) + + result = run_action(register(show_internal_note: true), + form_values('Send as internal note' => true)) + + expect(client).to have_received(:create_issue).with(hash_excluding('destination_metadata')) + expect(result[:message]).to include('Issue #12 created (internal, the requester was not contacted).') + end + + # The field is not on a form registered without the option, but the agent + # copies every key the request carried whether a field matched it or not. + # Honoured, it would leave the requester uncontacted on an action built to + # contact them. + it 'refuses an internal note submitted to an action that does not offer one' do + allow(client).to receive(:create_issue).and_return({ 'id' => 'i1', 'number' => 12 }) + + result = run_action(register, form_values('Send as internal note' => true)) + + expect(client).to have_received(:create_issue).with( + hash_including('destination_metadata' => hash_including('destination' => 'email')) + ) + expect(result[:message]).not_to include('internal') + end + + it 'sends no destination when the plugin itself is configured as internal' do + allow(client).to receive(:create_issue).and_return({ 'id' => 'i1' }) + + run_action(register(destination: 'internal'), form_values) + + expect(client).to have_received(:create_issue).with(hash_excluding('destination_metadata')) + end + + it 'imposes the configured priority over the form' do + allow(client).to receive(:create_issue).and_return({ 'id' => 'i1' }) + + run_action(register(priority_override: 'urgent'), form_values('Priority' => 'low')) + + expect(client).to have_received(:create_issue).with(hash_including('priority' => 'urgent')) + end + + it 'omits the priority when none was picked' do + allow(client).to receive(:create_issue).and_return({ 'id' => 'i1' }) + + run_action(register, form_values) + + expect(client).to have_received(:create_issue).with(hash_excluding('priority')) + end + + it 'falls back on the id when Pylon answers without a number' do + allow(client).to receive(:create_issue).and_return({ 'id' => 'i1' }) + + expect(run_action(register, form_values)[:message]).to include('Issue #i1 created') + end + + it 'refuses to post without a requester' do + allow(client).to receive(:create_issue) + + result = run_action(register, form_values('Requester email' => '')) + + expect(client).not_to have_received(:create_issue) + expect(result[:type]).to eq('Error') + expect(result[:message]).to include('Requester email is required.') + end + + it 'hands the operator the reason Pylon refused the creation' do + allow(client).to receive(:create_issue) + .and_raise(APIError.new('Pylon API call failed: create(issues): HTTP 400 account_id is required', + status: 400)) + + result = run_action(register, form_values) + + expect(result[:type]).to eq('Error') + expect(result[:message]).to include('account_id is required') + end + + it 'lets a Pylon failure that is not the operator\'s to fix stay an error' do + allow(client).to receive(:create_issue) + .and_raise(APIError.new('Pylon API call failed: create(issues): HTTP 503', status: 503)) + + expect { run_action(register, form_values) }.to raise_error(APIError, /503/) + end + end + + describe 'writing the issue id back on the host record' do + let(:collection) { instance_double(ForestAdminDatasourceCustomizer::Context::RelaxedWrappers::RelaxedCollection) } + let(:context_options) { { collection: collection, filter: :a_filter } } + + it 'updates the configured column' do + allow(client).to receive(:create_issue).and_return({ 'id' => 'i1', 'number' => 12 }) + allow(collection).to receive(:update) + + result = run_action(register(issue_id_field: 'pylon_issue_id'), form_values, context_options) + + expect(collection).to have_received(:update).with(:a_filter, { 'pylon_issue_id' => 'i1' }) + expect(result[:type]).to eq('Success') + end + + it 'writes nothing when no column is configured' do + allow(client).to receive(:create_issue).and_return({ 'id' => 'i1' }) + allow(collection).to receive(:update) + + run_action(register, form_values, context_options) + + expect(collection).not_to have_received(:update) + end + + # Pylon has no transaction: the issue exists, so the action succeeds and + # the failed writeback is a warning inside the success message. + it 'degrades the success message when the column cannot be written' do + allow(client).to receive(:create_issue).and_return({ 'id' => 'i1', 'number' => 12 }) + allow(collection).to receive(:update).and_raise(StandardError, 'column is read-only') + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + + result = run_action(register(issue_id_field: 'pylon_issue_id'), form_values, context_options) + + expect(result[:type]).to eq('Success') + expect(result[:message]).to include('Issue #12 created', 'could not store the issue id', 'read-only') + expect(ForestAdminDatasourcePylon.logger).to have_received(:warn) + .with(a_string_including('pylon_issue_id', 'read-only')) + end + end + + describe 'the record tokens a default is written with' do + let(:context) { FakeCreateContext.new(record: { 'email' => 'ada@acme.test', 'name' => 'Ada & Co ' }) } + + it 'keeps a token-free default as the literal it is' do + expect(register(default_subject: 'Outage').form[1][:default_value]).to eq('Outage') + end + + it 'interpolates the subject without escaping it' do + subject_field = register(default_subject: 'Outage for {{ record.name }}').form[1] + + expect(subject_field[:default_value].call(context)).to eq('Outage for Ada & Co ') + end + + # The message ships as body_html and is delivered as such: a record value + # carrying markup must not become markup. + it 'escapes the html of a value interpolated into the message' do + message_field = register(default_message: '

Hi {{ record.name }}

').form[2] + + expect(message_field[:default_value].call(context)).to eq('

Hi Ada & Co <boss>

') + end + + # A default that cannot be resolved must not take the form down with it: + # the operator gets the template with its tokens emptied, and types. + it 'logs and interpolates nothing when the record cannot be read' do + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + unreadable = instance_double( + ForestAdminDatasourceCustomizer::Decorators::Action::Context::ActionContextSingle + ) + allow(unreadable).to receive(:get_record).and_raise(StandardError, 'boom') + subject_field = register(default_subject: 'Outage for {{ record.name }}').form[1] + + expect(subject_field[:default_value].call(unreadable)).to eq('Outage for ') + expect(ForestAdminDatasourcePylon.logger).to have_received(:warn) + .with(a_string_including('token interpolation', 'boom')) + end + + it 'reads an empty string for a token the record has no value for' do + subject_field = register(default_subject: 'Hi {{ record.unknown }}!').form[1] + + expect(subject_field[:default_value].call(context)).to eq('Hi !') + end + + it 'resolves a lambda requester default against the record' do + field = register(requester_email_default: ->(record) { record['email'] }).form.first + + expect(field[:default_value].call(context)).to eq('ada@acme.test') + end + + it 'answers nothing when the requester resolver raises' do + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + field = register(requester_email_default: ->(_record) { raise 'boom' }).form.first + + expect(field[:default_value].call(context)).to be_nil + expect(ForestAdminDatasourcePylon.logger).to have_received(:warn).with(a_string_including('boom')) + end + end + + describe 'picking a template' do + let(:templates) { [{ title: 'Outage', content: '

Sorry {{ record.name }}

' }] } + let(:message_field) do + register(email_templates: templates).form.last[:elements].find { |field| field[:label] == 'Message' } + end + + def with_default(default_message) + register(action_name: "with default #{default_message}", email_templates: templates, + default_message: default_message) + .form.last[:elements].find { |field| field[:label] == 'Message' } + end + + def context_with_record + FakeCreateContext.new(record: { 'name' => 'Ada & Co' }) + end + + def template_context(changed:, chosen:, record: {}) + context = instance_double(ForestAdminDatasourceCustomizer::Decorators::Action::Context::ActionContextSingle) + allow(context).to receive(:field_changed?).with('Template').and_return(changed) + allow(context).to receive(:get_form_value).with('Template').and_return(chosen) + allow(context).to receive(:get_record).and_return(record) + context + end + + it 'fills the message with the chosen template, tokens escaped' do + value = message_field[:value].call(template_context(changed: true, chosen: 'Outage', + record: { 'name' => 'Ada & Co' })) + + expect(value).to eq('

Sorry Ada & Co

') + end + + it 'clears the message when the template is taken back and no default was configured' do + expect(message_field[:value].call(template_context(changed: true, chosen: 'No template'))).to eq('') + end + + # The wizard carries the configured default like the flat form does: the + # first page decides whether a template replaces it, not whether the + # operator is handed an empty required field. + it 'still fills the first render from the configured default' do + field = with_default('

Hi

') + + expect(field[:default_value]).to eq('

Hi

') + expect(field[:value]).to be_a(Proc) + end + + it 'interpolates the default of the first render, tokens escaped' do + field = with_default('

Hi {{ record.name }}

') + + expect(field[:default_value].call(context_with_record)).to eq('

Hi Ada & Co

') + end + + it 'restores the configured default when the template is taken back' do + value = with_default('

Hi

')[:value] + .call(template_context(changed: true, chosen: 'No template')) + + expect(value).to eq('

Hi

') + end + + it 'restores a default carrying tokens, interpolated' do + value = with_default('

Hi {{ record.name }}

')[:value] + .call(template_context(changed: true, chosen: 'No template', record: { 'name' => 'Ada & Co' })) + + expect(value).to eq('

Hi Ada & Co

') + end + + # Nil means "leave what the operator typed", which is what keeps their + # edits across the re-renders of the other fields. + it 'leaves the message alone while the template does not change' do + expect(message_field[:value].call(template_context(changed: false, chosen: 'Outage'))).to be_nil + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/query/condition_tree_translator_spec.rb b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/query/condition_tree_translator_spec.rb new file mode 100644 index 000000000..7cd28f477 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/query/condition_tree_translator_spec.rb @@ -0,0 +1,354 @@ +module ForestAdminDatasourcePylon + RSpec.describe Query::ConditionTreeTranslator do + def leaf(field, operator, value = nil) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def branch(aggregator, conditions) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + .new(aggregator, conditions) + end + + # Nests `depth` multi-condition branches: the outermost sits at depth 1, so + # the innermost branch sits at depth `depth`. + def nested(depth) + (1..depth).reduce(leaf('state', operators::EQUAL, 'new')) do |tree, _| + branch('And', [tree, leaf('team_id', operators::EQUAL, 'team-1')]) + end + end + + def translate(tree, api_filters: default_filters, timezone: nil) + described_class.call(tree, api_filters: api_filters, timezone: timezone) + end + + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + let(:default_filters) { Collections::Issue::ApiFilters::API_FILTERS } + + it 'translates a nil condition tree to no filter' do + expect(translate(nil)).to be_nil + end + + it 'rejects a node that is neither a leaf nor a branch' do + expect { translate(Object.new) } + .to raise_error(UnsupportedOperatorError, /Unknown condition node/) + end + + describe 'equality operators' do + it 'translates equal to a single-valued equals' do + expect(translate(leaf('state', operators::EQUAL, 'new'))) + .to eq('field' => 'state', 'operator' => 'equals', 'value' => 'new') + end + + it 'translates in and not_in to a values list' do + expect(translate(leaf('state', operators::IN, %w[new closed]))) + .to eq('field' => 'state', 'operator' => 'in', 'values' => %w[new closed]) + expect(translate(leaf('state', operators::NOT_IN, %w[closed]))) + .to eq('field' => 'state', 'operator' => 'not_in', 'values' => %w[closed]) + end + + # The toolkit rewrites not_equal into not_in, so the translator never has + # to spell the same filter twice. + it 'refuses not_equal, which the toolkit derives from not_in' do + expect { translate(leaf('state', operators::NOT_EQUAL, 'new')) } + .to raise_error(UnsupportedOperatorError, /Supported: equal, in, not_in/) + end + end + + describe 'presence operators' do + it 'translates present and blank without carrying a value' do + expect(translate(leaf('assignee_id', operators::PRESENT))) + .to eq('field' => 'assignee_id', 'operator' => 'is_set') + expect(translate(leaf('assignee_id', operators::BLANK))) + .to eq('field' => 'assignee_id', 'operator' => 'is_unset') + end + + # Left out of the map, `missing` is rewritten by the toolkit into + # `equal nil`, which no Pylon operator expresses: the field would refuse + # the very absence check it is one of the few to accept. + it 'translates missing to the same is_unset as blank' do + expect(translate(leaf('assignee_id', operators::MISSING))) + .to eq('field' => 'assignee_id', 'operator' => 'is_unset') + end + + # Pylon lists is_set / is_unset for the party ids and issue_type only. + it 'refuses presence on a field Pylon does not accept it for' do + expect { translate(leaf('state', operators::PRESENT)) } + .to raise_error(UnsupportedOperatorError, /Operator 'present' is not supported on field 'state'/) + end + end + + describe 'text operators' do + it 'maps both spellings of contains onto the single Pylon substring operator' do + expect(translate(leaf('title', operators::CONTAINS, 'boom'))) + .to eq('field' => 'title', 'operator' => 'string_contains', 'value' => 'boom') + expect(translate(leaf('body_html', operators::I_CONTAINS, 'boom'))) + .to eq('field' => 'body_html', 'operator' => 'string_contains', 'value' => 'boom') + end + + # Symmetrically with contains: Pylon documents no case semantics for its + # single substring operator, so leaving not_i_contains out would offer a + # case-insensitive "contains" the UI could not negate. + it 'maps both spellings of not_contains onto the single negated operator' do + expect(translate(leaf('title', operators::NOT_CONTAINS, 'boom'))) + .to eq('field' => 'title', 'operator' => 'string_does_not_contain', 'value' => 'boom') + expect(translate(leaf('body_html', operators::NOT_I_CONTAINS, 'boom'))) + .to eq('field' => 'body_html', 'operator' => 'string_does_not_contain', 'value' => 'boom') + end + + # A bare date is read off the operator, not off the shape of the string: a + # text field holding what looks like a date holds a value of its own. + it 'leaves a date-shaped value untouched on a field not compared as time' do + expect(translate(leaf('title', operators::I_CONTAINS, '2026-08-01'))) + .to include('value' => '2026-08-01') + end + + it 'refuses equal on a text field, which Pylon cannot match exactly' do + expect { translate(leaf('title', operators::EQUAL, 'Boom')) } + .to raise_error(UnsupportedOperatorError, /not supported on field 'title'/) + end + + # `tags` holds a list, so membership is matched against candidates rather + # than compared. + it 'translates tag membership with the list operators' do + expect(translate(leaf('tags', operators::IN, %w[urgent vip]))) + .to eq('field' => 'tags', 'operator' => 'in', 'values' => %w[urgent vip]) + expect(translate(leaf('tags', operators::NOT_IN, %w[urgent]))) + .to eq('field' => 'tags', 'operator' => 'not_in', 'values' => %w[urgent]) + end + + # Pylon accepts `contains` on a list field, and the map leaves it out: the + # column is typed Json, on which the toolkit refuses the operator before + # the translator ever sees it. + it 'refuses the substring operators on a list column' do + expect { translate(leaf('tags', operators::CONTAINS, 'urgent')) } + .to raise_error(UnsupportedOperatorError, /not supported on field 'tags'/) + end + end + + describe 'time operators' do + it 'translates the bare comparisons to the time bounds' do + expect(translate(leaf('created_at', operators::GREATER_THAN, '2026-08-01T00:00:00Z'))) + .to eq('field' => 'created_at', 'operator' => 'time_is_after', 'value' => '2026-08-01T00:00:00Z') + expect(translate(leaf('updated_at', operators::LESS_THAN, '2026-08-01T00:00:00Z'))) + .to eq('field' => 'updated_at', 'operator' => 'time_is_before', 'value' => '2026-08-01T00:00:00Z') + end + + it 'formats a Time value as an UTC timestamp' do + expect(translate(leaf('created_at', operators::GREATER_THAN, Time.utc(2026, 8, 7, 13, 6, 22)))) + .to include('value' => '2026-08-07T13:06:22Z') + end + + it 'reads a bare Date as midnight in the timezone of the caller' do + filter = translate(leaf('created_at', operators::GREATER_THAN, Date.new(2026, 8, 7)), + timezone: 'Europe/Paris') + + expect(filter).to include('value' => '2026-08-06T22:00:00Z') + end + + # Declaring only the bare comparisons on a Date column is enough: the + # toolkit rewrites every interval operator into the pair of bounds Pylon + # accepts, which is also why `time_range` never has to be emitted. + it 'translates an interval operator the toolkit rewrote into a pair of bounds' do + equivalent = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::ConditionTreeEquivalent + .get_equivalent_tree(leaf('created_at', operators::TODAY), + Collections::Issue::ApiFilters.forest_operators('created_at'), + 'Date', 'UTC') + + filter = translate(equivalent) + + expect(filter['operator']).to eq('and') + expect(filter['subfilters'].map { |sub| sub['operator'] }).to eq(%w[time_is_after time_is_before]) + end + + # A Dateonly column -- only a custom field is typed that way -- sends the + # date alone, where every native column sends `time_is_after` a timestamp. + # The bound is the one a Ruby Date already gets. + it 'reads a bare date string as midnight in the timezone of the caller' do + filter = translate(leaf('created_at', operators::GREATER_THAN, '2026-08-01'), timezone: 'Europe/Paris') + + expect(filter).to include('value' => '2026-07-31T22:00:00Z') + end + + it 'leaves a string no date can be read from to Pylon, which names what it refuses' do + expect(translate(leaf('created_at', operators::GREATER_THAN, '2026-13-45'))) + .to include('value' => '2026-13-45') + end + + it 'falls back to UTC and warns on a timezone it does not know' do + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + + filter = translate(leaf('created_at', operators::GREATER_THAN, Date.new(2026, 8, 7)), + timezone: 'Moon/Base') + + expect(filter).to include('value' => '2026-08-07T00:00:00Z') + expect(ForestAdminDatasourcePylon.logger).to have_received(:warn).with(%r{unknown timezone 'Moon/Base'}) + end + end + + # An issue is read with `type` / `resolution_time` / `latest_message_time` + # but filtered on the names below. + describe 'field renames' do + it 'renames the columns Pylon spells differently in a filter' do + expect(translate(leaf('type', operators::EQUAL, 'ticket'))).to include('field' => 'issue_type') + expect(translate(leaf('resolution_time', operators::GREATER_THAN, '2026-08-01T00:00:00Z'))) + .to include('field' => 'resolved_at') + expect(translate(leaf('latest_message_time', operators::LESS_THAN, '2026-08-01T00:00:00Z'))) + .to include('field' => 'latest_message_activity_at') + end + end + + describe 'branches' do + it 'translates an and into nested sub-filters' do + tree = branch('And', [leaf('state', operators::EQUAL, 'new'), leaf('team_id', operators::EQUAL, 'team-1')]) + + expect(translate(tree)).to eq( + 'operator' => 'and', + 'subfilters' => [{ 'field' => 'state', 'operator' => 'equals', 'value' => 'new' }, + { 'field' => 'team_id', 'operator' => 'equals', 'value' => 'team-1' }] + ) + end + + # Pylon nests sub-filters, so OR is expressed natively instead of being + # rejected the way the Zendesk query string forces. + it 'translates an or natively' do + tree = branch('Or', [leaf('state', operators::EQUAL, 'new'), leaf('title', operators::CONTAINS, 'boom')]) + + expect(translate(tree)['operator']).to eq('or') + expect(translate(tree)['subfilters'].size).to eq(2) + end + + it 'unwraps a branch carrying a single condition' do + tree = branch('And', [leaf('state', operators::EQUAL, 'new')]) + + expect(translate(tree)).to eq('field' => 'state', 'operator' => 'equals', 'value' => 'new') + end + + it 'accepts sub-filters nested up to the depth Pylon allows' do + expect(translate(nested(3))).to include('operator' => 'and') + end + + it 'refuses sub-filters nested deeper than Pylon allows' do + expect { translate(nested(4)) } + .to raise_error(UnsupportedOperatorError, /nested deeper than 3 levels/) + end + + it 'refuses a branch carrying no condition' do + expect { translate(branch('And', [])) } + .to raise_error(UnsupportedOperatorError, /carries no condition/) + end + + it 'refuses an aggregator it does not know' do + tree = branch('Xor', [leaf('state', operators::EQUAL, 'new'), leaf('team_id', operators::EQUAL, 'team-1')]) + + expect { translate(tree) }.to raise_error(UnsupportedOperatorError, /Unknown condition tree aggregator/) + end + + # The toolkit validates no aggregator, so the unwrap is the one place an + # unknown one could have slipped through unnoticed. + it 'refuses an aggregator it does not know even when it carries a lone condition' do + expect { translate(branch('Xor', [leaf('state', operators::EQUAL, 'new')])) } + .to raise_error(UnsupportedOperatorError, /Unknown condition tree aggregator/) + end + end + + describe 'guards' do + it 'refuses a field Pylon cannot filter on' do + expect { translate(leaf('number', operators::EQUAL, 12)) } + .to raise_error(UnsupportedOperatorError, /Pylon cannot filter on 'number'/) + end + + # An empty list would translate to a filter matching everything, turning + # "match nothing" into its exact opposite. + it 'refuses an empty list of values' do + expect { translate(leaf('state', operators::IN, [])) } + .to raise_error(UnsupportedOperatorError, /was given an empty list/) + end + + # Dropping the blanks would answer a different question: `not_in [nil, + # 'open']` was asked to exclude the blank records, and a narrowed + # `not_in ['open']` comes back including them. + it 'refuses a list holding a blank value rather than narrowing it' do + expect { translate(leaf('state', operators::IN, [nil, 'open'])) } + .to raise_error(UnsupportedOperatorError, /holding a blank value/) + expect { translate(leaf('state', operators::NOT_IN, ['open', ''])) } + .to raise_error(UnsupportedOperatorError, /holding a blank value/) + end + + # The agent advertises `present`, `blank` and `missing` on every field + # carrying an equality filter, and rewrites them into `not_in [nil, '']`, + # `in [nil, '']` and `equal nil`. On a field Pylon accepts no is_set / + # is_unset on, the refusal has to name the absence filter the operator set + # rather than the empty value it was rewritten into. + it 'refuses an absence filter on a field carrying no presence operator' do + expect { translate(leaf('state', operators::IN, [nil, ''])) } + .to raise_error(UnsupportedOperatorError, /cannot filter 'state' for absence/) + expect { translate(leaf('state', operators::NOT_IN, [nil, ''])) } + .to raise_error(UnsupportedOperatorError, /cannot filter 'state' for absence/) + expect { translate(leaf('state', operators::EQUAL, nil)) } + .to raise_error(UnsupportedOperatorError, /cannot filter 'state' for absence/) + end + + # `assignee_id` does carry is_set / is_unset, so a nil reaching the value + # comes from a scope or a segment written in Ruby rather than from a + # rewritten absence filter: the message names the operators to write it + # with instead. + it 'points a nil on a presence-filtered field at the presence operators' do + expect { translate(leaf('assignee_id', operators::EQUAL, nil)) } + .to raise_error(UnsupportedOperatorError, /use the PRESENT or BLANK operator/) + expect { translate(leaf('assignee_id', operators::IN, [nil, ''])) } + .to raise_error(UnsupportedOperatorError, /holding a blank value/) + end + + it 'refuses every filter when the collection declares none' do + expect { translate(leaf('state', operators::EQUAL, 'new'), api_filters: {}) } + .to raise_error(UnsupportedOperatorError, /Pylon cannot filter on 'state'/) + end + end + + # The filter travels as JSON, so only the date types need a wire format. + describe 'value types' do + let(:default_filters) do + { 'number_of_touches' => { ops: { operators::EQUAL => 'equals', operators::IN => 'in' } }, + 'customer_portal_visible' => { ops: { operators::EQUAL => 'equals' } } } + end + + it 'passes numbers and booleans through untouched' do + expect(translate(leaf('number_of_touches', operators::EQUAL, 12))).to include('value' => 12) + expect(translate(leaf('customer_portal_visible', operators::EQUAL, false))) + .to include('value' => false) + end + + # The agent casts every Number column with `to_f`, so an integer custom + # field would be filtered with `42.0` -- a form none of its values carry. + # `be` rather than `eq`: `12.0 == 12` holds in Ruby, and the form that + # travels to Pylon is the one thing this is about. + it 'sends an integer-valued float as the integer it is' do + expect(translate(leaf('number_of_touches', operators::EQUAL, 12.0))['value']).to be(12) + end + + it 'keeps what a decimal carries after the point' do + expect(translate(leaf('number_of_touches', operators::EQUAL, 12.5))['value']).to be(12.5) + end + + # `"1e999".to_f` is what the agent's own cast hands over. Left alone it + # reaches `to_i`, which raises, and then the JSON encoder, which raises + # too -- a 500 either way, on a value the operator typed. + it 'refuses a value the cast overflowed rather than raising on it' do + expect { translate(leaf('number_of_touches', operators::EQUAL, '1e999'.to_f)) } + .to raise_error(ForestAdminDatasourcePylon::UnsupportedOperatorError, /number_of_touches.*Infinity/m) + end + + it 'refuses a NaN the same way' do + expect { translate(leaf('number_of_touches', operators::EQUAL, Float::NAN)) } + .to raise_error(ForestAdminDatasourcePylon::UnsupportedOperatorError, /number_of_touches.*NaN/m) + end + + it 'refuses one inside a list too, the list being formatted the same way' do + expect(translate(leaf('number_of_touches', operators::IN, [1.0, 2.5]))['values']).to eq([1, 2.5]) + expect { translate(leaf('number_of_touches', operators::IN, [1.0, -Float::INFINITY])) } + .to raise_error(ForestAdminDatasourcePylon::UnsupportedOperatorError, /number_of_touches.*-Infinity/m) + end + end + end +end diff --git a/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/rate_limiter_spec.rb b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/rate_limiter_spec.rb new file mode 100644 index 000000000..b566bd078 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/rate_limiter_spec.rb @@ -0,0 +1,208 @@ +RSpec.describe ForestAdminDatasourcePylon::RateLimiter do + # A window of 6s rather than a minute, and a budget of 2 rather than 120, so a + # spec can saturate an endpoint in three lines and read the arithmetic off it. + let(:limit) { 2 } + let(:limits) do + rule = ForestAdminDatasourcePylon::RateLimits::Rule.new(name: 'get /things', limit: limit) + class_double(ForestAdminDatasourcePylon::RateLimits, for: rule) + end + + let(:now) { [0.0] } + let(:slept) { [] } + + def limiter(max_wait: 5.0, window: 6.0, rules: limits) + described_class.new(limits: rules, window: window, max_wait: max_wait, + clock: -> { now.first }, sleeper: ->(seconds) { slept << seconds }) + end + + def at(time) + now[0] = time + end + + describe '#acquire' do + it 'lets a request through while the window has room' do + subject = limiter + limit.times { subject.acquire(:get, '/things') } + + expect(slept).to be_empty + end + + # The window is full from t=0 to t=6, so the request asking at t=3 waits the + # 3s left of it rather than a fresh window. + it 'waits out what is left of the window when it is full' do + subject = limiter + subject.acquire(:get, '/things') + at(2.0) + subject.acquire(:get, '/things') + + at(3.0) + subject.acquire(:get, '/things') + + expect(slept).to eq([3.0]) + end + + # Two callers arriving on a full window take the two slots that free next, + # not the same one: a limiter handing both the first free slot would let them + # through together and spend twice the budget at that instant. + it 'spreads concurrent callers over distinct slots' do + subject = limiter + subject.acquire(:get, '/things') + at(2.0) + subject.acquire(:get, '/things') + + at(3.0) + subject.acquire(:get, '/things') + subject.acquire(:get, '/things') + + expect(slept).to eq([3.0, 5.0]) + end + + it 'lets a request through again once the window has rolled past' do + subject = limiter + limit.times { subject.acquire(:get, '/things') } + + at(7.0) + subject.acquire(:get, '/things') + + expect(slept).to be_empty + end + + describe 'when the wait would exceed max_wait' do + it 'lets the request through rather than queueing behind the window' do + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + subject = limiter(max_wait: 1.0) + limit.times { subject.acquire(:get, '/things') } + + subject.acquire(:get, '/things') + + expect(slept).to be_empty + end + + it 'warns, naming the endpoint and the budget it is at' do + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + subject = limiter(max_wait: 1.0) + limit.times { subject.acquire(:get, '/things') } + + subject.acquire(:get, '/things') + + expect(ForestAdminDatasourcePylon.logger) + .to have_received(:warn).with(%r{get /things is at its budget of 2 requests per 6s}) + end + + # What the warning reports is a saturation that lasts, so a line per request + # puts one on every request it describes: a stream 10% over a 120/min + # budget crosses the bound as the window fills and then bypasses almost + # everything, which is a thousand identical lines for a couple of minutes. + it 'warns once per window rather than once per request' do + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + subject = limiter(max_wait: 1.0) + limit.times { subject.acquire(:get, '/things') } + + 5.times { subject.acquire(:get, '/things') } + + expect(ForestAdminDatasourcePylon.logger).to have_received(:warn).once + end + + # Deduplicated, not silenced: a saturation still going a window later is + # news again, and the operator reading one line has to be able to tell the + # burst that passed from the stream that did not. + it 'warns again once a window has gone by' do + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + subject = limiter(max_wait: 1.0) + limit.times { subject.acquire(:get, '/things') } + subject.acquire(:get, '/things') + + at(7.0) + limit.times { subject.acquire(:get, '/things') } + subject.acquire(:get, '/things') + + expect(ForestAdminDatasourcePylon.logger).to have_received(:warn).twice + end + + # The slot it declined to wait for must not be booked: recording it would + # meter a request nobody made and push every later slot further out, so one + # burst past the bound would keep the window saturated indefinitely. + it 'books the slot it takes, not the one it refused' do + subject = limiter(max_wait: 1.0) + limit.times { subject.acquire(:get, '/things') } + subject.acquire(:get, '/things') + + at(7.0) + subject.acquire(:get, '/things') + + expect(slept).to be_empty + end + end + + # A booking of `now` — the request let through past the bound — lands before + # slots already reserved further out, so the list only reads as "the limit-th + # most recent booking" while it stays ordered. Here the fourth request goes + # out at t=3 while a slot is already booked at t=7: unsorted, the caller at + # t=6 reads that t=7 slot instead of the t=3 one and lets its request through + # against a window that had room for it 3s later. + it 'keeps its bookings in order, so a later caller reads the right slot' do + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + subject = limiter + at(1.0) + subject.acquire(:get, '/things') + + at(3.0) + 3.times { subject.acquire(:get, '/things') } + + at(6.0) + subject.acquire(:get, '/things') + + expect(slept).to eq([4.0, 3.0]) + end + + describe 'across endpoints' do + let(:limits) do + class_double(ForestAdminDatasourcePylon::RateLimits).tap do |stub| + allow(stub).to receive(:for) do |_method, path| + ForestAdminDatasourcePylon::RateLimits::Rule.new(name: path, limit: limit) + end + end + end + + # Pylon meters each endpoint on its own, so saturating one is no reason to + # hold back a request to another. + it 'meters each endpoint on its own window' do + subject = limiter + limit.times { subject.acquire(:get, '/things') } + + subject.acquire(:get, '/others') + + expect(slept).to be_empty + end + + # Each endpoint reports its own saturation too: one silenced by another's + # line would have the operator narrow the wrong budget. + it 'warns for each endpoint that saturates' do + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + subject = limiter(max_wait: 1.0) + + ['/things', '/others'].each { |path| (limit + 1).times { subject.acquire(:get, path) } } + + expect(ForestAdminDatasourcePylon.logger).to have_received(:warn).twice + end + end + end + + describe 'defaults' do + it 'meters over a minute, which is the unit Pylon documents' do + expect(described_class::WINDOW).to eq(60.0) + end + + # A bound above the Retry-After of a 429 would make the throttle the slower + # of the two paths it exists to shortcut. + it 'bounds the wait well under a full rate-limit window' do + expect(described_class::DEFAULT_MAX_WAIT).to be < described_class::WINDOW + end + + it 'reads the documented table unless handed another' do + subject = described_class.new(sleeper: ->(seconds) { slept << seconds }) + expect(subject.max_wait).to eq(described_class::DEFAULT_MAX_WAIT) + expect(subject.window).to eq(described_class::WINDOW) + end + end +end diff --git a/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/rate_limits_spec.rb b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/rate_limits_spec.rb new file mode 100644 index 000000000..c20e4b002 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/rate_limits_spec.rb @@ -0,0 +1,156 @@ +RSpec.describe ForestAdminDatasourcePylon::RateLimits do + def rule_for(method, path) = described_class.for(method, path) + + describe '.for' do + it 'reads the documented budget of the search endpoints' do + expect(rule_for(:post, '/issues/search').limit).to eq(120) + expect(rule_for(:post, '/accounts/search').limit).to eq(120) + expect(rule_for(:post, '/contacts/search').limit).to eq(120) + end + + it 'reads the documented budget of the single-record reads' do + expect(rule_for(:get, '/issues/abc-123').limit).to eq(300) + expect(rule_for(:get, '/accounts/acc_1').limit).to eq(300) + expect(rule_for(:get, '/contacts/con_1').limit).to eq(300) + expect(rule_for(:get, '/users/usr_1').limit).to eq(300) + expect(rule_for(:get, '/teams/team_1').limit).to eq(300) + end + + it 'reads the documented budget of the unpaginated lists' do + expect(rule_for(:get, '/users').limit).to eq(300) + expect(rule_for(:get, '/teams').limit).to eq(300) + expect(rule_for(:get, '/custom-fields').limit).to eq(300) + end + + it 'reads the documented budget of the writes' do + expect(rule_for(:post, '/issues').limit).to eq(30) + expect(rule_for(:patch, '/issues/abc').limit).to eq(120) + expect(rule_for(:delete, '/issues/abc').limit).to eq(120) + expect(rule_for(:post, '/teams').limit).to eq(30) + expect(rule_for(:patch, '/teams/team_1').limit).to eq(120) + expect(rule_for(:patch, '/users/usr_1').limit).to eq(120) + end + + # The budget of a write is no more uniform than that of a read: creating an + # account is granted ten times creating an issue, and patching one two and a + # half times patching an issue. A table spelling `/issues` and leaving the + # rest to the fallback would throttle these at a tenth of what Pylon grants. + it 'tells the write budget of one resource from another' do + expect(rule_for(:post, '/accounts').limit).to eq(300) + expect(rule_for(:patch, '/accounts/acc_1').limit).to eq(300) + expect(rule_for(:post, '/contacts').limit).to eq(300) + expect(rule_for(:patch, '/contacts/con_1').limit).to eq(300) + end + + # Deleting is the one write Pylon rates below the rest, and not everywhere: + # 30 a minute on an account or a contact, 120 on an issue. + it 'reads the documented budget of the deletes' do + expect(rule_for(:delete, '/accounts/acc_1').limit).to eq(30) + expect(rule_for(:delete, '/contacts/con_1').limit).to eq(30) + end + + it 'rates the identity endpoint the client calls on connection' do + expect(rule_for(:get, '/me').limit).to eq(300) + end + + # Every endpoint the client issues has to resolve to a rule: one left to the + # fallback is metered at 30 a minute whatever Pylon grants it, which is a + # throttle the datasource inflicts on itself. + it 'leaves no endpoint the client calls to the fallback' do + called = [ + [:get, '/me'], [:post, '/issues/search'], [:get, '/issues/abc'], + [:get, '/issues/abc/messages'], [:post, '/issues'], [:patch, '/issues/abc'], + [:delete, '/issues/abc'], [:post, '/accounts/search'], [:get, '/accounts'], + [:get, '/accounts/acc_1'], [:post, '/accounts'], [:patch, '/accounts/acc_1'], + [:delete, '/accounts/acc_1'], [:post, '/contacts/search'], [:get, '/contacts'], + [:get, '/contacts/con_1'], [:post, '/contacts'], [:patch, '/contacts/con_1'], + [:delete, '/contacts/con_1'], [:get, '/users'], [:get, '/users/usr_1'], + [:patch, '/users/usr_1'], [:get, '/teams'], [:get, '/teams/team_1'], + [:post, '/teams'], [:patch, '/teams/team_1'], [:get, '/custom-fields'] + ] + + undocumented = called.reject { |method, path| rule_for(method, path).name.include?('undocumented') } + expect(undocumented.size).to eq(called.size) + end + + # `/issues/{id}` and `/issues/{id}/messages` sit under the same prefix at + # budgets differing by a factor of two and a half, so a rule matching the + # prefix would meter the thread against the wrong window. + it 'tells a nested endpoint from the record it hangs off' do + expect(rule_for(:get, '/issues/abc/messages').limit).to eq(120) + expect(rule_for(:get, '/issues/abc').limit).to eq(300) + end + + # GET /issues is documented at a tenth of GET /issues/{id}: the verb is the + # same and the path differs by one segment, which is the whole difference. + it 'tells a collection endpoint from a record endpoint' do + expect(rule_for(:get, '/issues').limit).to eq(30) + expect(rule_for(:get, '/issues/abc').limit).to eq(300) + end + + # Reading an issue is granted two and a half times what patching it is, so a + # table keyed on the path alone would meter one of the two wrong. + it 'tells one verb from another on the same path' do + expect(rule_for(:get, '/issues/abc').limit).to eq(300) + expect(rule_for(:patch, '/issues/abc').limit).to eq(120) + end + + describe 'the bucket a rule names' do + # Pylon meters per endpoint, so two endpoints sharing a figure must not + # share a window: pooling them would throttle at half the granted budget. + it 'is distinct for two endpoints at the same budget' do + expect(rule_for(:get, '/users').name).not_to eq(rule_for(:get, '/teams').name) + end + + it 'is the same for two records of one endpoint' do + expect(rule_for(:get, '/issues/one').name).to eq(rule_for(:get, '/issues/two').name) + end + end + + # Pylon exposes no DELETE on a team and no verb at all on `/macros`, so + # neither can carry a documented figure for the table to read. + describe 'an endpoint the API reference does not rate' do + it 'falls back to the lowest budget documented anywhere' do + expect(rule_for(:delete, '/teams/team_1').limit).to eq(described_class::DEFAULT_LIMIT) + expect(rule_for(:get, '/macros').limit).to eq(described_class::DEFAULT_LIMIT) + end + + it 'says so in the name, so a log line does not read as a documented budget' do + expect(rule_for(:get, '/macros').name).to match(/undocumented/) + end + + # Keying the fallback on the path would open a window per record id, so a + # fan-out over a hundred records would meter as a hundred endpoints one + # request in — that is, as no limit at all. + it 'buckets by endpoint rather than by record' do + expect(rule_for(:delete, '/teams/one').name).to eq(rule_for(:delete, '/teams/two').name) + end + + it 'keeps two undocumented verbs on one path apart' do + expect(rule_for(:delete, '/teams/x').name).not_to eq(rule_for(:put, '/teams/x').name) + end + end + + describe 'the shape of the path it is handed' do + it 'accepts a string or a symbol verb' do + expect(rule_for('POST', '/issues/search').limit).to eq(120) + expect(rule_for(:post, '/issues/search').limit).to eq(120) + end + + it 'ignores a trailing slash' do + expect(rule_for(:get, '/users/').limit).to eq(300) + end + + it 'tolerates a path Faraday hands over without its leading slash' do + expect(rule_for(:get, 'users').limit).to eq(300) + end + end + end + + describe 'DEFAULT_LIMIT' do + # The fallback is only conservative if nothing documented sits below it. + it 'is no higher than the lowest rule in the table' do + expect(described_class::DEFAULT_LIMIT).to be <= described_class::RULES.map { |_v, _p, r| r.limit }.min + end + end +end diff --git a/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/retry_policy_spec.rb b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/retry_policy_spec.rb new file mode 100644 index 000000000..ed8dc0770 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/retry_policy_spec.rb @@ -0,0 +1,78 @@ +RSpec.describe ForestAdminDatasourcePylon::RetryPolicy do + describe '#initialize' do + it 'defaults the retry budget' do + policy = described_class.new + expect([policy.max_retries, policy.interval, policy.max_interval]) + .to eq([3, 0.5, described_class::DEFAULT_MAX_INTERVAL]) + end + + it 'keeps overridden values' do + policy = described_class.new(max_retries: 5, interval: 0.1, max_interval: 2) + expect([policy.max_retries, policy.interval, policy.max_interval]).to eq([5, 0.1, 2]) + end + end + + describe 'DEFAULT_MAX_INTERVAL' do + # faraday-retry gives up outright when Retry-After exceeds max_interval, and + # Pylon quotas are per-minute, so a cap below 60s silently disables the 429 + # retry on exactly the throttled endpoints it exists for. + it 'covers a full Pylon rate-limit window' do + expect(described_class::DEFAULT_MAX_INTERVAL).to be >= 60 + end + end + + describe '.boot' do + it 'retries once' do + expect(described_class.boot.max_retries).to eq(1) + end + + # The mirror image of DEFAULT_MAX_INTERVAL: faraday-retry abandons when + # Retry-After exceeds the cap, which is how a 429 costs a boot nothing. + it 'caps the wait below a Pylon rate-limit window' do + expect(described_class.boot.max_interval).to be < 60 + end + + it 'still absorbs a hiccup Pylon answered without a Retry-After' do + expect(described_class.boot.interval).to be_positive + end + end + + describe '#to_faraday_options' do + subject(:options) { described_class.new(max_retries: 2, interval: 0.1, max_interval: 7).to_faraday_options } + + it 'maps the budget onto faraday-retry keys' do + expect(options).to include(max: 2, interval: 0.1, max_interval: 7, backoff_factor: 2) + end + + it 'retries the throttling and transient-gateway statuses' do + expect(options[:retry_statuses]).to eq([429, 502, 503, 504]) + end + + it 'retries dropped connections on top of faraday-retry defaults' do + expect(options[:exceptions]).to include(Faraday::ConnectionFailed, Faraday::RetriableResponse) + end + + it 'limits blanket retries to the verbs that read' do + expect(options[:methods]).to eq(%i[get head options]) + expect(options[:methods]).not_to include(:post, :patch) + end + + # A 502 on the way back from a DELETE Pylon did perform would be replayed + # into a 404, which the write path surfaces as a deletion that failed when + # it landed. Only its 429 is retried, through RETRY_IF. + it 'never replays a delete on anything but a 429' do + expect(options[:methods]).not_to include(:delete) + end + end + + describe 'RETRY_IF' do + it 'allows retrying a non-idempotent verb when Pylon answered 429' do + expect(described_class::RETRY_IF.call({ status: 429 }, nil)).to be(true) + end + + it 'refuses to retry a non-idempotent verb on any other failure' do + expect(described_class::RETRY_IF.call({ status: 502 }, nil)).to be(false) + expect(described_class::RETRY_IF.call({ status: nil }, nil)).to be(false) + end + end +end diff --git a/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/schema/custom_fields_introspector_spec.rb b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/schema/custom_fields_introspector_spec.rb new file mode 100644 index 000000000..bef7d9daf --- /dev/null +++ b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/schema/custom_fields_introspector_spec.rb @@ -0,0 +1,280 @@ +RSpec.describe ForestAdminDatasourcePylon::Schema::CustomFieldsIntrospector do + let(:client) { instance_double(ForestAdminDatasourcePylon::Client) } + let(:introspector) { described_class.new(client) } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + + def definition(type, slug: 'severity', **extra) + { 'id' => 'cf_1', 'slug' => slug, 'label' => slug.capitalize, 'type' => type, + 'object_type' => 'issue', 'is_read_only' => false }.merge(extra) + end + + def select_metadata(*slugs) + { 'select_metadata' => { 'options' => slugs.map { |slug| { 'label' => slug.upcase, 'slug' => slug } } } } + end + + # `object_type` is what Pylon indexes its definitions by, and each accessor + # reads the type of the collection it feeds. + describe 'the object type of each accessor' do + it 'reads the definitions of the matching Pylon object type' do + %w[issue account contact].each do |object_type| + allow(client).to receive(:fetch_custom_fields).with(object_type).and_return([]) + end + + introspector.issue_custom_fields + introspector.account_custom_fields + introspector.contact_custom_fields + + expect(client).to have_received(:fetch_custom_fields).with('issue') + expect(client).to have_received(:fetch_custom_fields).with('account') + expect(client).to have_received(:fetch_custom_fields).with('contact') + end + + # Three calls in front of a Rails boot, each bounded per request: a Pylon + # that is down fails the two after the first the same way, so trying them + # spends the bound three times over to learn what the first one said. + it 'leaves the object types after a failure unread' do + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + allow(client).to receive(:fetch_custom_fields).with('issue').and_return(nil) + + introspector = described_class.new(client) + + expect(introspector.issue_custom_fields).to eq([]) + expect(introspector.account_custom_fields).to eq([]) + expect(introspector.contact_custom_fields).to eq([]) + expect(client).to have_received(:fetch_custom_fields).with('issue').once + expect(client).not_to have_received(:fetch_custom_fields).with('account') + expect(ForestAdminDatasourcePylon.logger).to have_received(:warn).with(/could not be read for issue/) + end + + # An organization that defined no custom field on one object type says + # nothing about the next, unlike a failure. + it 'keeps reading the object types after an empty one' do + allow(client).to receive(:fetch_custom_fields).with('issue').and_return([]) + allow(client).to receive(:fetch_custom_fields).with('account').and_return([]) + allow(client).to receive(:fetch_custom_fields).with('contact').and_return([]) + + introspector = described_class.new(client) + introspector.issue_custom_fields + introspector.account_custom_fields + + expect(client).to have_received(:fetch_custom_fields).with('account') + end + end + + describe 'column types' do + it 'maps every Pylon type onto the Forest column type holding it' do + definitions = %w[text url user number decimal boolean date datetime multiselect] + .map { |type| definition(type, slug: type) } + allow(client).to receive(:fetch_custom_fields).with('issue').and_return(definitions) + + types = introspector.issue_custom_fields.to_h { |cf| [cf[:column_name], cf[:schema].column_type] } + + expect(types).to eq('text' => 'String', 'url' => 'String', 'user' => 'String', + 'number' => 'Number', 'decimal' => 'Number', 'boolean' => 'Boolean', + 'date' => 'Dateonly', 'datetime' => 'Date', 'multiselect' => 'Json') + end + + # A type this datasource cannot map would filter and display wrong, which is + # worse for the operator than a column that is not there. + it 'skips a type it cannot map, and says which field it left out' do + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + allow(client).to receive(:fetch_custom_fields).with('account') + .and_return([definition('holographic', slug: 'tier')]) + + expect(introspector.account_custom_fields).to eq([]) + expect(ForestAdminDatasourcePylon.logger) + .to have_received(:warn).with(/'tier' on account has type "holographic".*skipping/) + end + + it 'skips a definition carrying no slug, which nothing could be read by' do + allow(client).to receive(:fetch_custom_fields).with('issue') + .and_return([definition('text', slug: ''), 'not a hash']) + + expect(introspector.issue_custom_fields).to eq([]) + end + end + + describe 'a select field' do + it 'advertises the option slugs, which are the values Pylon reads and filters' do + allow(client).to receive(:fetch_custom_fields).with('issue') + .and_return([definition('select', **select_metadata('p1', 'p2'))]) + + schema = introspector.issue_custom_fields.first[:schema] + + expect(schema.column_type).to eq('Enum') + expect(schema.enum_values).to eq(%w[p1 p2]) + end + + it 'reads a multiselect the same way' do + allow(client).to receive(:fetch_custom_fields).with('issue') + .and_return([definition('multiselect', **select_metadata('eu'))]) + + expect(introspector.issue_custom_fields.first[:schema].column_type).to eq('Json') + end + + # Forest refuses an Enum carrying no value, so a select whose options were + # all removed still shows what it holds instead of disappearing. + it 'falls back to String when every option is gone' do + allow(client).to receive(:fetch_custom_fields).with('issue') + .and_return([definition('select', **select_metadata)]) + + schema = introspector.issue_custom_fields.first[:schema] + + expect(schema.column_type).to eq('String') + expect(schema.enum_values).to eq([]) + expect(schema.filter_operators).to include(operators::I_CONTAINS) + end + + # `definition` flags the field writable, so this is Pylon saying yes and the + # fallback saying no: a select is written as the slug of one of its options, + # and a free-text editor on one offers no value the endpoint would accept. + it 'leaves that fallback read-only even where Pylon flags the field writable' do + allow(client).to receive(:fetch_custom_fields).with('issue') + .and_return([definition('select', **select_metadata)]) + + expect(introspector.issue_custom_fields.first[:schema].is_read_only).to be(true) + end + + it 'keeps a writable select writable while it still has options' do + allow(client).to receive(:fetch_custom_fields).with('issue') + .and_return([definition('select', **select_metadata('p1'))]) + + expect(introspector.issue_custom_fields.first[:schema].is_read_only).to be(false) + end + + it 'ignores an option with no slug rather than advertising a blank value' do + metadata = { 'select_metadata' => { 'options' => [{ 'label' => 'P1' }, { 'slug' => 'p2' }, 'nope'] } } + allow(client).to receive(:fetch_custom_fields).with('issue') + .and_return([definition('select', **metadata)]) + + expect(introspector.issue_custom_fields.first[:schema].enum_values).to eq(%w[p2]) + end + end + + describe 'filter operators' do + def operators_of(type, **extra) + allow(client).to receive(:fetch_custom_fields).with('issue').and_return([definition(type, **extra)]) + + introspector.issue_custom_fields.first[:schema].filter_operators + end + + it 'lets a text field be matched, listed, checked for presence and searched' do + expect(operators_of('text')).to eq([operators::EQUAL, operators::IN, operators::NOT_IN, + operators::PRESENT, operators::BLANK, operators::MISSING, + operators::CONTAINS, operators::I_CONTAINS, + operators::NOT_CONTAINS, operators::NOT_I_CONTAINS]) + end + + it 'gives a date field the bounds Pylon compares dates with' do + expect(operators_of('datetime')).to include(operators::GREATER_THAN, operators::LESS_THAN) + end + + # `Rules` grants a DATE or a DATEONLY column no array operator, so an + # advertised `in` would be refused by `ConditionTreeValidator` -- a filter the + # UI offers and the agent then answers with a 400. + it 'withholds the membership operators from a date, which the agent refuses on one' do + %w[date datetime].each do |type| + expect(operators_of(type)).to eq([operators::EQUAL, + operators::PRESENT, operators::BLANK, operators::MISSING, + operators::GREATER_THAN, operators::LESS_THAN]) + end + end + + # Pylon spells the bare comparisons `time_is_after` / `time_is_before` and + # documents nothing else, so a numeric range would travel as a time filter. + # + # The presence family carries MISSING next to PRESENT and BLANK: Pylon spells + # absence through `is_unset` alone, and a field left without MISSING would + # refuse the very filter its endpoint can answer. + it 'gives a number no comparison, only equality and presence' do + expect(operators_of('number')) + .to eq([operators::EQUAL, operators::IN, operators::NOT_IN, + operators::PRESENT, operators::BLANK, operators::MISSING]) + end + + it 'gives a boolean equality and presence' do + expect(operators_of('boolean')).to eq([operators::EQUAL, operators::IN, operators::NOT_IN, + operators::PRESENT, operators::BLANK, operators::MISSING]) + end + + # A membership filter is not part of what a custom field accepts, and no + # in-memory pass can stand in for one on a list. + it 'leaves a multiselect unfilterable' do + expect(operators_of('multiselect')).to eq([]) + end + + it 'keeps an enum to equality and presence' do + expect(operators_of('select', **select_metadata('p1'))) + .to eq([operators::EQUAL, operators::IN, operators::NOT_IN, + operators::PRESENT, operators::BLANK, operators::MISSING]) + end + end + + # No endpoint sorts, ever; read-only is Pylon's own call, which it makes for + # the fields an app or an integration syncs. + describe 'the schema every custom field gets' do + it 'is unsortable, and read-only when Pylon says so' do + allow(client).to receive(:fetch_custom_fields).with('issue') + .and_return([definition('text', 'is_read_only' => true)]) + + schema = introspector.issue_custom_fields.first[:schema] + + expect(schema.is_read_only).to be(true) + expect(schema.is_sortable).to be(false) + end + + it 'is writable when Pylon declares the field editable' do + allow(client).to receive(:fetch_custom_fields).with('issue') + .and_return([definition('text', 'is_read_only' => false)]) + + expect(introspector.issue_custom_fields.first[:schema].is_read_only).to be(false) + end + + # A definition carrying no flag is left read-only: this datasource + # advertises nothing an endpoint would refuse, and the fields Pylon syncs + # from an app are exactly the ones the flag tells apart, so reading its + # absence as "editable" would offer an editor whose every save is rejected. + it 'is read-only, and says so, when Pylon declares nothing' do + allow(ForestAdminDatasourcePylon.logger).to receive(:warn) + allow(client).to receive(:fetch_custom_fields).with('issue') + .and_return([definition('text', 'is_read_only' => nil)]) + + expect(introspector.issue_custom_fields.first[:schema].is_read_only).to be(true) + expect(ForestAdminDatasourcePylon.logger) + .to have_received(:warn).with(/carries no 'is_read_only' flag; leaving it read-only/) + end + + # `ColumnSchema` defaults this one to true, and the capabilities route turns + # `supportGroups` on as soon as a single field carries it: one custom field + # left groupable is the whole collection offering a chart `aggregate` raises + # on. Every native column declares it false for that reason. + it 'is not groupable, as Pylon exposes no aggregate endpoint' do + allow(client).to receive(:fetch_custom_fields).with('issue') + .and_return([definition('select', **select_metadata('p1'))]) + + expect(introspector.issue_custom_fields.first[:schema].is_groupable).to be(false) + end + + # The slug is both the key a read payload indexes the value by and the + # `field` a filter sends, so the column carries it unchanged. + it 'names the column after the Pylon slug, with nothing else to keep in step' do + allow(client).to receive(:fetch_custom_fields).with('issue') + .and_return([definition('text', slug: 'sev_level')]) + + expect(introspector.issue_custom_fields.first.keys).to eq(%i[column_name schema multi_value]) + expect(introspector.issue_custom_fields.first[:column_name]).to eq('sev_level') + end + + # Pylon writes a multiselect back through `values` and every other type + # through `value`, so the payload builder is told which one this is rather + # than guessing it from the Json column type. + it 'flags a multiselect as multi-valued, and nothing else' do + allow(client).to receive(:fetch_custom_fields).with('issue') + .and_return([definition('multiselect', **select_metadata('p1')), + definition('select', **select_metadata('p1')), + definition('text')]) + + expect(introspector.issue_custom_fields.map { |cf| cf[:multi_value] }).to eq([true, false, false]) + end + end +end diff --git a/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/throttle_spec.rb b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/throttle_spec.rb new file mode 100644 index 000000000..8bf80fc54 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/spec/forest_admin_datasource_pylon/throttle_spec.rb @@ -0,0 +1,97 @@ +RSpec.describe ForestAdminDatasourcePylon::Throttle do + let(:limiter) { instance_double(ForestAdminDatasourcePylon::RateLimiter, acquire: nil) } + let(:retry_policy) { ForestAdminDatasourcePylon::RetryPolicy.new(max_retries: 2, interval: 0) } + let(:configuration) do + ForestAdminDatasourcePylon::Configuration.new(api_key: 'k', retry_policy: retry_policy, rate_limiter: limiter) + end + let(:client) { ForestAdminDatasourcePylon::Client.new(configuration) } + let(:base) { configuration.url } + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + it 'asks for a slot before the request goes out' do + stub_request(:get, "#{base}/me").to_return(json('data' => { 'id' => 'org_1' })) + + client.me + + expect(limiter).to have_received(:acquire).with(:get, '/me').once + end + + it 'hands over the path rather than the whole url, which is what the table matches' do + stub_request(:get, "#{base}/issues/abc-123/messages").to_return(json('data' => [])) + + client.fetch_issue_messages('abc-123') + + expect(limiter).to have_received(:acquire).with(:get, '/issues/abc-123/messages') + end + + it 'meters a search under its own verb' do + stub_request(:post, "#{base}/issues/search").to_return(json('data' => [])) + + client.search_issues(limit: 10) + + expect(limiter).to have_received(:acquire).with(:post, '/issues/search') + end + + # The whole point of sitting inside `retry` rather than outside it: a replay + # spends the budget a first attempt did, so it has to wait for a slot too. + # Outside, this would run once for a request that reached Pylon twice. + it 'asks for a slot on every attempt, replays included' do + stub_request(:get, "#{base}/me") + .to_return(json({ 'message' => 'slow down' }, 429)) + .then.to_return(json('data' => { 'id' => 'org_1' })) + + client.me + + expect(WebMock).to have_requested(:get, "#{base}/me").twice + expect(limiter).to have_received(:acquire).twice + end + + # Faraday hands over the path of the resolved url, prefix included. Left on, + # every anchored rule in the table misses and the whole datasource meters in + # one fallback bucket at a tenth of the budget Pylon grants. + describe 'when the base url is mounted under a subpath' do + let(:configuration) do + ForestAdminDatasourcePylon::Configuration.new( + api_key: 'k', base_url: 'https://proxy.test/pylon/v1', retry_policy: retry_policy, rate_limiter: limiter + ) + end + + it 'meters the endpoint rather than the prefix in front of it' do + stub_request(:get, "#{base}/issues/abc-123").to_return(json('data' => { 'id' => 'abc-123' })) + + client.fetch_issue('abc-123') + + expect(limiter).to have_received(:acquire).with(:get, '/issues/abc-123') + end + + # The bucket the limiter ends up on is the point: an unstripped path resolves + # to `get /pylon (undocumented)` at 30 a minute, pooling every read together. + it 'lands on the documented bucket, not the fallback one' do + stub_request(:get, "#{base}/issues/abc-123").to_return(json('data' => { 'id' => 'abc-123' })) + paths = [] + allow(limiter).to receive(:acquire) { |_method, path| paths << path } + + client.fetch_issue('abc-123') + + rule = ForestAdminDatasourcePylon::RateLimits.for(:get, paths.first) + expect(rule.name).to eq('get /issues/:id') + expect(rule.limit).to eq(300) + end + end + + describe 'when the configuration declines a limiter' do + let(:configuration) do + ForestAdminDatasourcePylon::Configuration.new(api_key: 'k', retry_policy: retry_policy, rate_limiter: nil) + end + + it 'leaves the throttle out of the stack' do + stub_request(:get, "#{base}/me").to_return(json('data' => { 'id' => 'org_1' })) + + expect(client.me).to eq('id' => 'org_1') + expect(limiter).not_to have_received(:acquire) + end + end +end diff --git a/packages/forest_admin_datasource_pylon/spec/spec_helper.rb b/packages/forest_admin_datasource_pylon/spec/spec_helper.rb new file mode 100644 index 000000000..4998deb37 --- /dev/null +++ b/packages/forest_admin_datasource_pylon/spec/spec_helper.rb @@ -0,0 +1,59 @@ +require 'simplecov' +# JSON output is consumed by the qlty CI coverage step; HTML is for local +# inspection. simplecov-html and simplecov_json_formatter are required only +# in Gemfile-test, so guard the require for local Gemfile runs. +begin + require 'simplecov_json_formatter' + require 'simplecov-html' + SimpleCov.formatters = [SimpleCov::Formatter::JSONFormatter, SimpleCov::Formatter::HTMLFormatter] +rescue LoadError + # Local Gemfile run without the CI formatters; default text output is fine. +end + +SimpleCov.start do + add_filter '/spec/' + enable_coverage :branch + minimum_coverage 90 +end + +SimpleCov.coverage_dir 'coverage' + +require 'webmock/rspec' +require 'forest_admin_datasource_customizer' +require 'forest_admin_datasource_pylon' + +WebMock.disable_net_connect!(allow_localhost: true) + +# A datasource introspects the custom fields of its three object types while it +# registers its collections. A spec building one declares what those calls +# answer -- most of them, having nothing to do with custom fields, answer +# nothing. The base url is not taken from the datasource on purpose: reading it +# would build the datasource, and boot the introspection this stubs. +module PylonCustomFieldStubs + def stub_custom_fields(issue: [], account: [], contact: [], + base: ForestAdminDatasourcePylon::Configuration::DEFAULT_BASE_URL) + { 'issue' => issue, 'account' => account, 'contact' => contact }.each do |object_type, definitions| + stub_request(:get, "#{base}/custom-fields") + .with(query: { 'object_type' => object_type }) + .to_return(status: 200, body: { 'data' => definitions }.to_json, + headers: { 'Content-Type' => 'application/json' }) + end + end +end + +RSpec.configure do |config| + config.include PylonCustomFieldStubs + + config.expect_with :rspec do |c| + c.syntax = :expect + end + config.mock_with :rspec do |m| + m.verify_partial_doubles = true + end + config.disable_monkey_patching! + config.warnings = false + config.order = :random + Kernel.srand config.seed + + config.before { WebMock.reset! } +end diff --git a/packages/forest_admin_datasource_toolkit/lib/forest_admin_datasource_toolkit/components/query/condition_tree/nodes/condition_tree_branch.rb b/packages/forest_admin_datasource_toolkit/lib/forest_admin_datasource_toolkit/components/query/condition_tree/nodes/condition_tree_branch.rb index a8a92c24c..a9eddf7bb 100644 --- a/packages/forest_admin_datasource_toolkit/lib/forest_admin_datasource_toolkit/components/query/condition_tree/nodes/condition_tree_branch.rb +++ b/packages/forest_admin_datasource_toolkit/lib/forest_admin_datasource_toolkit/components/query/condition_tree/nodes/condition_tree_branch.rb @@ -34,11 +34,15 @@ def replace_leafs(&handler) ) end + # Recurses through `match`, not through `every_leaf` / `some_leaf`: + # those walk down to the leaves and would apply this branch's + # aggregator to a nested one's, reading `And(a, Or(b, c))` as + # `And(a, b, c)`. def match(record, collection, timezone) if @aggregator == 'And' - every_leaf { |condition| condition.match(record, collection, timezone) } + @conditions.all? { |condition| condition.match(record, collection, timezone) } else - some_leaf { |condition| condition.match(record, collection, timezone) } + @conditions.any? { |condition| condition.match(record, collection, timezone) } end end diff --git a/packages/forest_admin_datasource_toolkit/spec/lib/forest_admin_datasource_toolkit/components/query/condition_tree/nodes/condition_tree_spec.rb b/packages/forest_admin_datasource_toolkit/spec/lib/forest_admin_datasource_toolkit/components/query/condition_tree/nodes/condition_tree_spec.rb index 732698054..0dd3a67f6 100644 --- a/packages/forest_admin_datasource_toolkit/spec/lib/forest_admin_datasource_toolkit/components/query/condition_tree/nodes/condition_tree_spec.rb +++ b/packages/forest_admin_datasource_toolkit/spec/lib/forest_admin_datasource_toolkit/components/query/condition_tree/nodes/condition_tree_spec.rb @@ -120,6 +120,70 @@ module Nodes 'Europe/Paris')).to be_truthy end + it 'keeps the aggregator of a nested branch' do + collection = Collection.new(Datasource.new, 'myCollection') + collection.add_fields( + { + 'role' => ColumnSchema.new( + column_type: PrimitiveType::STRING, + filter_operators: [Operators::EQUAL] + ), + 'name' => ColumnSchema.new( + column_type: PrimitiveType::STRING, + filter_operators: [Operators::EQUAL] + ) + } + ) + tree = ConditionTreeBranch.new('And', [ + ConditionTreeLeaf.new('role', Operators::EQUAL, 'admin'), + ConditionTreeBranch.new('Or', [ + ConditionTreeLeaf.new('name', + Operators::EQUAL, + 'alice'), + ConditionTreeLeaf.new('name', + Operators::EQUAL, + 'carol') + ]) + ]) + + expect(tree.match({ 'role' => 'admin', 'name' => 'alice' }, collection, 'Europe/Paris')).to be_truthy + expect(tree.match({ 'role' => 'admin', 'name' => 'carol' }, collection, 'Europe/Paris')).to be_truthy + expect(tree.match({ 'role' => 'admin', 'name' => 'bob' }, collection, 'Europe/Paris')).to be_falsey + expect(tree.match({ 'role' => 'user', 'name' => 'alice' }, collection, 'Europe/Paris')).to be_falsey + end + + it 'stops at the first condition of a nested branch that settles it' do + collection = Collection.new(Datasource.new, 'myCollection') + collection.add_fields( + { + 'role' => ColumnSchema.new( + column_type: PrimitiveType::STRING, + filter_operators: [Operators::EQUAL] + ), + 'age' => ColumnSchema.new( + column_type: PrimitiveType::NUMBER, + filter_operators: [Operators::EQUAL] + ) + } + ) + # The shape a nil-unsafe comparison is guarded with: the presence + # check must keep the comparison from ever seeing the null. + tree = ConditionTreeBranch.new('Or', [ + ConditionTreeLeaf.new('role', Operators::EQUAL, 'ghost'), + ConditionTreeBranch.new('And', [ + ConditionTreeLeaf.new('age', + Operators::PRESENT), + ConditionTreeLeaf.new('age', + Operators::GREATER_THAN, + 20) + ]) + ]) + + expect(tree.match({ 'role' => 'user', 'age' => nil }, collection, 'Europe/Paris')).to be_falsey + expect(tree.match({ 'role' => 'user', 'age' => 30 }, collection, 'Europe/Paris')).to be_truthy + expect(tree.match({ 'role' => 'user', 'age' => 10 }, collection, 'Europe/Paris')).to be_falsey + end + it 'works with null value' do collection = Collection.new(Datasource.new, 'myCollection') collection.add_fields(