Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 2 additions & 10 deletions apps/sim/lib/api/server/routes/billing-read.ts
Original file line number Diff line number Diff line change
@@ -1,23 +1,15 @@
import { getPostgresCancellationReason } from '@sim/utils/errors'
import {
extendInternalErrorPolicy,
internalErrorResponse,
internalOrchestrationErrorPolicy,
} from '@/lib/api/server/routes/internal-json-route'
import { DatabaseReadDeadlineError, isTransientDatabaseReadError } from '@/lib/db/read-retry'
import { isDatabaseReadUnavailable } from '@/lib/db/read-retry'

/** Presents known temporary read failures as retryable 503s without masking permanent errors. */
export const internalBillingReadErrorPolicy = extendInternalErrorPolicy(
internalOrchestrationErrorPolicy,
(error) => {
const cancellationReason = getPostgresCancellationReason(error)
if (
!(error instanceof DatabaseReadDeadlineError) &&
!isTransientDatabaseReadError(error) &&
cancellationReason !== 'statement_timeout' &&
cancellationReason !== 'transaction_timeout'
)
return null
if (!isDatabaseReadUnavailable(error)) return null
return internalErrorResponse(
503,
{ error: 'Billing information is temporarily unavailable. Please try again.' },
Expand Down
15 changes: 14 additions & 1 deletion apps/sim/lib/db/read-retry.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { createLogger } from '@sim/logger'
import { getPostgresErrorCode } from '@sim/utils/errors'
import { getPostgresCancellationReason, getPostgresErrorCode } from '@sim/utils/errors'
import { sleep } from '@sim/utils/helpers'
import { backoffWithJitter } from '@sim/utils/retry'

Expand Down Expand Up @@ -43,6 +43,19 @@ export function isTransientDatabaseReadError(error: unknown): boolean {
return code !== undefined && TRANSIENT_READ_CODES.has(code)
}

/**
* Whether a failed read is temporary for the caller: a retryable driver error, an exhausted read
* deadline, or a statement or transaction timeout. Surfaces present these as retry-later responses;
* the timeouts stay out of {@link isTransientDatabaseReadError} because re-running a cancelled
* scan in-process only repeats its cost. Not interchangeable with `isRetryableInfrastructureError`:
* that one treats every 57014 cancel as retryable, including a user-requested one.
*/
export function isDatabaseReadUnavailable(error: unknown): boolean {
if (error instanceof DatabaseReadDeadlineError || isTransientDatabaseReadError(error)) return true
const cancellationReason = getPostgresCancellationReason(error)
return cancellationReason === 'statement_timeout' || cancellationReason === 'transaction_timeout'
}

/**
* Retries independent reads, rebuilding each statement or fresh read-only transaction
* after the previous attempt has fully rolled back. Never retry writes, locking reads,
Expand Down
43 changes: 42 additions & 1 deletion apps/sim/lib/logs/api/route-policies.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,55 @@ import {
authInternalDelegationMock,
authInternalDelegationMockFns,
} from '@sim/testing/mocks/auth-internal-delegation.mock'
import { DrizzleQueryError } from 'drizzle-orm/errors'
import { NextRequest } from 'next/server'
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'

vi.mock('@/lib/auth/internal-delegation', () => authInternalDelegationMock)
vi.unmock('@/lib/auth/internal')

import { generateInternalDelegationToken } from '@/lib/auth/internal'
import { internalLogsSessionOrExecutorAuth } from '@/lib/logs/api/route-policies'
import {
internalLogsSessionOrExecutorAuth,
v2LogErrorPolicies,
} from '@/lib/logs/api/route-policies'

function failedQuery(code: string, message: string): DrizzleQueryError {
return new DrizzleQueryError('select', [], Object.assign(new Error(message), { code }))
}

describe('v2 logs error policies', () => {
const policies = Object.entries(v2LogErrorPolicies)

it.each(policies)(
'%s answers a replica recovery conflict with a retryable 503',
async (_, policy) => {
const response = policy.render(
failedQuery('40001', 'canceling statement due to conflict with recovery')
)

expect(response?.status).toBe(503)
expect(response?.headers.get('Retry-After')).toBeTruthy()
expect(await response?.json()).toMatchObject({ error: { code: 'SERVICE_UNAVAILABLE' } })
}
)

it.each(policies)('%s answers a statement timeout with a retryable 503', (_, policy) => {
const response = policy.render(
failedQuery('57014', 'canceling statement due to statement timeout')
)

expect(response?.status).toBe(503)
expect(response?.headers.get('Retry-After')).toBeTruthy()
})

it.each(policies)('%s leaves a permanent database failure unhandled', (_, policy) => {
expect(policy.render(failedQuery('42P01', 'relation does not exist'))).toBeNull()
expect(
policy.render(failedQuery('57014', 'canceling statement due to user request'))
).toBeNull()
})
})

const mockGetSession = authMockFns.mockGetSession
const mockBindDelegation = authInternalDelegationMockFns.mockBindInternalExecutorDelegation
Expand Down
15 changes: 12 additions & 3 deletions apps/sim/lib/logs/api/route-policies.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
import {
createInternalSessionOrExecutorAuth,
createV2ResourceConcealmentPolicy,
v2OrchestrationErrorPolicy,
type V2ErrorPolicy,
} from '@/lib/api/server/routes'
import { isDatabaseReadUnavailable } from '@/lib/db/read-retry'
import { LOGS_DELEGATION_AUDIENCE } from '@/lib/logs/application/authorization'
import { v2CaughtOrchestrationError, v2Error } from '@/app/api/v2/lib/response'

const internalLogsSessionOrExecutorAuthBase = createInternalSessionOrExecutorAuth({
audience: LOGS_DELEGATION_AUDIENCE,
Expand All @@ -27,6 +29,12 @@ export const internalLogsSessionOrExecutorAuth = {
},
}

const renderV2LogError: V2ErrorPolicy['render'] = (error) => {
if (isDatabaseReadUnavailable(error))
return v2Error('SERVICE_UNAVAILABLE', 'Logs are temporarily unavailable. Please try again.')
return v2CaughtOrchestrationError(error)
}

/**
* `GET /logs` and `GET /billing/logs` both take a caller-named `workspaceId` and
* answer differently when the caller cannot reach it: this list projects the
Expand All @@ -49,8 +57,9 @@ export const internalLogsSessionOrExecutorAuth = {
* inconsistency for a larger one.
*/
export const v2LogErrorPolicies = {
default: v2OrchestrationErrorPolicy,
default: { render: renderV2LogError },
concealDetailAuthorization: createV2ResourceConcealmentPolicy({
notFoundMessage: 'Log not found',
render: renderV2LogError,
}),
} as const
} as const satisfies Record<string, V2ErrorPolicy>
Loading