Skip to content

Commit bc4160d

Browse files
Bill LeoutsakosBill Leoutsakos
Bill Leoutsakos
authored and
Bill Leoutsakos
committed
fix(cloudwatch): paginate log selectors
1 parent 349f40a commit bc4160d

7 files changed

Lines changed: 184 additions & 17 deletions

File tree

‎apps/sim/lib/internal/cloudwatch/client.ts‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -145,13 +145,23 @@ export interface DescribedLogStream {
145145
export async function describeLogStreams(
146146
client: CloudWatchLogsClient,
147147
logGroupName: string,
148-
options?: { prefix?: string; limit?: number; suppressTruncationLog?: boolean },
148+
options?: {
149+
prefix?: string
150+
limit?: number
151+
nextToken?: string
152+
suppressTruncationLog?: boolean
153+
},
149154
signal?: AbortSignal
150-
): Promise<{ logStreams: DescribedLogStream[]; truncated: boolean; pages: number }> {
155+
): Promise<{
156+
logStreams: DescribedLogStream[]
157+
truncated: boolean
158+
pages: number
159+
nextToken?: string
160+
}> {
151161
const hasPrefix = Boolean(options?.prefix)
152162
const totalLimit = options?.limit
153163
const logStreams: DescribedLogStream[] = []
154-
let nextToken: string | undefined
164+
let nextToken = options?.nextToken
155165
let pages = 0
156166
let truncated = false
157167

@@ -167,7 +177,7 @@ export async function describeLogStreams(
167177
? { orderBy: 'LogStreamName', logStreamNamePrefix: options!.prefix }
168178
: { orderBy: 'LastEventTime', descending: true }),
169179
limit: pageLimit,
170-
...(nextToken && { nextToken }),
180+
...(nextToken !== undefined ? { nextToken } : {}),
171181
})
172182

173183
const response = await client.send(command, { abortSignal: signal })
@@ -202,6 +212,7 @@ export async function describeLogStreams(
202212
logStreams: totalLimit !== undefined ? logStreams.slice(0, totalLimit) : logStreams,
203213
truncated,
204214
pages,
215+
...(nextToken !== undefined ? { nextToken } : {}),
205216
}
206217
}
207218

‎apps/sim/lib/selectors/manifest.test.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,11 @@ describe('selector manifest', () => {
5959
])
6060
})
6161

62+
it('declares both CloudWatch selectors as paginated', () => {
63+
expect(selectorManifest['cloudwatch.logGroups'].listMode).toBe('paginated')
64+
expect(selectorManifest['cloudwatch.logStreams'].listMode).toBe('paginated')
65+
})
66+
6267
/**
6368
* `serviceIds` names which credentials a selector accepts; the integration
6469
* allowlist has to judge which resource it *reaches*, and for a shared

‎apps/sim/lib/selectors/manifest.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -321,6 +321,7 @@ export const selectorManifest = {
321321
{
322322
readiness: { all: ['awsAccessKeyId', 'awsSecretAccessKey', 'awsRegion'] },
323323
sensitive: ['awsAccessKeyId', 'awsSecretAccessKey'],
324+
listMode: 'paginated',
324325
search: true,
325326
detail: true,
326327
}
@@ -332,6 +333,7 @@ export const selectorManifest = {
332333
all: ['awsAccessKeyId', 'awsSecretAccessKey', 'awsRegion', 'logGroupName'],
333334
},
334335
sensitive: ['awsAccessKeyId', 'awsSecretAccessKey'],
336+
listMode: 'paginated',
335337
search: true,
336338
detail: true,
337339
}

‎apps/sim/lib/selectors/server/providers/cloudwatch.test.ts‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,13 @@ function logGroupArgs(signal?: AbortSignal): ExecuteServerSelectorArgs {
3737
}
3838
}
3939

40+
function logStreamArgs(): ExecuteServerSelectorArgs {
41+
const args = logGroupArgs()
42+
args.selectorKey = 'cloudwatch.logStreams'
43+
args.context.logGroupName = '/aws/lambda/example'
44+
return args
45+
}
46+
4047
function cloudWatchError(status: number): CloudWatchLogsServiceException {
4148
return new CloudWatchLogsServiceException({
4249
name: 'CloudWatchLogsError',
@@ -45,7 +52,7 @@ function cloudWatchError(status: number): CloudWatchLogsServiceException {
4552
})
4653
}
4754

48-
describe('CloudWatch server selector adapter errors', () => {
55+
describe('CloudWatch server selector adapter', () => {
4956
beforeEach(() => vi.clearAllMocks())
5057

5158
it.each([
@@ -96,6 +103,47 @@ describe('CloudWatch server selector adapter errors', () => {
96103
)
97104
})
98105

106+
it.each([
107+
{
108+
selectorKey: 'cloudwatch.logGroups' as const,
109+
mockListing: mockListCloudWatchLogGroups,
110+
args: logGroupArgs,
111+
providerField: 'logGroupName' as const,
112+
binding: {},
113+
},
114+
{
115+
selectorKey: 'cloudwatch.logStreams' as const,
116+
mockListing: mockListCloudWatchLogStreams,
117+
args: logStreamArgs,
118+
providerField: 'logStreamName' as const,
119+
binding: { logGroupName: '/aws/lambda/example' },
120+
},
121+
])('forwards opaque cursors for $selectorKey', async (testCase) => {
122+
testCase.mockListing.mockResolvedValueOnce({
123+
items: [{ [testCase.providerField]: 'target' }],
124+
pages: 20,
125+
truncated: true,
126+
nextToken: 'opaque::next+=',
127+
})
128+
const args = testCase.args()
129+
args.request = { kind: 'list', search: 'target', cursor: 'opaque::start+=' }
130+
131+
await expect(
132+
cloudWatchSelectorAttachments[testCase.selectorKey].execute(args)
133+
).resolves.toEqual({
134+
kind: 'list',
135+
items: [{ id: 'target', label: 'target' }],
136+
nextCursor: 'opaque::next+=',
137+
})
138+
expect(testCase.mockListing).toHaveBeenCalledWith(
139+
expect.objectContaining({
140+
prefix: 'target',
141+
nextToken: 'opaque::start+=',
142+
...testCase.binding,
143+
})
144+
)
145+
})
146+
99147
it('rejects an invalid region before invoking the AWS listing helper', async () => {
100148
const args = logGroupArgs()
101149
args.context.awsRegion = 'not-a-region'

‎apps/sim/lib/selectors/server/providers/cloudwatch.ts‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -81,11 +81,12 @@ export const cloudWatchSelectorAttachments = {
8181
match?.logGroupName ? { id: match.logGroupName, label: match.logGroupName } : null
8282
)
8383
}
84-
const search = args.request.search
84+
const { search, cursor } = args.request
8585
const groups = await executeCloudWatchListing(args.signal, () =>
8686
listCloudWatchLogGroups({
8787
credentials: listingCredentials,
8888
prefix: search,
89+
nextToken: cursor,
8990
signal: args.signal,
9091
suppressTruncationLog: true,
9192
})
@@ -94,10 +95,7 @@ export const cloudWatchSelectorAttachments = {
9495
groups.items
9596
.filter((group) => group.logGroupName)
9697
.map((group) => ({ id: group.logGroupName, label: group.logGroupName })),
97-
undefined,
98-
groups.truncated
99-
? { truncated: { reason: 'provider-cap', pages: groups.pages } }
100-
: undefined
98+
groups.nextToken
10199
)
102100
},
103101
},
@@ -122,12 +120,13 @@ export const cloudWatchSelectorAttachments = {
122120
match?.logStreamName ? { id: match.logStreamName, label: match.logStreamName } : null
123121
)
124122
}
125-
const search = args.request.search
123+
const { search, cursor } = args.request
126124
const streams = await executeCloudWatchListing(args.signal, () =>
127125
listCloudWatchLogStreams({
128126
credentials: listingCredentials,
129127
logGroupName: args.context.logGroupName!,
130128
prefix: search,
129+
nextToken: cursor,
131130
signal: args.signal,
132131
suppressTruncationLog: true,
133132
})
@@ -136,10 +135,7 @@ export const cloudWatchSelectorAttachments = {
136135
streams.items
137136
.filter((stream) => stream.logStreamName)
138137
.map((stream) => ({ id: stream.logStreamName, label: stream.logStreamName })),
139-
undefined,
140-
streams.truncated
141-
? { truncated: { reason: 'provider-cap', pages: streams.pages } }
142-
: undefined
138+
streams.nextToken
143139
)
144140
},
145141
},

‎apps/sim/tools/cloudwatch/listing.test.ts‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
/**
2+
* @vitest-environment node
3+
*/
4+
import { beforeEach, describe, expect, it, vi } from 'vitest'
5+
6+
const mocks = vi.hoisted(() => ({
7+
createCloudWatchLogsClient: vi.fn(),
8+
destroy: vi.fn(),
9+
send: vi.fn(),
10+
}))
11+
12+
vi.mock('@/lib/internal/cloudwatch/client', async (importOriginal) => ({
13+
...(await importOriginal<typeof import('@/lib/internal/cloudwatch/client')>()),
14+
createCloudWatchLogsClient: mocks.createCloudWatchLogsClient,
15+
}))
16+
17+
import { listCloudWatchLogGroups, listCloudWatchLogStreams } from '@/tools/cloudwatch/listing'
18+
19+
const CREDENTIALS = {
20+
region: 'us-east-1',
21+
accessKeyId: 'access-key',
22+
secretAccessKey: 'secret-key',
23+
}
24+
25+
function mockTwentyPages(
26+
collection: 'logGroups' | 'logStreams',
27+
name: 'logGroupName' | 'logStreamName',
28+
finalToken: string
29+
) {
30+
let page = 0
31+
mocks.send.mockImplementation(async () => {
32+
page += 1
33+
return {
34+
[collection]: [{ [name]: `item-${page}` }],
35+
nextToken: page === 20 ? finalToken : `page-${page + 1}`,
36+
}
37+
})
38+
}
39+
40+
describe('CloudWatch shared listings', () => {
41+
beforeEach(() => {
42+
vi.clearAllMocks()
43+
mocks.createCloudWatchLogsClient.mockReturnValue({
44+
send: mocks.send,
45+
destroy: mocks.destroy,
46+
})
47+
})
48+
49+
it('continues log groups from an opaque token and exposes page twenty continuation', async () => {
50+
mockTwentyPages('logGroups', 'logGroupName', 'opaque::group/after-20+=')
51+
52+
const result = await listCloudWatchLogGroups({
53+
credentials: CREDENTIALS,
54+
prefix: '/aws/lambda/',
55+
nextToken: 'opaque::group/start+=',
56+
suppressTruncationLog: true,
57+
})
58+
59+
expect(result).toMatchObject({
60+
pages: 20,
61+
truncated: true,
62+
nextToken: 'opaque::group/after-20+=',
63+
})
64+
expect(result.items).toHaveLength(20)
65+
expect(mocks.send).toHaveBeenCalledTimes(20)
66+
expect(mocks.send.mock.calls[0]?.[0].input).toMatchObject({
67+
logGroupNamePrefix: '/aws/lambda/',
68+
limit: 50,
69+
nextToken: 'opaque::group/start+=',
70+
})
71+
})
72+
73+
it('continues streams within their group and prefix and exposes page twenty continuation', async () => {
74+
mockTwentyPages('logStreams', 'logStreamName', 'opaque::stream/after-20+=')
75+
76+
const result = await listCloudWatchLogStreams({
77+
credentials: CREDENTIALS,
78+
logGroupName: '/aws/lambda/example',
79+
prefix: '2026/09/',
80+
nextToken: 'opaque::stream/start+=',
81+
suppressTruncationLog: true,
82+
})
83+
84+
expect(result).toMatchObject({
85+
pages: 20,
86+
truncated: true,
87+
nextToken: 'opaque::stream/after-20+=',
88+
})
89+
expect(result.items).toHaveLength(20)
90+
expect(mocks.send).toHaveBeenCalledTimes(20)
91+
expect(mocks.send.mock.calls[0]?.[0].input).toMatchObject({
92+
logGroupName: '/aws/lambda/example',
93+
logStreamNamePrefix: '2026/09/',
94+
orderBy: 'LogStreamName',
95+
limit: 50,
96+
nextToken: 'opaque::stream/start+=',
97+
})
98+
})
99+
})

‎apps/sim/tools/cloudwatch/listing.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,19 +28,21 @@ export interface CloudWatchListingResult<T> {
2828
items: T[]
2929
truncated: boolean
3030
pages: number
31+
nextToken?: string
3132
}
3233

3334
export async function listCloudWatchLogGroups(input: {
3435
credentials: CloudWatchListingCredentials
3536
prefix?: string
3637
limit?: number
38+
nextToken?: string
3739
signal?: AbortSignal
3840
suppressTruncationLog?: boolean
3941
}): Promise<CloudWatchListingResult<DescribedLogGroup>> {
4042
const client = createCloudWatchLogsClient(input.credentials)
4143
try {
4244
const groups: DescribedLogGroup[] = []
43-
let nextToken: string | undefined
45+
let nextToken = input.nextToken
4446
let pages = 0
4547
let truncated = false
4648
for (let page = 0; page < MAX_PAGES; page += 1) {
@@ -50,7 +52,7 @@ export async function listCloudWatchLogGroups(input: {
5052
new DescribeLogGroupsCommand({
5153
...(input.prefix ? { logGroupNamePrefix: input.prefix } : {}),
5254
limit: Math.min(PAGE_SIZE, remaining),
53-
...(nextToken ? { nextToken } : {}),
55+
...(nextToken !== undefined ? { nextToken } : {}),
5456
}),
5557
input.signal ? { abortSignal: input.signal } : undefined
5658
)
@@ -80,6 +82,7 @@ export async function listCloudWatchLogGroups(input: {
8082
items: input.limit === undefined ? groups : groups.slice(0, input.limit),
8183
truncated,
8284
pages,
85+
...(nextToken !== undefined ? { nextToken } : {}),
8386
}
8487
} finally {
8588
client.destroy()
@@ -91,6 +94,7 @@ export async function listCloudWatchLogStreams(input: {
9194
logGroupName: string
9295
prefix?: string
9396
limit?: number
97+
nextToken?: string
9498
signal?: AbortSignal
9599
suppressTruncationLog?: boolean
96100
}): Promise<CloudWatchListingResult<DescribedLogStream>> {
@@ -102,6 +106,7 @@ export async function listCloudWatchLogStreams(input: {
102106
{
103107
prefix: input.prefix,
104108
limit: input.limit,
109+
...(input.nextToken !== undefined ? { nextToken: input.nextToken } : {}),
105110
...(input.suppressTruncationLog !== undefined
106111
? { suppressTruncationLog: input.suppressTruncationLog }
107112
: {}),
@@ -112,6 +117,7 @@ export async function listCloudWatchLogStreams(input: {
112117
items: result.logStreams,
113118
truncated: result.truncated ?? false,
114119
pages: result.pages ?? 0,
120+
...(result.nextToken !== undefined ? { nextToken: result.nextToken } : {}),
115121
}
116122
} finally {
117123
client.destroy()

0 commit comments

Comments
 (0)