fix(selectors): paginate SharePoint sites and lists (#7335)

Co-authored-by: Bill Leoutsakos <billleoutsakos@Bills-MacBook-Pro.local>

Author: BillLeoutsakosvl346

apps/sim/lib/selectors/manifest.ts

@@ -189,6 +189,7 @@ export const selectorManifest = {
   'pipedrive.pipelines': providerSelector([], { detail: true }),
   'sharepoint.lists': providerSelector(['siteId'], {
     readiness: { all: ['oauthCredential', 'siteId'] },
+    listMode: 'paginated',
     detail: true,
   }),
   'trello.boards': providerSelector([], { detail: true }),
@@ -254,7 +255,11 @@ export const selectorManifest = {
   }),
   'onedrive.files': providerSelector(['mimeType'], { listMode: 'paginated', detail: true }),
   'onedrive.folders': providerSelector(['driveId'], { listMode: 'paginated', detail: true }),
-  'sharepoint.sites': providerSelector([], { detail: true }),
+  'sharepoint.sites': providerSelector([], {
+    listMode: 'paginated',
+    search: true,
+    detail: true,
+  }),
   'microsoft.excel': providerSelector(['driveId'], {
     listMode: 'paginated',
     search: true,

apps/sim/lib/selectors/server/providers/sharepoint.test.ts

@@ -37,6 +37,21 @@ function detailArgs(
   }
 }
 
+function listArgs(
+  selectorKey: 'sharepoint.lists' | 'sharepoint.sites',
+  cursor?: string,
+  search?: string
+): ExecuteServerSelectorArgs {
+  return {
+    ...detailArgs(selectorKey, ''),
+    request: {
+      kind: 'list',
+      ...(cursor ? { cursor } : {}),
+      ...(search ? { search } : {}),
+    },
+  }
+}
+
 describe('SharePoint server selector adapter', () => {
   beforeEach(() => {
     vi.clearAllMocks()
@@ -46,6 +61,76 @@ describe('SharePoint server selector adapter', () => {
 
   afterAll(() => vi.unstubAllGlobals())
 
+  it.each([
+    {
+      selectorKey: 'sharepoint.sites' as const,
+      search: '  Engineering  ',
+      firstValue: { id: 'site-1', name: 'Engineering' },
+      firstItem: { id: 'site-1', label: 'Engineering' },
+      secondValue: { id: 'site-2', name: 'Operations' },
+      secondItem: { id: 'site-2', label: 'Operations' },
+      nextCursor: 'https://graph.microsoft.com/v1.0/sites?search=Engineering&$skiptoken=next',
+    },
+    {
+      selectorKey: 'sharepoint.lists' as const,
+      search: undefined,
+      firstValue: { id: 'list-1', displayName: 'Planning', list: { hidden: false } },
+      firstItem: { id: 'list-1', label: 'Planning' },
+      secondValue: { id: 'list-2', displayName: 'Operations', list: { hidden: false } },
+      secondItem: { id: 'list-2', label: 'Operations' },
+      nextCursor:
+        'https://graph.microsoft.com/v1.0/sites/contoso.sharepoint.com%2Csite%2Cweb/lists?$skiptoken=next',
+    },
+  ])('paginates $selectorKey only when its cursor is requested', async (testCase) => {
+    mockFetch
+      .mockResolvedValueOnce(
+        new Response(
+          JSON.stringify({ value: [testCase.firstValue], '@odata.nextLink': testCase.nextCursor }),
+          { status: 200 }
+        )
+      )
+      .mockResolvedValueOnce(
+        new Response(JSON.stringify({ value: [testCase.secondValue] }), { status: 200 })
+      )
+
+    const first = await sharepointSelectorAttachments[testCase.selectorKey].execute(
+      listArgs(testCase.selectorKey, undefined, testCase.search)
+    )
+
+    expect(first).toEqual({
+      kind: 'list',
+      items: [testCase.firstItem],
+      nextCursor: testCase.nextCursor,
+    })
+    expect(mockFetch).toHaveBeenCalledTimes(1)
+    if (testCase.search) {
+      expect(new URL(String(mockFetch.mock.calls[0]?.[0])).searchParams.get('search')).toBe(
+        'Engineering'
+      )
+    }
+
+    const second = await sharepointSelectorAttachments[testCase.selectorKey].execute(
+      listArgs(testCase.selectorKey, testCase.nextCursor, testCase.search)
+    )
+
+    expect(second).toEqual({ kind: 'list', items: [testCase.secondItem] })
+    expect(String(mockFetch.mock.calls[1]?.[0])).toBe(testCase.nextCursor)
+    expect(mockFetch).toHaveBeenCalledTimes(2)
+  })
+
+  it('rejects a Graph cursor for another SharePoint resource', async () => {
+    await expect(
+      sharepointSelectorAttachments['sharepoint.lists'].execute(
+        listArgs(
+          'sharepoint.lists',
+          'https://graph.microsoft.com/v1.0/sites/another-site/lists?$skiptoken=next'
+        )
+      )
+    ).rejects.toMatchObject({ name: 'SelectorContextUnavailableError' })
+    expect(mockResolveSelectorOAuthAccessToken).not.toHaveBeenCalled()
+    expect(mockFetch).not.toHaveBeenCalled()
+  })
+
   it('hydrates a selected list directly within its site', async () => {
     mockFetch.mockResolvedValueOnce(
       new Response(JSON.stringify({ id: 'list-1', displayName: 'Planning' }), { status: 200 })

apps/sim/lib/selectors/server/providers/sharepoint.ts

@@ -9,20 +9,19 @@ import {
   SelectorContextUnavailableError,
   SelectorOptionsUnavailableError,
 } from '@/lib/selectors/server/errors'
-import { flatSelectorResult } from '@/lib/selectors/server/providers/flat-results'
 import { fetchProviderJson } from '@/lib/selectors/server/providers/provider-http'
 import {
   detailSelectorResult,
   type ExecuteServerSelectorArgs,
+  listSelectorResult,
+  requireListRequest,
   type ServerSelectorAttachmentMap,
 } from '@/lib/selectors/server/types'
 import type { SafeSelectorOption } from '@/lib/selectors/types'
 import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils'
 
 type SharePointSelectorKey = Extract<ServerSelectorKey, 'sharepoint.lists' | 'sharepoint.sites'>
 
-const MAX_GRAPH_PAGES = 10
-
 const sharepointCredential = {
   kind: 'stored',
   field: 'oauthCredential',
@@ -44,24 +43,43 @@ async function graphToken(args: ExecuteServerSelectorArgs): Promise<string> {
   })
 }
 
-async function drainGraph<T>(
+interface GraphPage<T> {
+  items: T[]
+  nextCursor?: string
+}
+
+function graphPageUrl(cursor: string | undefined, initialUrl: string): string {
+  if (!cursor) return initialUrl
+  let cursorUrl: string
+  try {
+    cursorUrl = assertGraphNextPageUrl(cursor)
+  } catch {
+    throw new SelectorContextUnavailableError()
+  }
+  if (new URL(cursorUrl).pathname !== new URL(initialUrl).pathname) {
+    throw new SelectorContextUnavailableError()
+  }
+  return cursorUrl
+}
+
+async function fetchGraphPage<T>(
   args: ExecuteServerSelectorArgs,
   initialUrl: string
-): Promise<{ values: T[]; truncated: boolean }> {
+): Promise<GraphPage<T>> {
+  const request = requireListRequest(args.selectorKey, args.request)
+  const requestUrl = graphPageUrl(request.cursor, initialUrl)
   const token = await graphToken(args)
-  const values: T[] = []
-  let nextUrl: string | undefined = initialUrl
-  for (let page = 0; page < MAX_GRAPH_PAGES && nextUrl; page++) {
-    const data = await fetchProviderJson<{ value?: T[] } & Record<string, unknown>>(nextUrl, {
-      headers: { Authorization: `Bearer ${token}` },
-      signal: args.signal,
-      redirect: 'error',
-    })
-    if (Array.isArray(data.value)) values.push(...data.value)
-    const nextLink = getGraphNextPageUrl(data)
-    nextUrl = nextLink ? assertGraphNextPageUrl(nextLink) : undefined
+  const data = await fetchProviderJson<{ value?: T[] } & Record<string, unknown>>(requestUrl, {
+    headers: { Authorization: `Bearer ${token}` },
+    signal: args.signal,
+    redirect: 'error',
+  })
+  const nextLink = getGraphNextPageUrl(data)
+  const nextCursor = nextLink ? graphPageUrl(nextLink, initialUrl) : undefined
+  return {
+    items: Array.isArray(data.value) ? data.value : [],
+    ...(nextCursor ? { nextCursor } : {}),
   }
-  return { values, truncated: Boolean(nextUrl) }
 }
 
 function requireSiteId(value: string | undefined): string {
@@ -130,7 +148,7 @@ async function getSite(
 
 async function listLists(args: ExecuteServerSelectorArgs) {
   const siteId = requireSiteId(args.context.siteId)
-  const result = await drainGraph<{
+  const page = await fetchGraphPage<{
     id: string
     displayName: string
     list?: { hidden?: boolean }
@@ -139,21 +157,26 @@ async function listLists(args: ExecuteServerSelectorArgs) {
     `https://graph.microsoft.com/v1.0/sites/${encodeURIComponent(siteId)}/lists?$select=id,displayName,description,webUrl,list&$top=999`
   )
   return {
-    items: result.values
+    items: page.items
       .filter((list) => list.list?.hidden !== true)
       .map((list) => ({ id: list.id, label: list.displayName })),
-    truncated: result.truncated,
+    nextCursor: page.nextCursor,
   }
 }
 
 async function listSites(args: ExecuteServerSelectorArgs) {
-  const result = await drainGraph<{ id: string; name: string; displayName?: string }>(
+  const request = requireListRequest(args.selectorKey, args.request)
+  const url = new URL('https://graph.microsoft.com/v1.0/sites')
+  url.searchParams.set('search', request.search?.trim() || '*')
+  url.searchParams.set('$select', 'id,name,displayName,webUrl,createdDateTime,lastModifiedDateTime')
+  url.searchParams.set('$top', '999')
+  const page = await fetchGraphPage<{ id: string; name: string; displayName?: string }>(
     args,
-    'https://graph.microsoft.com/v1.0/sites?search=*&$select=id,name,displayName,webUrl,createdDateTime,lastModifiedDateTime&$top=999'
+    url.toString()
   )
   return {
-    items: result.values.map((site) => ({ id: site.id, label: site.displayName || site.name })),
-    truncated: result.truncated,
+    items: page.items.map((site) => ({ id: site.id, label: site.displayName || site.name })),
+    nextCursor: page.nextCursor,
   }
 }
 
@@ -165,15 +188,8 @@ export const sharepointSelectorAttachments = {
       if (args.request.kind === 'detail') {
         return detailSelectorResult(await getList(args, args.request.id))
       }
-      const result = await listLists(args)
-      return flatSelectorResult(
-        args.request,
-        result.items,
-        false,
-        result.truncated
-          ? { truncated: { reason: 'provider-cap', pages: MAX_GRAPH_PAGES } }
-          : undefined
-      )
+      const page = await listLists(args)
+      return listSelectorResult(page.items, page.nextCursor)
     },
   },
   'sharepoint.sites': {
@@ -183,15 +199,8 @@ export const sharepointSelectorAttachments = {
       if (args.request.kind === 'detail') {
         return detailSelectorResult(await getSite(args, args.request.id))
       }
-      const result = await listSites(args)
-      return flatSelectorResult(
-        args.request,
-        result.items,
-        false,
-        result.truncated
-          ? { truncated: { reason: 'provider-cap', pages: MAX_GRAPH_PAGES } }
-          : undefined
-      )
+      const page = await listSites(args)
+      return listSelectorResult(page.items, page.nextCursor)
     },
   },
 } satisfies ServerSelectorAttachmentMap<SharePointSelectorKey>