add cursor based pagination

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## Version 1.6.1 - Enhancement release - 2026-08-20
+
+- Adding support for cursor based pagination
+
 ## Version 1.6.0 - Enhancement release - 2026-08-20
 
 - Added supported Python versions: 3.12, 3.13, 3.14

custom-recipes/api-connect/Cobuild.md

@@ -4,4 +4,4 @@
 - API Connect automatically injects an API-key preset into the header or query parameter configured by the user in that preset. Do not add the API key again to request fields.
 - Use the dot-separated `extraction_key` when response rows are nested under a JSON path.
 - Keep `raw_output=true` when each response item should be preserved as raw JSON instead of flattened into columns.
-- Select the pagination mechanism from the target API's documentation; do not infer one from the endpoint shape. Relative next-page URLs require `next_page_url_base`, page pagination requires `extraction_key`, and offset or page pagination requires `skip_key`.
+- Select the pagination mechanism from the target API's documentation; do not infer one from the endpoint shape. Relative next-page URLs require `next_page_url_base`, page pagination requires `extraction_key`, offset or page pagination requires `skip_key`, cursor pagination requires `cursor_next_token_path` and `cursor_query_param`, and can optionally use `cursor_initial_token` to start from a known cursor.

custom-recipes/api-connect/recipe.json

@@ -222,6 +222,7 @@
             "selectChoices":[
                 {"value": "na", "label": "No pagination"},
                 {"value": "next_page", "label": "Next page URL provided"},
+                {"value": "cursor", "label": "Cursor pagination"},
                 {"value": "offset", "label": "Offset pagination"},
                 {"value": "page", "label": "Per page"}
             ]
@@ -279,6 +280,30 @@
             "defaultValue": null,
             "visibilityCondition": "model.pagination_type=='page'"
         },
+        {
+            "name": "cursor_next_token_path",
+            "label": "Key to next cursor token",
+            "description": "Dot separated key path to next cursor token",
+            "type": "STRING",
+            "defaultValue": null,
+            "visibilityCondition": "model.pagination_type=='cursor'"
+        },
+        {
+            "name": "cursor_query_param",
+            "label": "Query parameter",
+            "description": "Name of the url parameter that will contain the cursor token in following calls",
+            "type": "STRING",
+            "defaultValue": null,
+            "visibilityCondition": "model.pagination_type=='cursor'"
+        },
+        {
+            "name": "cursor_initial_token",
+            "label": "Initial cursor value",
+            "description": "Optional value sent in the query parameter on the first request",
+            "type": "STRING",
+            "defaultValue": null,
+            "visibilityCondition": "model.pagination_type=='cursor'"
+        },
         {
             "type": "SEPARATOR",
             "label": "Advanced"

plugin.json

@@ -1,6 +1,6 @@
 {
     "id": "api-connect",
-    "version": "1.6.0",
+    "version": "1.6.1",
     "meta": {
         "label": "API Connect",
         "description": "Retrieve data from any REST API",

python-connectors/api-connect_dataset/connector.json

@@ -178,6 +178,7 @@
             "selectChoices":[
                 {"value": "na", "label": "No pagination"},
                 {"value": "next_page", "label": "Next page URL provided"},
+                {"value": "cursor", "label": "Cursor pagination"},
                 {"value": "offset", "label": "Offset pagination"},
                 {"value": "page", "label": "Per page"}
             ]
@@ -235,6 +236,30 @@
             "defaultValue": null,
             "visibilityCondition": "model.pagination_type=='page'"
         },
+        {
+            "name": "cursor_next_token_path",
+            "label": "Key to next cursor token",
+            "description": "Dot separated key path to next cursor token",
+            "type": "STRING",
+            "defaultValue": null,
+            "visibilityCondition": "model.pagination_type=='cursor'"
+        },
+        {
+            "name": "cursor_query_param",
+            "label": "Query parameter",
+            "description": "Name of the url parameter that will contain the cursor token in following calls",
+            "type": "STRING",
+            "defaultValue": null,
+            "visibilityCondition": "model.pagination_type=='cursor'"
+        },
+        {
+            "name": "cursor_initial_token",
+            "label": "Initial cursor value",
+            "description": "Optional value sent in the query parameter on the first request",
+            "type": "STRING",
+            "defaultValue": null,
+            "visibilityCondition": "model.pagination_type=='cursor'"
+        },
         {
             "type": "SEPARATOR",
             "label": "Advanced"

python-lib/dku_utils.py

@@ -39,6 +39,7 @@ def get_endpoint_parameters(configuration):
         "requests_per_minute",
         "pagination_type",
         "next_page_url_key", "is_next_page_url_relative", "next_page_url_base",
+        "cursor_next_token_path", "cursor_query_param", "cursor_initial_token",
         "top_key", "skip_key", "maximum_number_rows",
         "use_mtls", "mtls_certificate_path", "mtls_key_path",
         "force_csv_parameters", "csv_delimiter",

python-lib/pagination.py

@@ -12,6 +12,10 @@ def __init__(self):
         self.next_page_url_base = None
         self.skip_key = None
         self.next_page_url = None
+        self.cursor_next_token_path = None
+        self.cursor_query_param = None
+        self.cursor_initial_token = None
+        self.cursor_query_param_value = None
         self.records_to_skip = None
         self.pagination_type = ""
         self.counting_key = None
@@ -25,28 +29,33 @@ def __init__(self):
         self.update_next_page = self.update_next_page_default
 
     def configure_paging(self, config=None, skip_key=None,
-                         next_page_key=None, next_page_url_base=None, pagination_type="na"):
+                         next_page_key=None, next_page_url_base=None,
+                         cursor_next_token_path=None, cursor_query_param=None,
+                         cursor_initial_token=None, pagination_type="na"):
         config = {} if config is None else config
         self.pagination_type = config.get("pagination_type", pagination_type)
+        self.update_next_page = self.update_next_page_default
         if self.pagination_type == "next_page":
             self.next_page_key = config.get("next_page_key", next_page_key)
             self.next_page_key = None if self.next_page_key == '' else self.next_page_key
             if next_page_url_base:
                 next_page_url_base = next_page_url_base.strip('/')
             self.next_page_url_base = next_page_url_base
-        elif self.pagination_type in ["offset", "page"]:
-            self.skip_key = config.get("skip_key", skip_key)
-        logger.info("configure_paging: self.pagination_type='{}', self.next_page_key='{}', self.next_page_url_base='{}', self.skip_key='{}'".format(
-                self.pagination_type, self.next_page_key, self.next_page_url_base, self.skip_key
-            ))
-        if self.pagination_type == "next_page":
             self.update_next_page = self.update_next_page_link
+        elif self.pagination_type == "cursor":
+            self.cursor_next_token_path = config.get("cursor_next_token_path", cursor_next_token_path)
+            self.cursor_query_param = config.get("cursor_query_param", cursor_query_param)
+            self.cursor_initial_token = config.get("cursor_initial_token", cursor_initial_token)
+            self.update_next_page = self.update_next_page_cursor
         elif self.pagination_type == "offset":
+            self.skip_key = config.get("skip_key", skip_key)
             self.update_next_page = self.update_next_page_offset
         elif self.pagination_type == "page":
+            self.skip_key = config.get("skip_key", skip_key)
             self.update_next_page = self.update_next_page_per_page
-        else:
-            self.update_next_page = self.update_next_page_default
+        logger.info("configure_paging: self.pagination_type='{}', self.next_page_key='{}', self.next_page_url_base='{}', self.skip_key='{}', self.cursor_next_token_path='{}', self.cursor_query_param='{}', self.cursor_initial_token='{}'".format(
+                self.pagination_type, self.next_page_key, self.next_page_url_base, self.skip_key, self.cursor_next_token_path, self.cursor_query_param, self.cursor_initial_token
+            ))
 
     def reset_paging(self, counting_key=None, url=None):
         self.records_to_skip = 0
@@ -57,6 +66,7 @@ def reset_paging(self, counting_key=None, url=None):
         else:
             self.next_page_number = 0
         self.next_page_url = url
+        self.cursor_query_param_value = self.cursor_initial_token
         self.is_last_batch_empty = False
         self.is_first_batch = True
         self.is_paging_started = True
@@ -127,6 +137,16 @@ def update_next_page_link(self, data, response_links=None):
                 next_link, self.next_page_url, self.params_must_be_blanked, self.next_page_number, self.counter
             ))
 
+    def update_next_page_cursor(self, data, response_links=None):
+        self.is_first_batch = False
+        self.counter += 1
+        self.cursor_query_param_value = None
+        self.data_is_list = False
+
+        if self.cursor_next_token_path and self.cursor_query_param:
+            self.cursor_query_param_value = extract_key_using_json_path(data, self.cursor_next_token_path)
+            logger.info("update_next_page_cursor:{}".format({self.cursor_query_param_value}))
+
     def update_next_page_default(self, data, response_links=None):
         self.is_first_batch = False
 
@@ -145,6 +165,15 @@ def has_next_page(self):
                 ret
             ))
             return ret
+        if self.pagination_type == "cursor":
+            ret = (self.cursor_query_param_value is not None) and (self.cursor_query_param_value != "")
+            logger.info("has_next_page:cursor_next_token_path={} cursor_query_param={} cursor_query_param_value={} -> {}".format(
+                self.cursor_next_token_path,
+                self.cursor_query_param,
+                self.cursor_query_param_value,
+                ret
+            ))
+            return ret
         if self.pagination_type in ["page", "offset"]:
             if self.counting_key:
                 #  There is a counting key and we already know the last batch was not empty
@@ -168,6 +197,10 @@ def get_params(self):
             ret.update({
                 self.skip_key: self.next_page_number if self.pagination_type == "page" else self.records_to_skip
             })
+        if self.cursor_query_param and self.cursor_query_param_value not in [None, ""]:
+            ret.update({
+                self.cursor_query_param: self.cursor_query_param_value
+            })
         return ret
 
     def get_next_page_url(self):

python-lib/rest_api_client.py

@@ -89,13 +89,19 @@ def __init__(self, credential, secure_credentials, endpoint, custom_key_values={
         next_page_url_base = endpoint.get("next_page_url_base", None) if is_next_page_url_relative else None
         next_page_url_base = format_template(next_page_url_base, **self.presets_variables)
         skip_key = endpoint.get("skip_key")
+        cursor_next_token_path = endpoint.get("cursor_next_token_path")
+        cursor_query_param = endpoint.get("cursor_query_param")
+        cursor_initial_token = format_template(endpoint.get("cursor_initial_token"), **self.presets_variables)
         pagination_type = endpoint.get("pagination_type", "na")
         if pagination_type == "next_page" and is_next_page_url_relative and not next_page_url_base:
             raise RestAPIClientError("Pagination's 'Next page URL' is relative but no 'Base URL to next page' has been set")
         self.pagination.configure_paging(
             skip_key=skip_key,
             next_page_key=next_page_url_key,
             next_page_url_base=next_page_url_base,
+            cursor_next_token_path=cursor_next_token_path,
+            cursor_query_param=cursor_query_param,
+            cursor_initial_token=cursor_initial_token,
             pagination_type=pagination_type
         )
         self.last_interaction = None