feat: support prompt caching and add tests

docs/release_notes.md

@@ -22,6 +22,11 @@
 - [OpenAI] You can now add multiple custom headers to an `OpenAiClient` at once via `.withHeaders()`.
 - [Orchestration] Added `GEMINI_3_1_PRO_PREVIEW_EA` and `MISTRAL_MEDIUM` to model list in `OrchestrationAiModel`.
 - [Orchestration] Reasoning content is now supported for users through `OrchestrationChatResponse.getReasoningText()` and `OrchestrationChatCompletionDelta.getDeltaReasoningText()` to see the thinking processes when using the reasoning models.
+- [Orchestration] Added support for prompt caching for new Claude models. The following messages are cacheable:
+  - `SystemMessage`
+  - `UserMessage`
+  - `ToolMessage`
+  - refer to `com.sap.ai.sdk.orchestration.PromptCachingConfig.PROMPT_CACHING_SUPPORT` for the models supporting caching
 
 ### 📈 Improvements
 

orchestration/src/main/java/com/sap/ai/sdk/orchestration/CacheControl.java

@@ -0,0 +1,19 @@
+package com.sap.ai.sdk.orchestration;
+
+import javax.annotation.Nonnull;
+import lombok.Getter;
+
+/** Represents CacheControl object, used in prompt caching API */
+@Getter
+public final class CacheControl {
+  private final String ttl;
+
+  /**
+   * Constructs cache control object with a ttl
+   *
+   * @param ttl time to live for the cache
+   */
+  public CacheControl(@Nonnull final String ttl) {
+    this.ttl = ttl;
+  }
+}

orchestration/src/main/java/com/sap/ai/sdk/orchestration/CacheablePrompt.java

@@ -0,0 +1,15 @@
+package com.sap.ai.sdk.orchestration;
+
+import javax.annotation.Nullable;
+
+/** Prompt objects supporting caching may implement this interface */
+public sealed interface CacheablePrompt permits TextItem {
+
+  /**
+   * Returns cache control for a given cacheable prompt (if set)
+   *
+   * @return cacheControl, nullable
+   */
+  @Nullable
+  CacheControl getCacheControl();
+}

orchestration/src/main/java/com/sap/ai/sdk/orchestration/ConfigToRequestTransformer.java

@@ -37,8 +37,11 @@ static CompletionRequestConfiguration toCompletionPostRequest(
       @Nonnull final OrchestrationModuleConfig config,
       @Nonnull final OrchestrationModuleConfig... fallbackConfigs) {
 
+    final var cachingConfig = resolveCachingConfig(config, fallbackConfigs);
     final UnaryOperator<OrchestrationModuleConfig> copyWithImmutableTemplateConfig =
-        c -> c.withTemplateConfig(toTemplateModuleConfig(prompt, c.getTemplateConfig()));
+        c ->
+            c.withTemplateConfig(
+                toTemplateModuleConfig(prompt, cachingConfig, c.getTemplateConfig()));
     val configList = Lists.asList(config, fallbackConfigs);
     val configsCopy = Lists.transform(configList, copyWithImmutableTemplateConfig::apply);
 
@@ -63,6 +66,14 @@ static CompletionRequestConfiguration toCompletionPostRequest(
   static PromptTemplatingModuleConfigPrompt toTemplateModuleConfig(
       @Nonnull final OrchestrationPrompt prompt,
       @Nullable final PromptTemplatingModuleConfigPrompt config) {
+    return toTemplateModuleConfig(prompt, PromptCachingConfig.noCaching(), config);
+  }
+
+  @Nonnull
+  static PromptTemplatingModuleConfigPrompt toTemplateModuleConfig(
+      @Nonnull final OrchestrationPrompt prompt,
+      @Nonnull final PromptCachingConfig cachingConfig,
+      @Nullable final PromptTemplatingModuleConfigPrompt config) {
     /*
      * Currently, we have to merge the prompt into the template configuration.
      * This works around the limitation that the template config is required.
@@ -78,8 +89,9 @@ static PromptTemplatingModuleConfigPrompt toTemplateModuleConfig(
     val messages = template.getTemplate();
     val messagesWithPrompt = new ArrayList<>(messages);
 
-    messagesWithPrompt.addAll(
-        prompt.getMessages().stream().map(Message::createChatMessage).toList());
+    final var promptMessages = withCachingConstraintsApplied(prompt.getMessages(), cachingConfig);
+
+    messagesWithPrompt.addAll(promptMessages.stream().map(Message::createChatMessage).toList());
     if (messagesWithPrompt.isEmpty()) {
       throw new IllegalStateException(
           "A prompt is required. Pass at least one message or configure a template with messages or a template reference.");
@@ -98,6 +110,75 @@ static PromptTemplatingModuleConfigPrompt toTemplateModuleConfig(
     return result;
   }
 
+  static PromptCachingConfig resolveCachingConfig(
+      @Nonnull final OrchestrationModuleConfig config,
+      @Nonnull final OrchestrationModuleConfig... fallbackConfigs) {
+
+    var modelName = "unknown";
+    if (config.getLlmConfig() != null) {
+      modelName = config.getLlmConfig().getName();
+    } else {
+      for (final var fallbackConfig : fallbackConfigs) {
+        if (fallbackConfig.getLlmConfig() != null) {
+          modelName = fallbackConfig.getLlmConfig().getName();
+          break;
+        }
+      }
+    }
+    return PromptCachingConfig.forModel(modelName);
+  }
+
+  static List<Message> withCachingConstraintsApplied(
+      @Nonnull final List<Message> promptMessages,
+      @Nonnull final PromptCachingConfig cachingConfig) {
+    final var outputMessages = new ArrayList<Message>(promptMessages.size());
+    @SuppressWarnings("PMD.LocalVariableShouldBeFinal") // it is a variable, cannot be final
+    var remainingCacheableCheckpoints = cachingConfig.getMaxCheckpointsPerRequest();
+    for (int i = 0; i < promptMessages.size(); i++) {
+      var message = promptMessages.get(i);
+      final var contentItems = new ArrayList<ContentItem>(message.content().items().size());
+      final var messageSupportsCache =
+          message instanceof UserMessage
+              || message instanceof SystemMessage
+              || message instanceof ToolMessage;
+      for (int j = 0; j < message.content().items().size(); j++) {
+        var contentItem = message.content().items().get(j);
+        if (contentItem instanceof TextItem textItem) {
+          final var existingCacheControl = textItem.getCacheControl();
+          if (existingCacheControl != null) {
+            if (remainingCacheableCheckpoints <= 0 || !messageSupportsCache) {
+              // skipping text control if there is already too many cache checkpoints
+              contentItem = new TextItem(textItem.text());
+            } else if (cachingConfig.getMinTokensPerCheckpoint()
+                > textItem.getText().chars().filter(c -> c == ' ').count() + 1) {
+              // to few tokens to cache, skipping to respect model limitation
+              contentItem = new TextItem(textItem.text());
+            } else {
+              remainingCacheableCheckpoints--;
+              if (!cachingConfig
+                  .getSupportedTTLValues()
+                  .matcher(existingCacheControl.getTtl())
+                  .matches()) {
+                contentItem =
+                    new TextItem(
+                        textItem.text(), new CacheControl(cachingConfig.getDefaultTTLValue()));
+              }
+            }
+          }
+        }
+        contentItems.add(contentItem);
+      }
+      if (message instanceof UserMessage) {
+        message = new UserMessage(new MessageContent(contentItems));
+      } else if (message instanceof SystemMessage) {
+        message = new SystemMessage(new MessageContent(contentItems));
+      }
+
+      outputMessages.add(message);
+    }
+    return outputMessages;
+  }
+
   @Nonnull
   static InnerModuleConfigs toModuleConfigs(@Nonnull final OrchestrationModuleConfig config) {
     return OrchestrationConfigModules.createInnerModuleConfigs(setupModuleConfigs(config));

orchestration/src/main/java/com/sap/ai/sdk/orchestration/Message.java

@@ -4,6 +4,7 @@
 import java.nio.file.Path;
 import java.util.List;
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 /** Interface representing convenience wrappers of chat message to the orchestration service. */
 public sealed interface Message permits AssistantMessage, SystemMessage, ToolMessage, UserMessage {
@@ -16,7 +17,21 @@ public sealed interface Message permits AssistantMessage, SystemMessage, ToolMes
    */
   @Nonnull
   static UserMessage user(@Nonnull final String message) {
-    return new UserMessage(message);
+    return user(message, null);
+  }
+
+  /**
+   * A convenience method to create a user message from a string.
+   *
+   * @since 1.23.0
+   * @param message the message content.
+   * @param cacheControl cache checkpoint configuration
+   * @return the user message.
+   */
+  @Nonnull
+  static UserMessage user(
+      @Nonnull final String message, @Nullable final CacheControl cacheControl) {
+    return new UserMessage(message, cacheControl);
   }
 
   /**
@@ -62,7 +77,22 @@ static AssistantMessage assistant(@Nonnull final String message) {
    */
   @Nonnull
   static SystemMessage system(@Nonnull final String message) {
-    return new SystemMessage(message);
+    return system(message, null);
+  }
+
+  /**
+   * A convenience method to create a system message from a string allowing to configure cache
+   * checkpoint
+   *
+   * @since 1.23.0
+   * @param message the message content
+   * @param cacheControl optional cache checkpoint configuration
+   * @return the system message
+   */
+  @Nonnull
+  static SystemMessage system(
+      @Nonnull final String message, @Nullable final CacheControl cacheControl) {
+    return new SystemMessage(message, cacheControl);
   }
 
   /**

orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationPrompt.java

@@ -7,6 +7,7 @@
 import java.util.Map;
 import java.util.TreeMap;
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import lombok.AccessLevel;
 import lombok.Getter;
 import lombok.Value;
@@ -31,7 +32,19 @@ public class OrchestrationPrompt {
    * @param message A user message.
    */
   public OrchestrationPrompt(@Nonnull final String message) {
-    messages.add(new UserMessage(message));
+    this(message, null);
+  }
+
+  /**
+   * Initialize a prompt with the given user message with optional cache checkpoint configuration
+   *
+   * @since 1.23.0
+   * @param message A user message.
+   * @param cacheControl optional cache checkpoint configuration
+   */
+  public OrchestrationPrompt(
+      @Nonnull final String message, @Nullable final CacheControl cacheControl) {
+    messages.add(new UserMessage(message, cacheControl));
   }
 
   /**

orchestration/src/main/java/com/sap/ai/sdk/orchestration/PromptCachingConfig.java

@@ -0,0 +1,108 @@
+package com.sap.ai.sdk.orchestration;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Pattern;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import lombok.Getter;
+
+/** Describes supported caching properties of a model */
+@Getter
+public final class PromptCachingConfig {
+
+  private static final PromptCachingConfig NOT_SUPPORTED =
+      new PromptCachingConfig(0, 0, "0m", "5m");
+
+  private static final Map<String, PromptCachingConfig> PROMPT_CACHING_SUPPORT =
+      Collections.unmodifiableMap(
+          new HashMap<>() {
+            {
+              put(
+                  OrchestrationAiModel.CLAUDE_4_5_OPUS.getName(),
+                  new PromptCachingConfig(4096, 4, "5m|1h", "5m"));
+              put(
+                  OrchestrationAiModel.CLAUDE_4_6_OPUS.getName(),
+                  new PromptCachingConfig(4096, 4, "5m|1h", "5m"));
+              put(
+                  OrchestrationAiModel.CLAUDE_4_5_SONNET.getName(),
+                  new PromptCachingConfig(4096, 4, "5m|1h", "5m"));
+              put(
+                  OrchestrationAiModel.CLAUDE_4_6_SONNET.getName(),
+                  new PromptCachingConfig(1024, 4, "5m|1h", "5m"));
+              put(
+                  OrchestrationAiModel.CLAUDE_4_5_HAIKU.getName(),
+                  new PromptCachingConfig(4096, 4, "5m|1h", "5m"));
+              put(
+                  OrchestrationAiModel.CLAUDE_4_OPUS.getName(),
+                  new PromptCachingConfig(4096, 4, "5m", "5m"));
+              put(
+                  OrchestrationAiModel.CLAUDE_4_7_OPUS.getName(),
+                  new PromptCachingConfig(4096, 4, "5m|1h", "5m"));
+              put(
+                  OrchestrationAiModel.CLAUDE_4_8_OPUS.getName(),
+                  new PromptCachingConfig(4096, 4, "5m|1h", "5m"));
+            }
+          });
+
+  /** Caching checkpoint can only be made for so few prompt input tokens and not fewer */
+  private final int minTokensPerCheckpoint;
+
+  /** Only up to this number of caching points can be created per request */
+  private final int maxCheckpointsPerRequest;
+
+  /** Pattern of supported TTL values, which can be passed */
+  private final Pattern supportedTTLValues;
+
+  /** Caching TTL value to use if TTL has not been explicitly specified */
+  private final String defaultTTLValue;
+
+  private PromptCachingConfig(
+      final int minTokensPerCheckpoint,
+      final int maxCheckpointsPerRequest,
+      final String ttlPattern,
+      final String defaultTTLValue) {
+    this.minTokensPerCheckpoint = minTokensPerCheckpoint;
+    this.maxCheckpointsPerRequest = maxCheckpointsPerRequest;
+    this.supportedTTLValues = Pattern.compile(ttlPattern);
+    this.defaultTTLValue = defaultTTLValue;
+  }
+
+  /**
+   * Factory method, returns instance of PromptCachingConfig with possible caching configurations.
+   * If model does not support caching, a special instance will be returned
+   *
+   * @param modelName - model name to use
+   * @return model prompt caching configuration
+   */
+  @Nonnull
+  public static PromptCachingConfig forModel(@Nullable final String modelName) {
+    if (modelName == null || modelName.isEmpty()) {
+      return NOT_SUPPORTED;
+    }
+    return PROMPT_CACHING_SUPPORT.getOrDefault(modelName, NOT_SUPPORTED);
+  }
+
+  /**
+   * Factory method, returns instance of PromptCachingConfig with possible caching configurations.
+   * If model does not support caching, a special instance will be returned
+   *
+   * @param model - model to use
+   * @return model prompt caching configuration
+   */
+  @Nonnull
+  public static PromptCachingConfig forModel(@Nonnull final OrchestrationAiModel model) {
+    return forModel(model.getName());
+  }
+
+  /**
+   * Explicit "no caching" configuration
+   *
+   * @return "no caching" prompt caching configuration
+   */
+  @Nonnull
+  public static PromptCachingConfig noCaching() {
+    return NOT_SUPPORTED;
+  }
+}