From 14fcb076eab7834c4b5af6810a0a09a8411a23a4 Mon Sep 17 00:00:00 2001 From: Claus Ibsen Date: Mon, 19 Oct 2026 10:03:55 +0200 Subject: [PATCH 1/2] CAMEL-25777: camel-core - the tokenizer EIP works in XML and YAML routes, as each tokenizer names itself The reifier finds the tokenizer by its name, and only the Java DSL builder set that name. The XML and YAML DSL create the tokenizer with the no-arg constructors and set its implementation only, so their routes failed to start with "FailedToCreateRouteException ... because: jar" (a NullPointerException for the null name). Each tokenizer implementation now sets its name in its constructor, as its builder does, and the tokenizer falls back to the name of its implementation. modelName of the LangChain4j tokenizers is optional, as its metadata says, but @XmlAttribute(required = true) made the XSD and the model require it, so an XML route without it failed XSD validation. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01STT6whBgK1AqsSsUKrnE8m --- .../models/langChain4jCharacterTokenizer.json | 2 +- .../models/langChain4jLineTokenizer.json | 2 +- .../models/langChain4jParagraphTokenizer.json | 2 +- .../models/langChain4jSentenceTokenizer.json | 2 +- .../models/langChain4jWordTokenizer.json | 2 +- .../camel/catalog/schemas/camel-spring.xsd | 2 +- .../camel/catalog/schemas/camel-xml-io.xsd | 2 +- .../LangChain4JTokenizerDefinitionTest.java | 79 +++++++++++++++++++ .../langChain4jCharacterTokenizer.json | 2 +- .../tokenizer/langChain4jLineTokenizer.json | 2 +- .../langChain4jParagraphTokenizer.json | 2 +- .../langChain4jSentenceTokenizer.json | 2 +- .../tokenizer/langChain4jWordTokenizer.json | 2 +- .../camel/model/TokenizerDefinition.java | 6 +- ...ngChain4jCharacterTokenizerDefinition.java | 2 + .../LangChain4jLineTokenizerDefinition.java | 2 + ...ngChain4jParagraphTokenizerDefinition.java | 2 + ...angChain4jSentenceTokenizerDefinition.java | 2 + .../LangChain4jTokenizerDefinition.java | 2 +- .../LangChain4jWordTokenizerDefinition.java | 2 + 20 files changed, 107 insertions(+), 14 deletions(-) create mode 100644 components/camel-ai/camel-langchain4j-tokenizer/src/test/java/org/apache/camel/component/langchain4j/tokenizer/LangChain4JTokenizerDefinitionTest.java diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/langChain4jCharacterTokenizer.json b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/langChain4jCharacterTokenizer.json index d968a5b47359b..3baf8ac1bc842 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/langChain4jCharacterTokenizer.json +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/langChain4jCharacterTokenizer.json @@ -17,6 +17,6 @@ "tokenizerType": { "index": 1, "kind": "attribute", "displayName": "Tokenizer Type", "group": "common", "required": true, "type": "enum", "javaType": "org.apache.camel.model.tokenizer.TokenizerType", "enum": [ "OPEN_AI", "AZURE", "QWEN" ], "deprecated": false, "autowired": false, "secret": false, "description": "The tokenizer type. Must be one of OPEN_AI, AZURE or QWEN." }, "maxTokens": { "index": 2, "kind": "attribute", "displayName": "Max Tokens", "group": "common", "required": true, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "The maximum number of tokens on each segment." }, "maxOverlap": { "index": 3, "kind": "attribute", "displayName": "Max Overlap", "group": "common", "required": true, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "The maximum number of tokens that can overlap in each segment." }, - "modelName": { "index": 4, "kind": "attribute", "displayName": "Model Name", "group": "common", "required": true, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The underlying model name used by the tokenizer. Providing this switches to computing segment sizes in terms of tokens." } + "modelName": { "index": 4, "kind": "attribute", "displayName": "Model Name", "group": "common", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The underlying model name used by the tokenizer. Providing this switches to computing segment sizes in terms of tokens." } } } diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/langChain4jLineTokenizer.json b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/langChain4jLineTokenizer.json index 843cf5419594b..364d2590da983 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/langChain4jLineTokenizer.json +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/langChain4jLineTokenizer.json @@ -17,6 +17,6 @@ "tokenizerType": { "index": 1, "kind": "attribute", "displayName": "Tokenizer Type", "group": "common", "required": true, "type": "enum", "javaType": "org.apache.camel.model.tokenizer.TokenizerType", "enum": [ "OPEN_AI", "AZURE", "QWEN" ], "deprecated": false, "autowired": false, "secret": false, "description": "The tokenizer type. Must be one of OPEN_AI, AZURE or QWEN." }, "maxTokens": { "index": 2, "kind": "attribute", "displayName": "Max Tokens", "group": "common", "required": true, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "The maximum number of tokens on each segment." }, "maxOverlap": { "index": 3, "kind": "attribute", "displayName": "Max Overlap", "group": "common", "required": true, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "The maximum number of tokens that can overlap in each segment." }, - "modelName": { "index": 4, "kind": "attribute", "displayName": "Model Name", "group": "common", "required": true, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The underlying model name used by the tokenizer. Providing this switches to computing segment sizes in terms of tokens." } + "modelName": { "index": 4, "kind": "attribute", "displayName": "Model Name", "group": "common", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The underlying model name used by the tokenizer. Providing this switches to computing segment sizes in terms of tokens." } } } diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/langChain4jParagraphTokenizer.json b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/langChain4jParagraphTokenizer.json index 4118c4afeb50c..c252e73498333 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/langChain4jParagraphTokenizer.json +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/langChain4jParagraphTokenizer.json @@ -17,6 +17,6 @@ "tokenizerType": { "index": 1, "kind": "attribute", "displayName": "Tokenizer Type", "group": "common", "required": true, "type": "enum", "javaType": "org.apache.camel.model.tokenizer.TokenizerType", "enum": [ "OPEN_AI", "AZURE", "QWEN" ], "deprecated": false, "autowired": false, "secret": false, "description": "The tokenizer type. Must be one of OPEN_AI, AZURE or QWEN." }, "maxTokens": { "index": 2, "kind": "attribute", "displayName": "Max Tokens", "group": "common", "required": true, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "The maximum number of tokens on each segment." }, "maxOverlap": { "index": 3, "kind": "attribute", "displayName": "Max Overlap", "group": "common", "required": true, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "The maximum number of tokens that can overlap in each segment." }, - "modelName": { "index": 4, "kind": "attribute", "displayName": "Model Name", "group": "common", "required": true, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The underlying model name used by the tokenizer. Providing this switches to computing segment sizes in terms of tokens." } + "modelName": { "index": 4, "kind": "attribute", "displayName": "Model Name", "group": "common", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The underlying model name used by the tokenizer. Providing this switches to computing segment sizes in terms of tokens." } } } diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/langChain4jSentenceTokenizer.json b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/langChain4jSentenceTokenizer.json index 1d8328d7fc916..869f54bc15785 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/langChain4jSentenceTokenizer.json +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/langChain4jSentenceTokenizer.json @@ -17,6 +17,6 @@ "tokenizerType": { "index": 1, "kind": "attribute", "displayName": "Tokenizer Type", "group": "common", "required": true, "type": "enum", "javaType": "org.apache.camel.model.tokenizer.TokenizerType", "enum": [ "OPEN_AI", "AZURE", "QWEN" ], "deprecated": false, "autowired": false, "secret": false, "description": "The tokenizer type. Must be one of OPEN_AI, AZURE or QWEN." }, "maxTokens": { "index": 2, "kind": "attribute", "displayName": "Max Tokens", "group": "common", "required": true, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "The maximum number of tokens on each segment." }, "maxOverlap": { "index": 3, "kind": "attribute", "displayName": "Max Overlap", "group": "common", "required": true, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "The maximum number of tokens that can overlap in each segment." }, - "modelName": { "index": 4, "kind": "attribute", "displayName": "Model Name", "group": "common", "required": true, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The underlying model name used by the tokenizer. Providing this switches to computing segment sizes in terms of tokens." } + "modelName": { "index": 4, "kind": "attribute", "displayName": "Model Name", "group": "common", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The underlying model name used by the tokenizer. Providing this switches to computing segment sizes in terms of tokens." } } } diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/langChain4jWordTokenizer.json b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/langChain4jWordTokenizer.json index 57e842b9d8145..b5d89c3e823b2 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/langChain4jWordTokenizer.json +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/langChain4jWordTokenizer.json @@ -17,6 +17,6 @@ "tokenizerType": { "index": 1, "kind": "attribute", "displayName": "Tokenizer Type", "group": "common", "required": true, "type": "enum", "javaType": "org.apache.camel.model.tokenizer.TokenizerType", "enum": [ "OPEN_AI", "AZURE", "QWEN" ], "deprecated": false, "autowired": false, "secret": false, "description": "The tokenizer type. Must be one of OPEN_AI, AZURE or QWEN." }, "maxTokens": { "index": 2, "kind": "attribute", "displayName": "Max Tokens", "group": "common", "required": true, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "The maximum number of tokens on each segment." }, "maxOverlap": { "index": 3, "kind": "attribute", "displayName": "Max Overlap", "group": "common", "required": true, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "The maximum number of tokens that can overlap in each segment." }, - "modelName": { "index": 4, "kind": "attribute", "displayName": "Model Name", "group": "common", "required": true, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The underlying model name used by the tokenizer. Providing this switches to computing segment sizes in terms of tokens." } + "modelName": { "index": 4, "kind": "attribute", "displayName": "Model Name", "group": "common", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The underlying model name used by the tokenizer. Providing this switches to computing segment sizes in terms of tokens." } } } diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camel-spring.xsd b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camel-spring.xsd index 278dd6cd0df9f..8edbc97f545fd 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camel-spring.xsd +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camel-spring.xsd @@ -14551,7 +14551,7 @@ The maximum number of tokens that can overlap in each segment. - + - + into + LangChain4jParagraphTokenizerDefinition paragraph = new LangChain4jParagraphTokenizerDefinition(); + paragraph.setMaxTokens("1024"); + paragraph.setMaxOverlap("10"); + paragraph.setTokenizerType("OPEN_AI"); + TokenizerDefinition tokenizer = new TokenizerDefinition(); + tokenizer.setTokenizerImplementation(paragraph); + + RouteDefinition route = from("direct:start"); + route.addOutput(tokenizer); + route.split().body().to("mock:result"); + } + }; + } +} diff --git a/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/tokenizer/langChain4jCharacterTokenizer.json b/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/tokenizer/langChain4jCharacterTokenizer.json index d968a5b47359b..3baf8ac1bc842 100644 --- a/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/tokenizer/langChain4jCharacterTokenizer.json +++ b/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/tokenizer/langChain4jCharacterTokenizer.json @@ -17,6 +17,6 @@ "tokenizerType": { "index": 1, "kind": "attribute", "displayName": "Tokenizer Type", "group": "common", "required": true, "type": "enum", "javaType": "org.apache.camel.model.tokenizer.TokenizerType", "enum": [ "OPEN_AI", "AZURE", "QWEN" ], "deprecated": false, "autowired": false, "secret": false, "description": "The tokenizer type. Must be one of OPEN_AI, AZURE or QWEN." }, "maxTokens": { "index": 2, "kind": "attribute", "displayName": "Max Tokens", "group": "common", "required": true, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "The maximum number of tokens on each segment." }, "maxOverlap": { "index": 3, "kind": "attribute", "displayName": "Max Overlap", "group": "common", "required": true, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "The maximum number of tokens that can overlap in each segment." }, - "modelName": { "index": 4, "kind": "attribute", "displayName": "Model Name", "group": "common", "required": true, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The underlying model name used by the tokenizer. Providing this switches to computing segment sizes in terms of tokens." } + "modelName": { "index": 4, "kind": "attribute", "displayName": "Model Name", "group": "common", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The underlying model name used by the tokenizer. Providing this switches to computing segment sizes in terms of tokens." } } } diff --git a/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/tokenizer/langChain4jLineTokenizer.json b/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/tokenizer/langChain4jLineTokenizer.json index 843cf5419594b..364d2590da983 100644 --- a/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/tokenizer/langChain4jLineTokenizer.json +++ b/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/tokenizer/langChain4jLineTokenizer.json @@ -17,6 +17,6 @@ "tokenizerType": { "index": 1, "kind": "attribute", "displayName": "Tokenizer Type", "group": "common", "required": true, "type": "enum", "javaType": "org.apache.camel.model.tokenizer.TokenizerType", "enum": [ "OPEN_AI", "AZURE", "QWEN" ], "deprecated": false, "autowired": false, "secret": false, "description": "The tokenizer type. Must be one of OPEN_AI, AZURE or QWEN." }, "maxTokens": { "index": 2, "kind": "attribute", "displayName": "Max Tokens", "group": "common", "required": true, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "The maximum number of tokens on each segment." }, "maxOverlap": { "index": 3, "kind": "attribute", "displayName": "Max Overlap", "group": "common", "required": true, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "The maximum number of tokens that can overlap in each segment." }, - "modelName": { "index": 4, "kind": "attribute", "displayName": "Model Name", "group": "common", "required": true, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The underlying model name used by the tokenizer. Providing this switches to computing segment sizes in terms of tokens." } + "modelName": { "index": 4, "kind": "attribute", "displayName": "Model Name", "group": "common", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The underlying model name used by the tokenizer. Providing this switches to computing segment sizes in terms of tokens." } } } diff --git a/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/tokenizer/langChain4jParagraphTokenizer.json b/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/tokenizer/langChain4jParagraphTokenizer.json index 4118c4afeb50c..c252e73498333 100644 --- a/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/tokenizer/langChain4jParagraphTokenizer.json +++ b/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/tokenizer/langChain4jParagraphTokenizer.json @@ -17,6 +17,6 @@ "tokenizerType": { "index": 1, "kind": "attribute", "displayName": "Tokenizer Type", "group": "common", "required": true, "type": "enum", "javaType": "org.apache.camel.model.tokenizer.TokenizerType", "enum": [ "OPEN_AI", "AZURE", "QWEN" ], "deprecated": false, "autowired": false, "secret": false, "description": "The tokenizer type. Must be one of OPEN_AI, AZURE or QWEN." }, "maxTokens": { "index": 2, "kind": "attribute", "displayName": "Max Tokens", "group": "common", "required": true, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "The maximum number of tokens on each segment." }, "maxOverlap": { "index": 3, "kind": "attribute", "displayName": "Max Overlap", "group": "common", "required": true, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "The maximum number of tokens that can overlap in each segment." }, - "modelName": { "index": 4, "kind": "attribute", "displayName": "Model Name", "group": "common", "required": true, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The underlying model name used by the tokenizer. Providing this switches to computing segment sizes in terms of tokens." } + "modelName": { "index": 4, "kind": "attribute", "displayName": "Model Name", "group": "common", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The underlying model name used by the tokenizer. Providing this switches to computing segment sizes in terms of tokens." } } } diff --git a/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/tokenizer/langChain4jSentenceTokenizer.json b/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/tokenizer/langChain4jSentenceTokenizer.json index 1d8328d7fc916..869f54bc15785 100644 --- a/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/tokenizer/langChain4jSentenceTokenizer.json +++ b/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/tokenizer/langChain4jSentenceTokenizer.json @@ -17,6 +17,6 @@ "tokenizerType": { "index": 1, "kind": "attribute", "displayName": "Tokenizer Type", "group": "common", "required": true, "type": "enum", "javaType": "org.apache.camel.model.tokenizer.TokenizerType", "enum": [ "OPEN_AI", "AZURE", "QWEN" ], "deprecated": false, "autowired": false, "secret": false, "description": "The tokenizer type. Must be one of OPEN_AI, AZURE or QWEN." }, "maxTokens": { "index": 2, "kind": "attribute", "displayName": "Max Tokens", "group": "common", "required": true, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "The maximum number of tokens on each segment." }, "maxOverlap": { "index": 3, "kind": "attribute", "displayName": "Max Overlap", "group": "common", "required": true, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "The maximum number of tokens that can overlap in each segment." }, - "modelName": { "index": 4, "kind": "attribute", "displayName": "Model Name", "group": "common", "required": true, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The underlying model name used by the tokenizer. Providing this switches to computing segment sizes in terms of tokens." } + "modelName": { "index": 4, "kind": "attribute", "displayName": "Model Name", "group": "common", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The underlying model name used by the tokenizer. Providing this switches to computing segment sizes in terms of tokens." } } } diff --git a/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/tokenizer/langChain4jWordTokenizer.json b/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/tokenizer/langChain4jWordTokenizer.json index 57e842b9d8145..b5d89c3e823b2 100644 --- a/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/tokenizer/langChain4jWordTokenizer.json +++ b/core/camel-core-model/src/generated/resources/META-INF/org/apache/camel/model/tokenizer/langChain4jWordTokenizer.json @@ -17,6 +17,6 @@ "tokenizerType": { "index": 1, "kind": "attribute", "displayName": "Tokenizer Type", "group": "common", "required": true, "type": "enum", "javaType": "org.apache.camel.model.tokenizer.TokenizerType", "enum": [ "OPEN_AI", "AZURE", "QWEN" ], "deprecated": false, "autowired": false, "secret": false, "description": "The tokenizer type. Must be one of OPEN_AI, AZURE or QWEN." }, "maxTokens": { "index": 2, "kind": "attribute", "displayName": "Max Tokens", "group": "common", "required": true, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "The maximum number of tokens on each segment." }, "maxOverlap": { "index": 3, "kind": "attribute", "displayName": "Max Overlap", "group": "common", "required": true, "type": "integer", "javaType": "java.lang.Integer", "deprecated": false, "autowired": false, "secret": false, "description": "The maximum number of tokens that can overlap in each segment." }, - "modelName": { "index": 4, "kind": "attribute", "displayName": "Model Name", "group": "common", "required": true, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The underlying model name used by the tokenizer. Providing this switches to computing segment sizes in terms of tokens." } + "modelName": { "index": 4, "kind": "attribute", "displayName": "Model Name", "group": "common", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "The underlying model name used by the tokenizer. Providing this switches to computing segment sizes in terms of tokens." } } } diff --git a/core/camel-core-model/src/main/java/org/apache/camel/model/TokenizerDefinition.java b/core/camel-core-model/src/main/java/org/apache/camel/model/TokenizerDefinition.java index 1994e9e521c17..c94a2cd2df230 100644 --- a/core/camel-core-model/src/main/java/org/apache/camel/model/TokenizerDefinition.java +++ b/core/camel-core-model/src/main/java/org/apache/camel/model/TokenizerDefinition.java @@ -66,9 +66,13 @@ public TokenizerDefinition(TokenizerImplementationDefinition tokenizerImplementa } /** - * Gets the tokenizer name + * Gets the tokenizer name, which is the name of the tokenizer implementation when not set: the XML and YAML DSL set + * only the implementation (CAMEL-25777) */ public String tokenizerName() { + if (tokenizerName == null && tokenizerImplementation != null) { + return tokenizerImplementation.tokenizerName(); + } return tokenizerName; } diff --git a/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jCharacterTokenizerDefinition.java b/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jCharacterTokenizerDefinition.java index 012fd9672920f..ea30e473a35d0 100644 --- a/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jCharacterTokenizerDefinition.java +++ b/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jCharacterTokenizerDefinition.java @@ -33,6 +33,8 @@ public class LangChain4jCharacterTokenizerDefinition extends LangChain4jTokenizerDefinition { public LangChain4jCharacterTokenizerDefinition() { + // the name the XML and YAML DSL do not set, as the builder does (CAMEL-25777) + setTokenizerName(toName("CharacterTokenizer")); } public LangChain4jCharacterTokenizerDefinition(LangChain4jTokenizerDefinition source) { diff --git a/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jLineTokenizerDefinition.java b/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jLineTokenizerDefinition.java index 11833a62e8a8e..2d8698560b3ae 100644 --- a/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jLineTokenizerDefinition.java +++ b/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jLineTokenizerDefinition.java @@ -33,6 +33,8 @@ public class LangChain4jLineTokenizerDefinition extends LangChain4jTokenizerDefinition { public LangChain4jLineTokenizerDefinition() { + // the name the XML and YAML DSL do not set, as the builder does (CAMEL-25777) + setTokenizerName(toName("LineTokenizer")); } public LangChain4jLineTokenizerDefinition(LangChain4jTokenizerDefinition source) { diff --git a/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jParagraphTokenizerDefinition.java b/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jParagraphTokenizerDefinition.java index 2a0a864a600f5..d91e9547d8c01 100644 --- a/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jParagraphTokenizerDefinition.java +++ b/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jParagraphTokenizerDefinition.java @@ -33,6 +33,8 @@ public class LangChain4jParagraphTokenizerDefinition extends LangChain4jTokenizerDefinition { public LangChain4jParagraphTokenizerDefinition() { + // the name the XML and YAML DSL do not set, as the builder does (CAMEL-25777) + setTokenizerName(toName("ParagraphTokenizer")); } public LangChain4jParagraphTokenizerDefinition(LangChain4jTokenizerDefinition source) { diff --git a/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jSentenceTokenizerDefinition.java b/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jSentenceTokenizerDefinition.java index 8e94a09f4adfa..8c4acc34bdc38 100644 --- a/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jSentenceTokenizerDefinition.java +++ b/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jSentenceTokenizerDefinition.java @@ -33,6 +33,8 @@ public class LangChain4jSentenceTokenizerDefinition extends LangChain4jTokenizerDefinition { public LangChain4jSentenceTokenizerDefinition() { + // the name the XML and YAML DSL do not set, as the builder does (CAMEL-25777) + setTokenizerName(toName("SentenceTokenizer")); } public LangChain4jSentenceTokenizerDefinition(LangChain4jTokenizerDefinition source) { diff --git a/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jTokenizerDefinition.java b/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jTokenizerDefinition.java index 596bf1e8d6599..93410763536ed 100644 --- a/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jTokenizerDefinition.java +++ b/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jTokenizerDefinition.java @@ -52,7 +52,7 @@ public class LangChain4jTokenizerDefinition extends TokenizerImplementationDefin description = "The maximum number of tokens that can overlap in each segment.") private String maxOverlap; - @XmlAttribute(required = true) + @XmlAttribute @Metadata(javaType = "java.lang.String", required = false, description = "The underlying model name used by the tokenizer." + " Providing this switches to computing segment sizes in terms of tokens.") diff --git a/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jWordTokenizerDefinition.java b/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jWordTokenizerDefinition.java index 7029eb5f59c5c..d30ddf15fe3cb 100644 --- a/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jWordTokenizerDefinition.java +++ b/core/camel-core-model/src/main/java/org/apache/camel/model/tokenizer/LangChain4jWordTokenizerDefinition.java @@ -33,6 +33,8 @@ public class LangChain4jWordTokenizerDefinition extends LangChain4jTokenizerDefinition { public LangChain4jWordTokenizerDefinition() { + // the name the XML and YAML DSL do not set, as the builder does (CAMEL-25777) + setTokenizerName(toName("WordTokenizer")); } public LangChain4jWordTokenizerDefinition(LangChain4jTokenizerDefinition source) { From 73e55b9303c639ac088f65f57144c9e332b45d4f Mon Sep 17 00:00:00 2001 From: Guillaume Nodet Date: Mon, 19 Oct 2026 08:46:47 +0000 Subject: [PATCH 2/2] Regen --- .../camel/catalog/schemas/camelYamlDsl-model.json | 15 +++++---------- .../resources/schema/camelYamlDsl-model.json | 15 +++++---------- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camelYamlDsl-model.json b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camelYamlDsl-model.json index bab314bd2401f..31cc9a2ad77f1 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camelYamlDsl-model.json +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camelYamlDsl-model.json @@ -19651,8 +19651,7 @@ "displayName" : "Model Name", "kind" : "attribute", "index" : 4, - "group" : "common", - "required" : true + "group" : "common" } ] }, "langChain4jLineTokenizer" : { @@ -19707,8 +19706,7 @@ "displayName" : "Model Name", "kind" : "attribute", "index" : 4, - "group" : "common", - "required" : true + "group" : "common" } ] }, "langChain4jParagraphTokenizer" : { @@ -19763,8 +19761,7 @@ "displayName" : "Model Name", "kind" : "attribute", "index" : 4, - "group" : "common", - "required" : true + "group" : "common" } ] }, "langChain4jSentenceTokenizer" : { @@ -19819,8 +19816,7 @@ "displayName" : "Model Name", "kind" : "attribute", "index" : 4, - "group" : "common", - "required" : true + "group" : "common" } ] }, "langChain4jTokenizer" : { @@ -19897,8 +19893,7 @@ "displayName" : "Model Name", "kind" : "attribute", "index" : 4, - "group" : "common", - "required" : true + "group" : "common" } ] }, "customTransformer" : { diff --git a/dsl/camel-yaml-dsl/camel-yaml-dsl/src/generated/resources/schema/camelYamlDsl-model.json b/dsl/camel-yaml-dsl/camel-yaml-dsl/src/generated/resources/schema/camelYamlDsl-model.json index bab314bd2401f..31cc9a2ad77f1 100644 --- a/dsl/camel-yaml-dsl/camel-yaml-dsl/src/generated/resources/schema/camelYamlDsl-model.json +++ b/dsl/camel-yaml-dsl/camel-yaml-dsl/src/generated/resources/schema/camelYamlDsl-model.json @@ -19651,8 +19651,7 @@ "displayName" : "Model Name", "kind" : "attribute", "index" : 4, - "group" : "common", - "required" : true + "group" : "common" } ] }, "langChain4jLineTokenizer" : { @@ -19707,8 +19706,7 @@ "displayName" : "Model Name", "kind" : "attribute", "index" : 4, - "group" : "common", - "required" : true + "group" : "common" } ] }, "langChain4jParagraphTokenizer" : { @@ -19763,8 +19761,7 @@ "displayName" : "Model Name", "kind" : "attribute", "index" : 4, - "group" : "common", - "required" : true + "group" : "common" } ] }, "langChain4jSentenceTokenizer" : { @@ -19819,8 +19816,7 @@ "displayName" : "Model Name", "kind" : "attribute", "index" : 4, - "group" : "common", - "required" : true + "group" : "common" } ] }, "langChain4jTokenizer" : { @@ -19897,8 +19893,7 @@ "displayName" : "Model Name", "kind" : "attribute", "index" : 4, - "group" : "common", - "required" : true + "group" : "common" } ] }, "customTransformer" : {