From 91e7209b316f92ee8a39277aef4402a14f121647 Mon Sep 17 00:00:00 2001 From: "d.svitak" Date: Tue, 20 Oct 2026 15:55:28 +0200 Subject: [PATCH 1/7] MIG-629 Create data definitions from the variable structures - initial impl --- .../common/DeployDataDefinitions.groovy | 26 ++++ .../repository/VariableStructureRepository.kt | 15 ++- ...V18__variable_structure_status_tracking.kt | 29 +++++ .../migration/service/ResourcePathProvider.kt | 27 +++- .../migration/service/deploy/DeployClient.kt | 5 +- .../service/deploy/EvolveDeployClient.kt | 5 +- .../service/deploy/InteractiveDeployClient.kt | 52 ++++++++ .../deploy/utility/ConflictDetector.kt | 5 + .../deploy/utility/DeploymentResult.kt | 20 ++- .../service/deploy/utility/PostProcess.kt | 4 +- .../DesignerDocumentObjectBuilder.kt | 8 +- .../InspireBaseTemplateBuilder.kt | 11 +- .../InspireVariableStructureBuilder.kt | 15 +++ .../migration/service/ipsclient/IpsClient.kt | 6 + .../migration/service/ipsclient/IpsService.kt | 7 +- .../VariableStructureRepositoryTest.kt | 34 +++++ .../persistence/upgrade/V18MigrationTest.kt | 116 ++++++++++++++++++ .../service/ResourcePathProviderTest.kt | 39 ++++++ .../service/deploy/DeployClientTest.kt | 2 +- .../deploy/DesignerDeployClientTest.kt | 8 +- .../service/deploy/EvolveDeployClientTest.kt | 5 +- .../deploy/InteractiveDeployClientTest.kt | 62 +++++++++- .../DesignerDocumentObjectBuilderTest.kt | 12 ++ .../InspireBaseTemplateBuilderTest.kt | 29 ++++- .../InspireVariableStructureBuilderTest.kt | 73 +++++++++++ .../migration/tools/TestObjectBuilders.kt | 2 +- .../com/quadient/wfdxml/WfdXmlBuilder.java | 8 ++ .../layout/ForwardReferencesExporter.java | 6 +- .../internal/module/layout/LayoutImpl.java | 11 ++ .../quadient/wfdxml/WfdXmlBuilderTest.groovy | 28 ++++- 30 files changed, 638 insertions(+), 32 deletions(-) create mode 100644 migration-examples/src/main/groovy/com/quadient/migration/example/common/DeployDataDefinitions.groovy create mode 100644 migration-library/src/main/kotlin/com/quadient/migration/persistence/upgrade/V18__variable_structure_status_tracking.kt create mode 100644 migration-library/src/test/kotlin/com/quadient/migration/persistence/upgrade/V18MigrationTest.kt create mode 100644 migration-library/src/test/kotlin/com/quadient/migration/service/inspirebuilder/InspireVariableStructureBuilderTest.kt diff --git a/migration-examples/src/main/groovy/com/quadient/migration/example/common/DeployDataDefinitions.groovy b/migration-examples/src/main/groovy/com/quadient/migration/example/common/DeployDataDefinitions.groovy new file mode 100644 index 00000000..63664a4f --- /dev/null +++ b/migration-examples/src/main/groovy/com/quadient/migration/example/common/DeployDataDefinitions.groovy @@ -0,0 +1,26 @@ +//! --- +//! displayName: Deploy Data Definitions +//! category: Deployment +//! description: Deploys one data definition per variable structure to Interactive +//! --- +package com.quadient.migration.example.common + +import groovy.transform.Field +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import static com.quadient.migration.example.common.util.InitMigration.initMigration + +@Field static Logger log = LoggerFactory.getLogger(this.class.name) +def migration = initMigration(this.binding) +deploymentResult = migration.deployClient.deployDataDefinitions() +for (def item : deploymentResult.deployed) { + log.info "Deployed data definition '${item.id}' to '${item.targetPath}'" +} +for (def item : deploymentResult.warnings) { + log.warn "Data definition '${item.id}' deployed with warning: ${item.message}" +} +for (def item : deploymentResult.errors) { + log.error "Data definition '${item.id}' failed to deploy: ${item.message}" +} +log.info "Deployment finished. Deployed ${deploymentResult.deployed.size()} data definitions with ${deploymentResult.warnings.size()} warnings and ${deploymentResult.errors.size()} errors" diff --git a/migration-library/src/main/kotlin/com/quadient/migration/api/repository/VariableStructureRepository.kt b/migration-library/src/main/kotlin/com/quadient/migration/api/repository/VariableStructureRepository.kt index 2cc444f5..d3472c92 100644 --- a/migration-library/src/main/kotlin/com/quadient/migration/api/repository/VariableStructureRepository.kt +++ b/migration-library/src/main/kotlin/com/quadient/migration/api/repository/VariableStructureRepository.kt @@ -7,6 +7,7 @@ import com.quadient.migration.api.dto.migrationmodel.VariableRef import com.quadient.migration.api.dto.migrationmodel.VariableStructure import com.quadient.migration.persistence.table.DocumentObjectTable import com.quadient.migration.persistence.table.VariableStructureTable +import com.quadient.migration.service.deploy.utility.ResourceType import com.quadient.migration.tools.concat import kotlin.time.Clock import kotlin.time.toJavaInstant @@ -19,8 +20,10 @@ import org.jetbrains.exposed.v1.jdbc.transactions.transaction import org.jetbrains.exposed.v1.jdbc.upsertReturning import kotlin.collections.map -class VariableStructureRepository(projectName: ProjectName) : - Repository(VariableStructureTable, projectName.name) { +class VariableStructureRepository( + projectName: ProjectName, + private val statusTrackingRepository: StatusTrackingRepository, +) : Repository(VariableStructureTable, projectName.name) { override fun fromDb(row: ResultRow): VariableStructure { return VariableStructure( @@ -49,6 +52,10 @@ class VariableStructureRepository(projectName: ProjectName) : val now = Clock.System.now() + if (existingItem == null) { + statusTrackingRepository.active(dto.id, ResourceType.VariableStructure) + } + table.upsertReturning(table.id, table.projectName) { it[VariableStructureTable.id] = dto.id it[VariableStructureTable.projectName] = this@VariableStructureRepository.projectName @@ -79,6 +86,10 @@ class VariableStructureRepository(projectName: ProjectName) : dtos.forEach { dto -> val existingItem = find(dto.id) + if (existingItem == null) { + statusTrackingRepository.active(dto.id, ResourceType.VariableStructure) + } + stmt.setString(index++, dto.id) stmt.setString(index++, this@VariableStructureRepository.projectName) stmt.setString(index++, dto.name) diff --git a/migration-library/src/main/kotlin/com/quadient/migration/persistence/upgrade/V18__variable_structure_status_tracking.kt b/migration-library/src/main/kotlin/com/quadient/migration/persistence/upgrade/V18__variable_structure_status_tracking.kt new file mode 100644 index 00000000..eb5273f1 --- /dev/null +++ b/migration-library/src/main/kotlin/com/quadient/migration/persistence/upgrade/V18__variable_structure_status_tracking.kt @@ -0,0 +1,29 @@ +package com.quadient.migration.persistence.upgrade + +import org.flywaydb.core.api.migration.BaseJavaMigration +import org.flywaydb.core.api.migration.Context + +/** + * Variable structures did not have status tracking until they became deployable (as data definitions). + * Creates an initial Active status record for every existing variable structure that has none, + * so they are picked up by the deployment. + */ +class V18__variable_structure_status_tracking : BaseJavaMigration() { + override fun migrate(context: Context) { + val connection = context.connection + connection.createStatement().use { stmt -> + stmt.execute( + """ + INSERT INTO status_tracking (id, resource_type, project_name, status_events) + SELECT vs.id, 'VariableStructure', vs.project_name, + jsonb_build_array(jsonb_build_object( + 'type', 'Active', + 'timestamp', to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') + )) + FROM variable_structure vs + ON CONFLICT (id, resource_type, project_name) DO NOTHING + """.trimIndent() + ) + } + } +} diff --git a/migration-library/src/main/kotlin/com/quadient/migration/service/ResourcePathProvider.kt b/migration-library/src/main/kotlin/com/quadient/migration/service/ResourcePathProvider.kt index 216ee809..95186e07 100644 --- a/migration-library/src/main/kotlin/com/quadient/migration/service/ResourcePathProvider.kt +++ b/migration-library/src/main/kotlin/com/quadient/migration/service/ResourcePathProvider.kt @@ -9,6 +9,7 @@ import com.quadient.migration.api.dto.migrationmodel.DisplayRule import com.quadient.migration.api.dto.migrationmodel.DocumentObject import com.quadient.migration.api.dto.migrationmodel.Image import com.quadient.migration.api.dto.migrationmodel.LiteralBaseTemplatePath +import com.quadient.migration.api.dto.migrationmodel.VariableStructure import com.quadient.migration.service.inspirebuilder.appendExtensionIfMissing import com.quadient.migration.shared.AttachmentType import com.quadient.migration.shared.DocumentObjectType @@ -18,6 +19,13 @@ import com.quadient.migration.shared.orDefault import com.quadient.migration.shared.toIcmPath interface ResourcePathProvider { + fun getDataDefinitionPath(variableStructure: VariableStructure): IcmPath = + error("Data definition deployment is supported only for Interactive output.") + fun getDataDefinitionSourceTemplatePath(): IcmPath = + error("Data definition deployment is supported only for Interactive output.") + fun getSourceBaseTemplatePath(): IcmPath = + error("Interactive base template source paths are not supported for Designer output.") + fun getDocumentObjectFileName(documentObject: DocumentObject): String fun getDocumentObjectPath(documentObject: DocumentObject): IcmPath @@ -156,6 +164,23 @@ class DesignerResourcePathProvider(private val projectConfig: ProjectConfig) : R } open class InteractiveResourcePathProvider(private val projectConfig: ProjectConfig) : ResourcePathProvider { + override fun getDataDefinitionPath(variableStructure: VariableStructure): IcmPath = IcmPath.root() + .join("Interactive").join(projectConfig.interactiveTenant).join("DataDefinitions") + .join(resolveTargetDir(projectConfig.defaultTargetFolder)) + .join("${variableStructure.nameOrId()}.jdd") + + override fun getDataDefinitionSourceTemplatePath(): IcmPath = IcmPath.root() + .join("Interactive").join(projectConfig.interactiveTenant).join("Sources").join("SourceTemplate.wfd") + + override fun getSourceBaseTemplatePath(): IcmPath { + val configuredPath = projectConfig.sourceBaseTemplatePath?.takeIf { it.isNotBlank() }?.toIcmPath() + require(configuredPath == null || configuredPath.isAbsolute()) { + "The configured source base template path '$configuredPath' is not absolute." + } + return configuredPath + ?: IcmPath.root().join("Interactive").join(projectConfig.interactiveTenant).join("Sources").join("SourceTemplate.wfd") + } + override fun getDocumentObjectFileName(documentObject: DocumentObject): String { val ext = when (documentObject.type) { DocumentObjectType.Snippet -> "jsd" @@ -318,4 +343,4 @@ open class InteractiveResourcePathProvider(private val projectConfig: ProjectCon } } -class EvolveResourcePathProvider(projectConfig: ProjectConfig) : InteractiveResourcePathProvider(projectConfig) \ No newline at end of file +class EvolveResourcePathProvider(projectConfig: ProjectConfig) : InteractiveResourcePathProvider(projectConfig) diff --git a/migration-library/src/main/kotlin/com/quadient/migration/service/deploy/DeployClient.kt b/migration-library/src/main/kotlin/com/quadient/migration/service/deploy/DeployClient.kt index 83264b4d..3d9f7af7 100644 --- a/migration-library/src/main/kotlin/com/quadient/migration/service/deploy/DeployClient.kt +++ b/migration-library/src/main/kotlin/com/quadient/migration/service/deploy/DeployClient.kt @@ -115,6 +115,9 @@ sealed class DeployClient( abstract fun deployBaseTemplates(): DeploymentResult + open fun deployDataDefinitions(): DeploymentResult = + error("Data definition deployment is supported only for Interactive output.") + fun deployDocumentObjects() = deployDocumentObjects(false) fun deployDocumentObjects(skipDependencies: Boolean): DeploymentResult { val documentObjects = if (projectConfig.getDocumentObjectsToProcess().isNotEmpty()) { @@ -411,4 +414,4 @@ sealed class DeployClient( return resources } -} \ No newline at end of file +} diff --git a/migration-library/src/main/kotlin/com/quadient/migration/service/deploy/EvolveDeployClient.kt b/migration-library/src/main/kotlin/com/quadient/migration/service/deploy/EvolveDeployClient.kt index d273cac1..0edc1a0d 100644 --- a/migration-library/src/main/kotlin/com/quadient/migration/service/deploy/EvolveDeployClient.kt +++ b/migration-library/src/main/kotlin/com/quadient/migration/service/deploy/EvolveDeployClient.kt @@ -31,6 +31,7 @@ import com.quadient.migration.service.deploy.utility.ResourceType import com.quadient.migration.service.deploy.utility.ResultTrackerImpl import com.quadient.migration.service.inspirebuilder.InspireDocumentObjectBuilder import com.quadient.migration.service.inspirebuilder.InspireBaseTemplateBuilder +import com.quadient.migration.service.inspirebuilder.InspireVariableStructureBuilder import com.quadient.migration.service.ipsclient.IpsClientException import com.quadient.migration.service.ipsclient.IpsService import com.quadient.migration.service.ipsclient.OperationResult @@ -70,6 +71,7 @@ class EvolveDeployClient( baseTemplateBuilder: InspireBaseTemplateBuilder, ipsService: IpsService, storage: Storage, + variableStructureBuilder: InspireVariableStructureBuilder, ) : InteractiveDeployClient( projectConfig, resourcePathProvider, @@ -94,6 +96,7 @@ class EvolveDeployClient( baseTemplateBuilder, ipsService, storage, + variableStructureBuilder, ) { private companion object { val STYLE_DEFINITION_MASTER = "icm://Interactive/StandardPackage/CompanyStyles/StyleDefinition.wfd".toIcmPath() @@ -618,4 +621,4 @@ class EvolveDeployClient( logger.error("Failed to set production approval state in local ICM for $paths: ${result.message}") } } -} \ No newline at end of file +} diff --git a/migration-library/src/main/kotlin/com/quadient/migration/service/deploy/InteractiveDeployClient.kt b/migration-library/src/main/kotlin/com/quadient/migration/service/deploy/InteractiveDeployClient.kt index 00bab945..299cd654 100644 --- a/migration-library/src/main/kotlin/com/quadient/migration/service/deploy/InteractiveDeployClient.kt +++ b/migration-library/src/main/kotlin/com/quadient/migration/service/deploy/InteractiveDeployClient.kt @@ -1,6 +1,7 @@ package com.quadient.migration.service.deploy import com.quadient.migration.api.ProjectConfig +import com.quadient.migration.api.InspireOutput import com.quadient.migration.api.repository.StatusTrackingRepository import com.quadient.migration.api.dto.migrationmodel.DocumentObject import com.quadient.migration.api.dto.migrationmodel.Attachment @@ -44,6 +45,7 @@ import com.quadient.migration.service.deploy.utility.RefInheritanceServiceImpl import com.quadient.migration.service.deploy.utility.ProgressReporterImpl import com.quadient.migration.service.inspirebuilder.InspireDocumentObjectBuilder import com.quadient.migration.service.inspirebuilder.InspireBaseTemplateBuilder +import com.quadient.migration.service.inspirebuilder.InspireVariableStructureBuilder import com.quadient.migration.service.ipsclient.IpsService import com.quadient.migration.service.ipsclient.OperationResult import com.quadient.migration.service.resolveTarget @@ -89,6 +91,7 @@ open class InteractiveDeployClient( protected val baseTemplateBuilder: InspireBaseTemplateBuilder, ipsService: IpsService, storage: Storage, + private val inspireVariableStructureBuilder: InspireVariableStructureBuilder, ) : DeployClient( projectConfig, metadataValidator, @@ -197,6 +200,55 @@ open class InteractiveDeployClient( return documentObject.internal != true && !documentObject.skip.skipped } + override fun deployDataDefinitions(): DeploymentResult { + check(projectConfig.inspireOutput == InspireOutput.Interactive) { + "Data definition deployment is supported only for Interactive output." + } + val tracker = ResultTrackerImpl(statusTrackingRepository, projectConfig.inspireOutput) + val definitions = variableStructureRepository.listAll() + logger.info("Found ${definitions.size} data definition(s) in the repository.") + + for (structure in definitions) { + val targetPath = resourcePathProvider.getDataDefinitionPath(structure) + + if (!shouldDeployObject(structure.id, ResourceType.VariableStructure, targetPath, tracker.deploymentResult)) { + logger.info("Skipping deployment of '${structure.id}' as it is not marked for deployment.") + continue + } + + val fileNameError = validateFileName( + structure.nameOrId(), ResourceType.VariableStructure, structure.id + ) + if (fileNameError != null) { + logger.error(fileNameError) + tracker.errorDataDefinition(structure.id, targetPath, fileNameError) + continue + } + + try { + val wfdXml = inspireVariableStructureBuilder.buildDataDefinition(structure) + when (val result = ipsService.deployDataDefinition( + resourcePathProvider.getDataDefinitionSourceTemplatePath(), wfdXml, targetPath + )) { + OperationResult.Success -> { + logger.debug("Deployment of data definition '${structure.nameOrId()}' to $targetPath is successful.") + tracker.deployedDataDefinition(structure.id, targetPath) + } + is OperationResult.Failure -> { + logger.error("Failed to deploy data definition '${structure.nameOrId()}' to $targetPath: ${result.message}") + tracker.errorDataDefinition(structure.id, targetPath, result.message) + } + } + } catch (e: Exception) { + val message = "Failed to deploy data definition '${structure.nameOrId()}' to $targetPath." + logger.error(message, e) + tracker.errorDataDefinition(structure.id, targetPath, e.message ?: message) + } + } + runPostProcessors(tracker.deploymentResult) + return tracker.deploymentResult + } + override fun deployBaseTemplates(): DeploymentResult { val tracker = ResultTrackerImpl(statusTrackingRepository, projectConfig.inspireOutput) diff --git a/migration-library/src/main/kotlin/com/quadient/migration/service/deploy/utility/ConflictDetector.kt b/migration-library/src/main/kotlin/com/quadient/migration/service/deploy/utility/ConflictDetector.kt index f129823e..b8b3a8f1 100644 --- a/migration-library/src/main/kotlin/com/quadient/migration/service/deploy/utility/ConflictDetector.kt +++ b/migration-library/src/main/kotlin/com/quadient/migration/service/deploy/utility/ConflictDetector.kt @@ -13,6 +13,7 @@ import com.quadient.migration.api.repository.DisplayRuleRepository import com.quadient.migration.api.repository.DocumentObjectRepository import com.quadient.migration.api.repository.ImageRepository import com.quadient.migration.api.repository.StatusTrackingRepository +import com.quadient.migration.api.repository.VariableStructureRepository import com.quadient.migration.data.Deployed import com.quadient.migration.service.ResourcePathProvider import com.quadient.migration.service.ipsclient.OperationResult @@ -27,6 +28,7 @@ class ConflictDetectorImpl( private val imageRepository: ImageRepository, private val attachmentRepository: AttachmentRepository, private val displayRuleRepository: DisplayRuleRepository, + private val variableStructureRepository: VariableStructureRepository, private val statusTrackingRepository: StatusTrackingRepository, private val resourcePathProvider: ResourcePathProvider, private val output: InspireOutput, @@ -105,6 +107,9 @@ class ConflictDetectorImpl( ResourceType.TextStyle -> resourcePathProvider.getStyleDefinitionPath() ResourceType.ParagraphStyle -> resourcePathProvider.getStyleDefinitionPath() + ResourceType.VariableStructure -> resourcePathProvider.getDataDefinitionPath( + variableStructureRepository.findOrFail(item.id) + ) ResourceType.BaseTemplate -> null } }.getOrNull() diff --git a/migration-library/src/main/kotlin/com/quadient/migration/service/deploy/utility/DeploymentResult.kt b/migration-library/src/main/kotlin/com/quadient/migration/service/deploy/utility/DeploymentResult.kt index 62d39217..8d8cf402 100644 --- a/migration-library/src/main/kotlin/com/quadient/migration/service/deploy/utility/DeploymentResult.kt +++ b/migration-library/src/main/kotlin/com/quadient/migration/service/deploy/utility/DeploymentResult.kt @@ -30,7 +30,7 @@ data class DeploymentInfo( ) enum class ResourceType { - DocumentObject, Image, Attachment, TextStyle, ParagraphStyle, DisplayRule, BaseTemplate + DocumentObject, Image, Attachment, TextStyle, ParagraphStyle, DisplayRule, BaseTemplate, VariableStructure } data class DeploymentError(val id: String, val message: String) @@ -54,6 +54,8 @@ interface ResultTracker { fun errorDisplayRule(id: String, path: IcmPath, message: String) fun deployedBaseTemplate(id: String, targetPath: IcmPath) fun errorBaseTemplate(id: String, targetPath: IcmPath?, message: String) + fun deployedDataDefinition(id: String, targetPath: IcmPath) + fun errorDataDefinition(id: String, targetPath: IcmPath?, message: String) } class ResultTrackerImpl( @@ -63,6 +65,20 @@ class ResultTrackerImpl( override val deploymentId: Uuid = Uuid.random(), override val timestamp: Instant = Clock.System.now(), ) : ResultTracker{ + override fun deployedDataDefinition(id: String, targetPath: IcmPath) { + statusTrackingRepository?.deployed( + id, deploymentId, timestamp, ResourceType.VariableStructure, targetPath, inspireOutput + ) + deploymentResult.deployed.add(DeploymentInfo(id, ResourceType.VariableStructure, targetPath)) + } + + override fun errorDataDefinition(id: String, targetPath: IcmPath?, message: String) { + statusTrackingRepository?.error( + id, deploymentId, timestamp, ResourceType.VariableStructure, targetPath, inspireOutput, message + ) + deploymentResult.errors.add(DeploymentError(id, message)) + } + constructor( statusTrackingRepository: StatusTrackingRepository?, inspireOutput: InspireOutput, @@ -236,4 +252,4 @@ class ResultTrackerImpl( ) deploymentResult.errors.add(DeploymentError(id, message)) } -} \ No newline at end of file +} diff --git a/migration-library/src/main/kotlin/com/quadient/migration/service/deploy/utility/PostProcess.kt b/migration-library/src/main/kotlin/com/quadient/migration/service/deploy/utility/PostProcess.kt index 9f9e7ee3..46adeb08 100644 --- a/migration-library/src/main/kotlin/com/quadient/migration/service/deploy/utility/PostProcess.kt +++ b/migration-library/src/main/kotlin/com/quadient/migration/service/deploy/utility/PostProcess.kt @@ -77,7 +77,7 @@ class PostProcessImpl( ResourceType.Attachment -> attachmentRepository.find(info.id) ResourceType.TextStyle -> textStyleRepository.find(info.id) ResourceType.ParagraphStyle -> paragraphStyleRepository.find(info.id) - ResourceType.BaseTemplate -> null + ResourceType.BaseTemplate, ResourceType.VariableStructure -> null } if (obj == null) { val msg = "Failed to run '$name' post processor, '${info.type}' with id '${info.id}' at path '${info.targetPath}'." @@ -88,4 +88,4 @@ class PostProcessImpl( info to obj }.mapNotNull { (info, obj) -> transform(info, obj) } } -} \ No newline at end of file +} diff --git a/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/DesignerDocumentObjectBuilder.kt b/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/DesignerDocumentObjectBuilder.kt index b849c78b..59846010 100644 --- a/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/DesignerDocumentObjectBuilder.kt +++ b/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/DesignerDocumentObjectBuilder.kt @@ -162,10 +162,14 @@ class DesignerDocumentObjectBuilder( } val documentObjectXml = builder.build() - return if (projectConfig.sourceBaseTemplatePath.isNullOrBlank()) { + val sourceBaseTemplatePath = projectConfig.sourceBaseTemplatePath?.takeIf { it.isNotBlank() }?.toIcmPath() + require(sourceBaseTemplatePath == null || sourceBaseTemplatePath.isAbsolute()) { + "The configured source base template path '$sourceBaseTemplatePath' is not absolute." + } + return if (sourceBaseTemplatePath == null) { documentObjectXml } else { - enrichLayoutWithSourceBaseTemplate(icmDataCache, documentObjectXml, projectConfig.sourceBaseTemplatePath.toIcmPath()) + enrichLayoutWithSourceBaseTemplate(icmDataCache, documentObjectXml, sourceBaseTemplatePath) } } diff --git a/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/InspireBaseTemplateBuilder.kt b/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/InspireBaseTemplateBuilder.kt index acbd708c..b1387cab 100644 --- a/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/InspireBaseTemplateBuilder.kt +++ b/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/InspireBaseTemplateBuilder.kt @@ -1,6 +1,5 @@ package com.quadient.migration.service.inspirebuilder -import com.quadient.migration.api.ProjectConfig import com.quadient.migration.api.dto.migrationmodel.BaseTemplate import com.quadient.migration.api.dto.migrationmodel.DocumentObject import com.quadient.migration.api.dto.migrationmodel.DocumentObjectRef @@ -13,7 +12,6 @@ import com.quadient.migration.service.ResourcePathProvider import com.quadient.migration.service.deploy.utility.RefInheritanceServiceImpl import com.quadient.migration.shared.DocumentObjectType import com.quadient.migration.shared.IcmPath -import com.quadient.migration.shared.toIcmPath import com.quadient.migration.tools.logger import com.quadient.wfdxml.WfdXmlBuilder import com.quadient.wfdxml.api.layoutnodes.Flow @@ -23,7 +21,6 @@ import com.quadient.wfdxml.api.layoutnodes.email.EmailComponentPlaceHolder import com.quadient.wfdxml.api.module.Layout class InspireBaseTemplateBuilder( - private val projectConfig: ProjectConfig, private val icmDataCache: IcmDataCache, private val resourcePathProvider: ResourcePathProvider, private val baseTemplateRepository: BaseTemplateRepository, @@ -92,13 +89,7 @@ class InspireBaseTemplateBuilder( enrichFromDocumentObjects(baseTemplate, layout) val baseTemplateXml = builder.build() - val sourceBaseTemplatePath = if (projectConfig.sourceBaseTemplatePath.isNullOrBlank()) { - IcmPath.root().join("Interactive").join("StandardPackage").join("Sources").join("SourceTemplate.wfd") - } else { - projectConfig.sourceBaseTemplatePath.toIcmPath() - } - - return enrichLayoutWithSourceBaseTemplate(icmDataCache, baseTemplateXml, sourceBaseTemplatePath) + return enrichLayoutWithSourceBaseTemplate(icmDataCache, baseTemplateXml, resourcePathProvider.getSourceBaseTemplatePath()) } fun findChannelModels(baseTemplate: BaseTemplate): BaseTemplateChannelModels { diff --git a/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/InspireVariableStructureBuilder.kt b/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/InspireVariableStructureBuilder.kt index 4c990efe..fdeee371 100644 --- a/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/InspireVariableStructureBuilder.kt +++ b/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/InspireVariableStructureBuilder.kt @@ -12,6 +12,7 @@ import com.quadient.migration.api.repository.VariableStructureRepository import com.quadient.wfdxml.api.layoutnodes.SheetNameType import com.quadient.wfdxml.api.layoutnodes.data.VariableKind import com.quadient.wfdxml.api.module.Layout +import com.quadient.wfdxml.WfdXmlBuilder import com.quadient.wfdxml.internal.data.WorkFlowTreeDefinition import com.quadient.wfdxml.internal.layoutnodes.data.WorkFlowTreeEnums.NodeOptionality import com.quadient.wfdxml.internal.layoutnodes.data.WorkFlowTreeEnums.NodeType.SUB_TREE @@ -22,6 +23,20 @@ class InspireVariableStructureBuilder( private val variableStructureRepository: VariableStructureRepository, private val projectConfig: ProjectConfig, ) { + fun buildDataDefinition(variableStructure: VariableStructure): String { + val builder = WfdXmlBuilder() + val layout = builder.addLayout() + val variableStructureModel = initVariableStructure(layout, variableStructure.id) + for ((id, pathData) in variableStructureModel.structure) { + val variable = variableRepository.findOrFail(id) + val path = pathData.path.resolve(variableStructureModel, variableRepository::findOrFail) + ?.takeIf { it.isNotBlank() } + ?: error("Variable '$id' has no resolvable path in structure '${variableStructureModel.id}'") + getOrCreateVariable(layout.data, pathData.name ?: variable.nameOrId(), variable, path) + } + return builder.buildDataDefinitionLayoutDelta() + } + fun initVariableStructure(layout: Layout, variableStructureId: String?): VariableStructure { val variableStructureId = variableStructureId ?: projectConfig.defaultVariableStructure diff --git a/migration-library/src/main/kotlin/com/quadient/migration/service/ipsclient/IpsClient.kt b/migration-library/src/main/kotlin/com/quadient/migration/service/ipsclient/IpsClient.kt index 04ae657b..358e21d1 100644 --- a/migration-library/src/main/kotlin/com/quadient/migration/service/ipsclient/IpsClient.kt +++ b/migration-library/src/main/kotlin/com/quadient/migration/service/ipsclient/IpsClient.kt @@ -200,6 +200,12 @@ class IpsClient(private val host: String, private val port: Int, private val tim return connection.readResponse() } + fun extractDataDefinition(workflowId: WorkFlowId, outputPath: String): IpsResult { + val command = """run $workflowId -e json -type datadefinition -f "$outputPath"""" + connection.writeLine(command).getOrElse { return IpsFailedWriteException(command, it).toIpsResult() } + return connection.readResponse() + } + fun extractJldStyleDefinition(workflowId: WorkFlowId, outputPath: String): IpsResult { val command = """run $workflowId -e json -type resources -f "$outputPath"""" diff --git a/migration-library/src/main/kotlin/com/quadient/migration/service/ipsclient/IpsService.kt b/migration-library/src/main/kotlin/com/quadient/migration/service/ipsclient/IpsService.kt index fc3e6aab..b76bcabe 100644 --- a/migration-library/src/main/kotlin/com/quadient/migration/service/ipsclient/IpsService.kt +++ b/migration-library/src/main/kotlin/com/quadient/migration/service/ipsclient/IpsService.kt @@ -1,6 +1,5 @@ package com.quadient.migration.service.ipsclient -import com.quadient.migration.api.FileEntry import com.quadient.migration.api.FileList import tools.jackson.dataformat.xml.XmlMapper import tools.jackson.module.kotlin.KotlinModule @@ -98,6 +97,12 @@ class IpsService(private val config: IpsConfig) : Closeable, IcmClient { } } + fun deployDataDefinition(sourceTemplatePath: IcmPath, xmlContent: String, outputPath: IcmPath): OperationResult { + return deployJld(sourceTemplatePath.toString(), "DocumentLayout", xmlContent) { workflowId -> + client.extractDataDefinition(workflowId, outputPath.toString()) + } + } + fun deployStyleJld(baseTemplate: IcmPath, xmlContent: String, outputPath: IcmPath): OperationResult { return deployJld(baseTemplate.toString(), "DocumentLayout", xmlContent) { wfId -> client.extractJldStyleDefinition(wfId, outputPath.toString()) diff --git a/migration-library/src/test/kotlin/com/quadient/migration/persistence/VariableStructureRepositoryTest.kt b/migration-library/src/test/kotlin/com/quadient/migration/persistence/VariableStructureRepositoryTest.kt index fd78339c..4cb55c02 100644 --- a/migration-library/src/test/kotlin/com/quadient/migration/persistence/VariableStructureRepositoryTest.kt +++ b/migration-library/src/test/kotlin/com/quadient/migration/persistence/VariableStructureRepositoryTest.kt @@ -1,6 +1,12 @@ package com.quadient.migration.persistence import com.quadient.migration.Postgres +import com.quadient.migration.api.InspireOutput +import com.quadient.migration.api.ProjectName +import com.quadient.migration.api.repository.StatusTrackingRepository +import com.quadient.migration.data.Active +import com.quadient.migration.service.deploy.utility.ResourceType +import com.quadient.migration.tools.aProjectConfig import com.quadient.migration.api.dto.migrationmodel.builder.VariableStructureBuilder import com.quadient.migration.tools.aVariableStructureRepository import com.quadient.migration.tools.shouldBeEqualTo @@ -10,6 +16,34 @@ import org.junit.jupiter.api.Test @Postgres class VariableStructureRepositoryTest { private val repo = aVariableStructureRepository() + private val statusRepo = StatusTrackingRepository(ProjectName(aProjectConfig().name)) + + @Test + fun `upsert creates active status only for new variable structure`() { + val dto = VariableStructureBuilder("vs-status").addVariable("var1", "Data.Test.Value").build() + + repo.upsert(dto) + repo.upsert(dto) + + val events = statusRepo.findEventsRelevantToOutput("vs-status", ResourceType.VariableStructure, InspireOutput.Interactive) + events.shouldBeOfSize(1) + (events.first() is Active).shouldBeEqualTo(true) + } + + @Test + fun `upsertBatch creates active status only for new variable structures`() { + val struct1 = VariableStructureBuilder("vs-batch1").addVariable("var1", "Data.Test.Value").build() + val struct2 = VariableStructureBuilder("vs-batch2").addVariable("var2", "Data.Test.Value").build() + + repo.upsertBatch(listOf(struct1, struct2)) + repo.upsertBatch(listOf(struct1, struct2)) + + for (id in listOf("vs-batch1", "vs-batch2")) { + val events = statusRepo.findEventsRelevantToOutput(id, ResourceType.VariableStructure, InspireOutput.Interactive) + events.shouldBeOfSize(1) + (events.first() is Active).shouldBeEqualTo(true) + } + } @Test fun `roundtrip is correct`() { diff --git a/migration-library/src/test/kotlin/com/quadient/migration/persistence/upgrade/V18MigrationTest.kt b/migration-library/src/test/kotlin/com/quadient/migration/persistence/upgrade/V18MigrationTest.kt new file mode 100644 index 00000000..3d6c70b1 --- /dev/null +++ b/migration-library/src/test/kotlin/com/quadient/migration/persistence/upgrade/V18MigrationTest.kt @@ -0,0 +1,116 @@ +package com.quadient.migration.persistence.upgrade + +import com.quadient.migration.Postgres.Companion.POSTGRES_CONTAINER +import com.quadient.migration.api.InspireOutput +import com.quadient.migration.api.ProjectName +import com.quadient.migration.api.repository.StatusTrackingRepository +import com.quadient.migration.data.Active +import com.quadient.migration.data.Deployed +import com.quadient.migration.service.deploy.utility.ResourceType +import org.flywaydb.core.Flyway +import org.jetbrains.exposed.v1.jdbc.Database +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertInstanceOf +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import org.testcontainers.postgresql.PostgreSQLContainer +import java.sql.Connection +import java.sql.DriverManager + +/** + * Integration test for V18 migration that backfills an Active status tracking record + * for every existing variable structure. + */ +class V18MigrationTest { + + @Test + fun `V18 migration creates active status for existing variable structures`() { + PostgreSQLContainer(POSTGRES_CONTAINER).use { postgres -> + postgres.start() + + Database.connect( + url = postgres.jdbcUrl, + driver = "org.postgresql.Driver", + user = postgres.username, + password = postgres.password + ) + + migrate(postgres, "17") + + val deployedEvent = + """[{"type":"Deployed","deploymentId":"00000000-0000-0000-0000-000000000001","timestamp":"2026-01-01T00:00:00Z","output":"Interactive","icmPath":"icm://x.jdd","data":{}}]""" + connection(postgres).use { conn -> + insertVariableStructure(conn, "vs1", "test") + insertVariableStructure(conn, "vs2", "other") + insertStatus(conn, "vs1", "DocumentObject", "test", deployedEvent) + } + + migrate(postgres, "18") + + val testRepo = StatusTrackingRepository(ProjectName("test")) + val otherRepo = StatusTrackingRepository(ProjectName("other")) + + assertInstanceOf( + Active::class.java, + testRepo.findLastEventRelevantToOutput("vs1", ResourceType.VariableStructure, InspireOutput.Interactive) + ) + assertInstanceOf( + Active::class.java, + otherRepo.findLastEventRelevantToOutput("vs2", ResourceType.VariableStructure, InspireOutput.Interactive) + ) + assertNull(otherRepo.findLastEventRelevantToOutput("vs1", ResourceType.VariableStructure, InspireOutput.Interactive)) + + // status of a different resource type with the same id is left untouched + assertInstanceOf( + Deployed::class.java, + testRepo.findLastEventRelevantToOutput("vs1", ResourceType.DocumentObject, InspireOutput.Interactive) + ) + + connection(postgres).use { conn -> + val rs = conn.createStatement() + .executeQuery("SELECT count(*) FROM status_tracking WHERE resource_type = 'VariableStructure'") + rs.next() + assertEquals(2, rs.getInt(1)) + } + } + } + + private fun migrate(postgres: PostgreSQLContainer, target: String) { + Flyway.configure() + .dataSource(postgres.jdbcUrl, postgres.username, postgres.password) + .locations("classpath:com/quadient/migration/persistence/upgrade") + .target(target) + .load() + .migrate() + } + + private fun insertVariableStructure(conn: Connection, id: String, projectName: String) { + conn.prepareStatement( + """ + INSERT INTO variable_structure (id, project_name, name, origin_locations, custom_fields, last_updated, created, structure) + VALUES (?, ?, ?, '{}', '{}', NOW(), NOW(), '{}') + """.trimIndent() + ).apply { + setString(1, id) + setString(2, projectName) + setString(3, id) + }.executeUpdate() + } + + private fun insertStatus(conn: Connection, id: String, type: String, projectName: String, events: String) { + conn.prepareStatement( + """ + INSERT INTO status_tracking (id, resource_type, project_name, status_events) + VALUES (?, ?, ?, ?::jsonb) + """.trimIndent() + ).apply { + setString(1, id) + setString(2, type) + setString(3, projectName) + setString(4, events) + }.executeUpdate() + } + + private fun connection(postgres: PostgreSQLContainer): Connection = + DriverManager.getConnection(postgres.jdbcUrl, postgres.username, postgres.password) +} diff --git a/migration-library/src/test/kotlin/com/quadient/migration/service/ResourcePathProviderTest.kt b/migration-library/src/test/kotlin/com/quadient/migration/service/ResourcePathProviderTest.kt index a2deb82b..9abe998b 100644 --- a/migration-library/src/test/kotlin/com/quadient/migration/service/ResourcePathProviderTest.kt +++ b/migration-library/src/test/kotlin/com/quadient/migration/service/ResourcePathProviderTest.kt @@ -25,6 +25,8 @@ import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.CsvSource +import org.junit.jupiter.params.provider.NullAndEmptySource +import org.junit.jupiter.params.provider.ValueSource class ResourcePathProviderTest { @@ -32,6 +34,43 @@ class ResourcePathProviderTest { inner class InteractiveResourcePathProviderTest { private fun aSubject(config: ProjectConfig) = InteractiveResourcePathProvider(config) + @ParameterizedTest + @NullAndEmptySource + @ValueSource(strings = [" "]) + fun `base template source defaults to the configured tenant`(sourcePath: String?) { + val config = aProjectConfig(interactiveTenant = "customTenant", sourceBaseTemplatePath = sourcePath) + + aSubject(config).getSourceBaseTemplatePath().toString() + .shouldBeEqualTo("icm://Interactive/customTenant/Sources/SourceTemplate.wfd") + } + + @Test + fun `base template source uses the absolute configured path unchanged`() { + val path = "icm://OtherFolder/Custom.wfd" + val config = aProjectConfig(interactiveTenant = "customTenant", sourceBaseTemplatePath = path) + + aSubject(config).getSourceBaseTemplatePath().toString().shouldBeEqualTo(path) + } + + @Test + fun `base template source must be absolute`() { + val config = aProjectConfig(sourceBaseTemplatePath = "Sources/Custom.wfd") + + val exception = assertThrows { aSubject(config).getSourceBaseTemplatePath() } + + exception.message.shouldBeEqualTo("The configured source base template path 'Sources/Custom.wfd' is not absolute.") + } + + @ParameterizedTest + @NullAndEmptySource + @ValueSource(strings = [" ", "icm://OtherFolder/Custom.wfd", "Sources/Custom.wfd"]) + fun `data definition source always uses the configured tenant and ignores the override`(sourcePath: String?) { + val config = aProjectConfig(interactiveTenant = "customTenant", sourceBaseTemplatePath = sourcePath) + + aSubject(config).getDataDefinitionSourceTemplatePath().toString() + .shouldBeEqualTo("icm://Interactive/customTenant/Sources/SourceTemplate.wfd") + } + @ParameterizedTest @CsvSource( // projectCfg.paths ,targetFolder ,defaultTargetFolder,expected diff --git a/migration-library/src/test/kotlin/com/quadient/migration/service/deploy/DeployClientTest.kt b/migration-library/src/test/kotlin/com/quadient/migration/service/deploy/DeployClientTest.kt index aa89317e..da4b8abd 100644 --- a/migration-library/src/test/kotlin/com/quadient/migration/service/deploy/DeployClientTest.kt +++ b/migration-library/src/test/kotlin/com/quadient/migration/service/deploy/DeployClientTest.kt @@ -88,7 +88,7 @@ class DeployClientTest { val projectConfig = aProjectConfig(output = InspireOutput.Designer) val postProcess = PostProcessImpl(ipsService, documentObjectRepository, imageRepository, attachmentRepository, displayRuleRepository, textStyleRepository, paragraphStyleRepository) val resourcePathProvider = DesignerResourcePathProvider(projectConfig) - val conflictDetector = ConflictDetectorImpl(documentObjectRepository, imageRepository, attachmentRepository, displayRuleRepository, statusTrackingRepository, resourcePathProvider, InspireOutput.Designer) + val conflictDetector = ConflictDetectorImpl(documentObjectRepository, imageRepository, attachmentRepository, displayRuleRepository, variableStructureRepository, statusTrackingRepository, resourcePathProvider, InspireOutput.Designer) val progressReporter = ProgressReporterImpl(documentObjectRepository, imageRepository, attachmentRepository, displayRuleRepository, documentObjectBuilder, statusTrackingRepository, resourcePathProvider, InspireOutput.Designer) val deployOrder = DeployOrderImpl(documentObjectRepository) val refInheritanceService = RefInheritanceServiceImpl(documentObjectRepository) diff --git a/migration-library/src/test/kotlin/com/quadient/migration/service/deploy/DesignerDeployClientTest.kt b/migration-library/src/test/kotlin/com/quadient/migration/service/deploy/DesignerDeployClientTest.kt index e8de68b9..da4b02d3 100644 --- a/migration-library/src/test/kotlin/com/quadient/migration/service/deploy/DesignerDeployClientTest.kt +++ b/migration-library/src/test/kotlin/com/quadient/migration/service/deploy/DesignerDeployClientTest.kt @@ -105,7 +105,7 @@ class DesignerDeployClientTest { val ipsService = mockk() val storage = mockk() val resourcePathProvider = mockk() - val conflictDetector = ConflictDetectorImpl(documentObjectRepository, imageRepository, attachmentRepository, displayRuleRepository, statusTrackingRepository, resourcePathProvider, InspireOutput.Designer) + val conflictDetector = ConflictDetectorImpl(documentObjectRepository, imageRepository, attachmentRepository, displayRuleRepository, variableStructureRepository, statusTrackingRepository, resourcePathProvider, InspireOutput.Designer) val progressReporter = ProgressReporterImpl(documentObjectRepository, imageRepository, attachmentRepository, displayRuleRepository, documentObjectBuilder, statusTrackingRepository, resourcePathProvider, InspireOutput.Designer) val deployOrder = DeployOrderImpl(documentObjectRepository) val refInheritanceService = RefInheritanceServiceImpl(documentObjectRepository) @@ -135,6 +135,12 @@ class DesignerDeployClientTest { storage ) + @Test + fun `Designer data definition deployment is unsupported`() { + assertThrows { subject().deployDataDefinitions() } + verify(exactly = 0) { variableStructureRepository.listAll() } + } + @BeforeEach fun setupAll() { every { documentObjectBuilder.shouldIncludeInternalDependency(any()) } answers { diff --git a/migration-library/src/test/kotlin/com/quadient/migration/service/deploy/EvolveDeployClientTest.kt b/migration-library/src/test/kotlin/com/quadient/migration/service/deploy/EvolveDeployClientTest.kt index ce46fdd8..863ce692 100644 --- a/migration-library/src/test/kotlin/com/quadient/migration/service/deploy/EvolveDeployClientTest.kt +++ b/migration-library/src/test/kotlin/com/quadient/migration/service/deploy/EvolveDeployClientTest.kt @@ -44,6 +44,7 @@ import com.quadient.migration.service.deploy.utility.DeployOrderImpl import com.quadient.migration.service.deploy.utility.RefInheritanceServiceImpl import com.quadient.migration.service.inspirebuilder.BaseTemplateChannelModels import com.quadient.migration.service.inspirebuilder.InspireBaseTemplateBuilder +import com.quadient.migration.service.inspirebuilder.InspireVariableStructureBuilder import com.quadient.migration.service.ipsclient.IpsClientException import com.quadient.migration.service.ipsclient.IpsService import com.quadient.migration.service.ipsclient.OperationResult @@ -94,6 +95,7 @@ class EvolveDeployClientTest { val caClient = mockk() val resourcePathProvider = mockk() val postProcess = mockk(relaxed = true) + val inspireVariableStructureBuilder = mockk() val evolveConfig = EvolveConfig( apiRetryDelayMs = 0L, @@ -119,7 +121,7 @@ class EvolveDeployClientTest { val deployOrder = DeployOrderImpl(documentObjectRepository) val refInheritanceService = RefInheritanceServiceImpl(documentObjectRepository) - val conflictDetector = ConflictDetectorImpl(documentObjectRepository, imageRepository, attachmentRepository, displayRuleRepository, statusTrackingRepository, resourcePathProvider, projectConfig.inspireOutput) + val conflictDetector = ConflictDetectorImpl(documentObjectRepository, imageRepository, attachmentRepository, displayRuleRepository, variableStructureRepository, statusTrackingRepository, resourcePathProvider, projectConfig.inspireOutput) val progressReporter = ProgressReporterImpl(documentObjectRepository, imageRepository, attachmentRepository, displayRuleRepository, documentObjectBuilder, statusTrackingRepository, resourcePathProvider, projectConfig.inspireOutput) private fun createSubject( @@ -151,6 +153,7 @@ class EvolveDeployClientTest { baseTemplateBuilder, ipsService, storage, + inspireVariableStructureBuilder, ) private val subject = createSubject() diff --git a/migration-library/src/test/kotlin/com/quadient/migration/service/deploy/InteractiveDeployClientTest.kt b/migration-library/src/test/kotlin/com/quadient/migration/service/deploy/InteractiveDeployClientTest.kt index cf17c675..36ad7b9c 100644 --- a/migration-library/src/test/kotlin/com/quadient/migration/service/deploy/InteractiveDeployClientTest.kt +++ b/migration-library/src/test/kotlin/com/quadient/migration/service/deploy/InteractiveDeployClientTest.kt @@ -1,5 +1,7 @@ package com.quadient.migration.service.deploy +import com.quadient.migration.api.dto.migrationmodel.builder.VariableStructureBuilder + import com.quadient.migration.api.InspireOutput import com.quadient.migration.api.dto.migrationmodel.Attachment import com.quadient.migration.api.dto.migrationmodel.AttachmentRef @@ -51,6 +53,7 @@ import com.quadient.migration.service.InteractiveResourcePathProvider import com.quadient.migration.service.deploy.utility.DeployOrderImpl import com.quadient.migration.service.deploy.utility.RefInheritanceServiceImpl import com.quadient.migration.service.inspirebuilder.InspireBaseTemplateBuilder +import com.quadient.migration.service.inspirebuilder.InspireVariableStructureBuilder import com.quadient.migration.service.ipsclient.IpsClientException import com.quadient.migration.service.ipsclient.IpsService import com.quadient.migration.service.ipsclient.OperationResult @@ -121,16 +124,17 @@ class InteractiveDeployClientTest { val tenant = config.interactiveTenant val postProcess = PostProcessImpl(ipsService, documentObjectRepository, imageRepository, attachmentRepository, displayRuleRepository, textStyleRepository, paragraphStyleRepository) val resourcePathProvider = mockk() - val conflictDetector = ConflictDetectorImpl(documentObjectRepository, imageRepository, attachmentRepository, displayRuleRepository, statusTrackingRepository, resourcePathProvider, config.inspireOutput) + val conflictDetector = ConflictDetectorImpl(documentObjectRepository, imageRepository, attachmentRepository, displayRuleRepository, variableStructureRepository, statusTrackingRepository, resourcePathProvider, config.inspireOutput) val progressReporter = ProgressReporterImpl(documentObjectRepository, imageRepository, attachmentRepository, displayRuleRepository, documentObjectBuilder, statusTrackingRepository, resourcePathProvider, config.inspireOutput) val deployOrder = DeployOrderImpl(documentObjectRepository) val refInheritanceService = RefInheritanceServiceImpl(documentObjectRepository) - private fun subject(selectedDocumentObjects: List = emptyList()) = InteractiveDeployClient( + private fun subject(selectedDocumentObjects: List = emptyList(), output: InspireOutput = InspireOutput.Interactive) = InteractiveDeployClient( aProjectConfig( targetDefaultFolder = "defaultFolder", baseTemplatePath = "icm://Interactive/tenant/BaseTemplates/templ.wfd", selectedDocumentObjects = selectedDocumentObjects, + output = output, ), resourcePathProvider, metadataValidator, @@ -154,6 +158,7 @@ class InteractiveDeployClientTest { baseTemplateBuilder, ipsService, storage, + InspireVariableStructureBuilder(variableRepository, variableStructureRepository, config), ) @BeforeEach @@ -166,6 +171,10 @@ class InteractiveDeployClientTest { every { ipsService.setProductionApprovalState(any>()) } returns OperationResult.Success every { documentObjectRepository.find(any()) } returns null every { documentObjectRepository.listAll() } returns emptyList() + every { variableStructureRepository.findOrFail(any()) } answers { + variableStructureRepository.listAll().first { it.id == firstArg() } + } + every { statusTrackingRepository.listAll() } returns emptyList() every { resourcePathProvider.getBaseTemplateFullPath(any(), any()) } answers { InteractiveResourcePathProvider(config).getBaseTemplateFullPath(firstArg(), secondArg()) } @@ -176,6 +185,53 @@ class InteractiveDeployClientTest { every { resourcePathProvider.getBaseTemplateFileName(any()) } answers { callOriginal() } } + @Test + fun `data definitions deploy one jdd per structure and record failures independently`() { + val first = VariableStructureBuilder("first").name("Customers").build() + val second = VariableStructureBuilder("second").build() + val third = VariableStructureBuilder("third").build() + val realPaths = InteractiveResourcePathProvider(config.copy(sourceBaseTemplatePath = "icm://Custom/Source.wfd")) + every { variableStructureRepository.listAll() } returns listOf(third, first, second) + every { statusTrackingRepository.findLastEventRelevantToOutput(any(), ResourceType.VariableStructure, InspireOutput.Interactive) } returns Active() + every { resourcePathProvider.getDataDefinitionPath(any()) } answers { realPaths.getDataDefinitionPath(firstArg()) } + every { resourcePathProvider.getDataDefinitionSourceTemplatePath() } returns realPaths.getDataDefinitionSourceTemplatePath() + every { statusTrackingRepository.deployed(any(), any(), any(), any(), any(), any(), any()) } returns aDeployedStatus("first") + every { statusTrackingRepository.error(any(), any(), any(), any(), any(), any(), any(), any()) } returns aErrorStatus("second") + every { ipsService.deployDataDefinition(any(), any(), realPaths.getDataDefinitionPath(first)) } returns OperationResult.Success + every { ipsService.deployDataDefinition(any(), any(), realPaths.getDataDefinitionPath(second)) } returns OperationResult.Failure("export failed") + every { ipsService.deployDataDefinition(any(), any(), realPaths.getDataDefinitionPath(third)) } throws + Exception("unexpected export failure") + + val result = subject().deployDataDefinitions() + + result.deployed.shouldBeEqualTo(mutableListOf(DeploymentInfo("first", ResourceType.VariableStructure, realPaths.getDataDefinitionPath(first)))) + result.errors.shouldBeEqualTo(mutableListOf( + DeploymentError("third", "unexpected export failure"), + DeploymentError("second", "export failed"), + )) + verify { ipsService.deployDataDefinition(IcmPath.from("icm://Interactive/$tenant/Sources/SourceTemplate.wfd"), match { it.contains("Def.SystemVariables") }, realPaths.getDataDefinitionPath(first)) } + verify(exactly = 0) { resourcePathProvider.getSourceBaseTemplatePath() } + verify { ipsService.setProductionApprovalState(listOf(realPaths.getDataDefinitionPath(first))) } + } + + @Test + fun `already deployed data definitions are skipped`() { + val structure = VariableStructureBuilder("deployed").name("Customers").build() + val realPaths = InteractiveResourcePathProvider(config) + val path = realPaths.getDataDefinitionPath(structure) + every { variableStructureRepository.listAll() } returns listOf(structure) + every { resourcePathProvider.getDataDefinitionPath(any()) } answers { realPaths.getDataDefinitionPath(firstArg()) } + every { statusTrackingRepository.findLastEventRelevantToOutput(structure.id, ResourceType.VariableStructure, InspireOutput.Interactive) } returns + Deployed(Uuid.random(), Clock.System.now(), InspireOutput.Interactive, path) + + val result = subject().deployDataDefinitions() + + result.deployed.shouldBeEmpty() + result.errors.shouldBeEmpty() + verify(exactly = 0) { resourcePathProvider.getDataDefinitionSourceTemplatePath() } + verify(exactly = 0) { ipsService.deployDataDefinition(any(), any(), any()) } + } + @Test fun `deployDocumentObjects deploys external document objects as standalone jld files and approves them`() { // given @@ -1767,4 +1823,4 @@ class InteractiveDeployClientTest { } } -} \ No newline at end of file +} diff --git a/migration-library/src/test/kotlin/com/quadient/migration/service/inspirebuilder/DesignerDocumentObjectBuilderTest.kt b/migration-library/src/test/kotlin/com/quadient/migration/service/inspirebuilder/DesignerDocumentObjectBuilderTest.kt index d86c533c..80f60519 100644 --- a/migration-library/src/test/kotlin/com/quadient/migration/service/inspirebuilder/DesignerDocumentObjectBuilderTest.kt +++ b/migration-library/src/test/kotlin/com/quadient/migration/service/inspirebuilder/DesignerDocumentObjectBuilderTest.kt @@ -80,6 +80,7 @@ import io.mockk.mockk import io.mockk.verify import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows class DesignerDocumentObjectBuilderTest { val documentObjectRepository = mockk() @@ -923,6 +924,17 @@ class DesignerDocumentObjectBuilderTest { contentFlow["FlowContent"]["P"]["T"][""].stringValue().shouldBeEqualTo("Text") } + @Test + fun `source base template must be absolute`() { + val block = aDocObj("B_1", Block, listOf(aParagraph(aText("Text")))).mock() + val subject = aSubject(config.copy(sourceBaseTemplatePath = "Sources/Custom.wfd")) + + val exception = assertThrows { subject.buildDocumentObject(block) } + + exception.message.shouldBeEqualTo("The configured source base template path 'Sources/Custom.wfd' is not absolute.") + verify(exactly = 0) { ipsService.wfd2xml(any()) } + } + @Test fun `loading of source base template is cached during single run`() { // given diff --git a/migration-library/src/test/kotlin/com/quadient/migration/service/inspirebuilder/InspireBaseTemplateBuilderTest.kt b/migration-library/src/test/kotlin/com/quadient/migration/service/inspirebuilder/InspireBaseTemplateBuilderTest.kt index 30e4632f..1ee44111 100644 --- a/migration-library/src/test/kotlin/com/quadient/migration/service/inspirebuilder/InspireBaseTemplateBuilderTest.kt +++ b/migration-library/src/test/kotlin/com/quadient/migration/service/inspirebuilder/InspireBaseTemplateBuilderTest.kt @@ -27,6 +27,7 @@ import com.quadient.migration.tools.shouldBeNull import com.quadient.migration.tools.shouldNotBeNull import io.mockk.every import io.mockk.mockk +import io.mockk.verify import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import tools.jackson.databind.JsonNode @@ -44,7 +45,6 @@ class InspireBaseTemplateBuilderTest { private val variableStructureRepository = mockk() private val refInheritanceService = RefInheritanceServiceImpl(documentObjectRepository) private val subject = InspireBaseTemplateBuilder( - config, icmDataCache, resourcePathProvider, baseTemplateRepository, @@ -72,6 +72,33 @@ class InspireBaseTemplateBuilderTest { every { documentObjectRepository.listAll() } returns emptyList() } + @Test + fun `base templates default to the tenant source template path`() { + subject.buildBaseTemplate(BaseTemplateBuilder("BT_1").build()) + verify { + ipsService.wfd2xml(IcmPath.from("icm://Interactive/${config.interactiveTenant}/Sources/SourceTemplate.wfd")) + } + } + + @Test + fun `base templates use the configured source template override`() { + val customPath = IcmPath.from("icm://Custom/Source.wfd") + val customConfig = config.copy(sourceBaseTemplatePath = customPath.toString()) + val customPaths = InteractiveResourcePathProvider(customConfig) + val subject = InspireBaseTemplateBuilder( + InteractiveIcmDataCache(ipsService, customPaths), + customPaths, + baseTemplateRepository, + documentObjectRepository, + InspireVariableStructureBuilder(variableRepository, variableStructureRepository, customConfig), + refInheritanceService, + ) + + subject.buildBaseTemplate(BaseTemplateBuilder("BT_1").build()) + + verify { ipsService.wfd2xml(customPath) } + } + @Test fun `buildBaseTemplate sets external styles layout when style definition exists`() { // given diff --git a/migration-library/src/test/kotlin/com/quadient/migration/service/inspirebuilder/InspireVariableStructureBuilderTest.kt b/migration-library/src/test/kotlin/com/quadient/migration/service/inspirebuilder/InspireVariableStructureBuilderTest.kt new file mode 100644 index 00000000..36ea791f --- /dev/null +++ b/migration-library/src/test/kotlin/com/quadient/migration/service/inspirebuilder/InspireVariableStructureBuilderTest.kt @@ -0,0 +1,73 @@ +package com.quadient.migration.service.inspirebuilder + +import com.quadient.migration.api.dto.migrationmodel.builder.VariableBuilder +import com.quadient.migration.api.dto.migrationmodel.builder.VariableStructureBuilder +import com.quadient.migration.api.repository.VariableRepository +import com.quadient.migration.api.repository.VariableStructureRepository +import com.quadient.migration.shared.DataType +import com.quadient.migration.tools.aProjectConfig +import com.quadient.migration.tools.shouldBeEqualTo +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.Test +import tools.jackson.dataformat.xml.XmlMapper + +class InspireVariableStructureBuilderTest { + private val variables = mockk() + private val structures = mockk() + private val subject = InspireVariableStructureBuilder(variables, structures, aProjectConfig()) + private val xmlMapper = XmlMapper() + + @Test + fun `data definition initializes array paths before declaring listed variables`() { + val array = VariableBuilder("clients").name("Clients").dataType(DataType.Array).build() + val name = VariableBuilder("name").name("OriginalName").dataType(DataType.String).defaultValue("Alice").build() + every { variables.findOrFail(array.id) } returns array + every { variables.findOrFail(name.id) } returns name + val structure = VariableStructureBuilder("structure") + .addVariable(name.id, array, "CustomerName") + .addVariable(array.id, "Data") + .build() + every { structures.findOrFail(structure.id) } returns structure + + val nodes = xmlMapper.readTree(subject.buildDataDefinition(structure))["Variable"].toList() + + nodes.count { it["Name"]?.stringValue() == "Clients" }.shouldBeEqualTo(1) + nodes.count { it["Name"]?.stringValue() == "Value" }.shouldBeEqualTo(1) + val customerName = nodes.first { it["Name"]?.stringValue() == "CustomerName" } + customerName["ParentId"].stringValue().shouldBeEqualTo("Data.Clients.Value") + nodes.first { it["Id"] == customerName["Id"] && it.has("Content") }["Content"].stringValue() + .shouldBeEqualTo("Alice") + } + + @Test + fun `data definition initializes subtree paths before declaring listed variables`() { + val subtree = VariableBuilder("customer").name("Customer").dataType(DataType.SubTree).build() + val number = VariableBuilder("number").name("Number").dataType(DataType.Integer).defaultValue("42").build() + every { variables.findOrFail(subtree.id) } returns subtree + every { variables.findOrFail(number.id) } returns number + val structure = VariableStructureBuilder("structure") + .addVariable(subtree.id, "Data") + .addVariable(number.id, subtree) + .build() + every { structures.findOrFail(structure.id) } returns structure + + val nodes = xmlMapper.readTree(subject.buildDataDefinition(structure))["Variable"].toList() + + nodes.count { it["Name"]?.stringValue() == "Customer" }.shouldBeEqualTo(1) + nodes.first { it["Name"]?.stringValue() == "Number" }["ParentId"].stringValue() + .shouldBeEqualTo("Data.Customer") + nodes.count { it["Content"]?.stringValue() == "42" }.shouldBeEqualTo(1) + } + + @Test + fun `empty data definition still exports system variables`() { + val structure = VariableStructureBuilder("empty").build() + every { structures.findOrFail(structure.id) } returns structure + + val nodes = xmlMapper.readTree(subject.buildDataDefinition(structure))["Variable"].toList() + + nodes.count { it["Name"]?.stringValue() == "PageCounter" }.shouldBeEqualTo(1) + nodes.count { it["Id"]?.stringValue() == "Def.SystemVariables" }.shouldBeEqualTo(1) + } +} diff --git a/migration-library/src/test/kotlin/com/quadient/migration/tools/TestObjectBuilders.kt b/migration-library/src/test/kotlin/com/quadient/migration/tools/TestObjectBuilders.kt index 32516caa..fc8a645d 100644 --- a/migration-library/src/test/kotlin/com/quadient/migration/tools/TestObjectBuilders.kt +++ b/migration-library/src/test/kotlin/com/quadient/migration/tools/TestObjectBuilders.kt @@ -526,7 +526,7 @@ fun aErrorStatus( private val statusRepo = StatusTrackingRepository(ProjectName(aProjectConfig().name)) fun aDocumentObjectRepository() = DocumentObjectRepository(ProjectName(aProjectConfig().name), statusRepo) fun aVariableRepository() = VariableRepository(ProjectName(aProjectConfig().name)) -fun aVariableStructureRepository() = VariableStructureRepository(ProjectName(aProjectConfig().name)) +fun aVariableStructureRepository() = VariableStructureRepository(ProjectName(aProjectConfig().name), statusRepo) fun aParaStyleRepository() = ParagraphStyleRepository(ProjectName(aProjectConfig().name), statusRepo) fun aTextStyleRepository() = TextStyleRepository(ProjectName(aProjectConfig().name), statusRepo) fun aDisplayRuleRepository() = DisplayRuleRepository(ProjectName(aProjectConfig().name), statusRepo) diff --git a/wfd-xml/impl/src/main/java/com/quadient/wfdxml/WfdXmlBuilder.java b/wfd-xml/impl/src/main/java/com/quadient/wfdxml/WfdXmlBuilder.java index a2da9cce..424821d8 100644 --- a/wfd-xml/impl/src/main/java/com/quadient/wfdxml/WfdXmlBuilder.java +++ b/wfd-xml/impl/src/main/java/com/quadient/wfdxml/WfdXmlBuilder.java @@ -79,6 +79,14 @@ public String buildStyleLayoutDelta() { return exporter.buildString(); } + public String buildDataDefinitionLayoutDelta() { + XmlExporter exporter = new XmlExporter(); + LayoutImpl layout = modules.stream().filter(LayoutImpl.class::isInstance).map(LayoutImpl.class::cast) + .findFirst().orElseThrow(() -> new IllegalStateException("No layout module found")); + layout.exportDataDefinitionLayoutDelta(exporter); + return exporter.buildString(); + } + public String buildLayoutDelta() { XmlExporter exporter = new XmlExporter(); diff --git a/wfd-xml/impl/src/main/java/com/quadient/wfdxml/internal/module/layout/ForwardReferencesExporter.java b/wfd-xml/impl/src/main/java/com/quadient/wfdxml/internal/module/layout/ForwardReferencesExporter.java index 914d1507..1a904b7a 100644 --- a/wfd-xml/impl/src/main/java/com/quadient/wfdxml/internal/module/layout/ForwardReferencesExporter.java +++ b/wfd-xml/impl/src/main/java/com/quadient/wfdxml/internal/module/layout/ForwardReferencesExporter.java @@ -45,6 +45,10 @@ public void exportForwardReferences(Boolean useExisting) { exportTree(layout, useExisting); } + public void exportForwardReferences(Tree tree, Boolean useExisting) { + exportTree(tree, useExisting); + } + private void exportTree(Tree tree, Boolean useExisting) { for (Object c : tree.children) { NodeImpl child = (NodeImpl) c; @@ -93,4 +97,4 @@ private void writeForwardReferenceToExporter(NodeImpl node, NodeImpl parent, Boo exporter.endElement(); } -} \ No newline at end of file +} diff --git a/wfd-xml/impl/src/main/java/com/quadient/wfdxml/internal/module/layout/LayoutImpl.java b/wfd-xml/impl/src/main/java/com/quadient/wfdxml/internal/module/layout/LayoutImpl.java index fe6b0031..95cc9e7a 100644 --- a/wfd-xml/impl/src/main/java/com/quadient/wfdxml/internal/module/layout/LayoutImpl.java +++ b/wfd-xml/impl/src/main/java/com/quadient/wfdxml/internal/module/layout/LayoutImpl.java @@ -393,6 +393,17 @@ public void exportStyleLayoutDelta(XmlExporter exporter) { exporter.endElement(); } + public void exportDataDefinitionLayoutDelta(XmlExporter exporter) { + for (DefNode defNode : defNodes.values()) { + exporter.getIdRegister().setObjectId(defNode.node, defNode.xmlId); + } + + exporter.beginElement("Layout"); + new ForwardReferencesExporter(this, defNodes, exporter).exportForwardReferences(data, true); + exportTree(data, exporter); + exporter.endElement(); + } + public void exportLayoutDelta(XmlExporter exporter) { for (DefNode defNode : defNodes.values()) { exporter.getIdRegister().setObjectId(defNode.node, defNode.xmlId); diff --git a/wfd-xml/impl/src/test/groovy/com/quadient/wfdxml/WfdXmlBuilderTest.groovy b/wfd-xml/impl/src/test/groovy/com/quadient/wfdxml/WfdXmlBuilderTest.groovy index 326e17a3..f03e0e33 100644 --- a/wfd-xml/impl/src/test/groovy/com/quadient/wfdxml/WfdXmlBuilderTest.groovy +++ b/wfd-xml/impl/src/test/groovy/com/quadient/wfdxml/WfdXmlBuilderTest.groovy @@ -2,11 +2,37 @@ package com.quadient.wfdxml import com.quadient.wfdxml.api.layoutnodes.Flow import com.quadient.wfdxml.api.module.Layout +import groovy.xml.XmlParser import spock.lang.Specification import static com.quadient.wfdxml.utils.AssertXml.assertXmlFileEquals class WfdXmlBuilderTest extends Specification { + def "data definition delta requires a layout"() { + when: + new WfdXmlBuilder().buildDataDefinitionLayoutDelta() + + then: + thrown(IllegalStateException) + } + + def "data definition exports only variables without changing the workflow"() { + given: + def builder = new WfdXmlBuilder() + def layout = builder.addLayout() + layout.addRoot() + layout.addFlow().setName("DocumentContent").setType(Flow.Type.SIMPLE) + layout.addPage() + layout.data.addVariable().setName("CustomerName") + + when: + Node xml = new XmlParser().parseText(builder.buildDataDefinitionLayoutDelta()) + + then: + xml.name() == "Layout" + xml.children().size() > 30 + xml.children().every { it.name() == "Variable" } + } def "empty workflow have correct xml declaration with version 1 and encoding UTF-8"() { given: @@ -58,4 +84,4 @@ class WfdXmlBuilderTest extends Specification { then: assertXmlFileEquals("com/quadient/wfdxml/workflow/SimpleDeltaLayout.xml", wfdXml) } -} \ No newline at end of file +} From 27f2ad0950e41d1e37d0ff47037cdba9df598f93 Mon Sep 17 00:00:00 2001 From: "d.svitak" Date: Wed, 21 Oct 2026 13:28:28 +0200 Subject: [PATCH 2/7] MIG-629 Create data definitions from the variable structures - The leaf variables are declared as data input variables instead of disconnected variables for data definition deployment --- .../inspirebuilder/InspireBuilderUtils.kt | 24 +++- .../InspireVariableStructureBuilder.kt | 85 ++++++----- .../migration/shared/JrdDefinition.kt | 3 +- .../inspirebuilder/InspireBuilderUtilsTest.kt | 22 +++ .../InspireVariableStructureBuilderTest.kt | 132 ++++++++++++++++-- 5 files changed, 215 insertions(+), 51 deletions(-) diff --git a/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/InspireBuilderUtils.kt b/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/InspireBuilderUtils.kt index 35391965..b67e7e14 100644 --- a/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/InspireBuilderUtils.kt +++ b/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/InspireBuilderUtils.kt @@ -55,19 +55,24 @@ fun removeValueFromVariablePath(path: String): String { }.joinToString(".") } -sealed class VariablePathPart(val name: String) { +sealed class VariablePathPart(val name: String, val dataType: DataTypeModel) { val children = mutableMapOf() } -class ArrayVariable(name: String) : VariablePathPart(name) -class SubtreeVariable(name: String) : VariablePathPart(name) +class ArrayVariable(name: String) : VariablePathPart(name, DataTypeModel.Array) +class SubtreeVariable(name: String) : VariablePathPart(name, DataTypeModel.SubTree) +class LeafVariable(name: String, dataType: DataTypeModel) : VariablePathPart(name, dataType) -fun buildVariableTree(paths: List): Map { +fun buildVariableTree( + paths: List, variableTypes: Map = emptyMap(), +): Map { val root = mutableMapOf() + val normalizedTypes = variableTypes.mapKeys { (path, _) -> removeValueFromVariablePath(path) } for (path in paths) { val parts = path.split(".") var currentMap = root + val currentPath = mutableListOf() var i = 0 while (i < parts.size) { @@ -78,11 +83,20 @@ fun buildVariableTree(paths: List): Map { continue } + currentPath.add(part) val isArray = (i + 1 < parts.size && parts[i + 1].lowercase() == "value") val variable = currentMap.getOrPut(part) { - if (isArray) ArrayVariable(part) else SubtreeVariable(part) + when (val type = normalizedTypes[currentPath.joinToString(".")]) { + DataTypeModel.Array -> ArrayVariable(part) + DataTypeModel.SubTree -> SubtreeVariable(part) + null -> if (isArray) ArrayVariable(part) else SubtreeVariable(part) + else -> LeafVariable(part, type) + } } + require(variable !is LeafVariable || i == parts.lastIndex) { + "Leaf variable '${variable.name}' cannot contain child variables" + } currentMap = variable.children i++ } diff --git a/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/InspireVariableStructureBuilder.kt b/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/InspireVariableStructureBuilder.kt index fdeee371..20e12488 100644 --- a/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/InspireVariableStructureBuilder.kt +++ b/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/InspireVariableStructureBuilder.kt @@ -9,13 +9,14 @@ import com.quadient.migration.api.dto.migrationmodel.VariableStringContent import com.quadient.migration.api.dto.migrationmodel.VariableStructure import com.quadient.migration.api.repository.VariableRepository import com.quadient.migration.api.repository.VariableStructureRepository +import com.quadient.migration.shared.DataType import com.quadient.wfdxml.api.layoutnodes.SheetNameType import com.quadient.wfdxml.api.layoutnodes.data.VariableKind import com.quadient.wfdxml.api.module.Layout import com.quadient.wfdxml.WfdXmlBuilder import com.quadient.wfdxml.internal.data.WorkFlowTreeDefinition import com.quadient.wfdxml.internal.layoutnodes.data.WorkFlowTreeEnums.NodeOptionality -import com.quadient.wfdxml.internal.layoutnodes.data.WorkFlowTreeEnums.NodeType.SUB_TREE +import com.quadient.wfdxml.internal.layoutnodes.data.WorkFlowTreeEnums.NodeType import kotlin.time.Clock class InspireVariableStructureBuilder( @@ -26,14 +27,7 @@ class InspireVariableStructureBuilder( fun buildDataDefinition(variableStructure: VariableStructure): String { val builder = WfdXmlBuilder() val layout = builder.addLayout() - val variableStructureModel = initVariableStructure(layout, variableStructure.id) - for ((id, pathData) in variableStructureModel.structure) { - val variable = variableRepository.findOrFail(id) - val path = pathData.path.resolve(variableStructureModel, variableRepository::findOrFail) - ?.takeIf { it.isNotBlank() } - ?: error("Variable '$id' has no resolvable path in structure '${variableStructureModel.id}'") - getOrCreateVariable(layout.data, pathData.name ?: variable.nameOrId(), variable, path) - } + initVariableStructure(layout, variableStructure, includeLeaves = true) return builder.buildDataDefinitionLayoutDelta() } @@ -50,44 +44,63 @@ class InspireVariableStructureBuilder( languageVariable = null, ) - val normalizedVariablePaths = variableStructureModel.structure.map { (variableId, variablePathData) -> - val literalPath = variablePathData.path.resolve(variableStructureModel, variableRepository::findOrFail) - ?: error("Variable '$variableId' referenced as array path has no resolvable literal path in structure") - removeDataFromVariablePath(literalPath) - }.filter { it.isNotBlank() }.filter { it != "SystemVariable" && !it.startsWith("SystemVariable.") } + initVariableStructure(layout, variableStructureModel) + return variableStructureModel + } - val variableTree = buildVariableTree(normalizedVariablePaths) + private fun initVariableStructure( + layout: Layout, variableStructure: VariableStructure, includeLeaves: Boolean = false, + ) { + val paths = variableStructure.structure.map { (id, pathData) -> + val path = pathData.path.resolve(variableStructure, variableRepository::findOrFail) + ?: error("Variable '$id' has no resolvable path in structure '${variableStructure.id}'") + if (includeLeaves) { + require(path.isNotBlank()) { + "Variable '$id' has no resolvable path in structure '${variableStructure.id}'" + } + val variable = variableRepository.findOrFail(id) + val name = pathData.name ?: variable.nameOrId() + removeDataFromVariablePath("$path.$name") to variable.dataType + } else { + removeDataFromVariablePath(path) to null + } + }.filter { (path, _) -> path.isNotBlank() && !isSystemPath(path) }.toMap() - val workflowTreeDefinition = WorkFlowTreeDefinition("Root", SUB_TREE, NodeOptionality.ARRAY).also { - buildVariablePathPart(it, variableTree) - } + val definition = WorkFlowTreeDefinition("Root", NodeType.SUB_TREE, NodeOptionality.ARRAY) + val variableTypes = paths.mapNotNull { (path, type) -> type?.let { path to it } }.toMap() + buildVariablePathPart(definition, buildVariableTree(paths.keys.toList(), variableTypes)) - val layoutData = layout.data - layoutData.importDataDefinition(workflowTreeDefinition) - if (variableTree.isNotEmpty() && variableTree.values.first() is ArrayVariable) { - layoutData.setRepeatedBy("Data.${variableTree.keys.first()}") + layout.data.importDataDefinition(definition) + val firstNode = definition.subNodes.firstOrNull() + if (firstNode?.nodeOptionality == NodeOptionality.ARRAY) { + layout.data.setRepeatedBy("Data.${firstNode.caption}") } - - return variableStructureModel } private fun buildVariablePathPart( - parentNode: WorkFlowTreeDefinition, currentMap: Map + parent: WorkFlowTreeDefinition, tree: Map, ) { - currentMap.forEach { - val variablePathPart = it.value - val optionality = - if (variablePathPart is ArrayVariable) NodeOptionality.ARRAY else NodeOptionality.MUST_EXIST - - val node = WorkFlowTreeDefinition(variablePathPart.name, SUB_TREE, optionality) - parentNode.addSubNode(node) - - if (variablePathPart.children.isNotEmpty()) { - buildVariablePathPart(node, variablePathPart.children) - } + for (part in tree.values) { + val optionality = if (part is ArrayVariable) NodeOptionality.ARRAY else NodeOptionality.MUST_EXIST + val node = WorkFlowTreeDefinition(part.name, part.dataType.toNodeType(), optionality) + parent.addSubNode(node) + buildVariablePathPart(node, part.children) } } + private fun isSystemPath(path: String) = path == "SystemVariable" || path.startsWith("SystemVariable.") + + private fun DataType.toNodeType(): NodeType = when (this) { + DataType.String -> NodeType.STRING + DataType.Integer -> NodeType.INT + DataType.Integer64 -> NodeType.INT64 + DataType.Double -> NodeType.DOUBLE + DataType.Currency -> NodeType.CURRENCY + DataType.Boolean -> NodeType.BOOL + DataType.DateTime -> NodeType.DATETIME + DataType.Array, DataType.SubTree -> NodeType.SUB_TREE + } + fun addEmailMetadataToPages(layout: Layout, documentObject: DocumentObject, variableStructure: VariableStructure) = addEmailMetadataToPages(layout, documentObject.options as? EmailOptions, variableStructure) diff --git a/migration-library/src/main/kotlin/com/quadient/migration/shared/JrdDefinition.kt b/migration-library/src/main/kotlin/com/quadient/migration/shared/JrdDefinition.kt index 787a0462..6f0fd7f2 100644 --- a/migration-library/src/main/kotlin/com/quadient/migration/shared/JrdDefinition.kt +++ b/migration-library/src/main/kotlin/com/quadient/migration/shared/JrdDefinition.kt @@ -7,12 +7,12 @@ import tools.jackson.databind.SerializationContext import tools.jackson.databind.annotation.JsonSerialize import tools.jackson.databind.json.JsonMapper import tools.jackson.databind.ser.std.StdSerializer -import com.quadient.migration.api.ProjectConfig import com.quadient.migration.api.dto.migrationmodel.DisplayRule import com.quadient.migration.api.dto.migrationmodel.Variable import com.quadient.migration.api.dto.migrationmodel.VariableStructure import com.quadient.migration.service.inspirebuilder.ArrayVariable import com.quadient.migration.service.inspirebuilder.InspireDocumentObjectBuilder +import com.quadient.migration.service.inspirebuilder.LeafVariable import com.quadient.migration.service.inspirebuilder.SubtreeVariable import com.quadient.migration.service.inspirebuilder.VariablePathPart import com.quadient.migration.service.inspirebuilder.buildVariableTree @@ -350,6 +350,7 @@ fun treeToNode( nodePath = currentPath, varType = when (part) { is ArrayVariable -> "Array" is SubtreeVariable -> "Subtree" + is LeafVariable -> part.dataType.toInteractiveDataType() } ) diff --git a/migration-library/src/test/kotlin/com/quadient/migration/service/inspirebuilder/InspireBuilderUtilsTest.kt b/migration-library/src/test/kotlin/com/quadient/migration/service/inspirebuilder/InspireBuilderUtilsTest.kt index 41923408..6b4191ba 100644 --- a/migration-library/src/test/kotlin/com/quadient/migration/service/inspirebuilder/InspireBuilderUtilsTest.kt +++ b/migration-library/src/test/kotlin/com/quadient/migration/service/inspirebuilder/InspireBuilderUtilsTest.kt @@ -1,13 +1,35 @@ package com.quadient.migration.service.inspirebuilder import com.quadient.migration.shared.Color +import com.quadient.migration.shared.DataType import com.quadient.migration.tools.shouldBeEqualTo import com.quadient.migration.tools.shouldBeNull import com.quadient.migration.tools.shouldNotBeNull import com.quadient.wfdxml.WfdXmlBuilder import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows class InspireBuilderUtilsTest { + @Test + fun `variable tree retains inferred arrays and deduplicates shared parents`() { + val tree = buildVariableTree(listOf("Clients.Value.Address", "Clients.Value.Address")) + + tree.size.shouldBeEqualTo(1) + tree.getValue("Clients").dataType.shouldBeEqualTo(DataType.Array) + tree.getValue("Clients").children.keys.toList().shouldBeEqualTo(listOf("Address")) + tree.getValue("Clients").children.getValue("Address").dataType.shouldBeEqualTo(DataType.SubTree) + } + + @Test + fun `variable tree rejects leaf parents regardless of path order`() { + val paths = listOf("Parent", "Parent.Child") + val types = mapOf("Parent" to DataType.String, "Parent.Child" to DataType.Integer) + + for (orderedPaths in listOf(paths, paths.reversed())) { + assertThrows { buildVariableTree(orderedPaths, types) } + } + } + @Test fun `getTextStyleByName returns null for non-existent style`() { // given diff --git a/migration-library/src/test/kotlin/com/quadient/migration/service/inspirebuilder/InspireVariableStructureBuilderTest.kt b/migration-library/src/test/kotlin/com/quadient/migration/service/inspirebuilder/InspireVariableStructureBuilderTest.kt index 36ea791f..c9a55b90 100644 --- a/migration-library/src/test/kotlin/com/quadient/migration/service/inspirebuilder/InspireVariableStructureBuilderTest.kt +++ b/migration-library/src/test/kotlin/com/quadient/migration/service/inspirebuilder/InspireVariableStructureBuilderTest.kt @@ -7,9 +7,16 @@ import com.quadient.migration.api.repository.VariableStructureRepository import com.quadient.migration.shared.DataType import com.quadient.migration.tools.aProjectConfig import com.quadient.migration.tools.shouldBeEqualTo +import com.quadient.wfdxml.WfdXmlBuilder +import com.quadient.wfdxml.internal.layoutnodes.data.LAVariableIface.VariableType +import com.quadient.wfdxml.internal.layoutnodes.data.VariableImpl import io.mockk.every import io.mockk.mockk import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.EnumSource +import tools.jackson.databind.JsonNode import tools.jackson.dataformat.xml.XmlMapper class InspireVariableStructureBuilderTest { @@ -19,7 +26,7 @@ class InspireVariableStructureBuilderTest { private val xmlMapper = XmlMapper() @Test - fun `data definition initializes array paths before declaring listed variables`() { + fun `data definition imports leaves and array paths together regardless of declaration order`() { val array = VariableBuilder("clients").name("Clients").dataType(DataType.Array).build() val name = VariableBuilder("name").name("OriginalName").dataType(DataType.String).defaultValue("Alice").build() every { variables.findOrFail(array.id) } returns array @@ -34,14 +41,14 @@ class InspireVariableStructureBuilderTest { nodes.count { it["Name"]?.stringValue() == "Clients" }.shouldBeEqualTo(1) nodes.count { it["Name"]?.stringValue() == "Value" }.shouldBeEqualTo(1) - val customerName = nodes.first { it["Name"]?.stringValue() == "CustomerName" } - customerName["ParentId"].stringValue().shouldBeEqualTo("Data.Clients.Value") - nodes.first { it["Id"] == customerName["Id"] && it.has("Content") }["Content"].stringValue() - .shouldBeEqualTo("Alice") + assertDataVariable(nodes, "CustomerName", "Value", "String") + assertDataVariable(nodes, "Clients", null, "Array") + assertDataVariable(nodes, "Value", "Clients", "SubTree", -1) + assertDataVariable(nodes, "Count", "Clients", "Int", -2) } @Test - fun `data definition initializes subtree paths before declaring listed variables`() { + fun `data definition imports leaves under subtrees`() { val subtree = VariableBuilder("customer").name("Customer").dataType(DataType.SubTree).build() val number = VariableBuilder("number").name("Number").dataType(DataType.Integer).defaultValue("42").build() every { variables.findOrFail(subtree.id) } returns subtree @@ -55,9 +62,100 @@ class InspireVariableStructureBuilderTest { val nodes = xmlMapper.readTree(subject.buildDataDefinition(structure))["Variable"].toList() nodes.count { it["Name"]?.stringValue() == "Customer" }.shouldBeEqualTo(1) - nodes.first { it["Name"]?.stringValue() == "Number" }["ParentId"].stringValue() - .shouldBeEqualTo("Data.Customer") - nodes.count { it["Content"]?.stringValue() == "42" }.shouldBeEqualTo(1) + assertDataVariable(nodes, "Customer", null, "SubTree") + assertDataVariable(nodes, "Number", "Customer", "Int") + } + + @ParameterizedTest + @EnumSource(DataType::class) + fun `data definition imports each model type as an input variable even without children`(type: DataType) { + val variable = VariableBuilder("variable").name("Variable").dataType(type).build() + every { variables.findOrFail(variable.id) } returns variable + val structure = VariableStructureBuilder("structure").addVariable(variable.id, "Data").build() + + val xml = xmlMapper.readTree(subject.buildDataDefinition(structure)) + val nodes = xml["Variable"].toList() + + assertDataVariable(nodes, "Variable", null, type.toInteractiveDataType()) + nodes.any { it["Type"]?.stringValue() == "Disconnected" }.shouldBeEqualTo(false) + if (type == DataType.Array) { + assertDataVariable(nodes, "Value", "Variable", "SubTree", -1) + assertDataVariable(nodes, "Count", "Variable", "Int", -2) + } + } + + @Test + fun `data definition supports nested arrays aliases and implicit parent paths`() { + val clients = VariableBuilder("clients").name("OriginalClients").dataType(DataType.Array).build() + val jobs = VariableBuilder("jobs").name("OriginalJobs").dataType(DataType.Array).build() + val salary = VariableBuilder("salary").name("Salary").dataType(DataType.Currency).build() + val city = VariableBuilder("city").name("City").dataType(DataType.String).build() + listOf(clients, jobs, salary, city).forEach { every { variables.findOrFail(it.id) } returns it } + val structure = VariableStructureBuilder("structure") + .addVariable(salary.id, jobs, "Pay") + .addVariable(city.id, "Data.Customers.Value.Address") + .addVariable(jobs.id, clients, "Positions") + .addVariable(clients.id, "Data", "Customers") + .build() + + val xml = xmlMapper.readTree(subject.buildDataDefinition(structure)) + val nodes = xml["Variable"].toList() + + assertDataVariable(nodes, "Customers", null, "Array") + val positions = assertDataVariable(nodes, "Positions", "Value", "Array") + val positionValue = nodes.single { it["Name"]?.stringValue() == "Value" && it["ParentId"] == positions["Id"] } + val pay = nodes.single { it["Name"]?.stringValue() == "Pay" } + pay["ParentId"].shouldBeEqualTo(positionValue["Id"]) + val payProperties = nodes.single { it["Id"] == pay["Id"] && it.has("Type") } + payProperties["Type"].stringValue().shouldBeEqualTo("DataVariable") + payProperties["VarType"].stringValue().shouldBeEqualTo("Currency") + assertDataVariable(nodes, "Address", "Value", "SubTree") + assertDataVariable(nodes, "City", "Address", "String") + } + + @Test + fun `data definition keeps built in system variables without duplicating their parent`() { + val pageCounter = VariableBuilder("page").name("PageCounter").dataType(DataType.Integer).build() + every { variables.findOrFail(pageCounter.id) } returns pageCounter + val structure = VariableStructureBuilder("structure") + .addVariable(pageCounter.id, "Data.SystemVariable") + .build() + + val nodes = xmlMapper.readTree(subject.buildDataDefinition(structure))["Variable"].toList() + + nodes.count { it["Id"]?.stringValue() == "Def.SystemVariables" }.shouldBeEqualTo(1) + assertDataVariable(nodes, "PageCounter", null, "Int", -11) + } + + @Test + fun `document initialization still imports only paths and leaves used variables disconnected`() { + val name = VariableBuilder("name").name("Name").dataType(DataType.String).defaultValue("Alice").build() + val structure = VariableStructureBuilder("structure") + .addVariable(name.id, "Data.Clients.Value") + .build() + every { structures.findOrFail(structure.id) } returns structure + val layout = WfdXmlBuilder().addLayout() + + subject.initVariableStructure(layout, structure.id).shouldBeEqualTo(structure) + + (layout.data.findVariable("Clients") as VariableImpl).type.shouldBeEqualTo(VariableType.DATA_VARIABLE) + assertThrows { layout.data.findVariable("Clients", "Name") } + val leaf = getOrCreateVariable(layout.data, name.nameOrId(), name, "Data.Clients.Value") + (leaf as VariableImpl).type.shouldBeEqualTo(VariableType.DISCONNECTED) + leaf.constantText.shouldBeEqualTo("Alice") + } + + @Test + fun `data definition rejects leaf variables used as parents`() { + val parent = VariableBuilder("parent").name("Parent").dataType(DataType.String).build() + val child = VariableBuilder("child").name("Child").dataType(DataType.String).build() + listOf(parent, child).forEach { every { variables.findOrFail(it.id) } returns it } + val structure = VariableStructureBuilder("structure") + .addVariable(parent.id, "Data") + .addVariable(child.id, "Data.Parent") + .build() + + assertThrows { subject.buildDataDefinition(structure) } } @Test @@ -70,4 +168,20 @@ class InspireVariableStructureBuilderTest { nodes.count { it["Name"]?.stringValue() == "PageCounter" }.shouldBeEqualTo(1) nodes.count { it["Id"]?.stringValue() == "Def.SystemVariables" }.shouldBeEqualTo(1) } + + private fun assertDataVariable( + nodes: List, name: String, parentName: String?, varType: String, insideFnc: Int = 0, + ): JsonNode { + val declaration = nodes.single { it["Name"]?.stringValue() == name } + if (parentName != null) { + nodes.any { it["Name"]?.stringValue() == parentName && it["Id"] == declaration["ParentId"] } + .shouldBeEqualTo(true) + } + val properties = nodes.single { it["Id"] == declaration["Id"] && it.has("Type") } + properties["Type"].stringValue().shouldBeEqualTo("DataVariable") + properties["VarType"].stringValue().shouldBeEqualTo(varType) + properties["InsideFnc"].stringValue().shouldBeEqualTo(insideFnc.toString()) + properties.has("Content").shouldBeEqualTo(false) + return declaration + } } From 3bf2fb50a8c65fa8ec8fb2c1c9f645928436068a Mon Sep 17 00:00:00 2001 From: "d.svitak" Date: Thu, 29 Oct 2026 07:49:53 +0100 Subject: [PATCH 3/7] MIG-629 Create data definitions from the variable structures - variables structures mapping, target folder introduced and reflected in deployed paths. --- .../mapping/VariableStructuresExport.groovy | 46 +++++++++++ .../mapping/VariableStructuresImport.groovy | 80 +++++++++++++++++++ .../common/mapping/VariablesExport.groovy | 2 +- .../common/mapping/VariablesImport.groovy | 5 +- .../mapping/VariablesMappingExportTest.groovy | 2 +- .../mapping/VariablesMappingImportTest.groovy | 55 ++++++++++++- .../api/dto/migrationmodel/Mapping.kt | 6 +- .../dto/migrationmodel/VariableStructure.kt | 1 + .../builder/VariableStructureBuilder.kt | 8 +- .../repository/VariableStructureRepository.kt | 7 +- .../migrationmodel/MappingEntity.kt | 9 ++- .../table/VariableStructureTable.kt | 1 + ...18__variable_structure_data_definition.kt} | 9 +-- .../migration/service/ResourcePathProvider.kt | 19 ++++- .../persistence/MappingRepositoryTest.kt | 3 +- .../VariableStructureRepositoryTest.kt | 3 +- .../persistence/upgrade/V18MigrationTest.kt | 32 +++++++- .../service/ResourcePathProviderTest.kt | 19 +++++ 18 files changed, 278 insertions(+), 29 deletions(-) create mode 100644 migration-examples/src/main/groovy/com/quadient/migration/example/common/mapping/VariableStructuresExport.groovy create mode 100644 migration-examples/src/main/groovy/com/quadient/migration/example/common/mapping/VariableStructuresImport.groovy rename migration-library/src/main/kotlin/com/quadient/migration/persistence/upgrade/{V18__variable_structure_status_tracking.kt => V18__variable_structure_data_definition.kt} (74%) diff --git a/migration-examples/src/main/groovy/com/quadient/migration/example/common/mapping/VariableStructuresExport.groovy b/migration-examples/src/main/groovy/com/quadient/migration/example/common/mapping/VariableStructuresExport.groovy new file mode 100644 index 00000000..b5bdad1d --- /dev/null +++ b/migration-examples/src/main/groovy/com/quadient/migration/example/common/mapping/VariableStructuresExport.groovy @@ -0,0 +1,46 @@ +//! --- +//! displayName: Export Variable Structures +//! category: Mapping +//! description: Creates a CSV file with variable structure details (name and target folder of the deployed data definition) from the migration project. The generated CSV columns can be updated and later imported back into the database using a dedicated import task. +//! target: gradle +//! --- +package com.quadient.migration.example.common.mapping + +import com.quadient.migration.api.Migration +import com.quadient.migration.api.dto.migrationmodel.VariableStructure +import com.quadient.migration.example.common.util.Csv +import com.quadient.migration.example.common.util.Mapping +import com.quadient.migration.service.deploy.utility.ResourceType + +import static com.quadient.migration.example.common.util.InitMigration.initMigration + +def migration = initMigration(this.binding) + +def variableStructuresPath = Mapping.csvPath(binding, migration.projectConfig.name, migration.projectConfig.subProjectId, "variable-structures") + +run(migration, variableStructuresPath.toFile()) + +static void run(Migration migration, File variableStructuresDstPath) { + List variableStructures = Mapping.collectSelectedOrAll(migration, VariableStructure) { migration.variableStructureRepository.listAll() } + + variableStructuresDstPath.createParentDirectories() + + variableStructuresDstPath.withWriter { writer -> + def headers = ["id", "name", "targetFolder", "status", Mapping.displayHeader("originLocations", true)] + writer.writeLine(headers.join(",")) + variableStructures.each { obj -> + def status = migration.statusTrackingRepository.findLastEventRelevantToOutput(obj.id, + ResourceType.VariableStructure, + migration.projectConfig.inspireOutput) + + def builder = new StringBuilder() + builder.append(Csv.serialize(obj.id)) + builder.append("," + Csv.serialize(obj.name)) + builder.append("," + Csv.serialize(obj.targetFolder)) + builder.append("," + Csv.serialize(status?.class?.simpleName)) + builder.append("," + Csv.serialize(obj.originLocations)) + + writer.writeLine(builder.toString()) + } + } +} diff --git a/migration-examples/src/main/groovy/com/quadient/migration/example/common/mapping/VariableStructuresImport.groovy b/migration-examples/src/main/groovy/com/quadient/migration/example/common/mapping/VariableStructuresImport.groovy new file mode 100644 index 00000000..caada519 --- /dev/null +++ b/migration-examples/src/main/groovy/com/quadient/migration/example/common/mapping/VariableStructuresImport.groovy @@ -0,0 +1,80 @@ +//! --- +//! displayName: Import Variable Structures +//! category: Mapping +//! description: Imports variable structure details (name and target folder of the deployed data definition) from CSV files into the migration project, applying any updates made to the columns during editing. +//! target: gradle +//! --- +package com.quadient.migration.example.common.mapping + +import com.quadient.migration.api.Migration +import com.quadient.migration.api.dto.migrationmodel.MappingItem +import com.quadient.migration.example.common.util.Csv +import com.quadient.migration.example.common.util.Mapping +import com.quadient.migration.service.deploy.utility.ResourceType + +import groovy.transform.Field +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import static com.quadient.migration.example.common.util.InitMigration.initMigration + +@Field static Logger log = LoggerFactory.getLogger(this.class.name) + +def migration = initMigration(this.binding) + +def variableStructuresPath = Mapping.csvPath(binding, migration.projectConfig.name, migration.projectConfig.subProjectId, "variable-structures") + +run(migration, variableStructuresPath.toFile()) + +static void run(Migration migration, File file) { + def deploymentId = UUID.randomUUID().toString() + def now = new Date().getTime() + def output = migration.projectConfig.inspireOutput + + def lines = file.readLines() + def columnNames = Csv.parseColumnNames(lines.removeFirst()).collect { Mapping.normalizeHeader(it) } + def total = lines.size() + + def mappings = new HashMap() + for (line in lines) { + def values = Csv.getCells(line, columnNames) + def id = values.get("id") + + def existingStructure = migration.variableStructureRepository.find(id) + if (existingStructure == null) { + throw new Exception("VariableStructure with id ${id} not found") + } + def existingMapping = migration.mappingRepository.getVariableStructureMapping(id) + + // The structure content is owned by the variables mapping, keep it as it is in the object + if (!existingMapping.mappings) existingMapping.mappings = new HashMap(existingStructure.structure) + if (existingMapping.languageVariable == null) existingMapping.languageVariable = existingStructure.languageVariable + + def status = migration.statusTrackingRepository.findLastEventRelevantToOutput(existingStructure.id, + ResourceType.VariableStructure, + migration.projectConfig.inspireOutput) + + def newName = Csv.deserialize(values.get("name"), String.class) + existingMapping.name = newName + + def newTargetFolder = Csv.deserialize(values.get("targetFolder"), String.class) + existingMapping.targetFolder = newTargetFolder + + def csvStatus = values.get("status") + if ((csvStatus == null || csvStatus == "") && status == null) { + migration.statusTrackingRepository.active(existingStructure.id, ResourceType.VariableStructure, [reason: "Manual"]) + } else if (csvStatus == "Active" && status?.class?.simpleName != "Active") { + migration.statusTrackingRepository.active(existingStructure.id, ResourceType.VariableStructure, [reason: "Manual"]) + } else if (csvStatus == "Deployed" && status?.class?.simpleName != "Deployed") { + migration.statusTrackingRepository.deployed(existingStructure.id, deploymentId, now, ResourceType.VariableStructure, null, output, [reason: "Manual"]) + } + + mappings[id] = existingMapping + if (total > 1000 && mappings.size() % 1000 == 0) { + log.info "Processed ${mappings.size()}/${total} mappings" + } + } + + Mapping.upsertBatched(migration.mappingRepository, mappings, "variable structure mappings", log) + migration.mappingRepository.applyAllVariableStructureMappings() +} diff --git a/migration-examples/src/main/groovy/com/quadient/migration/example/common/mapping/VariablesExport.groovy b/migration-examples/src/main/groovy/com/quadient/migration/example/common/mapping/VariablesExport.groovy index d99552b5..2a1e9c6e 100644 --- a/migration-examples/src/main/groovy/com/quadient/migration/example/common/mapping/VariablesExport.groovy +++ b/migration-examples/src/main/groovy/com/quadient/migration/example/common/mapping/VariablesExport.groovy @@ -1,5 +1,5 @@ //! --- -//! displayName: Export Variable Structure +//! displayName: Export Variables //! category: Mapping //! description: Creates a CSV report with variable structure from the migration project. The export is interactive, prompting the user to select or enter a variable structure to export. //! target: gradle diff --git a/migration-examples/src/main/groovy/com/quadient/migration/example/common/mapping/VariablesImport.groovy b/migration-examples/src/main/groovy/com/quadient/migration/example/common/mapping/VariablesImport.groovy index e08ad626..bece701f 100644 --- a/migration-examples/src/main/groovy/com/quadient/migration/example/common/mapping/VariablesImport.groovy +++ b/migration-examples/src/main/groovy/com/quadient/migration/example/common/mapping/VariablesImport.groovy @@ -1,5 +1,5 @@ //! --- -//! displayName: Import Variable Structure +//! displayName: Import Variables //! category: Mapping //! description: Imports variable structure from specified CSV files. The import is interactive, prompting the user to select variable structure CSV to import. //! target: gradle @@ -45,6 +45,9 @@ static void run(Migration migration, Path path) { migration.variableStructureRepository.upsert(new VariableStructureBuilder(structureId).build()) } + if (structureMapping.name == null) structureMapping.name = existingStructure?.name + if (structureMapping.targetFolder == null) structureMapping.targetFolder = existingStructure?.targetFolder + def languageVariableFound = false def mappings = new HashMap() for (line in lines) { diff --git a/migration-examples/src/test/groovy/com/quadient/migration/example/common/mapping/VariablesMappingExportTest.groovy b/migration-examples/src/test/groovy/com/quadient/migration/example/common/mapping/VariablesMappingExportTest.groovy index 0fdc990f..12a2f9f8 100644 --- a/migration-examples/src/test/groovy/com/quadient/migration/example/common/mapping/VariablesMappingExportTest.groovy +++ b/migration-examples/src/test/groovy/com/quadient/migration/example/common/mapping/VariablesMappingExportTest.groovy @@ -43,7 +43,7 @@ class VariablesMappingExportTest { when(migration.mappingRepository.getVariableStructureMapping(any())).thenReturn(new MappingItem.VariableStructure("", ["overridden": new VariablePathData("override/path", "overridden name"), "refVar" : new VariablePathData(new VariableRefPath("full"), "ref name")], - new VariableRef("full"))) + new VariableRef("full"), null)) VariablesExport.run(migration, mappingFile) diff --git a/migration-examples/src/test/groovy/com/quadient/migration/example/common/mapping/VariablesMappingImportTest.groovy b/migration-examples/src/test/groovy/com/quadient/migration/example/common/mapping/VariablesMappingImportTest.groovy index ad143b5d..09b61b6c 100644 --- a/migration-examples/src/test/groovy/com/quadient/migration/example/common/mapping/VariablesMappingImportTest.groovy +++ b/migration-examples/src/test/groovy/com/quadient/migration/example/common/mapping/VariablesMappingImportTest.groovy @@ -5,6 +5,7 @@ import com.quadient.migration.api.dto.migrationmodel.CustomFieldMap import com.quadient.migration.api.dto.migrationmodel.MappingItem import com.quadient.migration.api.dto.migrationmodel.Variable import com.quadient.migration.api.dto.migrationmodel.VariableRef +import com.quadient.migration.api.dto.migrationmodel.builder.VariableStructureBuilder import com.quadient.migration.example.Utils import com.quadient.migration.shared.DataType import com.quadient.migration.shared.VariablePathData @@ -68,7 +69,7 @@ class VariablesMappingImportTest { "withPathEmpty" : new VariablePathData("newPath", null), "withVariableRef" : new VariablePathData(new VariableRefPath("referencedVar"), null), "withVariableRefLegacy": new VariablePathData(new VariableRefPath("legacyRef"), null),] - , null)) + , null, null)) verify(migration.mappingRepository, times(1)).applyVariableStructureMapping("test") } @@ -135,7 +136,7 @@ class VariablesMappingImportTest { ["unchanged" : new VariablePathData("", null), "kept" : new VariablePathData("", null), "overridden": new VariablePathData("", "Overridden in structure")], - null)) + null, null)) } @Test @@ -164,7 +165,53 @@ class VariablesMappingImportTest { ["unchanged" : new VariablePathData("", null), "kept" : new VariablePathData("", null), "overridden": new VariablePathData("", "Overridden in structure")], - new VariableRef("overridden"))) + new VariableRef("overridden"), null)) + } + + @Test + void keepsStructureNameAndTargetFolderFromObjectWhenNotMapped() { + def migration = Utils.mockMigration() + Path mappingFile = Paths.get(dir.path, "testProject-variable-structure-test.csv") + def input = """\ + id,name,data_type,inspire_path,inspire_name,originLocations + var1,,String,path,,[] + """.stripIndent() + mappingFile.toFile().write(input) + + givenExistingVariable(migration, "var1", null, DataType.String, null) + when(migration.mappingRepository.getVariableMapping("var1")).thenReturn(new MappingItem.Variable(null, DataType.String)) + when(migration.variableStructureRepository.find("test")) + .thenReturn(new VariableStructureBuilder("test").name("structName").targetFolder("structFolder").build()) + when(migration.mappingRepository.getVariableStructureMapping("test")) + .thenReturn(new MappingItem.VariableStructure(null, [:], null, null)) + + VariablesImport.run(migration, mappingFile) + + verify(migration.mappingRepository, times(1)).upsert("test", + new MappingItem.VariableStructure("structName", ["var1": new VariablePathData("path", null)], null, "structFolder")) + } + + @Test + void keepsStructureNameAndTargetFolderFromExistingMapping() { + def migration = Utils.mockMigration() + Path mappingFile = Paths.get(dir.path, "testProject-variable-structure-test.csv") + def input = """\ + id,name,data_type,inspire_path,inspire_name,originLocations + var1,,String,path,,[] + """.stripIndent() + mappingFile.toFile().write(input) + + givenExistingVariable(migration, "var1", null, DataType.String, null) + when(migration.mappingRepository.getVariableMapping("var1")).thenReturn(new MappingItem.Variable(null, DataType.String)) + when(migration.variableStructureRepository.find("test")) + .thenReturn(new VariableStructureBuilder("test").name("objectName").targetFolder("objectFolder").build()) + when(migration.mappingRepository.getVariableStructureMapping("test")) + .thenReturn(new MappingItem.VariableStructure("mappedName", [:], null, "mappedFolder")) + + VariablesImport.run(migration, mappingFile) + + verify(migration.mappingRepository, times(1)).upsert("test", + new MappingItem.VariableStructure("mappedName", ["var1": new VariablePathData("path", null)], null, "mappedFolder")) } static void givenExistingVariable(Migration mig, String id, String name, DataType dataType, String inspirePath = null) { @@ -183,6 +230,6 @@ class VariablesMappingImportTest { } when(mig.mappingRepository.getVariableMapping(variableId)).thenReturn(new MappingItem.Variable(name, dataType)) - when(mig.mappingRepository.getVariableStructureMapping(any())).thenReturn(new MappingItem.VariableStructure(null, mappings, new VariableRef("empty"))) + when(mig.mappingRepository.getVariableStructureMapping(any())).thenReturn(new MappingItem.VariableStructure(null, mappings, new VariableRef("empty"), null)) } } diff --git a/migration-library/src/main/kotlin/com/quadient/migration/api/dto/migrationmodel/Mapping.kt b/migration-library/src/main/kotlin/com/quadient/migration/api/dto/migrationmodel/Mapping.kt index b1c03202..c261293e 100644 --- a/migration-library/src/main/kotlin/com/quadient/migration/api/dto/migrationmodel/Mapping.kt +++ b/migration-library/src/main/kotlin/com/quadient/migration/api/dto/migrationmodel/Mapping.kt @@ -90,8 +90,9 @@ sealed class MappingItem { data class VariableStructure( override var name: String?, - val mappings: MutableMap?, + var mappings: MutableMap?, var languageVariable: VariableRef?, + var targetFolder: String? = null, ) : MappingItem() data class DisplayRule( @@ -225,7 +226,8 @@ sealed class MappingItem { MappingItemEntity.VariableStructure( name = this.name, mappings = mappings, - languageVariable = this.languageVariable?.let { VariableEntityRef(it.id) } + languageVariable = this.languageVariable?.let { VariableEntityRef(it.id) }, + targetFolder = this.targetFolder, ) } diff --git a/migration-library/src/main/kotlin/com/quadient/migration/api/dto/migrationmodel/VariableStructure.kt b/migration-library/src/main/kotlin/com/quadient/migration/api/dto/migrationmodel/VariableStructure.kt index 25a87eb2..faf76159 100644 --- a/migration-library/src/main/kotlin/com/quadient/migration/api/dto/migrationmodel/VariableStructure.kt +++ b/migration-library/src/main/kotlin/com/quadient/migration/api/dto/migrationmodel/VariableStructure.kt @@ -12,6 +12,7 @@ data class VariableStructure @JvmOverloads constructor( val languageVariable: VariableRef?, override var created: Instant? = null, override var lastUpdated: Instant? = null, + var targetFolder: String? = null, ) : MigrationObject, RefValidatable { override fun collectRefs(): Set { return structure.keys.map { VariableRef(it) }.toSet() diff --git a/migration-library/src/main/kotlin/com/quadient/migration/api/dto/migrationmodel/builder/VariableStructureBuilder.kt b/migration-library/src/main/kotlin/com/quadient/migration/api/dto/migrationmodel/builder/VariableStructureBuilder.kt index bfef8d1f..a0b8d514 100644 --- a/migration-library/src/main/kotlin/com/quadient/migration/api/dto/migrationmodel/builder/VariableStructureBuilder.kt +++ b/migration-library/src/main/kotlin/com/quadient/migration/api/dto/migrationmodel/builder/VariableStructureBuilder.kt @@ -3,12 +3,15 @@ package com.quadient.migration.api.dto.migrationmodel.builder import com.quadient.migration.api.dto.migrationmodel.Variable import com.quadient.migration.api.dto.migrationmodel.VariableRef import com.quadient.migration.api.dto.migrationmodel.VariableStructure +import com.quadient.migration.api.dto.migrationmodel.builder.components.HasTargetFolder import com.quadient.migration.shared.DataType import com.quadient.migration.shared.LiteralPath import com.quadient.migration.shared.VariablePathData import com.quadient.migration.shared.VariableRefPath -class VariableStructureBuilder(id: String) : DtoBuilderBase(id) { +class VariableStructureBuilder(id: String) : DtoBuilderBase(id), + HasTargetFolder { + override var targetFolder: String? = null var structure = mutableMapOf() var languageVariable: VariableRef? = null @@ -144,7 +147,8 @@ class VariableStructureBuilder(id: String) : DtoBuilderBase?, val languageVariable: VariableEntityRef? = null, + val targetFolder: String? = null, ) : MappingItemEntity() { fun apply(item: VariableStructureModel): VariableStructureModel { return item.copy( name = name, structure = mappings ?: mutableMapOf(), - languageVariable = languageVariable?.let { VariableRef(it.id) }) + languageVariable = languageVariable?.let { VariableRef(it.id) }, + targetFolder = targetFolder, + ) } } @@ -472,7 +475,9 @@ sealed class MappingItemEntity { MappingItem.VariableStructure( name = this.name, mappings = this.mappings ?: mutableMapOf(), - languageVariable = this.languageVariable?.let { VariableRef(it.id) }) + languageVariable = this.languageVariable?.let { VariableRef(it.id) }, + targetFolder = this.targetFolder, + ) } is DisplayRule -> MappingItem.DisplayRule( diff --git a/migration-library/src/main/kotlin/com/quadient/migration/persistence/table/VariableStructureTable.kt b/migration-library/src/main/kotlin/com/quadient/migration/persistence/table/VariableStructureTable.kt index b1e1e4af..49c20451 100644 --- a/migration-library/src/main/kotlin/com/quadient/migration/persistence/table/VariableStructureTable.kt +++ b/migration-library/src/main/kotlin/com/quadient/migration/persistence/table/VariableStructureTable.kt @@ -7,4 +7,5 @@ import org.jetbrains.exposed.v1.json.jsonb object VariableStructureTable : MigrationObjectTable("variable_structure") { val structure = jsonb>("structure", Json) val languageVariable = varchar("language_variable", 255).nullable() + val targetFolder = varchar("target_folder", 255).nullable() } diff --git a/migration-library/src/main/kotlin/com/quadient/migration/persistence/upgrade/V18__variable_structure_status_tracking.kt b/migration-library/src/main/kotlin/com/quadient/migration/persistence/upgrade/V18__variable_structure_data_definition.kt similarity index 74% rename from migration-library/src/main/kotlin/com/quadient/migration/persistence/upgrade/V18__variable_structure_status_tracking.kt rename to migration-library/src/main/kotlin/com/quadient/migration/persistence/upgrade/V18__variable_structure_data_definition.kt index eb5273f1..93030ef5 100644 --- a/migration-library/src/main/kotlin/com/quadient/migration/persistence/upgrade/V18__variable_structure_status_tracking.kt +++ b/migration-library/src/main/kotlin/com/quadient/migration/persistence/upgrade/V18__variable_structure_data_definition.kt @@ -3,15 +3,12 @@ package com.quadient.migration.persistence.upgrade import org.flywaydb.core.api.migration.BaseJavaMigration import org.flywaydb.core.api.migration.Context -/** - * Variable structures did not have status tracking until they became deployable (as data definitions). - * Creates an initial Active status record for every existing variable structure that has none, - * so they are picked up by the deployment. - */ -class V18__variable_structure_status_tracking : BaseJavaMigration() { +class V18__variable_structure_data_definition : BaseJavaMigration() { override fun migrate(context: Context) { val connection = context.connection connection.createStatement().use { stmt -> + stmt.execute("ALTER TABLE variable_structure ADD COLUMN IF NOT EXISTS target_folder VARCHAR(255) NULL") + stmt.execute( """ INSERT INTO status_tracking (id, resource_type, project_name, status_events) diff --git a/migration-library/src/main/kotlin/com/quadient/migration/service/ResourcePathProvider.kt b/migration-library/src/main/kotlin/com/quadient/migration/service/ResourcePathProvider.kt index 95186e07..3ec32a4b 100644 --- a/migration-library/src/main/kotlin/com/quadient/migration/service/ResourcePathProvider.kt +++ b/migration-library/src/main/kotlin/com/quadient/migration/service/ResourcePathProvider.kt @@ -164,10 +164,21 @@ class DesignerResourcePathProvider(private val projectConfig: ProjectConfig) : R } open class InteractiveResourcePathProvider(private val projectConfig: ProjectConfig) : ResourcePathProvider { - override fun getDataDefinitionPath(variableStructure: VariableStructure): IcmPath = IcmPath.root() - .join("Interactive").join(projectConfig.interactiveTenant).join("DataDefinitions") - .join(resolveTargetDir(projectConfig.defaultTargetFolder)) - .join("${variableStructure.nameOrId()}.jdd") + override fun getDataDefinitionPath(variableStructure: VariableStructure): IcmPath { + val fileName = "${variableStructure.nameOrId()}.jdd" + + val targetFolder = variableStructure.targetFolder?.let { IcmPath.from(it) } + if (targetFolder?.isAbsolute() == true) { + return targetFolder.join(fileName) + } + + return IcmPath.root() + .join("Interactive") + .join(projectConfig.interactiveTenant) + .join("DataDefinitions") + .join(resolveTargetDir(projectConfig.defaultTargetFolder, targetFolder)) + .join(fileName) + } override fun getDataDefinitionSourceTemplatePath(): IcmPath = IcmPath.root() .join("Interactive").join(projectConfig.interactiveTenant).join("Sources").join("SourceTemplate.wfd") diff --git a/migration-library/src/test/kotlin/com/quadient/migration/persistence/MappingRepositoryTest.kt b/migration-library/src/test/kotlin/com/quadient/migration/persistence/MappingRepositoryTest.kt index 1acdc6e1..d1b71cc7 100644 --- a/migration-library/src/test/kotlin/com/quadient/migration/persistence/MappingRepositoryTest.kt +++ b/migration-library/src/test/kotlin/com/quadient/migration/persistence/MappingRepositoryTest.kt @@ -98,7 +98,7 @@ class MappingRepositoryTest { repo.upsert("varStructId", MappingItem.VariableStructure( name = "new name", mappings = mutableMapOf( "varId" to VariablePathData("somePath"), "anotherVarId" to VariablePathData("anotherPath") - ), languageVariable = null + ), languageVariable = null, targetFolder = "new folder" )) repo.applyVariableStructureMapping("varStructId") @@ -108,6 +108,7 @@ class MappingRepositoryTest { match { it.id == "varStructId" && it.name == "new name" && + it.targetFolder == "new folder" && it.structure == mapOf( "varId" to VariablePathData("somePath"), "anotherVarId" to VariablePathData("anotherPath") diff --git a/migration-library/src/test/kotlin/com/quadient/migration/persistence/VariableStructureRepositoryTest.kt b/migration-library/src/test/kotlin/com/quadient/migration/persistence/VariableStructureRepositoryTest.kt index 4cb55c02..7f5376db 100644 --- a/migration-library/src/test/kotlin/com/quadient/migration/persistence/VariableStructureRepositoryTest.kt +++ b/migration-library/src/test/kotlin/com/quadient/migration/persistence/VariableStructureRepositoryTest.kt @@ -48,7 +48,7 @@ class VariableStructureRepositoryTest { @Test fun `roundtrip is correct`() { val dto = VariableStructureBuilder("id").customFields(mutableMapOf("f1" to "val1")) - .originLocations(listOf("test1", "test2")).addVariable("var1", "Data.Test.Value") + .originLocations(listOf("test1", "test2")).targetFolder("folder").addVariable("var1", "Data.Test.Value") .addVariable("var2", "Data.Test.Value2").build() repo.upsert(dto) @@ -67,6 +67,7 @@ class VariableStructureRepositoryTest { .name("Variable Structure 1") .customFields(mutableMapOf("field1" to "value1")) .originLocations(listOf("origin1")) + .targetFolder("folder1") .addVariable("var1", "Data.Test.Value1") .addVariable("var2", "Data.Test.Value2") .build() diff --git a/migration-library/src/test/kotlin/com/quadient/migration/persistence/upgrade/V18MigrationTest.kt b/migration-library/src/test/kotlin/com/quadient/migration/persistence/upgrade/V18MigrationTest.kt index 3d6c70b1..db9b5ba8 100644 --- a/migration-library/src/test/kotlin/com/quadient/migration/persistence/upgrade/V18MigrationTest.kt +++ b/migration-library/src/test/kotlin/com/quadient/migration/persistence/upgrade/V18MigrationTest.kt @@ -18,8 +18,8 @@ import java.sql.Connection import java.sql.DriverManager /** - * Integration test for V18 migration that backfills an Active status tracking record - * for every existing variable structure. + * Integration test for V18 migration that adds the target folder to variable structures + * and backfills an Active status tracking record for every existing variable structure. */ class V18MigrationTest { @@ -75,6 +75,34 @@ class V18MigrationTest { } } + @Test + fun `V18 migration adds nullable target folder to variable structures`() { + PostgreSQLContainer(POSTGRES_CONTAINER).use { postgres -> + postgres.start() + + migrate(postgres, "17") + connection(postgres).use { conn -> insertVariableStructure(conn, "vs1", "test") } + + migrate(postgres, "18") + + connection(postgres).use { conn -> + conn.createStatement().executeUpdate("UPDATE variable_structure SET target_folder = 'folder' WHERE id = 'vs1'") + val rs = conn.createStatement().executeQuery( + "SELECT target_folder FROM variable_structure WHERE id = 'vs1'" + ) + rs.next() + assertEquals("folder", rs.getString(1)) + + insertVariableStructure(conn, "vs2", "test") + val rs2 = conn.createStatement().executeQuery( + "SELECT target_folder FROM variable_structure WHERE id = 'vs2'" + ) + rs2.next() + assertNull(rs2.getString(1)) + } + } + } + private fun migrate(postgres: PostgreSQLContainer, target: String) { Flyway.configure() .dataSource(postgres.jdbcUrl, postgres.username, postgres.password) diff --git a/migration-library/src/test/kotlin/com/quadient/migration/service/ResourcePathProviderTest.kt b/migration-library/src/test/kotlin/com/quadient/migration/service/ResourcePathProviderTest.kt index 9abe998b..3390a907 100644 --- a/migration-library/src/test/kotlin/com/quadient/migration/service/ResourcePathProviderTest.kt +++ b/migration-library/src/test/kotlin/com/quadient/migration/service/ResourcePathProviderTest.kt @@ -8,6 +8,7 @@ import com.quadient.migration.api.dto.migrationmodel.builder.BaseTemplateBuilder import com.quadient.migration.api.dto.migrationmodel.builder.DisplayRuleBuilder import com.quadient.migration.api.dto.migrationmodel.builder.DocumentObjectBuilder import com.quadient.migration.api.dto.migrationmodel.builder.ImageBuilder +import com.quadient.migration.api.dto.migrationmodel.builder.VariableStructureBuilder import com.quadient.migration.service.inspirebuilder.InteractiveDocumentObjectBuilder import com.quadient.migration.shared.AttachmentType import com.quadient.migration.shared.DocumentObjectType @@ -71,6 +72,24 @@ class ResourcePathProviderTest { .shouldBeEqualTo("icm://Interactive/customTenant/Sources/SourceTemplate.wfd") } + @ParameterizedTest + @CsvSource( + // name ,targetFolder ,defaultTargetFolder,expected + " , , ,icm://Interactive/tenant/DataDefinitions/VS_1.jdd", + "null ,null ,null ,icm://Interactive/tenant/DataDefinitions/VS_1.jdd", + "struct , , ,icm://Interactive/tenant/DataDefinitions/struct.jdd", + " ,relative , ,icm://Interactive/tenant/DataDefinitions/relative/VS_1.jdd", + " , ,default ,icm://Interactive/tenant/DataDefinitions/default/VS_1.jdd", + " ,subDir ,shouldIgnore ,icm://Interactive/tenant/DataDefinitions/subDir/VS_1.jdd", + "struct ,icm://absolute/,shouldIgnore ,icm://absolute/struct.jdd", + ) + fun `data definition path`(name: String?, targetFolder: String?, defaultTargetFolder: String?, expected: String) { + val config = aProjectConfig(targetDefaultFolder = defaultTargetFolder.nullToNull(), interactiveTenant = "tenant") + val structure = VariableStructureBuilder("VS_1").name(name.nullToNull()).targetFolder(targetFolder.nullToNull()).build() + + aSubject(config).getDataDefinitionPath(structure).toString().shouldBeEqualTo(expected) + } + @ParameterizedTest @CsvSource( // projectCfg.paths ,targetFolder ,defaultTargetFolder,expected From 37b07996ae78a50477ef7500972b4b1b9af93fa6 Mon Sep 17 00:00:00 2001 From: "d.svitak" Date: Thu, 29 Oct 2026 10:30:09 +0100 Subject: [PATCH 4/7] MIG-629 Create data definitions from the variable structures - add missing unit tests --- ...VariableStructuresMappingExportTest.groovy | 76 ++++++++++ ...VariableStructuresMappingImportTest.groovy | 140 ++++++++++++++++++ 2 files changed, 216 insertions(+) create mode 100644 migration-examples/src/test/groovy/com/quadient/migration/example/common/mapping/VariableStructuresMappingExportTest.groovy create mode 100644 migration-examples/src/test/groovy/com/quadient/migration/example/common/mapping/VariableStructuresMappingImportTest.groovy diff --git a/migration-examples/src/test/groovy/com/quadient/migration/example/common/mapping/VariableStructuresMappingExportTest.groovy b/migration-examples/src/test/groovy/com/quadient/migration/example/common/mapping/VariableStructuresMappingExportTest.groovy new file mode 100644 index 00000000..815fc9a4 --- /dev/null +++ b/migration-examples/src/test/groovy/com/quadient/migration/example/common/mapping/VariableStructuresMappingExportTest.groovy @@ -0,0 +1,76 @@ +package com.quadient.migration.example.common.mapping + +import com.quadient.migration.api.dto.migrationmodel.builder.DocumentObjectBuilder +import com.quadient.migration.api.dto.migrationmodel.builder.VariableStructureBuilder +import com.quadient.migration.data.Active +import com.quadient.migration.example.Utils +import com.quadient.migration.example.common.util.Mapping +import com.quadient.migration.shared.DocumentObjectType +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir + +import java.nio.file.Path +import java.nio.file.Paths + +import static org.mockito.ArgumentMatchers.any +import static org.mockito.Mockito.when + +class VariableStructuresMappingExportTest { + @TempDir + File dir + + @Test + void allPossibleFields() { + Path mappingFile = Paths.get(dir.path, "testProject.csv") + def migration = Utils.mockMigration() + + when(migration.variableStructureRepository.listAll()).thenReturn([ + new VariableStructureBuilder("empty").build(), + new VariableStructureBuilder("full").name("full").targetFolder("targetFolder1").originLocations(["foo", "bar"]).build(), + ]) + + when(migration.statusTrackingRepository.findLastEventRelevantToOutput(any(), any(), any())).thenReturn(new Active()) + + VariableStructuresExport.run(migration, mappingFile.toFile()) + + def expected = """\ + id,name,targetFolder,status,originLocations (read-only) + empty,,,Active,[] + full,full,targetFolder1,Active,[foo; bar] + """.stripIndent() + Assertions.assertEquals(expected, mappingFile.toFile().text.replaceAll("\\r\\n|\\r", "\n")) + } + + @Test + void exportsOnlySelectedVariableStructures() { + Path mappingFile = Paths.get(dir.path, "testProject.csv") + def migration = Utils.mockMigration() + + def referencedStructure = new VariableStructureBuilder("referenced").build() + def unselectedStructure = new VariableStructureBuilder("unselected").build() + def selectedDocObject = new DocumentObjectBuilder("doc1", DocumentObjectType.Block).build() + + when(migration.projectConfig.getDocumentObjectsToProcess()).thenReturn(["doc1"]) + when(migration.documentObjectRepository.listIds(["doc1"])).thenReturn([selectedDocObject]) + when(migration.referenceCollector.collectAllObjects(selectedDocObject)).thenReturn([referencedStructure] as Set) + when(migration.variableStructureRepository.listAll()).thenReturn([referencedStructure, unselectedStructure]) + + when(migration.statusTrackingRepository.findLastEventRelevantToOutput(any(), any(), any())).thenReturn(new Active()) + + VariableStructuresExport.run(migration, mappingFile.toFile()) + + def expected = """\ + id,name,targetFolder,status,originLocations (read-only) + referenced,,,Active,[] + """.stripIndent() + Assertions.assertEquals(expected, mappingFile.toFile().text.replaceAll("\\r\\n|\\r", "\n")) + } + + @Test + void csvFileIsNotPickedUpByVariablesImport() { + def fileName = "${Mapping.namePrefix("testProject", null)}-variable-structures.csv" + + Assertions.assertFalse(fileName.startsWith(Mapping.variableStructureFileNamePrefix("testProject", null))) + } +} diff --git a/migration-examples/src/test/groovy/com/quadient/migration/example/common/mapping/VariableStructuresMappingImportTest.groovy b/migration-examples/src/test/groovy/com/quadient/migration/example/common/mapping/VariableStructuresMappingImportTest.groovy new file mode 100644 index 00000000..cb0f0e34 --- /dev/null +++ b/migration-examples/src/test/groovy/com/quadient/migration/example/common/mapping/VariableStructuresMappingImportTest.groovy @@ -0,0 +1,140 @@ +package com.quadient.migration.example.common.mapping + +import com.quadient.migration.api.InspireOutput +import com.quadient.migration.api.dto.migrationmodel.MappingItem +import com.quadient.migration.api.dto.migrationmodel.VariableRef +import com.quadient.migration.api.dto.migrationmodel.builder.VariableStructureBuilder +import com.quadient.migration.data.Active +import com.quadient.migration.example.Utils +import com.quadient.migration.service.deploy.utility.ResourceType +import com.quadient.migration.shared.VariablePathData +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir + +import java.nio.file.Path +import java.nio.file.Paths + +import static org.mockito.ArgumentMatchers.any +import static org.mockito.ArgumentMatchers.anyLong +import static org.mockito.ArgumentMatchers.anyString +import static org.mockito.ArgumentMatchers.eq +import static org.mockito.Mockito.* + +class VariableStructuresMappingImportTest { + @TempDir + File dir + + @Test + void overridesAllMappableFields() { + def migration = Utils.mockMigration() + Path mappingFile = Paths.get(dir.path, "testProject.csv") + def input = """\ + id,name,targetFolder,status,originLocations (read-only) + unchanged,,,Active,[] + overridden,someName,overriddenFolder,Active,[some; location] + """.stripIndent() + mappingFile.toFile().write(input) + when(migration.variableStructureRepository.find("unchanged")).thenReturn(new VariableStructureBuilder("unchanged").build()) + when(migration.mappingRepository.getVariableStructureMapping("unchanged")).thenReturn(new MappingItem.VariableStructure(null, [:], null, null)) + when(migration.variableStructureRepository.find("overridden")) + .thenReturn(new VariableStructureBuilder("overridden").name("previousName").targetFolder("previousFolder").build()) + when(migration.mappingRepository.getVariableStructureMapping("overridden")) + .thenReturn(new MappingItem.VariableStructure("previousName", [:], null, "previousFolder")) + + VariableStructuresImport.run(migration, mappingFile.toFile()) + + verify(migration.mappingRepository, times(1)).upsertBatch([ + "unchanged" : new MappingItem.VariableStructure(null, [:], null, null), + "overridden": new MappingItem.VariableStructure("someName", [:], null, "overriddenFolder") + ]) + verify(migration.mappingRepository, times(1)).applyAllVariableStructureMappings() + } + + @Test + void keepsStructureFromObjectWhenNotMapped() { + def migration = Utils.mockMigration() + Path mappingFile = Paths.get(dir.path, "testProject.csv") + def input = """\ + id,name,targetFolder,status,originLocations (read-only) + struct,newName,newFolder,Active,[] + """.stripIndent() + mappingFile.toFile().write(input) + when(migration.variableStructureRepository.find("struct")).thenReturn(new VariableStructureBuilder("struct") + .addVariable("var1", "Data.Clients.Value") + .languageVariable("var1") + .build()) + when(migration.mappingRepository.getVariableStructureMapping("struct")).thenReturn(new MappingItem.VariableStructure(null, [:], null, null)) + + VariableStructuresImport.run(migration, mappingFile.toFile()) + + verify(migration.mappingRepository, times(1)).upsertBatch([ + "struct": new MappingItem.VariableStructure("newName", ["var1": new VariablePathData("Data.Clients.Value", null)], new VariableRef("var1"), "newFolder") + ]) + } + + @Test + void keepsStructureFromExistingMapping() { + def migration = Utils.mockMigration() + Path mappingFile = Paths.get(dir.path, "testProject.csv") + def input = """\ + id,name,targetFolder,status,originLocations (read-only) + struct,newName,newFolder,Active,[] + """.stripIndent() + mappingFile.toFile().write(input) + when(migration.variableStructureRepository.find("struct")).thenReturn(new VariableStructureBuilder("struct") + .addVariable("objectVar", "Data.Object") + .languageVariable("objectVar") + .build()) + when(migration.mappingRepository.getVariableStructureMapping("struct")).thenReturn(new MappingItem.VariableStructure(null, + ["mappedVar": new VariablePathData("Data.Mapped", null)], new VariableRef("mappedVar"), null)) + + VariableStructuresImport.run(migration, mappingFile.toFile()) + + verify(migration.mappingRepository, times(1)).upsertBatch([ + "struct": new MappingItem.VariableStructure("newName", ["mappedVar": new VariablePathData("Data.Mapped", null)], new VariableRef("mappedVar"), "newFolder") + ]) + } + + @Test + void failsForUnknownVariableStructure() { + def migration = Utils.mockMigration() + Path mappingFile = Paths.get(dir.path, "testProject.csv") + def input = """\ + id,name,targetFolder,status,originLocations (read-only) + missing,,,Active,[] + """.stripIndent() + mappingFile.toFile().write(input) + + def exception = Assertions.assertThrows(Exception) { VariableStructuresImport.run(migration, mappingFile.toFile()) } + + Assertions.assertEquals("VariableStructure with id missing not found", exception.message) + } + + @Test + void updatesVariableStructureStatus() { + def migration = Utils.mockMigration() + Path mappingFile = Paths.get(dir.path, "testProject.csv") + def input = """\ + id,name,targetFolder,status,originLocations (read-only) + activateNew,,,Active,[] + keepActive,,,Active,[] + deployExisting,,,Deployed,[] + """.stripIndent() + mappingFile.toFile().write(input) + for (id in ["activateNew", "keepActive", "deployExisting"]) { + when(migration.variableStructureRepository.find(id)).thenReturn(new VariableStructureBuilder(id).build()) + when(migration.mappingRepository.getVariableStructureMapping(id)).thenReturn(new MappingItem.VariableStructure(null, [:], null, null)) + } + + when(migration.statusTrackingRepository.findLastEventRelevantToOutput(eq("activateNew"), any(), any())).thenReturn(null) + when(migration.statusTrackingRepository.findLastEventRelevantToOutput(eq("keepActive"), any(), any())).thenReturn(new Active()) + when(migration.statusTrackingRepository.findLastEventRelevantToOutput(eq("deployExisting"), any(), any())).thenReturn(new Active()) + + VariableStructuresImport.run(migration, mappingFile.toFile()) + + verify(migration.statusTrackingRepository, times(1)).active(eq("activateNew"), eq(ResourceType.VariableStructure), any(Map.class)) + verify(migration.statusTrackingRepository, never()).active(eq("keepActive"), eq(ResourceType.VariableStructure), any(Map.class)) + verify(migration.statusTrackingRepository, times(1)).deployed(eq("deployExisting"), anyString(), anyLong(), eq(ResourceType.VariableStructure), eq((String) null), eq(InspireOutput.Interactive), eq(["reason": "Manual"])) + } +} From 52dee102900c836475b62f3210ddad8b023ab1d5 Mon Sep 17 00:00:00 2001 From: "d.svitak" Date: Fri, 30 Oct 2026 09:29:48 +0100 Subject: [PATCH 5/7] MIG-629 Create data definitions from the variable structures - linking base templates to data definitions during the deployment when available --- .../InspireBaseTemplateBuilder.kt | 56 +++++++-- .../InspireVariableStructureBuilder.kt | 4 + .../InspireBaseTemplateBuilderTest.kt | 107 ++++++++++++++++++ .../quadient/wfdxml/api/layoutnodes/Root.java | 1 + .../wfdxml/internal/layoutnodes/RootImpl.java | 10 ++ .../internal/layoutnodes/RootImplTest.groovy | 28 +++++ 6 files changed, 197 insertions(+), 9 deletions(-) create mode 100644 wfd-xml/impl/src/test/groovy/com/quadient/wfdxml/internal/layoutnodes/RootImplTest.groovy diff --git a/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/InspireBaseTemplateBuilder.kt b/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/InspireBaseTemplateBuilder.kt index b1387cab..d40eedcd 100644 --- a/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/InspireBaseTemplateBuilder.kt +++ b/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/InspireBaseTemplateBuilder.kt @@ -48,6 +48,12 @@ class InspireBaseTemplateBuilder( if (resolvedStyleDefinitionPath != null) { root.setExternalStylesLayout(resolvedStyleDefinitionPath.toString()) } + + val usages = baseTemplateRepository.findUsages(baseTemplate.id).filterIsInstance() + val channelModels = findChannelModels(usages) + val variableStructureId = resolveVariableStructureId(baseTemplate, usages, channelModels) + resolveDataDefinitionPath(baseTemplate, variableStructureId)?.let { root.setDataDefinition(it.toString()) } + resolveArialFont(layout, icmDataCache) var mainFlow: Flow? = null @@ -86,17 +92,20 @@ class InspireBaseTemplateBuilder( mainFlow?.let { layout.pages.setMainFlow(it) } - enrichFromDocumentObjects(baseTemplate, layout) + enrichFromDocumentObjects(channelModels, variableStructureId, layout) val baseTemplateXml = builder.build() return enrichLayoutWithSourceBaseTemplate(icmDataCache, baseTemplateXml, resourcePathProvider.getSourceBaseTemplatePath()) } fun findChannelModels(baseTemplate: BaseTemplate): BaseTemplateChannelModels { + return findChannelModels(baseTemplateRepository.findUsages(baseTemplate.id).filterIsInstance()) + } + + private fun findChannelModels(usages: List): BaseTemplateChannelModels { var emailModel: DocumentObject? = null var smsModel: DocumentObject? = null - val usages = baseTemplateRepository.findUsages(baseTemplate.id).filterIsInstance() for (usage in usages) { for (content in usage.content) { val referenced = (content as? DocumentObjectRef)?.id?.let(documentObjectRepository::find) ?: continue @@ -111,16 +120,45 @@ class InspireBaseTemplateBuilder( return BaseTemplateChannelModels(emailModel, smsModel) } - private fun enrichFromDocumentObjects(baseTemplate: BaseTemplate, layout: Layout) { - val (emailModel, smsModel) = findChannelModels(baseTemplate) - if (emailModel == null && smsModel == null) return + private fun resolveVariableStructureId( + baseTemplate: BaseTemplate, usages: List, channelModels: BaseTemplateChannelModels, + ): String? { + baseTemplate.variableStructureRef?.id?.let { return it } + + val candidates = listOfNotNull(channelModels.email, channelModels.sms) + usages + if (candidates.isEmpty()) return null + return refInheritanceService.apply(candidates).firstNotNullOfOrNull { it.variableStructureRef?.id } + } - val resolvedVariableStructureId = baseTemplate.variableStructureRef?.id - ?: listOfNotNull(emailModel, smsModel).firstNotNullOfOrNull { - refInheritanceService.apply(listOf(it)).first().variableStructureRef?.id + private fun resolveDataDefinitionPath(baseTemplate: BaseTemplate, variableStructureId: String?): IcmPath? { + val variableStructure = variableStructureBuilder.findVariableStructure(variableStructureId) + if (variableStructure == null) { + if (variableStructureId != null) { + logger.warn("Variable structure '$variableStructureId' of base template '${baseTemplate.nameOrId()}' not found, data definition is not linked.") } + return null + } + + val path = resourcePathProvider.getDataDefinitionPath(variableStructure) + val exists = try { + icmDataCache.fileExists(path) + } catch (e: Exception) { + throw RuntimeException("Failed to check for data definition existence", e) + } + if (!exists) { + logger.warn("Data definition '$path' of base template '${baseTemplate.nameOrId()}' does not exist, data definition is not linked.") + return null + } + return path + } + + private fun enrichFromDocumentObjects( + channelModels: BaseTemplateChannelModels, variableStructureId: String?, layout: Layout, + ) { + val (emailModel, smsModel) = channelModels + if (emailModel == null && smsModel == null) return - val variableStructure = variableStructureBuilder.initVariableStructure(layout, resolvedVariableStructureId) + val variableStructure = variableStructureBuilder.initVariableStructure(layout, variableStructureId) if (smsModel != null) { val smsFlow = layout.addFlow().setSectionFlow(true).setWebEditingType(SECTION) diff --git a/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/InspireVariableStructureBuilder.kt b/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/InspireVariableStructureBuilder.kt index 20e12488..b5ec5a3a 100644 --- a/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/InspireVariableStructureBuilder.kt +++ b/migration-library/src/main/kotlin/com/quadient/migration/service/inspirebuilder/InspireVariableStructureBuilder.kt @@ -31,6 +31,10 @@ class InspireVariableStructureBuilder( return builder.buildDataDefinitionLayoutDelta() } + fun findVariableStructure(variableStructureId: String?): VariableStructure? { + return (variableStructureId ?: projectConfig.defaultVariableStructure)?.let { variableStructureRepository.find(it) } + } + fun initVariableStructure(layout: Layout, variableStructureId: String?): VariableStructure { val variableStructureId = variableStructureId ?: projectConfig.defaultVariableStructure diff --git a/migration-library/src/test/kotlin/com/quadient/migration/service/inspirebuilder/InspireBaseTemplateBuilderTest.kt b/migration-library/src/test/kotlin/com/quadient/migration/service/inspirebuilder/InspireBaseTemplateBuilderTest.kt index 1ee44111..1977eccf 100644 --- a/migration-library/src/test/kotlin/com/quadient/migration/service/inspirebuilder/InspireBaseTemplateBuilderTest.kt +++ b/migration-library/src/test/kotlin/com/quadient/migration/service/inspirebuilder/InspireBaseTemplateBuilderTest.kt @@ -3,6 +3,8 @@ package com.quadient.migration.service.inspirebuilder import com.quadient.migration.api.dto.migrationmodel.EmailOptions import com.quadient.migration.api.dto.migrationmodel.SmsOptions import com.quadient.migration.api.dto.migrationmodel.StringValue +import com.quadient.migration.api.dto.migrationmodel.VariableStructure +import com.quadient.migration.api.dto.migrationmodel.VariableStructureRef import com.quadient.migration.api.dto.migrationmodel.builder.BaseTemplateBuilder import com.quadient.migration.api.dto.migrationmodel.builder.DocumentObjectBuilder import com.quadient.migration.api.dto.migrationmodel.builder.EmailObjectBuilder @@ -22,6 +24,7 @@ import com.quadient.migration.shared.millimeters import com.quadient.migration.tools.aProjectConfig import com.quadient.migration.tools.model.aDocObj import com.quadient.migration.tools.model.aDocumentObjectRef +import com.quadient.migration.tools.model.aVariableStructure import com.quadient.migration.tools.shouldBeEqualTo import com.quadient.migration.tools.shouldBeNull import com.quadient.migration.tools.shouldNotBeNull @@ -70,6 +73,8 @@ class InspireBaseTemplateBuilderTest { every { ipsService.gatherFontData(any()) } returns "Arial,Regular,icm://Fonts/arial.ttf;" every { baseTemplateRepository.findUsages(any()) } returns emptyList() every { documentObjectRepository.listAll() } returns emptyList() + every { documentObjectRepository.find(any()) } returns null + every { variableStructureRepository.find(any()) } returns null } @Test @@ -126,6 +131,101 @@ class InspireBaseTemplateBuilderTest { result["Root"]["ExternalStylesLayout"].shouldBeNull() } + @Test + fun `buildBaseTemplate links data definition of explicitly referenced variable structure`() { + // given + val variableStructure = aVariableStructure("VS_1", name = "Structure") + val dataDefinitionPath = givenDeployedDataDefinition(variableStructure) + val baseTemplate = BaseTemplateBuilder("BT_1").variableStructureRef("VS_1").build() + + // when + val result = subject.buildBaseTemplate(baseTemplate).let { xmlMapper.readTree(it.trimIndent()) }["Layout"]["Layout"] + + // then + result["Root"]["DataDefinition"].stringValue().shouldBeEqualTo("VCSLocation,$dataDefinitionPath") + } + + @Test + fun `buildBaseTemplate links data definition of variable structure inherited from template usage`() { + // given + val variableStructure = aVariableStructure("VS_1", name = "Structure") + val dataDefinitionPath = givenDeployedDataDefinition(variableStructure) + val baseTemplate = BaseTemplateBuilder("BT_1").build() + val template = aDocObj("T_1", DocumentObjectType.Template).copy(variableStructureRef = VariableStructureRef("VS_1")) + every { baseTemplateRepository.findUsages(baseTemplate.id) } returns listOf(template) + every { documentObjectRepository.find(template.id) } returns template + + // when + val result = subject.buildBaseTemplate(baseTemplate).let { xmlMapper.readTree(it.trimIndent()) }["Layout"]["Layout"] + + // then + result["Root"]["DataDefinition"].stringValue().shouldBeEqualTo("VCSLocation,$dataDefinitionPath") + } + + @Test + fun `buildBaseTemplate prefers explicitly referenced variable structure over usages`() { + // given + val variableStructure = aVariableStructure("VS_1", name = "Structure") + val dataDefinitionPath = givenDeployedDataDefinition(variableStructure) + givenDeployedDataDefinition(aVariableStructure("VS_2", name = "Other")) + val baseTemplate = BaseTemplateBuilder("BT_1").variableStructureRef("VS_1").build() + val template = aDocObj("T_1", DocumentObjectType.Template).copy(variableStructureRef = VariableStructureRef("VS_2")) + every { baseTemplateRepository.findUsages(baseTemplate.id) } returns listOf(template) + + // when + val result = subject.buildBaseTemplate(baseTemplate).let { xmlMapper.readTree(it.trimIndent()) }["Layout"]["Layout"] + + // then + result["Root"]["DataDefinition"].stringValue().shouldBeEqualTo("VCSLocation,$dataDefinitionPath") + } + + @Test + fun `buildBaseTemplate links data definition of default variable structure`() { + // given + val customConfig = config.copy(defaultVariableStructure = "VS_1") + val subject = InspireBaseTemplateBuilder( + icmDataCache, + resourcePathProvider, + baseTemplateRepository, + documentObjectRepository, + InspireVariableStructureBuilder(variableRepository, variableStructureRepository, customConfig), + refInheritanceService, + ) + val dataDefinitionPath = givenDeployedDataDefinition(aVariableStructure("VS_1", name = "Structure")) + + // when + val result = subject.buildBaseTemplate(BaseTemplateBuilder("BT_1").build()) + .let { xmlMapper.readTree(it.trimIndent()) }["Layout"]["Layout"] + + // then + result["Root"]["DataDefinition"].stringValue().shouldBeEqualTo("VCSLocation,$dataDefinitionPath") + } + + @Test + fun `buildBaseTemplate does not link data definition when it is not deployed`() { + // given + every { variableStructureRepository.find("VS_1") } returns aVariableStructure("VS_1", name = "Structure") + val baseTemplate = BaseTemplateBuilder("BT_1").variableStructureRef("VS_1").build() + + // when + val result = subject.buildBaseTemplate(baseTemplate).let { xmlMapper.readTree(it.trimIndent()) }["Layout"]["Layout"] + + // then + result["Root"]["DataDefinition"].shouldBeNull() + } + + @Test + fun `buildBaseTemplate does not link data definition when variable structure does not exist`() { + // given + val baseTemplate = BaseTemplateBuilder("BT_1").variableStructureRef("VS_missing").build() + + // when + val result = subject.buildBaseTemplate(baseTemplate).let { xmlMapper.readTree(it.trimIndent()) }["Layout"]["Layout"] + + // then + result["Root"]["DataDefinition"].shouldBeNull() + } + @Test fun `buildBaseTemplate redirects Arial font to ICM location`() { // given @@ -433,6 +533,13 @@ class InspireBaseTemplateBuilderTest { result["Flow"].shouldBeNull() } + private fun givenDeployedDataDefinition(variableStructure: VariableStructure): IcmPath { + val path = resourcePathProvider.getDataDefinitionPath(variableStructure) + every { variableStructureRepository.find(variableStructure.id) } returns variableStructure + every { ipsService.fileExists(path) } returns true + return path + } + private fun sheetNameScript(result: JsonNode, index: Int): String { val calculatedScripts = result["Variable"].filter { it["Script"] != null }.map { it["Script"].stringValue() } return calculatedScripts[index] diff --git a/wfd-xml/api/src/main/java/com/quadient/wfdxml/api/layoutnodes/Root.java b/wfd-xml/api/src/main/java/com/quadient/wfdxml/api/layoutnodes/Root.java index 4acca4f2..4cb31682 100644 --- a/wfd-xml/api/src/main/java/com/quadient/wfdxml/api/layoutnodes/Root.java +++ b/wfd-xml/api/src/main/java/com/quadient/wfdxml/api/layoutnodes/Root.java @@ -7,6 +7,7 @@ public interface Root { Root setAllowRuntimeModifications(boolean allowRuntimeModifications); Root setExternalStylesLayout(String vcsLocation); + Root setDataDefinition(String vcsLocation); Root setSubject(String subject); Root addLockedWebNode(Node node); diff --git a/wfd-xml/impl/src/main/java/com/quadient/wfdxml/internal/layoutnodes/RootImpl.java b/wfd-xml/impl/src/main/java/com/quadient/wfdxml/internal/layoutnodes/RootImpl.java index 57bb7ac9..29ded5a6 100644 --- a/wfd-xml/impl/src/main/java/com/quadient/wfdxml/internal/layoutnodes/RootImpl.java +++ b/wfd-xml/impl/src/main/java/com/quadient/wfdxml/internal/layoutnodes/RootImpl.java @@ -12,6 +12,7 @@ public class RootImpl implements Root, XmlExportable { private boolean allowRuntimeModifications = false; private String externalStylesLayoutVcsLocation = null; + private String dataDefinitionVcsLocation = null; private String subject = null; private final List> lockedWebNodes = new ArrayList<>(); @@ -27,6 +28,12 @@ public Root setExternalStylesLayout(String vcsLocation) { return this; } + @Override + public Root setDataDefinition(String vcsLocation) { + this.dataDefinitionVcsLocation = vcsLocation; + return this; + } + @Override public Root setSubject(String subject) { this.subject = subject; @@ -52,6 +59,9 @@ public void export(XmlExporter exporter) { if (externalStylesLayoutVcsLocation != null) { exporter.addElementWithStringData("ExternalStylesLayout", "VCSLocation," + externalStylesLayoutVcsLocation); } + if (dataDefinitionVcsLocation != null) { + exporter.addElementWithStringData("DataDefinition", "VCSLocation," + dataDefinitionVcsLocation); + } if (subject != null) { exporter.addElementWithStringData("Subject", subject); } diff --git a/wfd-xml/impl/src/test/groovy/com/quadient/wfdxml/internal/layoutnodes/RootImplTest.groovy b/wfd-xml/impl/src/test/groovy/com/quadient/wfdxml/internal/layoutnodes/RootImplTest.groovy new file mode 100644 index 00000000..276c7c1c --- /dev/null +++ b/wfd-xml/impl/src/test/groovy/com/quadient/wfdxml/internal/layoutnodes/RootImplTest.groovy @@ -0,0 +1,28 @@ +package com.quadient.wfdxml.internal.layoutnodes + +import com.quadient.wfdxml.internal.xml.export.XmlExporter +import spock.lang.Specification + +import static com.quadient.wfdxml.utils.AssertXml.assertXmlEqualsWrapRoot + +class RootImplTest extends Specification { + XmlExporter exporter = new XmlExporter() + + def "export Root with external styles layout and data definition"() { + given: + RootImpl root = new RootImpl() + root.setAllowRuntimeModifications(true) + .setExternalStylesLayout("icm://Interactive/tenant/CompanyStyles/styles.jld") + .setDataDefinition("icm://Interactive/tenant/DataDefinitions/structure.jdd") + + when: + root.export(exporter) + + then: + assertXmlEqualsWrapRoot(exporter.buildString(), """ + True + VCSLocation,icm://Interactive/tenant/CompanyStyles/styles.jld + VCSLocation,icm://Interactive/tenant/DataDefinitions/structure.jdd + """) + } +} From ab427a6c51ae26933b0a6766e40f0dee0d31621f Mon Sep 17 00:00:00 2001 From: "d.svitak" Date: Fri, 30 Oct 2026 13:34:27 +0100 Subject: [PATCH 6/7] MIG-629 Create data definitions from the variable structures - update changelog --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e9f4394..9ed18316 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,10 +10,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) - Support base template deployment in evolve - Reworked migration-app, it now serves as a replacement for Gradle task runner +- DeployDataDefinitions task that deploys data definitions based on the variable structures to Interactive +- VariableStructuresExport/Import task that allows to modify variable structure data including name and target folder ### Changed -- deployStyles with interactive output no longer deploys a wfd file, instead it uses the default StyleDefinition.wfd +- DeployStyles with interactive output no longer deploys a wfd file, instead it uses the default StyleDefinition.wfd +- DeployBaseTemplates task links base templates to their corresponding data definitions if available ### Fixed From b24cf517b3e2783a95eefdb263cadaf7a4b45e89 Mon Sep 17 00:00:00 2001 From: "d.svitak" Date: Mon, 2 Nov 2026 14:15:05 +0100 Subject: [PATCH 7/7] MIG-629 Create data definitions from the variable structures - add agent skill for generating sample data (json) based on variable structure --- .agents/skills/generate-sample-data/SKILL.md | 113 +++++++++++ .../generate-sample-data/sample-data.mjs | 181 ++++++++++++++++++ .../variable-structure.sql | 26 +++ CLAUDE.md | 4 + migration-examples/.gitignore | 1 + 5 files changed, 325 insertions(+) create mode 100644 .agents/skills/generate-sample-data/SKILL.md create mode 100644 .agents/skills/generate-sample-data/sample-data.mjs create mode 100644 .agents/skills/generate-sample-data/variable-structure.sql create mode 100644 CLAUDE.md diff --git a/.agents/skills/generate-sample-data/SKILL.md b/.agents/skills/generate-sample-data/SKILL.md new file mode 100644 index 00000000..2f0d85b1 --- /dev/null +++ b/.agents/skills/generate-sample-data/SKILL.md @@ -0,0 +1,113 @@ +--- +name: generate-sample-data +description: Generate sample/test JSON data matching a migration variable structure (the data definition deployed to Inspire). Use when the user asks for sample data, test data, input data or example JSON for a variable structure, data definition or the current migration project. +compatibility: Requires Node.js; DB access via psql, Docker or the Gradle VariablesExport task +--- + +# Generate sample data for a variable structure + +Optional inputs from the request: variable structure id, array sizes, output path, project config file +(overrides `active-project-config`). Array sizes: `--top-array-size` = records of the outermost arrays (not +nested in another array, typically `Clients`), `--array-size` = records of nested arrays. Both default to 3; +a single unqualified size ("arrays of 5") sets both, "one client" / "one top-level record" sets only the top one. + +Produces a JSON file whose shape matches the variable structure of the **current migration project**, filled +with plausible values. Two helpers live next to this file: + +- `variable-structure.sql` – reads one structure from the working DB (raw entries, no resolution). +- `sample-data.mjs` – resolves paths, builds the tree, prints the field list or writes the final JSON + (accepts the SQL output **or** the `VariablesExport` CSV, so both sources share one resolution logic). + +`SKILL_DIR` below = the directory of this file (`.agents/skills/generate-sample-data`). Write intermediate +files to the scratchpad directory, not into the repo. + +## 1. Resolve the current project + +All config lives in `migration-examples/src/main/resources/`. + +1. Project config file: `--project-config` argument if given, else the first non-empty line not starting with + `#` in `active-project-config` (no extension), else `project-config.toml`. +2. Project name = `name` in that file. Every DB query **must** filter `project_name = `. +3. Structure id: argument if given → `defaultVariableStructure` in the project config (if not commented out) + → the only structure of the project → otherwise list the project's structures and ask the user. + +## 2. Load the structure – fallback order + +DB settings are `[dbConfig]` in `migration-config.toml` (host, port, dbName, user, password). Never echo the +password; pass it via the `PGPASSWORD` env var. Commands below use POSIX shell syntax – adapt them to the +shell in use (e.g. `$env:PGPASSWORD`, `gradlew.bat` in PowerShell/cmd). Use the first option that works: + +1. **`psql` on PATH** (native install, Windows service, remote host): + ```bash + PGPASSWORD='' psql -h -p -U -d -X -At -v ON_ERROR_STOP=1 \ + -v proj='' -v vs='' -f "$SKILL_DIR/variable-structure.sql" > "$SCRATCH/structure.json" + ``` +2. **`psql` of a local PostgreSQL install that is not on PATH** – look in the platform's usual install + location (newest version) and run the same command with it, e.g. `C:/Program Files/PostgreSQL/*/bin` + (Windows), `/usr/lib/postgresql/*/bin` or `/usr/pgsql-*/bin` (Linux), + `/opt/homebrew/opt/postgresql*/bin` or `/Applications/Postgres.app/Contents/Versions/latest/bin` (macOS). +3. **Docker container** publishing the configured port – find it with + `docker ps --filter publish= --format '{{.Names}}'` (fallback: any running container whose image is + `postgres*`), then pipe the SQL to the psql inside it: + ```bash + docker exec -i -e PGPASSWORD='' psql -U -d -X -At -v ON_ERROR_STOP=1 \ + -v proj='' -v vs='' < "$SKILL_DIR/variable-structure.sql" > "$SCRATCH/structure.json" + ``` +4. **Gradle CSV export** (uses the project's own JDBC connection – works whenever the migration scripts work, + no psql needed). From the repo root: + ```bash + ./gradlew :migration-examples:VariablesExport -PscriptArgs="--variable-structure-id " + ``` + The CSV lands in `migration-examples/mapping/` and its name ends with `.csv` (take the newest + match). Rows without `inspire_path` are not in the structure and are ignored by the script. Caveat: if the + project config selects specific document objects, the export may contain only their variables. +5. **User-provided CSV** in the same `VariablesExport` format – ask for it if everything above fails, and + report what failed for each option. + +To list structures (step 1.3) with options 1–3, run +`select id, name from variable_structure where project_name = ''` the same way. + +## 3. Check the field list + +```bash +node "$SKILL_DIR/sample-data.mjs" "$SCRATCH/structure.json" # or the .csv +``` + +Prints the tree (`Name[]` = Array, `Name{}` = SubTree, then type and variable id) plus `WARN` lines on stderr +(unresolvable references, missing variable rows). An empty structure or warnings are worth telling the user. + +How the structure works (the script implements it; know it to interpret the output): +- Each structure entry's `path` is the **parent** path; the field name is appended. Field name = + structure entry `name` → variable `name` → variable id. +- A path is a literal (`Data.Clients.Value`) or a reference to another variable (`$clientsArray` / + `{"variableId": …}`) meaning "inside that variable" – Array parents add `.Value`; references chain. +- `Data` is the root and is dropped. `X.Value` = X is an array; another segment = subtree. An explicit + `Array`/`SubTree` variable at that path wins. + +## 4. Choose values and build the JSON + +Write `$SCRATCH/values.json` mapping **variable id** → value, or a list of values. Each time a field is emitted +the next list item is used (cycling), so for a field inside arrays give as many distinct values as it will be +emitted (product of the enclosing array sizes, e.g. with defaults 3 for `Clients[]`, 9 for `Clients[].Jobs[]`; +with `--top-array-size 1` 1 and 3) +– records should differ from each other, and related fields in the same record should fit together +(first name / last name / email, city / state / zip). Unlisted ids get the defaults below. + +Values by type: +- `String` – realistic for the name (`DRV_RecipientFirstName` → "John", `DRV_City` → "New York", + `*Email*` → "john.smith@example.com", `*Zip*` → "10001", `*_Indicator`, `*Flag*` → "Y"). If unsure: `"string"`. +- `Integer` – whole number fitting the name (counts small, ids/numbers longer). Unsure: `0`. +- `Currency` – JSON number with up to two decimals, e.g. `1250.75`. Unsure: `0`. +- `DateTime` – ISO string; `"2026-03-15"` for dates, `"2026-03-15T14:30:00"` when the name implies time. +- `Boolean` – `true` unless the name clearly implies otherwise. +- Language variable (printed in the header) – a language code like `"en"`. + +Unless the user says otherwise every field gets a value and every array 3 records. Then: + +```bash +node "$SKILL_DIR/sample-data.mjs" "$SCRATCH/structure.json" --values "$SCRATCH/values.json" \ + --top-array-size 3 --array-size 3 --out "" +``` + +Default output: `migration-examples/sample-data/_.json` (git-ignored) unless the user +named a path. Show the user the path, a short excerpt, and any warnings. diff --git a/.agents/skills/generate-sample-data/sample-data.mjs b/.agents/skills/generate-sample-data/sample-data.mjs new file mode 100644 index 00000000..ceea3b4d --- /dev/null +++ b/.agents/skills/generate-sample-data/sample-data.mjs @@ -0,0 +1,181 @@ +#!/usr/bin/env node +// Builds sample JSON data for a migration variable structure. +// +// Usage: +// node sample-data.mjs [--values values.json] [--array-size 3] [--top-array-size N] [--out out.json] +// +// --array-size records per nested array (default 3) +// --top-array-size records per outermost array, i.e. not nested in another array (default = --array-size) +// +// JSON produced by variable-structure.sql, or the CSV produced by the VariablesExport gradle task. +// Without --values: prints the resolved field list (one line per leaf) so values can be chosen. +// With --values: writes the sample data JSON. values.json maps variable id -> value or list of values; +// a list is consumed in order each time the field is emitted (cycling), so fields +// inside arrays get distinct values. Missing ids fall back to a per-type default. + +import fs from "node:fs"; + +const args = process.argv.slice(2); +const opt = (name, dflt) => { + const i = args.indexOf(name); + if (i < 0) return dflt; + const v = args[i + 1]; + args.splice(i, 2); + return v; +}; +const valuesFile = opt("--values"); +const arraySize = Number(opt("--array-size", "3")); +const topArraySize = Number(opt("--top-array-size", String(arraySize))); +const outFile = opt("--out"); +const inputFile = args[0]; +if (!inputFile) { + console.error("Usage: node sample-data.mjs [--values values.json] [--array-size 3] [--top-array-size N] [--out out.json]"); + process.exit(2); +} + +const warnings = []; + +// ---- input ------------------------------------------------------------------------------------- + +function splitCsvLine(line) { + const cols = []; + let cur = "", inQuotes = false, quoted = false; + for (let i = 0; i < line.length; i++) { + const c = line[i]; + if (inQuotes) { + if (c === '"' && line[i + 1] === '"') { cur += '"'; i++; } + else if (c === '"') inQuotes = false; + else cur += c; + } else if (c === ",") { cols.push(cur); cur = ""; quoted = false; } + else if (cur.length === 0 && c === '"' && !quoted) { inQuotes = true; quoted = true; } + else cur += c; + } + cols.push(cur); + return cols; +} + +// VariablesExport CSV: id,name,data_type,inspire_path,inspire_name,language_variable,originLocations +// Variables without inspire_path are not part of the structure. +function readCsv(text) { + const lines = text.split(/\r?\n/).filter(l => l.trim() !== ""); + const header = splitCsvLine(lines[0]).map(h => h.trim()); + const col = n => header.indexOf(n); + const variables = []; + let languageVariable = null; + for (const line of lines.slice(1)) { + const c = splitCsvLine(line); + const id = c[col("id")], path = c[col("inspire_path")]; + if (!path) continue; + if ((c[col("language_variable")] || "").trim().toLowerCase() === "true") languageVariable = id; + variables.push({ + id, + name: c[col("inspire_name")] || c[col("name")] || id, + dataType: c[col("data_type")] || null, + literalPath: path.startsWith("$") ? null : path, + refId: path.startsWith("$") ? path.slice(1) : null, + }); + } + return { structureId: null, languageVariable, variables }; +} + +const raw = fs.readFileSync(inputFile, "utf8").replace(/^/, ""); +const input = inputFile.toLowerCase().endsWith(".csv") ? readCsv(raw) : JSON.parse(raw); +const byId = new Map(input.variables.map(v => [v.id, v])); + +// ---- path resolution (mirrors VariablePath.resolve in InspireDocumentObjectBuilder.kt) ----------- + +function parentPath(v, seen = new Set()) { + if (v.literalPath != null) return v.literalPath; + if (seen.has(v.id)) return null; + seen.add(v.id); + const parent = byId.get(v.refId); + if (!parent) return null; + const pp = parentPath(parent, seen); + if (!pp) return null; + if (parent.dataType === "Array") return `${pp}.${parent.name}.Value`; + if (parent.dataType === "SubTree") return `${pp}.${parent.name}`; + warnings.push(`'${v.id}' references '${parent.id}' of type ${parent.dataType}; only Array/SubTree can be referenced`); + return null; +} + +const resolved = []; +for (const v of input.variables) { + const pp = parentPath(v); + if (!pp) { warnings.push(`'${v.id}' has an unresolvable path - skipped`); continue; } + if (!v.dataType) warnings.push(`'${v.id}' has no variable record - treated as String`); + // segments without "Data" root and "Value" markers; isArray marks segments followed by ".Value" + const parts = `${pp}.${v.name}`.split("."); + const segs = []; + parts.forEach((p, i) => { + if (p.toLowerCase() === "value") return; + if (i === 0 && p === "Data") return; + segs.push({ name: p, isArray: parts[i + 1]?.toLowerCase() === "value" }); + }); + resolved.push({ ...v, dataType: v.dataType || "String", segs }); +} + +// ---- tree (mirrors buildVariableTree in InspireBuilderUtils.kt) --------------------------------- + +const root = { kind: "SubTree", children: new Map() }; +const explicitType = new Map(resolved.map(v => [v.segs.map(s => s.name).join("."), v])); +for (const v of resolved) { + let node = root; + v.segs.forEach((s, i) => { + const key = v.segs.slice(0, i + 1).map(x => x.name).join("."); + let child = node.children.get(s.name); + if (!child) { + const ex = explicitType.get(key); + const kind = ex ? ex.dataType : (s.isArray ? "Array" : "SubTree"); + child = { kind, id: ex?.id, path: key, children: new Map() }; + node.children.set(s.name, child); + } + if (child.kind !== "Array" && child.kind !== "SubTree" && i < v.segs.length - 1) { + throw new Error(`Leaf variable '${key}' cannot contain child variables (via '${v.id}')`); + } + node = child; + }); +} + +// ---- output ------------------------------------------------------------------------------------- + +if (!valuesFile) { + const lines = []; + const walk = (node, depth) => { + for (const [name, c] of node.children) { + const container = c.kind === "Array" ? "[]" : c.kind === "SubTree" ? "{}" : ""; + lines.push(`${" ".repeat(depth)}${name}${container} ${c.kind}${c.id ? ` id=${c.id}` : ""}`); + walk(c, depth + 1); + } + }; + walk(root, 0); + console.log(`structure: ${input.structureId ?? "(csv)"} languageVariable: ${input.languageVariable ?? "-"}`); + console.log(lines.join("\n")); +} else { + const values = JSON.parse(fs.readFileSync(valuesFile, "utf8")); + const counters = new Map(); + const today = new Date().toISOString().slice(0, 10); + const defaults = { String: "string", Integer: 0, Currency: 0.0, Boolean: true, DateTime: today }; + const leafValue = node => { + const n = counters.get(node.path) ?? 0; + counters.set(node.path, n + 1); + const v = node.id != null ? values[node.id] : undefined; + if (Array.isArray(v)) return v.length ? v[n % v.length] : defaults[node.kind]; + if (v !== undefined) return v; + return defaults[node.kind] ?? "string"; + }; + // inArray: false until the first (outermost) array, which gets topArraySize records + const build = (node, inArray = false) => { + const obj = {}; + for (const [name, c] of node.children) { + if (c.kind === "Array") obj[name] = Array.from({ length: inArray ? arraySize : topArraySize }, () => build(c, true)); + else if (c.kind === "SubTree") obj[name] = build(c, inArray); + else obj[name] = leafValue(c); + } + return obj; + }; + const json = JSON.stringify(build(root), null, 2) + "\n"; + if (outFile) { fs.writeFileSync(outFile, json); console.log(`written ${outFile}`); } + else process.stdout.write(json); +} + +for (const w of [...new Set(warnings)]) console.error(`WARN ${w}`); diff --git a/.agents/skills/generate-sample-data/variable-structure.sql b/.agents/skills/generate-sample-data/variable-structure.sql new file mode 100644 index 00000000..57eecba1 --- /dev/null +++ b/.agents/skills/generate-sample-data/variable-structure.sql @@ -0,0 +1,26 @@ +-- Raw variable structure entries for sample-data.mjs (path resolution happens in the script). +-- psql variables: :'proj' = project name, :'vs' = variable structure id. +-- Output: one JSON document { project, structureId, structureName, languageVariable, variables: [...] } +-- variables[]: { id, name, dataType, literalPath, refId } +-- name = structure entry name -> variable name -> variable id (same fallback as the deploy) +-- exactly one of literalPath / refId is set; dataType is null when the variable row is missing +SELECT jsonb_build_object( + 'project', vs.project_name, + 'structureId', vs.id, + 'structureName', vs.name, + 'languageVariable', vs.language_variable, + 'variables', coalesce(jsonb_agg(jsonb_build_object( + 'id', e.key, + 'name', coalesce(e.value ->> 'name', v.name, e.key), + 'dataType', v.data_type, + 'literalPath', CASE jsonb_typeof(e.value -> 'path') + WHEN 'string' THEN e.value ->> 'path' + ELSE e.value -> 'path' ->> 'path' END, + 'refId', e.value -> 'path' ->> 'variableId' + ) ORDER BY e.key) FILTER (WHERE e.key IS NOT NULL), '[]'::jsonb)) +FROM variable_structure vs + LEFT JOIN LATERAL jsonb_each(vs.structure) e ON TRUE + LEFT JOIN variable v ON v.id = e.key AND v.project_name = vs.project_name +WHERE vs.project_name = :'proj' + AND vs.id = :'vs' +GROUP BY vs.project_name, vs.id, vs.name, vs.language_variable; diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..f4c97360 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,4 @@ +# CLAUDE.md + +Agent skills for this repository live in `.agents/skills//SKILL.md` (tool-neutral location). +When a task matches a skill's `description`, read that `SKILL.md` and follow it. diff --git a/migration-examples/.gitignore b/migration-examples/.gitignore index 4d0441ef..5bf4ae25 100644 --- a/migration-examples/.gitignore +++ b/migration-examples/.gitignore @@ -3,6 +3,7 @@ logs/ **/report/ **/hierarchy/ **/document-objects/ +**/sample-data/ !src/main/groovy/com/quadient/migration/example/common/mapping/ !src/main/groovy/com/quadient/migration/example/common/mapping/*.groovy !src/main/groovy/com/quadient/migration/example/common/report/