Mig 629 create data definitions from the variable structures

CHANGELOG.md

@@ -10,10 +10,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/)
 
 - Support base template deployment in evolve
 - Reworked migration-app, it now serves as a replacement for Gradle task runner
+- DeployDataDefinitions task that deploys data definitions based on the variable structures to Interactive
+- VariableStructuresExport/Import task that allows to modify variable structure data including name and target folder
 
 ### Changed
 
-- deployStyles with interactive output no longer deploys a wfd file, instead it uses the default StyleDefinition.wfd
+- DeployStyles with interactive output no longer deploys a wfd file, instead it uses the default StyleDefinition.wfd
+- DeployBaseTemplates task links base templates to their corresponding data definitions if available
 
 ### Fixed
 

migration-examples/src/main/groovy/com/quadient/migration/example/common/DeployDataDefinitions.groovy

@@ -0,0 +1,26 @@
+//! ---
+//! displayName: Deploy Data Definitions
+//! category: Deployment
+//! description: Deploys one data definition per variable structure to Interactive
+//! ---
+package com.quadient.migration.example.common
+
+import groovy.transform.Field
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+
+import static com.quadient.migration.example.common.util.InitMigration.initMigration
+
+@Field static Logger log = LoggerFactory.getLogger(this.class.name)
+def migration = initMigration(this.binding)
+deploymentResult = migration.deployClient.deployDataDefinitions()
+for (def item : deploymentResult.deployed) {
+    log.info "Deployed data definition '${item.id}' to '${item.targetPath}'"
+}
+for (def item : deploymentResult.warnings) {
+    log.warn "Data definition '${item.id}' deployed with warning: ${item.message}"
+}
+for (def item : deploymentResult.errors) {
+    log.error "Data definition '${item.id}' failed to deploy: ${item.message}"
+}
+log.info "Deployment finished. Deployed ${deploymentResult.deployed.size()} data definitions with ${deploymentResult.warnings.size()} warnings and ${deploymentResult.errors.size()} errors"

migration-examples/src/main/groovy/com/quadient/migration/example/common/mapping/VariableStructuresExport.groovy

@@ -0,0 +1,46 @@
+//! ---
+//! displayName: Export Variable Structures
+//! category: Mapping
+//! description: Creates a CSV file with variable structure details (name and target folder of the deployed data definition) from the migration project. The generated CSV columns can be updated and later imported back into the database using a dedicated import task.
+//! target: gradle
+//! ---
+package com.quadient.migration.example.common.mapping
+
+import com.quadient.migration.api.Migration
+import com.quadient.migration.api.dto.migrationmodel.VariableStructure
+import com.quadient.migration.example.common.util.Csv
+import com.quadient.migration.example.common.util.Mapping
+import com.quadient.migration.service.deploy.utility.ResourceType
+
+import static com.quadient.migration.example.common.util.InitMigration.initMigration
+
+def migration = initMigration(this.binding)
+
+def variableStructuresPath = Mapping.csvPath(binding, migration.projectConfig.name, migration.projectConfig.subProjectId, "variable-structures")
+
+run(migration, variableStructuresPath.toFile())
+
+static void run(Migration migration, File variableStructuresDstPath) {
+    List<VariableStructure> variableStructures = Mapping.collectSelectedOrAll(migration, VariableStructure) { migration.variableStructureRepository.listAll() }
+
+    variableStructuresDstPath.createParentDirectories()
+
+    variableStructuresDstPath.withWriter { writer ->
+        def headers = ["id", "name", "targetFolder", "status", Mapping.displayHeader("originLocations", true)]
+        writer.writeLine(headers.join(","))
+        variableStructures.each { obj ->
+            def status = migration.statusTrackingRepository.findLastEventRelevantToOutput(obj.id,
+                    ResourceType.VariableStructure,
+                    migration.projectConfig.inspireOutput)
+
+            def builder = new StringBuilder()
+            builder.append(Csv.serialize(obj.id))
+            builder.append("," + Csv.serialize(obj.name))
+            builder.append("," + Csv.serialize(obj.targetFolder))
+            builder.append("," + Csv.serialize(status?.class?.simpleName))
+            builder.append("," + Csv.serialize(obj.originLocations))
+
+            writer.writeLine(builder.toString())
+        }
+    }
+}

migration-examples/src/main/groovy/com/quadient/migration/example/common/mapping/VariableStructuresImport.groovy

@@ -0,0 +1,80 @@
+//! ---
+//! displayName: Import Variable Structures
+//! category: Mapping
+//! description: Imports variable structure details (name and target folder of the deployed data definition) from CSV files into the migration project, applying any updates made to the columns during editing.
+//! target: gradle
+//! ---
+package com.quadient.migration.example.common.mapping
+
+import com.quadient.migration.api.Migration
+import com.quadient.migration.api.dto.migrationmodel.MappingItem
+import com.quadient.migration.example.common.util.Csv
+import com.quadient.migration.example.common.util.Mapping
+import com.quadient.migration.service.deploy.utility.ResourceType
+
+import groovy.transform.Field
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+
+import static com.quadient.migration.example.common.util.InitMigration.initMigration
+
+@Field static Logger log = LoggerFactory.getLogger(this.class.name)
+
+def migration = initMigration(this.binding)
+
+def variableStructuresPath = Mapping.csvPath(binding, migration.projectConfig.name, migration.projectConfig.subProjectId, "variable-structures")
+
+run(migration, variableStructuresPath.toFile())
+
+static void run(Migration migration, File file) {
+    def deploymentId = UUID.randomUUID().toString()
+    def now = new Date().getTime()
+    def output = migration.projectConfig.inspireOutput
+
+    def lines = file.readLines()
+    def columnNames = Csv.parseColumnNames(lines.removeFirst()).collect { Mapping.normalizeHeader(it) }
+    def total = lines.size()
+
+    def mappings = new HashMap<String, MappingItem>()
+    for (line in lines) {
+        def values = Csv.getCells(line, columnNames)
+        def id = values.get("id")
+
+        def existingStructure = migration.variableStructureRepository.find(id)
+        if (existingStructure == null) {
+            throw new Exception("VariableStructure with id ${id} not found")
+        }
+        def existingMapping = migration.mappingRepository.getVariableStructureMapping(id)
+
+        // The structure content is owned by the variables mapping, keep it as it is in the object
+        if (!existingMapping.mappings) existingMapping.mappings = new HashMap(existingStructure.structure)
+        if (existingMapping.languageVariable == null) existingMapping.languageVariable = existingStructure.languageVariable
+
+        def status = migration.statusTrackingRepository.findLastEventRelevantToOutput(existingStructure.id,
+            ResourceType.VariableStructure,
+            migration.projectConfig.inspireOutput)
+
+        def newName = Csv.deserialize(values.get("name"), String.class)
+        existingMapping.name = newName
+
+        def newTargetFolder = Csv.deserialize(values.get("targetFolder"), String.class)
+        existingMapping.targetFolder = newTargetFolder
+
+        def csvStatus = values.get("status")
+        if ((csvStatus == null || csvStatus == "") && status == null) {
+            migration.statusTrackingRepository.active(existingStructure.id, ResourceType.VariableStructure, [reason: "Manual"])
+        } else if (csvStatus == "Active" && status?.class?.simpleName != "Active") {
+            migration.statusTrackingRepository.active(existingStructure.id, ResourceType.VariableStructure, [reason: "Manual"])
+        } else if (csvStatus == "Deployed" && status?.class?.simpleName != "Deployed") {
+            migration.statusTrackingRepository.deployed(existingStructure.id, deploymentId, now, ResourceType.VariableStructure, null, output, [reason: "Manual"])
+        }
+
+        mappings[id] = existingMapping
+        if (total > 1000 && mappings.size() % 1000 == 0) {
+            log.info "Processed ${mappings.size()}/${total} mappings"
+        }
+    }
+
+    Mapping.upsertBatched(migration.mappingRepository, mappings, "variable structure mappings", log)
+    migration.mappingRepository.applyAllVariableStructureMappings()
+}

migration-examples/src/main/groovy/com/quadient/migration/example/common/mapping/VariablesExport.groovy

@@ -1,5 +1,5 @@
 //! ---
-//! displayName: Export Variable Structure
+//! displayName: Export Variables
 //! category: Mapping
 //! description: Creates a CSV report with variable structure from the migration project. The export is interactive, prompting the user to select or enter a variable structure to export.
 //! target: gradle

migration-examples/src/main/groovy/com/quadient/migration/example/common/mapping/VariablesImport.groovy

@@ -1,5 +1,5 @@
 //! ---
-//! displayName: Import Variable Structure
+//! displayName: Import Variables
 //! category: Mapping
 //! description: Imports variable structure from specified CSV files. The import is interactive, prompting the user to select variable structure CSV to import.
 //! target: gradle
@@ -45,6 +45,9 @@ static void run(Migration migration, Path path) {
         migration.variableStructureRepository.upsert(new VariableStructureBuilder(structureId).build())
     }
 
+    if (structureMapping.name == null) structureMapping.name = existingStructure?.name
+    if (structureMapping.targetFolder == null) structureMapping.targetFolder = existingStructure?.targetFolder
+
     def languageVariableFound = false
     def mappings = new HashMap<String, MappingItem>()
     for (line in lines) {

migration-examples/src/test/groovy/com/quadient/migration/example/common/mapping/VariableStructuresMappingExportTest.groovy

@@ -0,0 +1,76 @@
+package com.quadient.migration.example.common.mapping
+
+import com.quadient.migration.api.dto.migrationmodel.builder.DocumentObjectBuilder
+import com.quadient.migration.api.dto.migrationmodel.builder.VariableStructureBuilder
+import com.quadient.migration.data.Active
+import com.quadient.migration.example.Utils
+import com.quadient.migration.example.common.util.Mapping
+import com.quadient.migration.shared.DocumentObjectType
+import org.junit.jupiter.api.Assertions
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.io.TempDir
+
+import java.nio.file.Path
+import java.nio.file.Paths
+
+import static org.mockito.ArgumentMatchers.any
+import static org.mockito.Mockito.when
+
+class VariableStructuresMappingExportTest {
+    @TempDir
+    File dir
+
+    @Test
+    void allPossibleFields() {
+        Path mappingFile = Paths.get(dir.path, "testProject.csv")
+        def migration = Utils.mockMigration()
+
+        when(migration.variableStructureRepository.listAll()).thenReturn([
+            new VariableStructureBuilder("empty").build(),
+            new VariableStructureBuilder("full").name("full").targetFolder("targetFolder1").originLocations(["foo", "bar"]).build(),
+        ])
+
+        when(migration.statusTrackingRepository.findLastEventRelevantToOutput(any(), any(), any())).thenReturn(new Active())
+
+        VariableStructuresExport.run(migration, mappingFile.toFile())
+
+        def expected = """\
+            id,name,targetFolder,status,originLocations (read-only)
+            empty,,,Active,[]
+            full,full,targetFolder1,Active,[foo; bar]
+            """.stripIndent()
+        Assertions.assertEquals(expected, mappingFile.toFile().text.replaceAll("\\r\\n|\\r", "\n"))
+    }
+
+    @Test
+    void exportsOnlySelectedVariableStructures() {
+        Path mappingFile = Paths.get(dir.path, "testProject.csv")
+        def migration = Utils.mockMigration()
+
+        def referencedStructure = new VariableStructureBuilder("referenced").build()
+        def unselectedStructure = new VariableStructureBuilder("unselected").build()
+        def selectedDocObject = new DocumentObjectBuilder("doc1", DocumentObjectType.Block).build()
+
+        when(migration.projectConfig.getDocumentObjectsToProcess()).thenReturn(["doc1"])
+        when(migration.documentObjectRepository.listIds(["doc1"])).thenReturn([selectedDocObject])
+        when(migration.referenceCollector.collectAllObjects(selectedDocObject)).thenReturn([referencedStructure] as Set)
+        when(migration.variableStructureRepository.listAll()).thenReturn([referencedStructure, unselectedStructure])
+
+        when(migration.statusTrackingRepository.findLastEventRelevantToOutput(any(), any(), any())).thenReturn(new Active())
+
+        VariableStructuresExport.run(migration, mappingFile.toFile())
+
+        def expected = """\
+            id,name,targetFolder,status,originLocations (read-only)
+            referenced,,,Active,[]
+            """.stripIndent()
+        Assertions.assertEquals(expected, mappingFile.toFile().text.replaceAll("\\r\\n|\\r", "\n"))
+    }
+
+    @Test
+    void csvFileIsNotPickedUpByVariablesImport() {
+        def fileName = "${Mapping.namePrefix("testProject", null)}-variable-structures.csv"
+
+        Assertions.assertFalse(fileName.startsWith(Mapping.variableStructureFileNamePrefix("testProject", null)))
+    }
+}