From 63d002c7a93d8cc07a7cfa260c1515ddb8d88b97 Mon Sep 17 00:00:00 2001 From: "shan.wu" Date: Tue, 9 Jun 2026 14:48:00 +0800 Subject: [PATCH] [storage]: support ceph multi image cache pools PROBLEM: Ceph primary storage previously assumed one image cache pool for one primary storage. When a VM root volume is created on another pool, the root volume may be cloned from the image cache snapshot in the default pool. Some customer Ceph clusters do not support cross-pool RBD clone, so VM creation, reimage, and change image can fail when the selected volume pool differs from the image cache pool. SOLUTION: Add a Ceph image cache pool strategy. The default strategy keeps the old behavior and uses the default image cache pool. The volume-pool strategy selects the target root volume pool when that physical pool also has the ImageCache role. Otherwise it falls back to the default image cache pool. The existing-cache strategy reuses an existing cache first, preferring the default pool cache and then another valid cache record. Image cache preparation now selects the target cache pool before capacity allocation. If the selected pool already has a valid cache snapshot, the existing cache is reused. If the selected cache record is stale, only that cache record and its refs are removed and the flow retries. If another pool has a valid cache for the same image, the snapshot is copied into the selected pool with RBD cp/deep cp. The normal create-snapshot and protect-snapshot flow then creates a new cache in the selected pool. If no usable cache exists, the image is downloaded through the existing backup storage flow. Same-Ceph backup storage is copied into the selected pool when needed to avoid later cross-pool clone. The final root volume operation still clones from the prepared image cache snapshot. The data-copy step is only used to materialize the cache in the selected pool. Snapshot reuse images keep their original volumeSnapshotReuse cache record and do not fall back to ordinary backup-storage download. Reinit can still select existing cache records by image UUID when the source ImageVO has already been deleted. TEST: Verified on root@172.20.13.237 under /root/zstack-workspace/zstack. Ran the changed Ceph integration case through TestCaseStabilityTest: CephPrimaryStorageVolumePoolsCase. Result: BUILD SUCCESS. Tests run: 1, Failures: 0, Errors: 0. The case covers default strategy compatibility, volume-pool strategy, existing-cache strategy, same-Ceph backup storage copy to selected pool, stale selected cache cleanup, copying cache from another pool, image cache cleanup with refs, reimage, and multi-ImageCache-pool setup in the common Ceph primary storage spec. Resolves: ZSTAC-62608 Change-Id: Ie498c8af87c1a0248429cc81980b135725894778 --- conf/globalConfig/ceph.xml | 8 + ...loadVolumeTemplateToPrimaryStorageMsg.java | 9 + .../zstack/storage/ceph/CephGlobalConfig.java | 2 + .../APIAddCephPrimaryStoragePoolMsg.java | 2 +- .../primary/CephImageCachePoolSelector.java | 165 ++++ .../primary/CephImageCachePoolStrategy.java | 7 + .../ceph/primary/CephPrimaryStorageBase.java | 424 ++++++--- .../sdk/AddCephPrimaryStoragePoolAction.java | 2 +- .../storage/ceph/CephOperationCase.groovy | 7 +- .../CephPrimaryStorageVolumePoolsCase.groovy | 830 +++++++++++++++++- .../CephOpenSourcePoolCapacityCase.groovy | 2 +- .../CephSandStonePoolCapacityCase.groovy | 1 + .../capacity/CephXskyPoolCapacityCase.groovy | 1 + .../testlib/CephPrimaryStoragePoolSpec.groovy | 4 +- .../testlib/CephPrimaryStorageSpec.groovy | 82 +- 15 files changed, 1363 insertions(+), 183 deletions(-) create mode 100644 plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephImageCachePoolSelector.java create mode 100644 plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephImageCachePoolStrategy.java diff --git a/conf/globalConfig/ceph.xml b/conf/globalConfig/ceph.xml index ccde2ee2806..58eb4f50332 100755 --- a/conf/globalConfig/ceph.xml +++ b/conf/globalConfig/ceph.xml @@ -16,6 +16,14 @@ java.lang.Long + + imageCache.poolStrategy + strategy to select ceph image cache pool when preparing root volume image cache. + ceph + DefaultImageCachePool + java.lang.String + + trash.cleanup.interval interval to cleanup image trash on primary storage, in seconds. diff --git a/header/src/main/java/org/zstack/header/storage/primary/DownloadVolumeTemplateToPrimaryStorageMsg.java b/header/src/main/java/org/zstack/header/storage/primary/DownloadVolumeTemplateToPrimaryStorageMsg.java index d4286ab8176..ff43efe1a33 100644 --- a/header/src/main/java/org/zstack/header/storage/primary/DownloadVolumeTemplateToPrimaryStorageMsg.java +++ b/header/src/main/java/org/zstack/header/storage/primary/DownloadVolumeTemplateToPrimaryStorageMsg.java @@ -7,6 +7,7 @@ public class DownloadVolumeTemplateToPrimaryStorageMsg extends NeedReplyMessage private String primaryStorageUuid; private ImageSpec templateSpec; private String hostUuid; + private String targetVolumeInstallUrl; @Override public String getPrimaryStorageUuid() { @@ -32,4 +33,12 @@ public String getHostUuid() { public void setHostUuid(String hostUuid) { this.hostUuid = hostUuid; } + + public String getTargetVolumeInstallUrl() { + return targetVolumeInstallUrl; + } + + public void setTargetVolumeInstallUrl(String targetVolumeInstallUrl) { + this.targetVolumeInstallUrl = targetVolumeInstallUrl; + } } diff --git a/plugin/ceph/src/main/java/org/zstack/storage/ceph/CephGlobalConfig.java b/plugin/ceph/src/main/java/org/zstack/storage/ceph/CephGlobalConfig.java index c3a42f79013..88d2664c046 100755 --- a/plugin/ceph/src/main/java/org/zstack/storage/ceph/CephGlobalConfig.java +++ b/plugin/ceph/src/main/java/org/zstack/storage/ceph/CephGlobalConfig.java @@ -14,6 +14,8 @@ public class CephGlobalConfig { @GlobalConfigValidation(numberGreaterThan = 1) public static GlobalConfig IMAGE_CACHE_CLEANUP_INTERVAL = new GlobalConfig(CATEGORY, "imageCache.cleanup.interval"); + @GlobalConfigValidation(validValues = {"DefaultImageCachePool", "PreferVolumePool", "PreferExistingCache"}) + public static GlobalConfig IMAGE_CACHE_POOL_STRATEGY = new GlobalConfig(CATEGORY, "imageCache.poolStrategy"); @GlobalConfigValidation public static GlobalConfig PRIMARY_STORAGE_DELETE_POOL = new GlobalConfig(CATEGORY, "primaryStorage.deletePool"); @GlobalConfigValidation(numberGreaterThan = 0) diff --git a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/APIAddCephPrimaryStoragePoolMsg.java b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/APIAddCephPrimaryStoragePoolMsg.java index 458a3060d51..a6550f225c9 100755 --- a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/APIAddCephPrimaryStoragePoolMsg.java +++ b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/APIAddCephPrimaryStoragePoolMsg.java @@ -28,7 +28,7 @@ public class APIAddCephPrimaryStoragePoolMsg extends APICreateMessage implements private String aliasName; @APIParam(maxLength = 2048, required = false) private String description; - @APIParam(validValues = {"Root", "Data"}) + @APIParam(validValues = {"Root", "Data", "ImageCache"}) private String type; private boolean isCreate; diff --git a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephImageCachePoolSelector.java b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephImageCachePoolSelector.java new file mode 100644 index 00000000000..40132b84769 --- /dev/null +++ b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephImageCachePoolSelector.java @@ -0,0 +1,165 @@ +package org.zstack.storage.ceph.primary; + +import org.apache.commons.lang.StringUtils; +import org.zstack.core.db.Q; +import org.zstack.header.image.ImageConstant; +import org.zstack.header.image.ImageInventory; +import org.zstack.header.storage.primary.ImageCacheVO; +import org.zstack.header.storage.primary.ImageCacheVO_; +import org.zstack.storage.ceph.CephGlobalConfig; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.List; + +final class CephImageCachePoolSelector { + private static final String CEPH_INSTALL_URL_PREFIX = "ceph://"; + static final String SNAPSHOT_REUSE_POOL_NAME = "snapshot-reuse"; + + private static final CLogger logger = Utils.getLogger(CephImageCachePoolSelector.class); + + private final String primaryStorageUuid; + private final String defaultPoolName; + + CephImageCachePoolSelector(String primaryStorageUuid, String defaultPoolName) { + this.primaryStorageUuid = primaryStorageUuid; + this.defaultPoolName = defaultPoolName; + } + + Selection select(ImageInventory image, String targetVolumeInstallUrl) { + if (isSnapshotReuseImage(image)) { + return new Selection(CephImageCachePoolStrategy.DefaultImageCachePool, + SNAPSHOT_REUSE_POOL_NAME, findSnapshotReuseImageCache(image)); + } + + return select(image.getUuid(), targetVolumeInstallUrl, getStrategy()); + } + + Selection select(ImageInventory image, String targetVolumeInstallUrl, CephImageCachePoolStrategy strategy) { + if (isSnapshotReuseImage(image)) { + return new Selection(strategy, SNAPSHOT_REUSE_POOL_NAME, findSnapshotReuseImageCache(image)); + } + + return select(image.getUuid(), targetVolumeInstallUrl, strategy); + } + + Selection select(String imageUuid, String targetVolumeInstallUrl) { + return select(imageUuid, targetVolumeInstallUrl, getStrategy()); + } + + private Selection select(String imageUuid, String targetVolumeInstallUrl, CephImageCachePoolStrategy strategy) { + List caches = listImageCaches(imageUuid); + String poolName = selectPool(targetVolumeInstallUrl, caches, strategy); + return new Selection(strategy, poolName, findCacheInPool(caches, poolName)); + } + + private Selection selectInPool(String imageUuid, String poolName, CephImageCachePoolStrategy strategy) { + return new Selection(strategy, poolName, findCacheInPool(listImageCaches(imageUuid), poolName)); + } + + Selection selectInPool(ImageInventory image, String poolName, CephImageCachePoolStrategy strategy) { + if (isSnapshotReuseImage(image)) { + return new Selection(strategy, poolName, findSnapshotReuseImageCache(image)); + } + + return selectInPool(image.getUuid(), poolName, strategy); + } + + List listImageCaches(String imageUuid) { + return Q.New(ImageCacheVO.class) + .eq(ImageCacheVO_.primaryStorageUuid, primaryStorageUuid) + .eq(ImageCacheVO_.imageUuid, imageUuid) + .list(); + } + + static boolean isCephImageCacheRecord(ImageCacheVO cache) { + return cache != null && cache.getInstallUrl() != null && cache.getInstallUrl().startsWith(CEPH_INSTALL_URL_PREFIX); + } + + static boolean isSnapshotReuseImage(ImageInventory image) { + return image != null && StringUtils.startsWith(image.getUrl(), ImageConstant.SNAPSHOT_REUSE_IMAGE_SCHEMA); + } + + static String getPoolName(String installUrl) { + if (StringUtils.isBlank(installUrl) || !installUrl.startsWith(CEPH_INSTALL_URL_PREFIX)) { + return null; + } + + String path = installUrl.substring(CEPH_INSTALL_URL_PREFIX.length()); + int index = path.indexOf("/"); + return index > 0 ? path.substring(0, index) : null; + } + + private CephImageCachePoolStrategy getStrategy() { + String strategy = CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.value(String.class); + try { + return CephImageCachePoolStrategy.valueOf(strategy); + } catch (RuntimeException e) { + logger.warn(String.format("invalid ceph image cache pool strategy[%s], use DefaultImageCachePool", strategy)); + return CephImageCachePoolStrategy.DefaultImageCachePool; + } + } + + private String selectPool(String targetVolumeInstallUrl, List caches, + CephImageCachePoolStrategy strategy) { + if (strategy == CephImageCachePoolStrategy.PreferVolumePool) { + String targetPoolName = getPoolName(targetVolumeInstallUrl); + return hasImageCachePoolRole(targetPoolName) ? targetPoolName : defaultPoolName; + } + + if (strategy == CephImageCachePoolStrategy.PreferExistingCache) { + boolean defaultPoolCacheExists = caches.stream() + .anyMatch(c -> defaultPoolName.equals(getPoolName(c.getInstallUrl()))); + if (defaultPoolCacheExists) { + return defaultPoolName; + } + + return caches.stream() + .map(c -> getPoolName(c.getInstallUrl())) + .filter(StringUtils::isNotBlank) + .findFirst() + .orElse(defaultPoolName); + } + + return defaultPoolName; + } + + private ImageCacheVO findCacheInPool(List caches, String poolName) { + return caches.stream() + .filter(c -> poolName.equals(getPoolName(c.getInstallUrl()))) + .findFirst() + .orElse(null); + } + + private ImageCacheVO findSnapshotReuseImageCache(ImageInventory image) { + return Q.New(ImageCacheVO.class) + .eq(ImageCacheVO_.primaryStorageUuid, primaryStorageUuid) + .eq(ImageCacheVO_.imageUuid, image.getUuid()) + .eq(ImageCacheVO_.installUrl, image.getUrl()) + .find(); + } + + private boolean hasImageCachePoolRole(String poolName) { + if (StringUtils.isBlank(poolName)) { + return false; + } + + return Q.New(CephPrimaryStoragePoolVO.class) + .eq(CephPrimaryStoragePoolVO_.primaryStorageUuid, primaryStorageUuid) + .eq(CephPrimaryStoragePoolVO_.poolName, poolName) + .eq(CephPrimaryStoragePoolVO_.type, CephPrimaryStoragePoolType.ImageCache.toString()) + .isExists(); + } + + static final class Selection { + final CephImageCachePoolStrategy strategy; + final String poolName; + final ImageCacheVO cache; + + Selection(CephImageCachePoolStrategy strategy, String poolName, ImageCacheVO cache) { + this.strategy = strategy; + this.poolName = poolName; + this.cache = cache; + } + } +} diff --git a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephImageCachePoolStrategy.java b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephImageCachePoolStrategy.java new file mode 100644 index 00000000000..decb0e4033b --- /dev/null +++ b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephImageCachePoolStrategy.java @@ -0,0 +1,7 @@ +package org.zstack.storage.ceph.primary; + +public enum CephImageCachePoolStrategy { + DefaultImageCachePool, + PreferVolumePool, + PreferExistingCache +} diff --git a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java index 53816200465..3328d14bfe7 100755 --- a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java +++ b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java @@ -825,6 +825,7 @@ public static class CpCmd extends AgentCommand implements HasThreadContext { String srcPath; String dstPath; boolean shareable; + boolean skipIfExisting; } public static class CpRsp extends AgentResponse { @@ -2155,21 +2156,90 @@ protected void handle(final InstantiateVolumeOnPrimaryStorageMsg msg) { } } + private CephImageCachePoolSelector getImageCachePoolSelector() { + return new CephImageCachePoolSelector(self.getUuid(), getDefaultImageCachePoolName()); + } + + private void checkImageCacheBits(ImageCacheVO cache, ReturnValueCompletion completion) { + CheckIsBitsExistingCmd cmd = new CheckIsBitsExistingCmd(); + cmd.setInstallPath(ImageCacheUtil.getImageCachePath(cache.getInstallUrl())); + httpCall(CHECK_BITS_PATH, cmd, CheckIsBitsExistingRsp.class, new ReturnValueCompletion(completion) { + @Override + public void success(CheckIsBitsExistingRsp returnValue) { + completion.success(returnValue.isExisting()); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + private void removeImageCacheRecord(ImageCacheVO cache) { + logger.debug(String.format("remove stale ceph image cache[imageUuid:%s, installUrl:%s]", cache.getImageUuid(), cache.getInstallUrl())); + new SQLBatch() { + @Override + protected void scripts() { + sql(ImageCacheVolumeRefVO.class).eq(ImageCacheVolumeRefVO_.imageCacheId, cache.getId()).delete(); + boolean deleted = sql("delete from ImageCacheVO c where c.id = :id") + .param("id", cache.getId()).execute() > 0; + if (deleted && CephImageCachePoolSelector.isCephImageCacheRecord(cache)) { + osdHelper.releaseAvailableCapacity(cache.getInstallUrl(), cache.getSize()); + } + } + }.execute(); + } + + private void findUsableSourceCache(List candidates, int index, boolean cleanupStale, + ReturnValueCompletion completion) { + if (index >= candidates.size()) { + completion.success(null); + return; + } + + ImageCacheVO candidate = candidates.get(index); + checkImageCacheBits(candidate, new ReturnValueCompletion(completion) { + @Override + public void success(Boolean existing) { + if (existing) { + completion.success(candidate); + } else { + if (cleanupStale) { + removeImageCacheRecord(candidate); + } + findUsableSourceCache(candidates, index + 1, cleanupStale, completion); + } + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + private boolean shouldCopyCephBackupStorageImageToSelectedPool(String backupStorageUuid, String backupStorageInstallPath, + String selectedImageCachePoolName) { + String backupStoragePoolName = CephImageCachePoolSelector.getPoolName(backupStorageInstallPath); + if (StringUtils.isBlank(backupStoragePoolName) || StringUtils.isBlank(selectedImageCachePoolName) + || backupStoragePoolName.equals(selectedImageCachePoolName)) { + return false; + } + + CephBackupStorageVO cephBS = dbf.findByUuid(backupStorageUuid, CephBackupStorageVO.class); + return cephBS != null && getSelf().getFsid().equals(cephBS.getFsid()); + } + class DownloadToCache { ImageSpec image; VolumeSnapshotInventory snapshot; boolean incremental; - private void doDownload(final ReturnValueCompletion completion) { - ImageCacheVO cache = Q.New(ImageCacheVO.class) - .eq(ImageCacheVO_.primaryStorageUuid, self.getUuid()) - .eq(ImageCacheVO_.imageUuid, image.getInventory().getUuid()) - .find(); - if (cache != null) { - completion.success(cache); - return; - } + String targetVolumeInstallUrl; + + private void doDownload(String selectedImageCachePoolName, ImageCacheVO sourceCache, final ReturnValueCompletion completion) { final FlowChain chain = FlowChainBuilder.newShareFlowChain(); - chain.setName(String.format("prepare-image-cache-ceph-%s", self.getUuid())); + chain.setName(String.format("prepare-image-cache-ceph-%s-%s", self.getUuid(), selectedImageCachePoolName)); chain.then(new ShareFlow() { String cachePath; String snapshotPath; @@ -2191,6 +2261,9 @@ public void run(final FlowTrigger trigger, Map data) { amsg.setPurpose(PrimaryStorageAllocationPurpose.DownloadImage.toString()); amsg.setImageUuid(image.getInventory().getUuid()); amsg.setNoOverProvisioning(true); + if (StringUtils.isNotBlank(selectedImageCachePoolName)) { + amsg.setRequiredInstallUri(String.format("ceph://%s/", selectedImageCachePoolName)); + } bus.makeLocalServiceId(amsg, PrimaryStorageConstant.SERVICE_ID); bus.send(amsg, new CloudBusCallBack(trigger) { @Override @@ -2224,7 +2297,7 @@ public void rollback(FlowRollback trigger, Map data) { }); flow(new Flow() { - String __name__ = "download-from-" + (snapshot != null ? "volume" : "backup-storage"); + String __name__ = "download-from-" + (sourceCache != null ? "image-cache" : (snapshot != null ? "volume" : "backup-storage")); boolean deleteOnRollback; String dstPath; @@ -2233,7 +2306,9 @@ public void rollback(FlowRollback trigger, Map data) { public void run(final FlowTrigger trigger, Map data) { dstPath = makeVolumeInstallPathByTargetPool(image.getInventory().getUuid(), getTargetPoolNameFromAllocatedUrl(allocatedInstall)); - if (snapshot != null) { + if (sourceCache != null) { + copyFromImageCache(trigger); + } else if (snapshot != null) { if (incremental) { incrementalCreateFromVolumeSnapshot(trigger); } else { @@ -2262,36 +2337,39 @@ public void fail(ErrorCode errorCode) { }); } + private void copyFromImageCache(FlowTrigger trigger) { + String destPoolName = CephImageCachePoolSelector.getPoolName(dstPath); + boolean imageShouldExistsOnCephBs = destPoolName != null && destPoolName.equals(Q.New(CephBackupStorageVO.class) + .eq(CephBackupStorageVO_.fsid, getSelf().getFsid()) + .select(CephBackupStorageVO_.poolName) + .findValue()); + copyToCache(trigger, ImageCacheUtil.getImageCachePath(sourceCache.getInstallUrl()), imageShouldExistsOnCephBs); + } - private void createFromVolumeSnapshot(FlowTrigger trigger) { - deleteOnRollback = true; - CpCmd cmd = new CpCmd(); - cmd.srcPath = snapshot.getPrimaryStorageInstallPath(); - cmd.dstPath = dstPath; - cmd.shareable = false; - httpCall(CP_PATH, cmd, CpRsp.class, new ReturnValueCompletion(trigger) { - @Override - public void success(CpRsp rsp) { - if (rsp.actualSize != null) { - actualSize = rsp.actualSize; - } - cachePath = rsp.installPath; - trigger.next(); - } - @Override - public void fail(ErrorCode errorCode) { - trigger.fail(errorCode); - } - }); + private void createFromVolumeSnapshot(FlowTrigger trigger) { + copyToCache(trigger, snapshot.getPrimaryStorageInstallPath(), false); } private void downloadFromBackupStorage(FlowTrigger trigger) { + if (image.getSelectedBackupStorage() == null) { + trigger.fail(operr("cannot find backupstorage to download image [%s] to primarystorage [%s] due to lack of Ready and accessible image", + image.getInventory().getUuid(), getSelf().getUuid())); + return; + } + + String backupStorageUuid = image.getSelectedBackupStorage().getBackupStorageUuid(); + String backupStorageInstallPath = image.getSelectedBackupStorage().getInstallPath(); + if (shouldCopyCephBackupStorageImageToSelectedPool(backupStorageUuid, backupStorageInstallPath, selectedImageCachePoolName)) { + copyToCache(trigger, backupStorageInstallPath, false); + return; + } + MediatorDownloadParam param = new MediatorDownloadParam(); param.setImage(image); param.setInstallPath(dstPath); param.setPrimaryStorageUuid(self.getUuid()); - BackupStorageMediator mediator = getBackupStorageMediator(image.getSelectedBackupStorage().getBackupStorageUuid()); + BackupStorageMediator mediator = getBackupStorageMediator(backupStorageUuid); mediator.param = param; deleteOnRollback = mediator.deleteWhenRollbackDownload(); @@ -2309,6 +2387,31 @@ public void fail(ErrorCode errorCode) { }); } + private void copyToCache(FlowTrigger trigger, String srcPath, boolean skipIfExisting) { + deleteOnRollback = !skipIfExisting; + CpCmd cmd = new CpCmd(); + cmd.resourceUuid = image.getInventory().getUuid(); + cmd.srcPath = srcPath; + cmd.dstPath = dstPath; + cmd.shareable = false; + cmd.skipIfExisting = skipIfExisting; + httpCall(CP_PATH, cmd, CpRsp.class, new ReturnValueCompletion(trigger) { + @Override + public void success(CpRsp rsp) { + if (rsp.actualSize != null) { + actualSize = rsp.actualSize; + } + cachePath = StringUtils.isNotBlank(rsp.installPath) ? rsp.installPath : dstPath; + trigger.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + trigger.fail(errorCode); + } + }); + } + @Override public void rollback(FlowRollback trigger, Map data) { if (deleteOnRollback && cachePath != null) { @@ -2480,10 +2583,19 @@ public void handle(ErrorCode errCode, Map data) { } void download(final ReturnValueCompletion completion) { + CephImageCachePoolSelector selector = getImageCachePoolSelector(); + submitDownloadTask(selector, selector.select(image.getInventory(), targetVolumeInstallUrl), completion); + } + + private void submitDownloadTask(CephImageCachePoolSelector selector, + CephImageCachePoolSelector.Selection selection, + ReturnValueCompletion completion) { + String syncSignature = String.format("ceph-p-%s-download-image-%s-%s", self.getUuid(), + image.getInventory().getUuid(), selection.poolName); thdf.chainSubmit(new ChainTask(completion) { @Override public String getSyncSignature() { - return String.format("ceph-p-%s-download-image-%s", self.getUuid(), image.getInventory().getUuid()); + return syncSignature; } private void checkEncryptImageCache(ImageCacheVO cacheVO, final SyncTaskChain chain) { @@ -2513,49 +2625,25 @@ public void fail(ErrorCode errorCode) { @Override public void run(final SyncTaskChain chain) { - ImageCacheVO cache = Q.New(ImageCacheVO.class) - .eq(ImageCacheVO_.primaryStorageUuid, self.getUuid()) - .eq(ImageCacheVO_.imageUuid, image.getInventory().getUuid()) - .find(); - - if (cache != null) { - final CheckIsBitsExistingCmd cmd = new CheckIsBitsExistingCmd(); - cmd.setInstallPath(ImageCacheUtil.getImageCachePath(cache.getInstallUrl())); - httpCall(CHECK_BITS_PATH, cmd, CheckIsBitsExistingRsp.class, new ReturnValueCompletion(chain) { + prepareInSelectedPool(chain, selector.selectInPool(image.getInventory(), selection.poolName, + selection.strategy)); + } + + private void prepareInSelectedPool(final SyncTaskChain chain, + CephImageCachePoolSelector.Selection currentSelection) { + if (currentSelection.cache != null) { + checkImageCacheBits(currentSelection.cache, new ReturnValueCompletion(chain) { @Override - public void success(CheckIsBitsExistingRsp returnValue) { - if (returnValue.isExisting()) { - logger.debug("image has been existing"); - checkEncryptImageCache(cache, chain); - return; + public void success(Boolean existing) { + if (existing) { + logger.debug(String.format("image cache[installUrl:%s] has been existing", currentSelection.cache.getInstallUrl())); + checkEncryptImageCache(currentSelection.cache, chain); } else { - logger.debug("image not found, remove vo and re-download"); - SimpleQuery q = dbf.createQuery(ImageCacheVO.class); - q.add(ImageCacheVO_.primaryStorageUuid, Op.EQ, self.getUuid()); - q.add(ImageCacheVO_.imageUuid, Op.EQ, image.getInventory().getUuid()); - ImageCacheVO cvo = q.find(); - - ReleasePrimaryStorageSpaceMsg imsg = new ReleasePrimaryStorageSpaceMsg(); - imsg.setDiskSize(cvo.getSize()); - imsg.setPrimaryStorageUuid(cvo.getPrimaryStorageUuid()); - imsg.setAllocatedInstallUrl(cvo.getInstallUrl()); - bus.makeTargetServiceIdByResourceUuid(imsg, PrimaryStorageConstant.SERVICE_ID, cvo.getPrimaryStorageUuid()); - bus.send(imsg); - dbf.remove(cvo); - - doDownload(new ReturnValueCompletion(chain) { - @Override - public void success(ImageCacheVO returnValue) { - completion.success(returnValue); - chain.next(); - } - - @Override - public void fail(ErrorCode errorCode) { - completion.fail(errorCode); - chain.next(); - } - }); + logger.debug(String.format("image cache[installUrl:%s] not found, remove vo and re-prepare", currentSelection.cache.getInstallUrl())); + removeImageCacheRecord(currentSelection.cache); + submitDownloadTask(selector, selector.select(image.getInventory(), targetVolumeInstallUrl, + selection.strategy), completion); + chain.next(); } } @@ -2565,22 +2653,50 @@ public void fail(ErrorCode errorCode) { chain.next(); } }); + return; + } - } else { - doDownload(new ReturnValueCompletion(chain) { - @Override - public void success(ImageCacheVO returnValue) { - completion.success(returnValue); - chain.next(); - } + if (CephImageCachePoolSelector.isSnapshotReuseImage(image.getInventory())) { + completion.fail(operr("cannot find snapshot reuse image cache[imageUuid:%s, installUrl:%s] on primary storage[uuid:%s]", + image.getInventory().getUuid(), image.getInventory().getUrl(), self.getUuid())); + chain.next(); + return; + } - @Override - public void fail(ErrorCode errorCode) { - completion.fail(errorCode); - chain.next(); + List sourceCandidates = selector.listImageCaches(image.getInventory().getUuid()).stream() + .filter(CephImageCachePoolSelector::isCephImageCacheRecord) + .filter(c -> !StringUtils.equals(currentSelection.poolName, + CephImageCachePoolSelector.getPoolName(c.getInstallUrl()))) + .collect(Collectors.toList()); + findUsableSourceCache(sourceCandidates, 0, true, new ReturnValueCompletion(chain) { + @Override + public void success(ImageCacheVO sourceCache) { + if (sourceCache != null && currentSelection.strategy == CephImageCachePoolStrategy.PreferExistingCache) { + checkEncryptImageCache(sourceCache, chain); + return; } - }); - } + + doDownload(currentSelection.poolName, sourceCache, new ReturnValueCompletion(chain) { + @Override + public void success(ImageCacheVO returnValue) { + completion.success(returnValue); + chain.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + chain.next(); + } + }); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + chain.next(); + } + }); } @Override @@ -2615,6 +2731,7 @@ public void run(final FlowTrigger trigger, Map data) { dmsg.setPrimaryStorageUuid(msg.getPrimaryStorageUuid()); dmsg.setHostUuid(msg.getDestHost().getUuid()); dmsg.setTemplateSpec(ispec); + dmsg.setTargetVolumeInstallUrl(volumePath); bus.makeTargetServiceIdByResourceUuid(dmsg, PrimaryStorageConstant.SERVICE_ID, dmsg.getPrimaryStorageUuid()); bus.send(dmsg, new CloudBusCallBack(trigger) { @Override @@ -2768,9 +2885,7 @@ public void success(DeleteRsp ret) { @Override protected void handle(DownloadVolumeTemplateToPrimaryStorageMsg msg) { final DownloadVolumeTemplateToPrimaryStorageReply reply = new DownloadVolumeTemplateToPrimaryStorageReply(); - DownloadToCache downloadToCache = new DownloadToCache(); - downloadToCache.image = msg.getTemplateSpec(); - downloadToCache.download(new ReturnValueCompletion(msg) { + downloadImageToCache(msg.getTemplateSpec(), msg.getTargetVolumeInstallUrl(), new ReturnValueCompletion(msg) { @Override public void success(ImageCacheVO cache) { reply.setImageCache(ImageCacheInventory.valueOf(cache)); @@ -2785,6 +2900,14 @@ public void fail(ErrorCode errorCode) { }); } + private void downloadImageToCache(ImageSpec image, String targetVolumeInstallUrl, + ReturnValueCompletion completion) { + DownloadToCache downloadToCache = new DownloadToCache(); + downloadToCache.image = image; + downloadToCache.targetVolumeInstallUrl = targetVolumeInstallUrl; + downloadToCache.download(completion); + } + @Override protected void handle(DeleteBitsOnPrimaryStorageMsg msg) { inQueue().name(String.format("delete-bits-on-primarystorage-%s", self.getUuid())) @@ -4730,12 +4853,19 @@ public void done() { private void deleteImageCacheOnPrimaryStorage(DeleteImageCacheOnPrimaryStorageMsg msg, final NoErrorCompletion completion) { DeleteImageCacheOnPrimaryStorageReply reply = new DeleteImageCacheOnPrimaryStorageReply(); + String installPath = msg.getInstallPath(); + if (StringUtils.isBlank(installPath) || !installPath.startsWith("ceph://")) { + logger.debug(String.format("skip deleting non-ceph image cache[installUrl:%s] on ceph primary storage[uuid:%s]", installPath, self.getUuid())); + bus.reply(msg, reply); + completion.done(); + return; + } DeleteImageCacheCmd cmd = new DeleteImageCacheCmd(); cmd.setFsId(getSelf().getFsid()); cmd.setUuid(self.getUuid()); - cmd.imagePath = getVolumePathFromSnapshot(msg.getInstallPath()); - cmd.snapshotPath = msg.getInstallPath(); + cmd.imagePath = getVolumePathFromSnapshot(installPath); + cmd.snapshotPath = installPath; httpCall(DELETE_IMAGE_CACHE, cmd, AgentResponse.class, new ReturnValueCompletion(msg) { @Override public void success(AgentResponse rsp) { @@ -5280,6 +5410,19 @@ public void handle(ErrorCode errCode, Map data) { } + private ImageSpec makeImageSpec(String imageUuid, ImageCacheVO sourceCache) { + ImageInventory inventory = new ImageInventory(); + inventory.setUuid(imageUuid); + inventory.setActualSize(sourceCache.getSize()); + inventory.setSize(sourceCache.getSize()); + inventory.setMediaType(sourceCache.getMediaType() == null ? + ImageMediaType.RootVolumeTemplate.toString() : sourceCache.getMediaType().toString()); + + ImageSpec imageSpec = new ImageSpec(); + imageSpec.setInventory(inventory); + return imageSpec; + } + private ImageSpec makeImageSpec(VolumeInventory volume) { ImageVO image = dbf.findByUuid(volume.getRootImageUuid(), ImageVO.class); if (image == null) { @@ -5317,6 +5460,66 @@ private ImageSpec makeImageSpec(VolumeInventory volume) { return imageSpec; } + private void prepareImageCacheForReinitRootVolume(VolumeInventory volume, String targetVolumeInstallUrl, + ReturnValueCompletion completion) { + CephImageCachePoolSelector selector = getImageCachePoolSelector(); + CephImageCachePoolSelector.Selection selection = selector.select(volume.getRootImageUuid(), targetVolumeInstallUrl); + if (selection.cache == null) { + downloadImageCacheForReinitRootVolume(volume, targetVolumeInstallUrl, completion); + return; + } + + checkImageCacheBits(selection.cache, new ReturnValueCompletion(completion) { + @Override + public void success(Boolean existing) { + if (existing) { + completion.success(selection.cache); + } else { + downloadImageCacheForReinitRootVolume(volume, targetVolumeInstallUrl, completion); + } + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + private void downloadImageCacheForReinitRootVolume(VolumeInventory volume, String targetVolumeInstallUrl, + ReturnValueCompletion completion) { + CephImageCachePoolSelector selector = getImageCachePoolSelector(); + CephImageCachePoolSelector.Selection selection = selector.select(volume.getRootImageUuid(), targetVolumeInstallUrl); + List sourceCandidates = selector.listImageCaches(volume.getRootImageUuid()).stream() + .filter(CephImageCachePoolSelector::isCephImageCacheRecord) + .filter(c -> !StringUtils.equals(selection.poolName, + CephImageCachePoolSelector.getPoolName(c.getInstallUrl()))) + .collect(Collectors.toList()); + findUsableSourceCache(sourceCandidates, 0, false, new ReturnValueCompletion(completion) { + @Override + public void success(ImageCacheVO sourceCache) { + ImageSpec imageSpec; + if (sourceCache != null) { + imageSpec = makeImageSpec(volume.getRootImageUuid(), sourceCache); + } else { + try { + imageSpec = makeImageSpec(volume); + } catch (OperationFailureException e) { + completion.fail(e.getErrorCode()); + return; + } + } + + downloadImageToCache(imageSpec, targetVolumeInstallUrl, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + protected void handle(final ReInitRootVolumeFromTemplateOnPrimaryStorageMsg msg) { final ReInitRootVolumeFromTemplateOnPrimaryStorageReply reply = new ReInitRootVolumeFromTemplateOnPrimaryStorageReply(); @@ -5335,30 +5538,19 @@ public void setup() { @Override public void run(final FlowTrigger trigger, Map data) { - installUrl = Q.New(ImageCacheVO.class).eq(ImageCacheVO_.imageUuid, msg.getVolume().getRootImageUuid()). - eq(ImageCacheVO_.primaryStorageUuid, msg.getPrimaryStorageUuid()).select(ImageCacheVO_.installUrl).findValue(); - - if (installUrl != null) { - trigger.next(); - return; - } - - DownloadVolumeTemplateToPrimaryStorageMsg dmsg = new DownloadVolumeTemplateToPrimaryStorageMsg(); - dmsg.setTemplateSpec(makeImageSpec(msg.getVolume())); - dmsg.setPrimaryStorageUuid(msg.getPrimaryStorageUuid()); - bus.makeTargetServiceIdByResourceUuid(dmsg, PrimaryStorageConstant.SERVICE_ID, dmsg.getPrimaryStorageUuid()); - bus.send(dmsg, new CloudBusCallBack(trigger) { - @Override - public void run(MessageReply reply) { - if (!reply.isSuccess()) { - trigger.fail(reply.getError()); - return; - } + prepareImageCacheForReinitRootVolume(msg.getVolume(), volumePath, + new ReturnValueCompletion(trigger) { + @Override + public void success(ImageCacheVO cache) { + installUrl = ImageCacheUtil.getImageCachePath(cache.getInstallUrl()); + trigger.next(); + } - installUrl = ((DownloadVolumeTemplateToPrimaryStorageReply) reply).getImageCache().getInstallUrl(); - trigger.next(); - } - }); + @Override + public void fail(ErrorCode errorCode) { + trigger.fail(errorCode); + } + }); } }); diff --git a/sdk/src/main/java/org/zstack/sdk/AddCephPrimaryStoragePoolAction.java b/sdk/src/main/java/org/zstack/sdk/AddCephPrimaryStoragePoolAction.java index cfe72255f1f..98be1847771 100644 --- a/sdk/src/main/java/org/zstack/sdk/AddCephPrimaryStoragePoolAction.java +++ b/sdk/src/main/java/org/zstack/sdk/AddCephPrimaryStoragePoolAction.java @@ -37,7 +37,7 @@ public Result throwExceptionIfError() { @Param(required = false, maxLength = 2048, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.lang.String description; - @Param(required = true, validValues = {"Root","Data"}, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + @Param(required = true, validValues = {"Root","Data","ImageCache"}, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.lang.String type; @Param(required = false) diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/ceph/CephOperationCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/ceph/CephOperationCase.groovy index b131d740241..35e93ff096b 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/ceph/CephOperationCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/ceph/CephOperationCase.groovy @@ -183,9 +183,10 @@ class CephOperationCase extends SubCase { ps = env.inventoryByName("ceph-pri") as CephPrimaryStorageInventory bs = env.inventoryByName("ceph-bk") as CephBackupStorageInventory - assert ps.pools.securityPolicy == [DataSecurityPolicy.Copy.toString()] * 3 - assert ps.pools.diskUtilization == [0.33f] * 3 - assert ps.pools.replicatedSize == [3] * 3 + assert ps.pools.size() == 3 + assert ps.pools.securityPolicy.every { it == DataSecurityPolicy.Copy.toString() } + assert ps.pools.diskUtilization.every { it == 0.33f } + assert ps.pools.replicatedSize.every { it == 3 } assert bs.poolSecurityPolicy == DataSecurityPolicy.ErasureCode.toString() assert bs.poolDiskUtilization == 0.67f diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/CephPrimaryStorageVolumePoolsCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/CephPrimaryStorageVolumePoolsCase.groovy index 6f95c49e3e7..4d04ca629b7 100755 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/CephPrimaryStorageVolumePoolsCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/CephPrimaryStorageVolumePoolsCase.groovy @@ -2,13 +2,25 @@ package org.zstack.test.integration.storage.primary.ceph import org.springframework.http.HttpEntity import org.zstack.compute.vm.VmSystemTags +import org.zstack.core.Platform +import org.zstack.core.db.DatabaseFacade import org.zstack.core.db.Q +import org.zstack.header.image.ImageConstant.ImageMediaType +import org.zstack.header.storage.primary.ImageCacheInventory +import org.zstack.header.storage.primary.ImageCacheShadowVO +import org.zstack.header.storage.primary.ImageCacheShadowVO_ +import org.zstack.header.storage.primary.ImageCacheVolumeRefVO +import org.zstack.header.storage.primary.ImageCacheVO +import org.zstack.header.storage.primary.ImageCacheVO_ +import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceTreeVO import org.zstack.header.volume.VolumeVO import org.zstack.header.volume.VolumeVO_ +import org.zstack.storage.ceph.CephGlobalConfig import org.zstack.kvm.KVMConstant import org.zstack.sdk.* import org.zstack.storage.ceph.CephSystemTags import org.zstack.storage.ceph.primary.CephPrimaryStorageBase +import org.zstack.storage.ceph.primary.CephImageCachePoolStrategy import org.zstack.storage.ceph.primary.CephPrimaryStoragePoolVO import org.zstack.storage.ceph.primary.CephPrimaryStoragePoolVO_ import org.zstack.test.integration.storage.StorageTest @@ -29,7 +41,9 @@ class CephPrimaryStorageVolumePoolsCase extends SubCase { String NEW_ROOT_POOL_NAME = "new_root_pool" String ROOT_POOL_TYPE = "Root" String DATA_POOL_TYPE = "Data" + String IMAGE_CACHE_POOL_TYPE = "ImageCache" String NEW_DATA_POOL_NAME = "new_data_pool" + String ROOT_ONLY_POOL_NAME = "root_only_pool" @Override void setup() { @@ -85,10 +99,21 @@ class CephPrimaryStorageVolumePoolsCase extends SubCase { type = ROOT_POOL_TYPE } + pool { + poolName = NEW_ROOT_POOL_NAME + type = IMAGE_CACHE_POOL_TYPE + isCreate = false + } + pool { poolName = NEW_DATA_POOL_NAME type = DATA_POOL_TYPE } + + pool { + poolName = ROOT_ONLY_POOL_NAME + type = ROOT_POOL_TYPE + } } attachBackupStorage("ceph-bk") @@ -103,6 +128,27 @@ class CephPrimaryStorageVolumePoolsCase extends SubCase { name = "image" url = "http://zstack.org/download/test.qcow2" } + + image { + name = "image2" + url = "http://zstack.org/download/test2.qcow2" + } + + image { + name = "image3" + url = "http://zstack.org/download/test3.qcow2" + } + + image { + name = "image4" + url = "http://zstack.org/download/test4.qcow2" + } + + image { + name = "image5" + url = "http://zstack.org/download/test5.qcow2" + } + } diskOffering { @@ -140,6 +186,37 @@ class CephPrimaryStorageVolumePoolsCase extends SubCase { InstanceOfferingInventory instanceOffering2 ImageInventory image + void ensureNewRootPoolIsImageCachePool() { + if (Q.New(CephPrimaryStoragePoolVO.class) + .eq(CephPrimaryStoragePoolVO_.primaryStorageUuid, primaryStorage.uuid) + .eq(CephPrimaryStoragePoolVO_.poolName, NEW_ROOT_POOL_NAME) + .eq(CephPrimaryStoragePoolVO_.type, IMAGE_CACHE_POOL_TYPE) + .isExists()) { + return + } + + addCephPrimaryStoragePool { + primaryStorageUuid = primaryStorage.uuid + poolName = NEW_ROOT_POOL_NAME + type = IMAGE_CACHE_POOL_TYPE + isCreate = false + } + } + + void restoreCheckBitsSimulator() { + env.simulator(CephPrimaryStorageBase.CHECK_BITS_PATH) { + CephPrimaryStorageBase.CheckIsBitsExistingRsp rsp = new CephPrimaryStorageBase.CheckIsBitsExistingRsp() + rsp.setExisting(true) + return rsp + } + env.afterSimulator(CephPrimaryStorageBase.CHECK_BITS_PATH) { rsp, HttpEntity e, EnvSpec spec -> + CephPrimaryStorageBase.CheckIsBitsExistingCmd cmd = json(e.body, CephPrimaryStorageBase.CheckIsBitsExistingCmd.class) + VFS vfs = CephPrimaryStorageSpec.vfs(cmd, spec) + vfs.Assert(vfs.isFile(CephPrimaryStorageSpec.cephPathToVFSPath(cmd.installPath)), "cannot find ${cmd.installPath}") + return rsp + } + } + void testCreateDataVolumeInPool() { CephPrimaryStorageBase.CreateEmptyVolumeCmd cmd = null @@ -250,15 +327,15 @@ class CephPrimaryStorageVolumePoolsCase extends SubCase { acmd = json(e.body, CephPrimaryStorageBase.AddPoolCmd.class) } - AddCephPrimaryStoragePoolAction a = new AddCephPrimaryStoragePoolAction() - a.isCreate = true - a.poolName = LOW_POOL_NAME - a.primaryStorageUuid = primaryStorage.uuid - a.type = DATA_POOL_TYPE - a.sessionId = adminSession() - def res = a.call() + expect(AssertionError.class) { + addCephPrimaryStoragePool { + isCreate = true + poolName = LOW_POOL_NAME + primaryStorageUuid = primaryStorage.uuid + type = DATA_POOL_TYPE + } + } - assert res.error != null assert !Q.New(CephPrimaryStoragePoolVO.class).eq(CephPrimaryStoragePoolVO_.poolName, LOW_POOL_NAME).isExists() assert acmd != null assert acmd.isCreate @@ -278,22 +355,640 @@ class CephPrimaryStorageVolumePoolsCase extends SubCase { } void testAddSameCephPool() { - AddCephPrimaryStoragePoolAction action = new AddCephPrimaryStoragePoolAction() - action.primaryStorageUuid = primaryStorage.uuid - action.poolName = HIGH_POOL_NAME - action.sessionId = adminSession() - action.type = DATA_POOL_TYPE - def ret = action.call() - - AddCephPrimaryStoragePoolAction rootPoolAction = new AddCephPrimaryStoragePoolAction() - rootPoolAction.primaryStorageUuid = primaryStorage.uuid - rootPoolAction.poolName = NEW_ROOT_POOL_NAME - rootPoolAction.sessionId = adminSession() - rootPoolAction.type = ROOT_POOL_TYPE - def rootRet = rootPoolAction.call() - - assert ret.error != null - assert rootRet.error != null + expect(AssertionError.class) { + addCephPrimaryStoragePool { + primaryStorageUuid = primaryStorage.uuid + poolName = HIGH_POOL_NAME + type = DATA_POOL_TYPE + } + } + + expect(AssertionError.class) { + addCephPrimaryStoragePool { + primaryStorageUuid = primaryStorage.uuid + poolName = NEW_ROOT_POOL_NAME + type = ROOT_POOL_TYPE + } + } + } + + void testPreferVolumePoolImageCacheStrategy() { + ensureNewRootPoolIsImageCachePool() + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.PreferVolumePool.toString()) + + CephPrimaryStorageBase.CpCmd cpCmd = null + CephPrimaryStorageBase.CloneCmd cloneCmd = null + env.hijackSimulator(CephPrimaryStorageBase.CP_PATH) { rsp, HttpEntity e -> + cpCmd = json(e.body, CephPrimaryStorageBase.CpCmd.class) + return rsp + } + env.preSimulator(CephPrimaryStorageBase.CLONE_PATH) { HttpEntity e -> + cloneCmd = json(e.body, CephPrimaryStorageBase.CloneCmd.class) + } + + try { + VmInstanceInventory imageCachePoolVm = createVmInstance { + name = "image-cache-pool-vm" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = vm.imageUuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + rootVolumeSystemTags = [CephSystemTags.USE_CEPH_ROOT_POOL.instantiateTag([(CephSystemTags.USE_CEPH_ROOT_POOL_TOKEN): NEW_ROOT_POOL_NAME])] + } as VmInstanceInventory + + assert cpCmd != null + assert cpCmd.dstPath.contains(NEW_ROOT_POOL_NAME) + assert !cpCmd.skipIfExisting + assert cloneCmd != null + assert cloneCmd.srcPath.contains(NEW_ROOT_POOL_NAME) + + List caches = Q.New(ImageCacheVO.class) + .eq(ImageCacheVO_.primaryStorageUuid, primaryStorage.uuid) + .eq(ImageCacheVO_.imageUuid, vm.imageUuid) + .list() + assert caches.find { it.installUrl.contains(NEW_ROOT_POOL_NAME) } != null + assert caches.size() >= 2 + + List queriedCaches = queryImageCache { + conditions = asList("primaryStorageUuid=${primaryStorage.uuid}".toString(), "imageUuid=${vm.imageUuid}".toString()) + } + assert queriedCaches.size() >= 2 + + destroyVmInstance { + uuid = imageCachePoolVm.uuid + } + expungeVmInstance { + uuid = imageCachePoolVm.uuid + } + } finally { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + } + } + + void testDefaultImageCachePoolStrategyUsesDefaultPool() { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + String defaultImageCachePoolName = CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL.getTokenByResourceUuid(primaryStorage.uuid, CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL_TOKEN) + + List existingCaches = Q.New(ImageCacheVO.class) + .eq(ImageCacheVO_.primaryStorageUuid, primaryStorage.uuid) + .eq(ImageCacheVO_.imageUuid, vm.imageUuid) + .list() + assert !existingCaches.isEmpty() + + CephPrimaryStorageBase.CloneCmd cloneCmd = null + env.preSimulator(CephPrimaryStorageBase.CLONE_PATH) { HttpEntity e -> + cloneCmd = json(e.body, CephPrimaryStorageBase.CloneCmd.class) + } + + VmInstanceInventory defaultStrategyVm = createVmInstance { + name = "default-image-cache-pool-vm" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = vm.imageUuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + rootVolumeSystemTags = [CephSystemTags.USE_CEPH_ROOT_POOL.instantiateTag([(CephSystemTags.USE_CEPH_ROOT_POOL_TOKEN): NEW_ROOT_POOL_NAME])] + } as VmInstanceInventory + + assert cloneCmd != null + assert existingCaches.find { it.installUrl == cloneCmd.srcPath } != null + assert cloneCmd.srcPath.contains(defaultImageCachePoolName) + assert cloneCmd.dstPath.contains(NEW_ROOT_POOL_NAME) + + destroyVmInstance { + uuid = defaultStrategyVm.uuid + } + expungeVmInstance { + uuid = defaultStrategyVm.uuid + } + } + + void testDefaultStrategyCopiesCephBackupStorageImageToDefaultPool() { + ImageInventory image5 = env.inventoryByName("image5") as ImageInventory + String defaultImageCachePoolName = CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL.getTokenByResourceUuid(primaryStorage.uuid, CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL_TOKEN) + String backupStorageInstallPath = image5.backupStorageRefs[0].installPath + CephPrimaryStorageBase.CpCmd cpCmd = null + CephPrimaryStorageBase.CloneCmd cloneCmd = null + VmInstanceInventory defaultStrategyVm = null + + env.hijackSimulator(CephPrimaryStorageBase.CP_PATH) { rsp, HttpEntity e -> + cpCmd = json(e.body, CephPrimaryStorageBase.CpCmd.class) + return rsp + } + env.preSimulator(CephPrimaryStorageBase.CLONE_PATH) { HttpEntity e -> + cloneCmd = json(e.body, CephPrimaryStorageBase.CloneCmd.class) + } + + try { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + defaultStrategyVm = createVmInstance { + name = "default-strategy-copy-ceph-bs-image-vm" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = image5.uuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + rootVolumeSystemTags = [CephSystemTags.USE_CEPH_ROOT_POOL.instantiateTag([(CephSystemTags.USE_CEPH_ROOT_POOL_TOKEN): NEW_ROOT_POOL_NAME])] + } as VmInstanceInventory + + assert cpCmd != null + assert cpCmd.srcPath == backupStorageInstallPath + assert cpCmd.dstPath.contains(defaultImageCachePoolName) + assert cloneCmd != null + assert cloneCmd.srcPath.contains(defaultImageCachePoolName) + assert cloneCmd.dstPath.contains(NEW_ROOT_POOL_NAME) + } finally { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + if (defaultStrategyVm != null) { + destroyVmInstance { + uuid = defaultStrategyVm.uuid + } + expungeVmInstance { + uuid = defaultStrategyVm.uuid + } + } + } + } + + void testPreferVolumePoolFallbackToDefaultPool() { + String defaultImageCachePoolName = CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL.getTokenByResourceUuid(primaryStorage.uuid, CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL_TOKEN) + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.PreferVolumePool.toString()) + + CephPrimaryStorageBase.CloneCmd cloneCmd = null + env.preSimulator(CephPrimaryStorageBase.CLONE_PATH) { HttpEntity e -> + cloneCmd = json(e.body, CephPrimaryStorageBase.CloneCmd.class) + } + + try { + VmInstanceInventory fallbackVm = createVmInstance { + name = "prefer-volume-pool-fallback-vm" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = vm.imageUuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + rootVolumeSystemTags = [CephSystemTags.USE_CEPH_ROOT_POOL.instantiateTag([(CephSystemTags.USE_CEPH_ROOT_POOL_TOKEN): ROOT_ONLY_POOL_NAME])] + } as VmInstanceInventory + + assert cloneCmd != null + assert cloneCmd.srcPath.contains(defaultImageCachePoolName) + assert cloneCmd.dstPath.contains(ROOT_ONLY_POOL_NAME) + + destroyVmInstance { + uuid = fallbackVm.uuid + } + expungeVmInstance { + uuid = fallbackVm.uuid + } + } finally { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + } + } + + void testPreferExistingCacheStrategyPrefersDefaultPoolCache() { + ensureNewRootPoolIsImageCachePool() + + String defaultImageCachePoolName = CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL.getTokenByResourceUuid(primaryStorage.uuid, CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL_TOKEN) + VmInstanceInventory defaultCacheVm = null + CephPrimaryStorageBase.CloneCmd cloneCmd = null + + try { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.PreferVolumePool.toString()) + defaultCacheVm = createVmInstance { + name = "default-cache-vm" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = vm.imageUuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + } as VmInstanceInventory + + ImageCacheVO defaultCache = Q.New(ImageCacheVO.class) + .eq(ImageCacheVO_.primaryStorageUuid, primaryStorage.uuid) + .eq(ImageCacheVO_.imageUuid, vm.imageUuid) + .like(ImageCacheVO_.installUrl, String.format("ceph://%s/%%", defaultImageCachePoolName)) + .find() + assert defaultCache != null + + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.PreferExistingCache.toString()) + env.preSimulator(CephPrimaryStorageBase.CLONE_PATH) { HttpEntity e -> + cloneCmd = json(e.body, CephPrimaryStorageBase.CloneCmd.class) + } + + VmInstanceInventory imageCachePoolVm = createVmInstance { + name = "prefer-existing-cache-vm" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = vm.imageUuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + rootVolumeSystemTags = [CephSystemTags.USE_CEPH_ROOT_POOL.instantiateTag([(CephSystemTags.USE_CEPH_ROOT_POOL_TOKEN): NEW_ROOT_POOL_NAME])] + } as VmInstanceInventory + + assert cloneCmd != null + assert cloneCmd.srcPath.contains(defaultImageCachePoolName) + + destroyVmInstance { + uuid = imageCachePoolVm.uuid + } + expungeVmInstance { + uuid = imageCachePoolVm.uuid + } + } finally { + if (defaultCacheVm != null) { + destroyVmInstance { + uuid = defaultCacheVm.uuid + } + expungeVmInstance { + uuid = defaultCacheVm.uuid + } + } + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + } + } + + void testPreferExistingCacheStrategyUsesNonDefaultPoolCache() { + ensureNewRootPoolIsImageCachePool() + ImageInventory image2 = env.inventoryByName("image2") as ImageInventory + String defaultRootPoolName = CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_ROOT_VOLUME_POOL.getTokenByResourceUuid(primaryStorage.uuid, CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_ROOT_VOLUME_POOL_TOKEN) + VmInstanceInventory nonDefaultCacheVm = null + VmInstanceInventory preferExistingVm = null + + try { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.PreferVolumePool.toString()) + nonDefaultCacheVm = createVmInstance { + name = "non-default-cache-vm" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = image2.uuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + rootVolumeSystemTags = [CephSystemTags.USE_CEPH_ROOT_POOL.instantiateTag([(CephSystemTags.USE_CEPH_ROOT_POOL_TOKEN): NEW_ROOT_POOL_NAME])] + } as VmInstanceInventory + + assert Q.New(ImageCacheVO.class) + .eq(ImageCacheVO_.primaryStorageUuid, primaryStorage.uuid) + .eq(ImageCacheVO_.imageUuid, image2.uuid) + .like(ImageCacheVO_.installUrl, String.format("ceph://%s/%%", NEW_ROOT_POOL_NAME)) + .isExists() + + destroyVmInstance { + uuid = nonDefaultCacheVm.uuid + } + expungeVmInstance { + uuid = nonDefaultCacheVm.uuid + } + nonDefaultCacheVm = null + + CephPrimaryStorageBase.CloneCmd cloneCmd = null + env.preSimulator(CephPrimaryStorageBase.CLONE_PATH) { HttpEntity e -> + cloneCmd = json(e.body, CephPrimaryStorageBase.CloneCmd.class) + } + + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.PreferExistingCache.toString()) + preferExistingVm = createVmInstance { + name = "prefer-existing-non-default-cache-vm" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = image2.uuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + } as VmInstanceInventory + + assert cloneCmd != null + assert cloneCmd.srcPath.contains(NEW_ROOT_POOL_NAME) + assert cloneCmd.dstPath.contains(defaultRootPoolName) + } finally { + if (preferExistingVm != null) { + destroyVmInstance { + uuid = preferExistingVm.uuid + } + expungeVmInstance { + uuid = preferExistingVm.uuid + } + } + if (nonDefaultCacheVm != null) { + destroyVmInstance { + uuid = nonDefaultCacheVm.uuid + } + expungeVmInstance { + uuid = nonDefaultCacheVm.uuid + } + } + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + } + } + + void testPreferExistingCacheStrategySkipsStaleDefaultPoolCache() { + ensureNewRootPoolIsImageCachePool() + ImageInventory image2 = env.inventoryByName("image2") as ImageInventory + String defaultImageCachePoolName = CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL.getTokenByResourceUuid(primaryStorage.uuid, CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL_TOKEN) + String defaultRootPoolName = CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_ROOT_VOLUME_POOL.getTokenByResourceUuid(primaryStorage.uuid, CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_ROOT_VOLUME_POOL_TOKEN) + VmInstanceInventory defaultCacheVm = null + VmInstanceInventory nonDefaultCacheVm = null + VmInstanceInventory skipStaleDefaultVm = null + ImageCacheVO staleDefaultCache = null + ImageCacheVO nonDefaultCache = null + ImageCacheVolumeRefVO staleRef = null + CephPrimaryStorageBase.CloneCmd cloneCmd = null + + try { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + defaultCacheVm = createVmInstance { + name = "prefer-existing-stale-default-source-vm" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = image2.uuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + } as VmInstanceInventory + destroyVmInstance { + uuid = defaultCacheVm.uuid + } + expungeVmInstance { + uuid = defaultCacheVm.uuid + } + defaultCacheVm = null + + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.PreferVolumePool.toString()) + nonDefaultCacheVm = createVmInstance { + name = "prefer-existing-non-default-source-vm" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = image2.uuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + rootVolumeSystemTags = [CephSystemTags.USE_CEPH_ROOT_POOL.instantiateTag([(CephSystemTags.USE_CEPH_ROOT_POOL_TOKEN): NEW_ROOT_POOL_NAME])] + } as VmInstanceInventory + destroyVmInstance { + uuid = nonDefaultCacheVm.uuid + } + expungeVmInstance { + uuid = nonDefaultCacheVm.uuid + } + nonDefaultCacheVm = null + + staleDefaultCache = Q.New(ImageCacheVO.class) + .eq(ImageCacheVO_.primaryStorageUuid, primaryStorage.uuid) + .eq(ImageCacheVO_.imageUuid, image2.uuid) + .like(ImageCacheVO_.installUrl, String.format("ceph://%s/%%", defaultImageCachePoolName)) + .find() + nonDefaultCache = Q.New(ImageCacheVO.class) + .eq(ImageCacheVO_.primaryStorageUuid, primaryStorage.uuid) + .eq(ImageCacheVO_.imageUuid, image2.uuid) + .like(ImageCacheVO_.installUrl, String.format("ceph://%s/%%", NEW_ROOT_POOL_NAME)) + .find() + assert nonDefaultCache != null + + if (staleDefaultCache == null) { + staleDefaultCache = new ImageCacheVO() + staleDefaultCache.setPrimaryStorageUuid(primaryStorage.uuid) + staleDefaultCache.setImageUuid(image2.uuid) + staleDefaultCache.setInstallUrl(nonDefaultCache.installUrl.replaceFirst(String.format("ceph://%s/", NEW_ROOT_POOL_NAME), String.format("ceph://%s/", defaultImageCachePoolName))) + staleDefaultCache.setMediaType(nonDefaultCache.mediaType) + staleDefaultCache.setSize(nonDefaultCache.size) + staleDefaultCache.setMd5sum(nonDefaultCache.md5sum) + staleDefaultCache = bean(DatabaseFacade.class).persistAndRefresh(staleDefaultCache) + } + + staleRef = new ImageCacheVolumeRefVO() + staleRef.setImageCacheId(staleDefaultCache.id) + staleRef.setPrimaryStorageUuid(primaryStorage.uuid) + staleRef.setVolumeUuid(vm.rootVolumeUuid) + bean(DatabaseFacade.class).persist(staleRef) + + env.simulator(CephPrimaryStorageBase.CHECK_BITS_PATH) { HttpEntity e -> + CephPrimaryStorageBase.CheckIsBitsExistingCmd cmd = json(e.body, CephPrimaryStorageBase.CheckIsBitsExistingCmd.class) + CephPrimaryStorageBase.CheckIsBitsExistingRsp rsp = new CephPrimaryStorageBase.CheckIsBitsExistingRsp() + rsp.setExisting(!cmd.installPath.contains(defaultImageCachePoolName)) + return rsp + } + env.afterSimulator(CephPrimaryStorageBase.CHECK_BITS_PATH) { rsp, HttpEntity e -> + return rsp + } + env.preSimulator(CephPrimaryStorageBase.CLONE_PATH) { HttpEntity e -> + cloneCmd = json(e.body, CephPrimaryStorageBase.CloneCmd.class) + } + + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.PreferExistingCache.toString()) + skipStaleDefaultVm = createVmInstance { + name = "prefer-existing-skip-stale-default-cache-vm" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = image2.uuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + } as VmInstanceInventory + + assert dbFindById(staleDefaultCache.id, ImageCacheVO.class) == null + assert dbFindById(staleRef.id, ImageCacheVolumeRefVO.class) == null + assert dbFindById(nonDefaultCache.id, ImageCacheVO.class) != null + assert cloneCmd != null + assert cloneCmd.srcPath.contains(NEW_ROOT_POOL_NAME) + assert cloneCmd.dstPath.contains(defaultRootPoolName) + } finally { + if (skipStaleDefaultVm != null) { + destroyVmInstance { + uuid = skipStaleDefaultVm.uuid + } + expungeVmInstance { + uuid = skipStaleDefaultVm.uuid + } + } + if (nonDefaultCacheVm != null) { + destroyVmInstance { + uuid = nonDefaultCacheVm.uuid + } + expungeVmInstance { + uuid = nonDefaultCacheVm.uuid + } + } + if (defaultCacheVm != null) { + destroyVmInstance { + uuid = defaultCacheVm.uuid + } + expungeVmInstance { + uuid = defaultCacheVm.uuid + } + } + restoreCheckBitsSimulator() + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + } + } + + void testMissingSelectedCacheBitsCopiesFromOtherPool() { + ensureNewRootPoolIsImageCachePool() + ImageInventory image3 = env.inventoryByName("image3") as ImageInventory + String defaultImageCachePoolName = CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL.getTokenByResourceUuid(primaryStorage.uuid, CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL_TOKEN) + VFS vfs = CephPrimaryStorageSpec.vfs1("7ff218d9-f525-435f-8a40-3618d1772a64", env) + + ImageCacheVO sourceCache = new ImageCacheVO() + sourceCache.setPrimaryStorageUuid(primaryStorage.uuid) + sourceCache.setImageUuid(image3.uuid) + sourceCache.setInstallUrl(String.format("ceph://%s/%s-source@%s-source", defaultImageCachePoolName, image3.uuid, image3.uuid)) + sourceCache.setMediaType(ImageMediaType.RootVolumeTemplate) + sourceCache.setSize(SizeUnit.GIGABYTE.toByte(1)) + sourceCache.setMd5sum("not calculated") + sourceCache = bean(DatabaseFacade.class).persistAndRefresh(sourceCache) + + ImageCacheVO staleCache = new ImageCacheVO() + staleCache.setPrimaryStorageUuid(primaryStorage.uuid) + staleCache.setImageUuid(image3.uuid) + staleCache.setInstallUrl(String.format("ceph://%s/%s-stale@%s-stale", NEW_ROOT_POOL_NAME, image3.uuid, image3.uuid)) + staleCache.setMediaType(ImageMediaType.RootVolumeTemplate) + staleCache.setSize(SizeUnit.GIGABYTE.toByte(1)) + staleCache.setMd5sum("not calculated") + staleCache = bean(DatabaseFacade.class).persistAndRefresh(staleCache) + + String sourceCacheVolumePath = CephPrimaryStorageSpec.cephPathToVFSPath(sourceCache.installUrl.split("@")[0]) + String sourceCachePath = CephPrimaryStorageSpec.cephPathToVFSPath(sourceCache.installUrl) + vfs.createDirectories(String.format("/%s", defaultImageCachePoolName)) + if (!vfs.exists(sourceCacheVolumePath)) { + vfs.createCephRaw(sourceCacheVolumePath, 0L) + } + if (!vfs.exists(sourceCachePath)) { + vfs.createCephRaw(sourceCachePath, 0L) + } + String staleCachePath = CephPrimaryStorageSpec.cephPathToVFSPath(staleCache.installUrl) + String staleCacheVolumePath = CephPrimaryStorageSpec.cephPathToVFSPath(staleCache.installUrl.split("@")[0]) + if (vfs.exists(staleCachePath)) { + vfs.delete(staleCachePath) + } + if (vfs.exists(staleCacheVolumePath)) { + vfs.delete(staleCacheVolumePath) + } + + CephPrimaryStorageBase.CpCmd cpCmd = null + CephPrimaryStorageBase.CloneCmd cloneCmd = null + env.simulator(CephPrimaryStorageBase.CHECK_BITS_PATH) { HttpEntity e -> + CephPrimaryStorageBase.CheckIsBitsExistingCmd cmd = json(e.body, CephPrimaryStorageBase.CheckIsBitsExistingCmd.class) + CephPrimaryStorageBase.CheckIsBitsExistingRsp rsp = new CephPrimaryStorageBase.CheckIsBitsExistingRsp() + rsp.setExisting(!cmd.installPath.contains(NEW_ROOT_POOL_NAME)) + return rsp + } + env.afterSimulator(CephPrimaryStorageBase.CHECK_BITS_PATH) { rsp, HttpEntity e -> + return rsp + } + env.hijackSimulator(CephPrimaryStorageBase.CP_PATH) { rsp, HttpEntity e -> + cpCmd = json(e.body, CephPrimaryStorageBase.CpCmd.class) + return rsp + } + env.preSimulator(CephPrimaryStorageBase.CLONE_PATH) { HttpEntity e -> + cloneCmd = json(e.body, CephPrimaryStorageBase.CloneCmd.class) + } + + try { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.PreferVolumePool.toString()) + VmInstanceInventory staleCacheVm = createVmInstance { + name = "missing-selected-cache-bits-vm" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = image3.uuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + rootVolumeSystemTags = [CephSystemTags.USE_CEPH_ROOT_POOL.instantiateTag([(CephSystemTags.USE_CEPH_ROOT_POOL_TOKEN): NEW_ROOT_POOL_NAME])] + } as VmInstanceInventory + + assert dbFindById(staleCache.id, ImageCacheVO.class) == null + assert cpCmd != null + assert cpCmd.srcPath.contains(defaultImageCachePoolName) + assert cpCmd.dstPath.contains(NEW_ROOT_POOL_NAME) + assert cloneCmd != null + assert cloneCmd.srcPath.contains(NEW_ROOT_POOL_NAME) + + destroyVmInstance { + uuid = staleCacheVm.uuid + } + expungeVmInstance { + uuid = staleCacheVm.uuid + } + } finally { + restoreCheckBitsSimulator() + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + } + } + + void testCleanupImageCacheKeepsReferencedCacheOnly() { + restoreCheckBitsSimulator() + String cleanupImageUuid = "cleanup-image-cache-image" + String reuseImageUuid = "cleanup-image-cache-reuse-image" + String reuseTreeUuid = null + String defaultImageCachePoolName = CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL.getTokenByResourceUuid(primaryStorage.uuid, CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL_TOKEN) + + try { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + + ImageCacheVO protectedCache = new ImageCacheVO() + protectedCache.setPrimaryStorageUuid(primaryStorage.uuid) + protectedCache.setImageUuid(cleanupImageUuid) + protectedCache.setInstallUrl(String.format("ceph://%s/%s-protected@%s-protected", NEW_ROOT_POOL_NAME, cleanupImageUuid, cleanupImageUuid)) + protectedCache.setMediaType(ImageMediaType.RootVolumeTemplate) + protectedCache.setSize(SizeUnit.GIGABYTE.toByte(1)) + protectedCache.setMd5sum("not calculated") + protectedCache = bean(DatabaseFacade.class).persistAndRefresh(protectedCache) + + ImageCacheVO deletedCache = new ImageCacheVO() + deletedCache.setPrimaryStorageUuid(primaryStorage.uuid) + deletedCache.setImageUuid(cleanupImageUuid) + deletedCache.setInstallUrl(String.format("ceph://%s/%s-deleted@%s-deleted", defaultImageCachePoolName, cleanupImageUuid, cleanupImageUuid)) + deletedCache.setMediaType(ImageMediaType.RootVolumeTemplate) + deletedCache.setSize(SizeUnit.GIGABYTE.toByte(1)) + deletedCache.setMd5sum("not calculated") + deletedCache = bean(DatabaseFacade.class).persistAndRefresh(deletedCache) + + ImageCacheVO reuseCache = new ImageCacheVO() + reuseCache.setPrimaryStorageUuid(primaryStorage.uuid) + reuseCache.setImageUuid(reuseImageUuid) + reuseCache.setInstallUrl(String.format("volumeSnapshotReuse://%s", reuseImageUuid)) + reuseCache.setMediaType(ImageMediaType.RootVolumeTemplate) + reuseCache.setSize(SizeUnit.GIGABYTE.toByte(1)) + reuseCache.setMd5sum("not calculated") + reuseCache = bean(DatabaseFacade.class).persistAndRefresh(reuseCache) + + VolumeSnapshotReferenceTreeVO reuseTree = new VolumeSnapshotReferenceTreeVO() + reuseTree.setUuid(Platform.getUuid()) + reuseTree.setRootImageUuid(reuseImageUuid) + reuseTree.setRootInstallUrl(reuseCache.installUrl) + reuseTree.setPrimaryStorageUuid(primaryStorage.uuid) + bean(DatabaseFacade.class).persist(reuseTree) + reuseTreeUuid = reuseTree.uuid + + VFS vfs = CephPrimaryStorageSpec.vfs1("7ff218d9-f525-435f-8a40-3618d1772a64", env) + vfs.createDirectories(String.format("/%s", NEW_ROOT_POOL_NAME)) + vfs.createDirectories(String.format("/%s", defaultImageCachePoolName)) + [protectedCache, deletedCache].each { ImageCacheVO cache -> + String imagePath = CephPrimaryStorageSpec.cephPathToVFSPath(cache.installUrl.split("@")[0]) + String snapshotPath = CephPrimaryStorageSpec.cephPathToVFSPath(cache.installUrl) + if (!vfs.exists(imagePath)) { + vfs.createCephRaw(imagePath, 0L) + } + if (!vfs.exists(snapshotPath)) { + vfs.createCephRaw(snapshotPath, 0L) + } + } + + ImageCacheVolumeRefVO ref = new ImageCacheVolumeRefVO() + ref.setImageCacheId(protectedCache.id) + ref.setPrimaryStorageUuid(primaryStorage.uuid) + ref.setVolumeUuid(vm.rootVolumeUuid) + bean(DatabaseFacade.class).persist(ref) + + List deleteCmds = [] + env.preSimulator(CephPrimaryStorageBase.DELETE_IMAGE_CACHE) { HttpEntity e -> + deleteCmds.add(json(e.body, CephPrimaryStorageBase.DeleteImageCacheCmd.class)) + } + + cleanUpImageCacheOnPrimaryStorage { + uuid = primaryStorage.uuid + force = true + } + + retryInSecs { + assert dbFindById(protectedCache.id, ImageCacheVO.class) != null + assert dbFindById(deletedCache.id, ImageCacheVO.class) == null + assert dbFindById(reuseCache.id, ImageCacheVO.class) != null + assert !Q.New(ImageCacheShadowVO.class).eq(ImageCacheShadowVO_.installUrl, deletedCache.installUrl).isExists() + assert !vfs.exists(CephPrimaryStorageSpec.cephPathToVFSPath(deletedCache.installUrl.split("@")[0])) + assert !vfs.exists(CephPrimaryStorageSpec.cephPathToVFSPath(deletedCache.installUrl)) + assert deleteCmds.find { it.snapshotPath == deletedCache.installUrl } != null + assert deleteCmds.find { it.snapshotPath == protectedCache.installUrl } == null + assert deleteCmds.find { it.snapshotPath == reuseCache.installUrl } == null + } + } finally { + if (reuseTreeUuid != null) { + bean(DatabaseFacade.class).removeByPrimaryKey(reuseTreeUuid, VolumeSnapshotReferenceTreeVO.class) + } + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + } } void testAddCephPoolWithChinese(){ @@ -330,6 +1025,14 @@ class CephPrimaryStorageVolumePoolsCase extends SubCase { } void testReimageVmAndAllocatePool() { + ensureNewRootPoolIsImageCachePool() + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.PreferVolumePool.toString()) + + CephPrimaryStorageBase.CloneCmd cloneCmd = null + env.preSimulator(CephPrimaryStorageBase.CLONE_PATH) { HttpEntity e -> + cloneCmd = json(e.body, CephPrimaryStorageBase.CloneCmd.class) + } + L3NetworkSpec l3Spec = env.specByName("l3") as L3NetworkSpec VmInstanceInventory new_root_pool_vm = createVmInstance { name = "new_root_pool_vm" @@ -345,12 +1048,77 @@ class CephPrimaryStorageVolumePoolsCase extends SubCase { uuid = new_root_pool_vm.uuid } - reimageVmInstance { - vmInstanceUuid = new_root_pool_vm.uuid + cloneCmd = null + try { + reimageVmInstance { + vmInstanceUuid = new_root_pool_vm.uuid + } + } finally { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) } String VolumeInstallPath = Q.New(VolumeVO.class).select(VolumeVO_.installPath).eq(VolumeVO_.uuid, new_root_pool_vm.rootVolumeUuid).findValue() assert VolumeInstallPath.contains(NEW_ROOT_POOL_NAME) + assert cloneCmd != null + assert cloneCmd.srcPath.contains(NEW_ROOT_POOL_NAME) + } + + void testReimageVmCopiesImageToSelectedPoolAfterImageDeleted() { + ensureNewRootPoolIsImageCachePool() + ImageInventory image4 = env.inventoryByName("image4") as ImageInventory + VmInstanceInventory reimageVm = null + CephPrimaryStorageBase.CpCmd cpCmd = null + CephPrimaryStorageBase.CloneCmd cloneCmd = null + env.hijackSimulator(CephPrimaryStorageBase.CP_PATH) { rsp, HttpEntity e -> + cpCmd = json(e.body, CephPrimaryStorageBase.CpCmd.class) + return rsp + } + env.preSimulator(CephPrimaryStorageBase.CLONE_PATH) { HttpEntity e -> + cloneCmd = json(e.body, CephPrimaryStorageBase.CloneCmd.class) + } + + try { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + reimageVm = createVmInstance { + name = "reimage-vm-after-image-deleted" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = image4.uuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + rootVolumeSystemTags = [CephSystemTags.USE_CEPH_ROOT_POOL.instantiateTag([(CephSystemTags.USE_CEPH_ROOT_POOL_TOKEN): NEW_ROOT_POOL_NAME])] + } as VmInstanceInventory + + stopVmInstance { + uuid = reimageVm.uuid + } + + deleteImage { + uuid = image4.uuid + } + + cpCmd = null + cloneCmd = null + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.PreferVolumePool.toString()) + reimageVmInstance { + vmInstanceUuid = reimageVm.uuid + } + + assert cpCmd != null + assert cpCmd.dstPath.contains(NEW_ROOT_POOL_NAME) + assert cloneCmd != null + assert cloneCmd.srcPath.contains(NEW_ROOT_POOL_NAME) + } finally { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + + if (reimageVm != null) { + destroyVmInstance { + uuid = reimageVm.uuid + } + expungeVmInstance { + uuid = reimageVm.uuid + } + } + } } void testCreateVmInstanceWithCustomDiskOffering() { @@ -433,8 +1201,18 @@ class CephPrimaryStorageVolumePoolsCase extends SubCase { testAddPoolWithCheckExistenceFailure() testQueryPool() testAddSameCephPool() + testPreferVolumePoolImageCacheStrategy() + testDefaultImageCachePoolStrategyUsesDefaultPool() + testDefaultStrategyCopiesCephBackupStorageImageToDefaultPool() + testPreferVolumePoolFallbackToDefaultPool() + testPreferExistingCacheStrategyPrefersDefaultPoolCache() + testPreferExistingCacheStrategyUsesNonDefaultPoolCache() + testPreferExistingCacheStrategySkipsStaleDefaultPoolCache() + testMissingSelectedCacheBitsCopiesFromOtherPool() + testCleanupImageCacheKeepsReferencedCacheOnly() testAddCephPoolWithChinese() testReimageVmAndAllocatePool() + testReimageVmCopiesImageToSelectedPoolAfterImageDeleted() testCreateVmInstanceWithCustomDiskOffering() } } diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/capacity/CephOpenSourcePoolCapacityCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/capacity/CephOpenSourcePoolCapacityCase.groovy index 09c74368ad3..edf752b09b1 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/capacity/CephOpenSourcePoolCapacityCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/capacity/CephOpenSourcePoolCapacityCase.groovy @@ -212,4 +212,4 @@ class CephOpenSourcePoolCapacityCase extends SubCase { assert bsCapacity.totalCapacity == 106300440576 // 99G env.cleanSimulatorAndMessageHandlers() } -} \ No newline at end of file +} diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/sandstone/capacity/CephSandStonePoolCapacityCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/sandstone/capacity/CephSandStonePoolCapacityCase.groovy index df1fc1d7d0e..0eb92a7cfd7 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/sandstone/capacity/CephSandStonePoolCapacityCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/sandstone/capacity/CephSandStonePoolCapacityCase.groovy @@ -163,4 +163,5 @@ class CephSandStonePoolCapacityCase extends SubCase { assert afterBs.availableCapacity == bs.availableCapacity + addSize assert afterBs.totalCapacity == bs.totalCapacity + addSize } + } diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/xsky/capacity/CephXskyPoolCapacityCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/xsky/capacity/CephXskyPoolCapacityCase.groovy index 356599909f6..f23ed4a3c3a 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/xsky/capacity/CephXskyPoolCapacityCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/xsky/capacity/CephXskyPoolCapacityCase.groovy @@ -205,4 +205,5 @@ class CephXskyPoolCapacityCase extends SubCase { SQL.New(VolumeSnapshotVO.class).eq(VolumeSnapshotVO_.uuid, rootSnapshot.uuid) .set(VolumeSnapshotVO_.primaryStorageInstallPath, volumeSnapshotInstallPath).update() } + } diff --git a/testlib/src/main/java/org/zstack/testlib/CephPrimaryStoragePoolSpec.groovy b/testlib/src/main/java/org/zstack/testlib/CephPrimaryStoragePoolSpec.groovy index c0cfec41607..12d6eb293b4 100755 --- a/testlib/src/main/java/org/zstack/testlib/CephPrimaryStoragePoolSpec.groovy +++ b/testlib/src/main/java/org/zstack/testlib/CephPrimaryStoragePoolSpec.groovy @@ -13,6 +13,8 @@ class CephPrimaryStoragePoolSpec extends Spec { String description @SpecParam(required = true) String type + @SpecParam + boolean isCreate = true CephPrimaryStoragePoolInventory inventory @@ -40,7 +42,7 @@ class CephPrimaryStoragePoolSpec extends Spec { delegate.resourceUuid = uuid delegate.sessionId = sessionId delegate.type = type - delegate.isCreate = true + delegate.isCreate = isCreate } return id(poolName, inventory.uuid) diff --git a/testlib/src/main/java/org/zstack/testlib/CephPrimaryStorageSpec.groovy b/testlib/src/main/java/org/zstack/testlib/CephPrimaryStorageSpec.groovy index e8f90f1d110..9521d8dec87 100755 --- a/testlib/src/main/java/org/zstack/testlib/CephPrimaryStorageSpec.groovy +++ b/testlib/src/main/java/org/zstack/testlib/CephPrimaryStorageSpec.groovy @@ -37,8 +37,10 @@ class CephPrimaryStorageSpec extends PrimaryStorageSpec { String rootVolumePoolName = "pri-v-r-" + Platform.getUuid() @SpecParam String dataVolumePoolName = "pri-v-d-" + Platform.getUuid() - @SpecParam - String imageCachePoolName = "pri-c-" + Platform.getUuid() + @SpecParam + String imageCachePoolName = "pri-c-" + Platform.getUuid() + @SpecParam + String extraImageCachePoolName CephPrimaryStorageSpec(EnvSpec envSpec) { super(envSpec) @@ -78,9 +80,10 @@ class CephPrimaryStorageSpec extends PrimaryStorageSpec { } class CephPrimaryStorageStruct { - String rootVolumePoolName - String dataVolumePoolName - String imageCachePoolName + String rootVolumePoolName + String dataVolumePoolName + String imageCachePoolName + String extraImageCachePoolName } static class Simulators implements Simulator { @@ -496,18 +499,19 @@ class CephPrimaryStorageSpec extends PrimaryStorageSpec { return new CephPrimaryStorageBase.AgentResponse() } - simulator(CephPrimaryStorageBase.ADD_POOL_PATH) { HttpEntity entity, EnvSpec spec -> - def cmd = JSONObjectUtil.toObject(entity.body, CephPrimaryStorageBase.AddPoolCmd.class) + simulator(CephPrimaryStorageBase.ADD_POOL_PATH) { HttpEntity entity, EnvSpec spec -> + def cmd = JSONObjectUtil.toObject(entity.body, CephPrimaryStorageBase.AddPoolCmd.class) CephPrimaryStorageSpec cspec = spec.specByUuid(cmd.uuid) CephPrimaryStorageBase.AddPoolRsp rsp = new CephPrimaryStorageBase.AddPoolRsp() - rsp.totalCapacity = cspec.totalCapacity - rsp.availableCapacity = cspec.availableCapacity - long rootSize = cspec.availableCapacity / 3 - long dataSize = cspec.availableCapacity / 3 - long cacheSize = cspec.totalCapacity - rootSize - dataSize - rsp.setAvailableCapacity(SizeUnit.GIGABYTE.toByte(100)) - rsp.setTotalCapacity(SizeUnit.GIGABYTE.toByte(100)) + def newPoolCapacity = SizeUnit.GIGABYTE.toByte(100) + rsp.totalCapacity = newPoolCapacity + rsp.availableCapacity = newPoolCapacity + long rootSize = cspec.availableCapacity / 3 + long dataSize = cspec.availableCapacity / 3 + long cacheSize = cspec.totalCapacity - rootSize - dataSize + rsp.setAvailableCapacity(newPoolCapacity) + rsp.setTotalCapacity(newPoolCapacity) List poolCapacities = [ new CephPoolCapacity( name: cspec.rootVolumePoolName, @@ -539,17 +543,17 @@ class CephPrimaryStorageSpec extends PrimaryStorageSpec { diskUtilization: 0.33, relatedOsds: 'osd.3' ), - new CephPoolCapacity( - name: cmd.poolName, - availableCapacity: SizeUnit.GIGABYTE.toByte(100), - usedCapacity: 0, - totalCapacity: SizeUnit.GIGABYTE.toByte(100), - securityPolicy: DataSecurityPolicy.Copy.toString(), - replicatedSize: 3, - diskUtilization: 0.33, - relatedOsds: "osd.4" - ) - ] + new CephPoolCapacity( + name: cmd.poolName, + availableCapacity: newPoolCapacity, + usedCapacity: 0, + totalCapacity: newPoolCapacity, + securityPolicy: DataSecurityPolicy.Copy.toString(), + replicatedSize: 3, + diskUtilization: 0.33, + relatedOsds: 'osd.4' + ) + ] rsp.setPoolCapacities(poolCapacities) return rsp @@ -568,7 +572,7 @@ class CephPrimaryStorageSpec extends PrimaryStorageSpec { return } - if (f.parent?.setVolumeChainInstallPaths() == snapshotPath) { + if (f.parent?.pathString() == snapshotPath) { children.add(f.pathString()) } } @@ -820,14 +824,24 @@ class CephPrimaryStorageSpec extends PrimaryStorageSpec { delegate.imageCachePoolName = imageCachePoolName } as PrimaryStorageInventory - postCreate { - inventory = queryCephPrimaryStorage { - conditions=["uuid=${inventory.uuid}".toString()] - }[0] - } - - return id(name, inventory.uuid) - } + postCreate { + inventory = queryCephPrimaryStorage { + conditions=["uuid=${inventory.uuid}".toString()] + }[0] + } + + if (extraImageCachePoolName != null) { + addCephPrimaryStoragePool { + delegate.primaryStorageUuid = inventory.uuid + delegate.poolName = extraImageCachePoolName + delegate.type = CephPrimaryStoragePoolType.ImageCache.toString() + delegate.isCreate = true + delegate.sessionId = sessionId + } + } + + return id(name, inventory.uuid) + } CephPrimaryStoragePoolSpec pool(@DelegatesTo(strategy = Closure.DELEGATE_FIRST, value = CephPrimaryStoragePoolSpec.class) Closure c) { def spec = new CephPrimaryStoragePoolSpec(envSpec)