diff --git a/conf/globalConfig/ceph.xml b/conf/globalConfig/ceph.xml index ccde2ee2806..58eb4f50332 100755 --- a/conf/globalConfig/ceph.xml +++ b/conf/globalConfig/ceph.xml @@ -16,6 +16,14 @@ java.lang.Long + + imageCache.poolStrategy + strategy to select ceph image cache pool when preparing root volume image cache. + ceph + DefaultImageCachePool + java.lang.String + + trash.cleanup.interval interval to cleanup image trash on primary storage, in seconds. diff --git a/header/src/main/java/org/zstack/header/storage/primary/DownloadVolumeTemplateToPrimaryStorageMsg.java b/header/src/main/java/org/zstack/header/storage/primary/DownloadVolumeTemplateToPrimaryStorageMsg.java index d4286ab8176..ff43efe1a33 100644 --- a/header/src/main/java/org/zstack/header/storage/primary/DownloadVolumeTemplateToPrimaryStorageMsg.java +++ b/header/src/main/java/org/zstack/header/storage/primary/DownloadVolumeTemplateToPrimaryStorageMsg.java @@ -7,6 +7,7 @@ public class DownloadVolumeTemplateToPrimaryStorageMsg extends NeedReplyMessage private String primaryStorageUuid; private ImageSpec templateSpec; private String hostUuid; + private String targetVolumeInstallUrl; @Override public String getPrimaryStorageUuid() { @@ -32,4 +33,12 @@ public String getHostUuid() { public void setHostUuid(String hostUuid) { this.hostUuid = hostUuid; } + + public String getTargetVolumeInstallUrl() { + return targetVolumeInstallUrl; + } + + public void setTargetVolumeInstallUrl(String targetVolumeInstallUrl) { + this.targetVolumeInstallUrl = targetVolumeInstallUrl; + } } diff --git a/plugin/ceph/src/main/java/org/zstack/storage/ceph/CephGlobalConfig.java b/plugin/ceph/src/main/java/org/zstack/storage/ceph/CephGlobalConfig.java index c3a42f79013..88d2664c046 100755 --- a/plugin/ceph/src/main/java/org/zstack/storage/ceph/CephGlobalConfig.java +++ b/plugin/ceph/src/main/java/org/zstack/storage/ceph/CephGlobalConfig.java @@ -14,6 +14,8 @@ public class CephGlobalConfig { @GlobalConfigValidation(numberGreaterThan = 1) public static GlobalConfig IMAGE_CACHE_CLEANUP_INTERVAL = new GlobalConfig(CATEGORY, "imageCache.cleanup.interval"); + @GlobalConfigValidation(validValues = {"DefaultImageCachePool", "PreferVolumePool", "PreferExistingCache"}) + public static GlobalConfig IMAGE_CACHE_POOL_STRATEGY = new GlobalConfig(CATEGORY, "imageCache.poolStrategy"); @GlobalConfigValidation public static GlobalConfig PRIMARY_STORAGE_DELETE_POOL = new GlobalConfig(CATEGORY, "primaryStorage.deletePool"); @GlobalConfigValidation(numberGreaterThan = 0) diff --git a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/APIAddCephPrimaryStoragePoolMsg.java b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/APIAddCephPrimaryStoragePoolMsg.java index 458a3060d51..a6550f225c9 100755 --- a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/APIAddCephPrimaryStoragePoolMsg.java +++ b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/APIAddCephPrimaryStoragePoolMsg.java @@ -28,7 +28,7 @@ public class APIAddCephPrimaryStoragePoolMsg extends APICreateMessage implements private String aliasName; @APIParam(maxLength = 2048, required = false) private String description; - @APIParam(validValues = {"Root", "Data"}) + @APIParam(validValues = {"Root", "Data", "ImageCache"}) private String type; private boolean isCreate; diff --git a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephImageCachePoolSelector.java b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephImageCachePoolSelector.java new file mode 100644 index 00000000000..40132b84769 --- /dev/null +++ b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephImageCachePoolSelector.java @@ -0,0 +1,165 @@ +package org.zstack.storage.ceph.primary; + +import org.apache.commons.lang.StringUtils; +import org.zstack.core.db.Q; +import org.zstack.header.image.ImageConstant; +import org.zstack.header.image.ImageInventory; +import org.zstack.header.storage.primary.ImageCacheVO; +import org.zstack.header.storage.primary.ImageCacheVO_; +import org.zstack.storage.ceph.CephGlobalConfig; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.List; + +final class CephImageCachePoolSelector { + private static final String CEPH_INSTALL_URL_PREFIX = "ceph://"; + static final String SNAPSHOT_REUSE_POOL_NAME = "snapshot-reuse"; + + private static final CLogger logger = Utils.getLogger(CephImageCachePoolSelector.class); + + private final String primaryStorageUuid; + private final String defaultPoolName; + + CephImageCachePoolSelector(String primaryStorageUuid, String defaultPoolName) { + this.primaryStorageUuid = primaryStorageUuid; + this.defaultPoolName = defaultPoolName; + } + + Selection select(ImageInventory image, String targetVolumeInstallUrl) { + if (isSnapshotReuseImage(image)) { + return new Selection(CephImageCachePoolStrategy.DefaultImageCachePool, + SNAPSHOT_REUSE_POOL_NAME, findSnapshotReuseImageCache(image)); + } + + return select(image.getUuid(), targetVolumeInstallUrl, getStrategy()); + } + + Selection select(ImageInventory image, String targetVolumeInstallUrl, CephImageCachePoolStrategy strategy) { + if (isSnapshotReuseImage(image)) { + return new Selection(strategy, SNAPSHOT_REUSE_POOL_NAME, findSnapshotReuseImageCache(image)); + } + + return select(image.getUuid(), targetVolumeInstallUrl, strategy); + } + + Selection select(String imageUuid, String targetVolumeInstallUrl) { + return select(imageUuid, targetVolumeInstallUrl, getStrategy()); + } + + private Selection select(String imageUuid, String targetVolumeInstallUrl, CephImageCachePoolStrategy strategy) { + List caches = listImageCaches(imageUuid); + String poolName = selectPool(targetVolumeInstallUrl, caches, strategy); + return new Selection(strategy, poolName, findCacheInPool(caches, poolName)); + } + + private Selection selectInPool(String imageUuid, String poolName, CephImageCachePoolStrategy strategy) { + return new Selection(strategy, poolName, findCacheInPool(listImageCaches(imageUuid), poolName)); + } + + Selection selectInPool(ImageInventory image, String poolName, CephImageCachePoolStrategy strategy) { + if (isSnapshotReuseImage(image)) { + return new Selection(strategy, poolName, findSnapshotReuseImageCache(image)); + } + + return selectInPool(image.getUuid(), poolName, strategy); + } + + List listImageCaches(String imageUuid) { + return Q.New(ImageCacheVO.class) + .eq(ImageCacheVO_.primaryStorageUuid, primaryStorageUuid) + .eq(ImageCacheVO_.imageUuid, imageUuid) + .list(); + } + + static boolean isCephImageCacheRecord(ImageCacheVO cache) { + return cache != null && cache.getInstallUrl() != null && cache.getInstallUrl().startsWith(CEPH_INSTALL_URL_PREFIX); + } + + static boolean isSnapshotReuseImage(ImageInventory image) { + return image != null && StringUtils.startsWith(image.getUrl(), ImageConstant.SNAPSHOT_REUSE_IMAGE_SCHEMA); + } + + static String getPoolName(String installUrl) { + if (StringUtils.isBlank(installUrl) || !installUrl.startsWith(CEPH_INSTALL_URL_PREFIX)) { + return null; + } + + String path = installUrl.substring(CEPH_INSTALL_URL_PREFIX.length()); + int index = path.indexOf("/"); + return index > 0 ? path.substring(0, index) : null; + } + + private CephImageCachePoolStrategy getStrategy() { + String strategy = CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.value(String.class); + try { + return CephImageCachePoolStrategy.valueOf(strategy); + } catch (RuntimeException e) { + logger.warn(String.format("invalid ceph image cache pool strategy[%s], use DefaultImageCachePool", strategy)); + return CephImageCachePoolStrategy.DefaultImageCachePool; + } + } + + private String selectPool(String targetVolumeInstallUrl, List caches, + CephImageCachePoolStrategy strategy) { + if (strategy == CephImageCachePoolStrategy.PreferVolumePool) { + String targetPoolName = getPoolName(targetVolumeInstallUrl); + return hasImageCachePoolRole(targetPoolName) ? targetPoolName : defaultPoolName; + } + + if (strategy == CephImageCachePoolStrategy.PreferExistingCache) { + boolean defaultPoolCacheExists = caches.stream() + .anyMatch(c -> defaultPoolName.equals(getPoolName(c.getInstallUrl()))); + if (defaultPoolCacheExists) { + return defaultPoolName; + } + + return caches.stream() + .map(c -> getPoolName(c.getInstallUrl())) + .filter(StringUtils::isNotBlank) + .findFirst() + .orElse(defaultPoolName); + } + + return defaultPoolName; + } + + private ImageCacheVO findCacheInPool(List caches, String poolName) { + return caches.stream() + .filter(c -> poolName.equals(getPoolName(c.getInstallUrl()))) + .findFirst() + .orElse(null); + } + + private ImageCacheVO findSnapshotReuseImageCache(ImageInventory image) { + return Q.New(ImageCacheVO.class) + .eq(ImageCacheVO_.primaryStorageUuid, primaryStorageUuid) + .eq(ImageCacheVO_.imageUuid, image.getUuid()) + .eq(ImageCacheVO_.installUrl, image.getUrl()) + .find(); + } + + private boolean hasImageCachePoolRole(String poolName) { + if (StringUtils.isBlank(poolName)) { + return false; + } + + return Q.New(CephPrimaryStoragePoolVO.class) + .eq(CephPrimaryStoragePoolVO_.primaryStorageUuid, primaryStorageUuid) + .eq(CephPrimaryStoragePoolVO_.poolName, poolName) + .eq(CephPrimaryStoragePoolVO_.type, CephPrimaryStoragePoolType.ImageCache.toString()) + .isExists(); + } + + static final class Selection { + final CephImageCachePoolStrategy strategy; + final String poolName; + final ImageCacheVO cache; + + Selection(CephImageCachePoolStrategy strategy, String poolName, ImageCacheVO cache) { + this.strategy = strategy; + this.poolName = poolName; + this.cache = cache; + } + } +} diff --git a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephImageCachePoolStrategy.java b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephImageCachePoolStrategy.java new file mode 100644 index 00000000000..decb0e4033b --- /dev/null +++ b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephImageCachePoolStrategy.java @@ -0,0 +1,7 @@ +package org.zstack.storage.ceph.primary; + +public enum CephImageCachePoolStrategy { + DefaultImageCachePool, + PreferVolumePool, + PreferExistingCache +} diff --git a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java index 53816200465..3328d14bfe7 100755 --- a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java +++ b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java @@ -825,6 +825,7 @@ public static class CpCmd extends AgentCommand implements HasThreadContext { String srcPath; String dstPath; boolean shareable; + boolean skipIfExisting; } public static class CpRsp extends AgentResponse { @@ -2155,21 +2156,90 @@ protected void handle(final InstantiateVolumeOnPrimaryStorageMsg msg) { } } + private CephImageCachePoolSelector getImageCachePoolSelector() { + return new CephImageCachePoolSelector(self.getUuid(), getDefaultImageCachePoolName()); + } + + private void checkImageCacheBits(ImageCacheVO cache, ReturnValueCompletion completion) { + CheckIsBitsExistingCmd cmd = new CheckIsBitsExistingCmd(); + cmd.setInstallPath(ImageCacheUtil.getImageCachePath(cache.getInstallUrl())); + httpCall(CHECK_BITS_PATH, cmd, CheckIsBitsExistingRsp.class, new ReturnValueCompletion(completion) { + @Override + public void success(CheckIsBitsExistingRsp returnValue) { + completion.success(returnValue.isExisting()); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + private void removeImageCacheRecord(ImageCacheVO cache) { + logger.debug(String.format("remove stale ceph image cache[imageUuid:%s, installUrl:%s]", cache.getImageUuid(), cache.getInstallUrl())); + new SQLBatch() { + @Override + protected void scripts() { + sql(ImageCacheVolumeRefVO.class).eq(ImageCacheVolumeRefVO_.imageCacheId, cache.getId()).delete(); + boolean deleted = sql("delete from ImageCacheVO c where c.id = :id") + .param("id", cache.getId()).execute() > 0; + if (deleted && CephImageCachePoolSelector.isCephImageCacheRecord(cache)) { + osdHelper.releaseAvailableCapacity(cache.getInstallUrl(), cache.getSize()); + } + } + }.execute(); + } + + private void findUsableSourceCache(List candidates, int index, boolean cleanupStale, + ReturnValueCompletion completion) { + if (index >= candidates.size()) { + completion.success(null); + return; + } + + ImageCacheVO candidate = candidates.get(index); + checkImageCacheBits(candidate, new ReturnValueCompletion(completion) { + @Override + public void success(Boolean existing) { + if (existing) { + completion.success(candidate); + } else { + if (cleanupStale) { + removeImageCacheRecord(candidate); + } + findUsableSourceCache(candidates, index + 1, cleanupStale, completion); + } + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + private boolean shouldCopyCephBackupStorageImageToSelectedPool(String backupStorageUuid, String backupStorageInstallPath, + String selectedImageCachePoolName) { + String backupStoragePoolName = CephImageCachePoolSelector.getPoolName(backupStorageInstallPath); + if (StringUtils.isBlank(backupStoragePoolName) || StringUtils.isBlank(selectedImageCachePoolName) + || backupStoragePoolName.equals(selectedImageCachePoolName)) { + return false; + } + + CephBackupStorageVO cephBS = dbf.findByUuid(backupStorageUuid, CephBackupStorageVO.class); + return cephBS != null && getSelf().getFsid().equals(cephBS.getFsid()); + } + class DownloadToCache { ImageSpec image; VolumeSnapshotInventory snapshot; boolean incremental; - private void doDownload(final ReturnValueCompletion completion) { - ImageCacheVO cache = Q.New(ImageCacheVO.class) - .eq(ImageCacheVO_.primaryStorageUuid, self.getUuid()) - .eq(ImageCacheVO_.imageUuid, image.getInventory().getUuid()) - .find(); - if (cache != null) { - completion.success(cache); - return; - } + String targetVolumeInstallUrl; + + private void doDownload(String selectedImageCachePoolName, ImageCacheVO sourceCache, final ReturnValueCompletion completion) { final FlowChain chain = FlowChainBuilder.newShareFlowChain(); - chain.setName(String.format("prepare-image-cache-ceph-%s", self.getUuid())); + chain.setName(String.format("prepare-image-cache-ceph-%s-%s", self.getUuid(), selectedImageCachePoolName)); chain.then(new ShareFlow() { String cachePath; String snapshotPath; @@ -2191,6 +2261,9 @@ public void run(final FlowTrigger trigger, Map data) { amsg.setPurpose(PrimaryStorageAllocationPurpose.DownloadImage.toString()); amsg.setImageUuid(image.getInventory().getUuid()); amsg.setNoOverProvisioning(true); + if (StringUtils.isNotBlank(selectedImageCachePoolName)) { + amsg.setRequiredInstallUri(String.format("ceph://%s/", selectedImageCachePoolName)); + } bus.makeLocalServiceId(amsg, PrimaryStorageConstant.SERVICE_ID); bus.send(amsg, new CloudBusCallBack(trigger) { @Override @@ -2224,7 +2297,7 @@ public void rollback(FlowRollback trigger, Map data) { }); flow(new Flow() { - String __name__ = "download-from-" + (snapshot != null ? "volume" : "backup-storage"); + String __name__ = "download-from-" + (sourceCache != null ? "image-cache" : (snapshot != null ? "volume" : "backup-storage")); boolean deleteOnRollback; String dstPath; @@ -2233,7 +2306,9 @@ public void rollback(FlowRollback trigger, Map data) { public void run(final FlowTrigger trigger, Map data) { dstPath = makeVolumeInstallPathByTargetPool(image.getInventory().getUuid(), getTargetPoolNameFromAllocatedUrl(allocatedInstall)); - if (snapshot != null) { + if (sourceCache != null) { + copyFromImageCache(trigger); + } else if (snapshot != null) { if (incremental) { incrementalCreateFromVolumeSnapshot(trigger); } else { @@ -2262,36 +2337,39 @@ public void fail(ErrorCode errorCode) { }); } + private void copyFromImageCache(FlowTrigger trigger) { + String destPoolName = CephImageCachePoolSelector.getPoolName(dstPath); + boolean imageShouldExistsOnCephBs = destPoolName != null && destPoolName.equals(Q.New(CephBackupStorageVO.class) + .eq(CephBackupStorageVO_.fsid, getSelf().getFsid()) + .select(CephBackupStorageVO_.poolName) + .findValue()); + copyToCache(trigger, ImageCacheUtil.getImageCachePath(sourceCache.getInstallUrl()), imageShouldExistsOnCephBs); + } - private void createFromVolumeSnapshot(FlowTrigger trigger) { - deleteOnRollback = true; - CpCmd cmd = new CpCmd(); - cmd.srcPath = snapshot.getPrimaryStorageInstallPath(); - cmd.dstPath = dstPath; - cmd.shareable = false; - httpCall(CP_PATH, cmd, CpRsp.class, new ReturnValueCompletion(trigger) { - @Override - public void success(CpRsp rsp) { - if (rsp.actualSize != null) { - actualSize = rsp.actualSize; - } - cachePath = rsp.installPath; - trigger.next(); - } - @Override - public void fail(ErrorCode errorCode) { - trigger.fail(errorCode); - } - }); + private void createFromVolumeSnapshot(FlowTrigger trigger) { + copyToCache(trigger, snapshot.getPrimaryStorageInstallPath(), false); } private void downloadFromBackupStorage(FlowTrigger trigger) { + if (image.getSelectedBackupStorage() == null) { + trigger.fail(operr("cannot find backupstorage to download image [%s] to primarystorage [%s] due to lack of Ready and accessible image", + image.getInventory().getUuid(), getSelf().getUuid())); + return; + } + + String backupStorageUuid = image.getSelectedBackupStorage().getBackupStorageUuid(); + String backupStorageInstallPath = image.getSelectedBackupStorage().getInstallPath(); + if (shouldCopyCephBackupStorageImageToSelectedPool(backupStorageUuid, backupStorageInstallPath, selectedImageCachePoolName)) { + copyToCache(trigger, backupStorageInstallPath, false); + return; + } + MediatorDownloadParam param = new MediatorDownloadParam(); param.setImage(image); param.setInstallPath(dstPath); param.setPrimaryStorageUuid(self.getUuid()); - BackupStorageMediator mediator = getBackupStorageMediator(image.getSelectedBackupStorage().getBackupStorageUuid()); + BackupStorageMediator mediator = getBackupStorageMediator(backupStorageUuid); mediator.param = param; deleteOnRollback = mediator.deleteWhenRollbackDownload(); @@ -2309,6 +2387,31 @@ public void fail(ErrorCode errorCode) { }); } + private void copyToCache(FlowTrigger trigger, String srcPath, boolean skipIfExisting) { + deleteOnRollback = !skipIfExisting; + CpCmd cmd = new CpCmd(); + cmd.resourceUuid = image.getInventory().getUuid(); + cmd.srcPath = srcPath; + cmd.dstPath = dstPath; + cmd.shareable = false; + cmd.skipIfExisting = skipIfExisting; + httpCall(CP_PATH, cmd, CpRsp.class, new ReturnValueCompletion(trigger) { + @Override + public void success(CpRsp rsp) { + if (rsp.actualSize != null) { + actualSize = rsp.actualSize; + } + cachePath = StringUtils.isNotBlank(rsp.installPath) ? rsp.installPath : dstPath; + trigger.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + trigger.fail(errorCode); + } + }); + } + @Override public void rollback(FlowRollback trigger, Map data) { if (deleteOnRollback && cachePath != null) { @@ -2480,10 +2583,19 @@ public void handle(ErrorCode errCode, Map data) { } void download(final ReturnValueCompletion completion) { + CephImageCachePoolSelector selector = getImageCachePoolSelector(); + submitDownloadTask(selector, selector.select(image.getInventory(), targetVolumeInstallUrl), completion); + } + + private void submitDownloadTask(CephImageCachePoolSelector selector, + CephImageCachePoolSelector.Selection selection, + ReturnValueCompletion completion) { + String syncSignature = String.format("ceph-p-%s-download-image-%s-%s", self.getUuid(), + image.getInventory().getUuid(), selection.poolName); thdf.chainSubmit(new ChainTask(completion) { @Override public String getSyncSignature() { - return String.format("ceph-p-%s-download-image-%s", self.getUuid(), image.getInventory().getUuid()); + return syncSignature; } private void checkEncryptImageCache(ImageCacheVO cacheVO, final SyncTaskChain chain) { @@ -2513,49 +2625,25 @@ public void fail(ErrorCode errorCode) { @Override public void run(final SyncTaskChain chain) { - ImageCacheVO cache = Q.New(ImageCacheVO.class) - .eq(ImageCacheVO_.primaryStorageUuid, self.getUuid()) - .eq(ImageCacheVO_.imageUuid, image.getInventory().getUuid()) - .find(); - - if (cache != null) { - final CheckIsBitsExistingCmd cmd = new CheckIsBitsExistingCmd(); - cmd.setInstallPath(ImageCacheUtil.getImageCachePath(cache.getInstallUrl())); - httpCall(CHECK_BITS_PATH, cmd, CheckIsBitsExistingRsp.class, new ReturnValueCompletion(chain) { + prepareInSelectedPool(chain, selector.selectInPool(image.getInventory(), selection.poolName, + selection.strategy)); + } + + private void prepareInSelectedPool(final SyncTaskChain chain, + CephImageCachePoolSelector.Selection currentSelection) { + if (currentSelection.cache != null) { + checkImageCacheBits(currentSelection.cache, new ReturnValueCompletion(chain) { @Override - public void success(CheckIsBitsExistingRsp returnValue) { - if (returnValue.isExisting()) { - logger.debug("image has been existing"); - checkEncryptImageCache(cache, chain); - return; + public void success(Boolean existing) { + if (existing) { + logger.debug(String.format("image cache[installUrl:%s] has been existing", currentSelection.cache.getInstallUrl())); + checkEncryptImageCache(currentSelection.cache, chain); } else { - logger.debug("image not found, remove vo and re-download"); - SimpleQuery q = dbf.createQuery(ImageCacheVO.class); - q.add(ImageCacheVO_.primaryStorageUuid, Op.EQ, self.getUuid()); - q.add(ImageCacheVO_.imageUuid, Op.EQ, image.getInventory().getUuid()); - ImageCacheVO cvo = q.find(); - - ReleasePrimaryStorageSpaceMsg imsg = new ReleasePrimaryStorageSpaceMsg(); - imsg.setDiskSize(cvo.getSize()); - imsg.setPrimaryStorageUuid(cvo.getPrimaryStorageUuid()); - imsg.setAllocatedInstallUrl(cvo.getInstallUrl()); - bus.makeTargetServiceIdByResourceUuid(imsg, PrimaryStorageConstant.SERVICE_ID, cvo.getPrimaryStorageUuid()); - bus.send(imsg); - dbf.remove(cvo); - - doDownload(new ReturnValueCompletion(chain) { - @Override - public void success(ImageCacheVO returnValue) { - completion.success(returnValue); - chain.next(); - } - - @Override - public void fail(ErrorCode errorCode) { - completion.fail(errorCode); - chain.next(); - } - }); + logger.debug(String.format("image cache[installUrl:%s] not found, remove vo and re-prepare", currentSelection.cache.getInstallUrl())); + removeImageCacheRecord(currentSelection.cache); + submitDownloadTask(selector, selector.select(image.getInventory(), targetVolumeInstallUrl, + selection.strategy), completion); + chain.next(); } } @@ -2565,22 +2653,50 @@ public void fail(ErrorCode errorCode) { chain.next(); } }); + return; + } - } else { - doDownload(new ReturnValueCompletion(chain) { - @Override - public void success(ImageCacheVO returnValue) { - completion.success(returnValue); - chain.next(); - } + if (CephImageCachePoolSelector.isSnapshotReuseImage(image.getInventory())) { + completion.fail(operr("cannot find snapshot reuse image cache[imageUuid:%s, installUrl:%s] on primary storage[uuid:%s]", + image.getInventory().getUuid(), image.getInventory().getUrl(), self.getUuid())); + chain.next(); + return; + } - @Override - public void fail(ErrorCode errorCode) { - completion.fail(errorCode); - chain.next(); + List sourceCandidates = selector.listImageCaches(image.getInventory().getUuid()).stream() + .filter(CephImageCachePoolSelector::isCephImageCacheRecord) + .filter(c -> !StringUtils.equals(currentSelection.poolName, + CephImageCachePoolSelector.getPoolName(c.getInstallUrl()))) + .collect(Collectors.toList()); + findUsableSourceCache(sourceCandidates, 0, true, new ReturnValueCompletion(chain) { + @Override + public void success(ImageCacheVO sourceCache) { + if (sourceCache != null && currentSelection.strategy == CephImageCachePoolStrategy.PreferExistingCache) { + checkEncryptImageCache(sourceCache, chain); + return; } - }); - } + + doDownload(currentSelection.poolName, sourceCache, new ReturnValueCompletion(chain) { + @Override + public void success(ImageCacheVO returnValue) { + completion.success(returnValue); + chain.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + chain.next(); + } + }); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + chain.next(); + } + }); } @Override @@ -2615,6 +2731,7 @@ public void run(final FlowTrigger trigger, Map data) { dmsg.setPrimaryStorageUuid(msg.getPrimaryStorageUuid()); dmsg.setHostUuid(msg.getDestHost().getUuid()); dmsg.setTemplateSpec(ispec); + dmsg.setTargetVolumeInstallUrl(volumePath); bus.makeTargetServiceIdByResourceUuid(dmsg, PrimaryStorageConstant.SERVICE_ID, dmsg.getPrimaryStorageUuid()); bus.send(dmsg, new CloudBusCallBack(trigger) { @Override @@ -2768,9 +2885,7 @@ public void success(DeleteRsp ret) { @Override protected void handle(DownloadVolumeTemplateToPrimaryStorageMsg msg) { final DownloadVolumeTemplateToPrimaryStorageReply reply = new DownloadVolumeTemplateToPrimaryStorageReply(); - DownloadToCache downloadToCache = new DownloadToCache(); - downloadToCache.image = msg.getTemplateSpec(); - downloadToCache.download(new ReturnValueCompletion(msg) { + downloadImageToCache(msg.getTemplateSpec(), msg.getTargetVolumeInstallUrl(), new ReturnValueCompletion(msg) { @Override public void success(ImageCacheVO cache) { reply.setImageCache(ImageCacheInventory.valueOf(cache)); @@ -2785,6 +2900,14 @@ public void fail(ErrorCode errorCode) { }); } + private void downloadImageToCache(ImageSpec image, String targetVolumeInstallUrl, + ReturnValueCompletion completion) { + DownloadToCache downloadToCache = new DownloadToCache(); + downloadToCache.image = image; + downloadToCache.targetVolumeInstallUrl = targetVolumeInstallUrl; + downloadToCache.download(completion); + } + @Override protected void handle(DeleteBitsOnPrimaryStorageMsg msg) { inQueue().name(String.format("delete-bits-on-primarystorage-%s", self.getUuid())) @@ -4730,12 +4853,19 @@ public void done() { private void deleteImageCacheOnPrimaryStorage(DeleteImageCacheOnPrimaryStorageMsg msg, final NoErrorCompletion completion) { DeleteImageCacheOnPrimaryStorageReply reply = new DeleteImageCacheOnPrimaryStorageReply(); + String installPath = msg.getInstallPath(); + if (StringUtils.isBlank(installPath) || !installPath.startsWith("ceph://")) { + logger.debug(String.format("skip deleting non-ceph image cache[installUrl:%s] on ceph primary storage[uuid:%s]", installPath, self.getUuid())); + bus.reply(msg, reply); + completion.done(); + return; + } DeleteImageCacheCmd cmd = new DeleteImageCacheCmd(); cmd.setFsId(getSelf().getFsid()); cmd.setUuid(self.getUuid()); - cmd.imagePath = getVolumePathFromSnapshot(msg.getInstallPath()); - cmd.snapshotPath = msg.getInstallPath(); + cmd.imagePath = getVolumePathFromSnapshot(installPath); + cmd.snapshotPath = installPath; httpCall(DELETE_IMAGE_CACHE, cmd, AgentResponse.class, new ReturnValueCompletion(msg) { @Override public void success(AgentResponse rsp) { @@ -5280,6 +5410,19 @@ public void handle(ErrorCode errCode, Map data) { } + private ImageSpec makeImageSpec(String imageUuid, ImageCacheVO sourceCache) { + ImageInventory inventory = new ImageInventory(); + inventory.setUuid(imageUuid); + inventory.setActualSize(sourceCache.getSize()); + inventory.setSize(sourceCache.getSize()); + inventory.setMediaType(sourceCache.getMediaType() == null ? + ImageMediaType.RootVolumeTemplate.toString() : sourceCache.getMediaType().toString()); + + ImageSpec imageSpec = new ImageSpec(); + imageSpec.setInventory(inventory); + return imageSpec; + } + private ImageSpec makeImageSpec(VolumeInventory volume) { ImageVO image = dbf.findByUuid(volume.getRootImageUuid(), ImageVO.class); if (image == null) { @@ -5317,6 +5460,66 @@ private ImageSpec makeImageSpec(VolumeInventory volume) { return imageSpec; } + private void prepareImageCacheForReinitRootVolume(VolumeInventory volume, String targetVolumeInstallUrl, + ReturnValueCompletion completion) { + CephImageCachePoolSelector selector = getImageCachePoolSelector(); + CephImageCachePoolSelector.Selection selection = selector.select(volume.getRootImageUuid(), targetVolumeInstallUrl); + if (selection.cache == null) { + downloadImageCacheForReinitRootVolume(volume, targetVolumeInstallUrl, completion); + return; + } + + checkImageCacheBits(selection.cache, new ReturnValueCompletion(completion) { + @Override + public void success(Boolean existing) { + if (existing) { + completion.success(selection.cache); + } else { + downloadImageCacheForReinitRootVolume(volume, targetVolumeInstallUrl, completion); + } + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + + private void downloadImageCacheForReinitRootVolume(VolumeInventory volume, String targetVolumeInstallUrl, + ReturnValueCompletion completion) { + CephImageCachePoolSelector selector = getImageCachePoolSelector(); + CephImageCachePoolSelector.Selection selection = selector.select(volume.getRootImageUuid(), targetVolumeInstallUrl); + List sourceCandidates = selector.listImageCaches(volume.getRootImageUuid()).stream() + .filter(CephImageCachePoolSelector::isCephImageCacheRecord) + .filter(c -> !StringUtils.equals(selection.poolName, + CephImageCachePoolSelector.getPoolName(c.getInstallUrl()))) + .collect(Collectors.toList()); + findUsableSourceCache(sourceCandidates, 0, false, new ReturnValueCompletion(completion) { + @Override + public void success(ImageCacheVO sourceCache) { + ImageSpec imageSpec; + if (sourceCache != null) { + imageSpec = makeImageSpec(volume.getRootImageUuid(), sourceCache); + } else { + try { + imageSpec = makeImageSpec(volume); + } catch (OperationFailureException e) { + completion.fail(e.getErrorCode()); + return; + } + } + + downloadImageToCache(imageSpec, targetVolumeInstallUrl, completion); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + protected void handle(final ReInitRootVolumeFromTemplateOnPrimaryStorageMsg msg) { final ReInitRootVolumeFromTemplateOnPrimaryStorageReply reply = new ReInitRootVolumeFromTemplateOnPrimaryStorageReply(); @@ -5335,30 +5538,19 @@ public void setup() { @Override public void run(final FlowTrigger trigger, Map data) { - installUrl = Q.New(ImageCacheVO.class).eq(ImageCacheVO_.imageUuid, msg.getVolume().getRootImageUuid()). - eq(ImageCacheVO_.primaryStorageUuid, msg.getPrimaryStorageUuid()).select(ImageCacheVO_.installUrl).findValue(); - - if (installUrl != null) { - trigger.next(); - return; - } - - DownloadVolumeTemplateToPrimaryStorageMsg dmsg = new DownloadVolumeTemplateToPrimaryStorageMsg(); - dmsg.setTemplateSpec(makeImageSpec(msg.getVolume())); - dmsg.setPrimaryStorageUuid(msg.getPrimaryStorageUuid()); - bus.makeTargetServiceIdByResourceUuid(dmsg, PrimaryStorageConstant.SERVICE_ID, dmsg.getPrimaryStorageUuid()); - bus.send(dmsg, new CloudBusCallBack(trigger) { - @Override - public void run(MessageReply reply) { - if (!reply.isSuccess()) { - trigger.fail(reply.getError()); - return; - } + prepareImageCacheForReinitRootVolume(msg.getVolume(), volumePath, + new ReturnValueCompletion(trigger) { + @Override + public void success(ImageCacheVO cache) { + installUrl = ImageCacheUtil.getImageCachePath(cache.getInstallUrl()); + trigger.next(); + } - installUrl = ((DownloadVolumeTemplateToPrimaryStorageReply) reply).getImageCache().getInstallUrl(); - trigger.next(); - } - }); + @Override + public void fail(ErrorCode errorCode) { + trigger.fail(errorCode); + } + }); } }); diff --git a/sdk/src/main/java/org/zstack/sdk/AddCephPrimaryStoragePoolAction.java b/sdk/src/main/java/org/zstack/sdk/AddCephPrimaryStoragePoolAction.java index cfe72255f1f..98be1847771 100644 --- a/sdk/src/main/java/org/zstack/sdk/AddCephPrimaryStoragePoolAction.java +++ b/sdk/src/main/java/org/zstack/sdk/AddCephPrimaryStoragePoolAction.java @@ -37,7 +37,7 @@ public Result throwExceptionIfError() { @Param(required = false, maxLength = 2048, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.lang.String description; - @Param(required = true, validValues = {"Root","Data"}, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + @Param(required = true, validValues = {"Root","Data","ImageCache"}, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.lang.String type; @Param(required = false) diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/ceph/CephOperationCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/ceph/CephOperationCase.groovy index b131d740241..35e93ff096b 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/ceph/CephOperationCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/ceph/CephOperationCase.groovy @@ -183,9 +183,10 @@ class CephOperationCase extends SubCase { ps = env.inventoryByName("ceph-pri") as CephPrimaryStorageInventory bs = env.inventoryByName("ceph-bk") as CephBackupStorageInventory - assert ps.pools.securityPolicy == [DataSecurityPolicy.Copy.toString()] * 3 - assert ps.pools.diskUtilization == [0.33f] * 3 - assert ps.pools.replicatedSize == [3] * 3 + assert ps.pools.size() == 3 + assert ps.pools.securityPolicy.every { it == DataSecurityPolicy.Copy.toString() } + assert ps.pools.diskUtilization.every { it == 0.33f } + assert ps.pools.replicatedSize.every { it == 3 } assert bs.poolSecurityPolicy == DataSecurityPolicy.ErasureCode.toString() assert bs.poolDiskUtilization == 0.67f diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/CephPrimaryStorageVolumePoolsCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/CephPrimaryStorageVolumePoolsCase.groovy index 6f95c49e3e7..4d04ca629b7 100755 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/CephPrimaryStorageVolumePoolsCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/CephPrimaryStorageVolumePoolsCase.groovy @@ -2,13 +2,25 @@ package org.zstack.test.integration.storage.primary.ceph import org.springframework.http.HttpEntity import org.zstack.compute.vm.VmSystemTags +import org.zstack.core.Platform +import org.zstack.core.db.DatabaseFacade import org.zstack.core.db.Q +import org.zstack.header.image.ImageConstant.ImageMediaType +import org.zstack.header.storage.primary.ImageCacheInventory +import org.zstack.header.storage.primary.ImageCacheShadowVO +import org.zstack.header.storage.primary.ImageCacheShadowVO_ +import org.zstack.header.storage.primary.ImageCacheVolumeRefVO +import org.zstack.header.storage.primary.ImageCacheVO +import org.zstack.header.storage.primary.ImageCacheVO_ +import org.zstack.header.storage.snapshot.reference.VolumeSnapshotReferenceTreeVO import org.zstack.header.volume.VolumeVO import org.zstack.header.volume.VolumeVO_ +import org.zstack.storage.ceph.CephGlobalConfig import org.zstack.kvm.KVMConstant import org.zstack.sdk.* import org.zstack.storage.ceph.CephSystemTags import org.zstack.storage.ceph.primary.CephPrimaryStorageBase +import org.zstack.storage.ceph.primary.CephImageCachePoolStrategy import org.zstack.storage.ceph.primary.CephPrimaryStoragePoolVO import org.zstack.storage.ceph.primary.CephPrimaryStoragePoolVO_ import org.zstack.test.integration.storage.StorageTest @@ -29,7 +41,9 @@ class CephPrimaryStorageVolumePoolsCase extends SubCase { String NEW_ROOT_POOL_NAME = "new_root_pool" String ROOT_POOL_TYPE = "Root" String DATA_POOL_TYPE = "Data" + String IMAGE_CACHE_POOL_TYPE = "ImageCache" String NEW_DATA_POOL_NAME = "new_data_pool" + String ROOT_ONLY_POOL_NAME = "root_only_pool" @Override void setup() { @@ -85,10 +99,21 @@ class CephPrimaryStorageVolumePoolsCase extends SubCase { type = ROOT_POOL_TYPE } + pool { + poolName = NEW_ROOT_POOL_NAME + type = IMAGE_CACHE_POOL_TYPE + isCreate = false + } + pool { poolName = NEW_DATA_POOL_NAME type = DATA_POOL_TYPE } + + pool { + poolName = ROOT_ONLY_POOL_NAME + type = ROOT_POOL_TYPE + } } attachBackupStorage("ceph-bk") @@ -103,6 +128,27 @@ class CephPrimaryStorageVolumePoolsCase extends SubCase { name = "image" url = "http://zstack.org/download/test.qcow2" } + + image { + name = "image2" + url = "http://zstack.org/download/test2.qcow2" + } + + image { + name = "image3" + url = "http://zstack.org/download/test3.qcow2" + } + + image { + name = "image4" + url = "http://zstack.org/download/test4.qcow2" + } + + image { + name = "image5" + url = "http://zstack.org/download/test5.qcow2" + } + } diskOffering { @@ -140,6 +186,37 @@ class CephPrimaryStorageVolumePoolsCase extends SubCase { InstanceOfferingInventory instanceOffering2 ImageInventory image + void ensureNewRootPoolIsImageCachePool() { + if (Q.New(CephPrimaryStoragePoolVO.class) + .eq(CephPrimaryStoragePoolVO_.primaryStorageUuid, primaryStorage.uuid) + .eq(CephPrimaryStoragePoolVO_.poolName, NEW_ROOT_POOL_NAME) + .eq(CephPrimaryStoragePoolVO_.type, IMAGE_CACHE_POOL_TYPE) + .isExists()) { + return + } + + addCephPrimaryStoragePool { + primaryStorageUuid = primaryStorage.uuid + poolName = NEW_ROOT_POOL_NAME + type = IMAGE_CACHE_POOL_TYPE + isCreate = false + } + } + + void restoreCheckBitsSimulator() { + env.simulator(CephPrimaryStorageBase.CHECK_BITS_PATH) { + CephPrimaryStorageBase.CheckIsBitsExistingRsp rsp = new CephPrimaryStorageBase.CheckIsBitsExistingRsp() + rsp.setExisting(true) + return rsp + } + env.afterSimulator(CephPrimaryStorageBase.CHECK_BITS_PATH) { rsp, HttpEntity e, EnvSpec spec -> + CephPrimaryStorageBase.CheckIsBitsExistingCmd cmd = json(e.body, CephPrimaryStorageBase.CheckIsBitsExistingCmd.class) + VFS vfs = CephPrimaryStorageSpec.vfs(cmd, spec) + vfs.Assert(vfs.isFile(CephPrimaryStorageSpec.cephPathToVFSPath(cmd.installPath)), "cannot find ${cmd.installPath}") + return rsp + } + } + void testCreateDataVolumeInPool() { CephPrimaryStorageBase.CreateEmptyVolumeCmd cmd = null @@ -250,15 +327,15 @@ class CephPrimaryStorageVolumePoolsCase extends SubCase { acmd = json(e.body, CephPrimaryStorageBase.AddPoolCmd.class) } - AddCephPrimaryStoragePoolAction a = new AddCephPrimaryStoragePoolAction() - a.isCreate = true - a.poolName = LOW_POOL_NAME - a.primaryStorageUuid = primaryStorage.uuid - a.type = DATA_POOL_TYPE - a.sessionId = adminSession() - def res = a.call() + expect(AssertionError.class) { + addCephPrimaryStoragePool { + isCreate = true + poolName = LOW_POOL_NAME + primaryStorageUuid = primaryStorage.uuid + type = DATA_POOL_TYPE + } + } - assert res.error != null assert !Q.New(CephPrimaryStoragePoolVO.class).eq(CephPrimaryStoragePoolVO_.poolName, LOW_POOL_NAME).isExists() assert acmd != null assert acmd.isCreate @@ -278,22 +355,640 @@ class CephPrimaryStorageVolumePoolsCase extends SubCase { } void testAddSameCephPool() { - AddCephPrimaryStoragePoolAction action = new AddCephPrimaryStoragePoolAction() - action.primaryStorageUuid = primaryStorage.uuid - action.poolName = HIGH_POOL_NAME - action.sessionId = adminSession() - action.type = DATA_POOL_TYPE - def ret = action.call() - - AddCephPrimaryStoragePoolAction rootPoolAction = new AddCephPrimaryStoragePoolAction() - rootPoolAction.primaryStorageUuid = primaryStorage.uuid - rootPoolAction.poolName = NEW_ROOT_POOL_NAME - rootPoolAction.sessionId = adminSession() - rootPoolAction.type = ROOT_POOL_TYPE - def rootRet = rootPoolAction.call() - - assert ret.error != null - assert rootRet.error != null + expect(AssertionError.class) { + addCephPrimaryStoragePool { + primaryStorageUuid = primaryStorage.uuid + poolName = HIGH_POOL_NAME + type = DATA_POOL_TYPE + } + } + + expect(AssertionError.class) { + addCephPrimaryStoragePool { + primaryStorageUuid = primaryStorage.uuid + poolName = NEW_ROOT_POOL_NAME + type = ROOT_POOL_TYPE + } + } + } + + void testPreferVolumePoolImageCacheStrategy() { + ensureNewRootPoolIsImageCachePool() + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.PreferVolumePool.toString()) + + CephPrimaryStorageBase.CpCmd cpCmd = null + CephPrimaryStorageBase.CloneCmd cloneCmd = null + env.hijackSimulator(CephPrimaryStorageBase.CP_PATH) { rsp, HttpEntity e -> + cpCmd = json(e.body, CephPrimaryStorageBase.CpCmd.class) + return rsp + } + env.preSimulator(CephPrimaryStorageBase.CLONE_PATH) { HttpEntity e -> + cloneCmd = json(e.body, CephPrimaryStorageBase.CloneCmd.class) + } + + try { + VmInstanceInventory imageCachePoolVm = createVmInstance { + name = "image-cache-pool-vm" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = vm.imageUuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + rootVolumeSystemTags = [CephSystemTags.USE_CEPH_ROOT_POOL.instantiateTag([(CephSystemTags.USE_CEPH_ROOT_POOL_TOKEN): NEW_ROOT_POOL_NAME])] + } as VmInstanceInventory + + assert cpCmd != null + assert cpCmd.dstPath.contains(NEW_ROOT_POOL_NAME) + assert !cpCmd.skipIfExisting + assert cloneCmd != null + assert cloneCmd.srcPath.contains(NEW_ROOT_POOL_NAME) + + List caches = Q.New(ImageCacheVO.class) + .eq(ImageCacheVO_.primaryStorageUuid, primaryStorage.uuid) + .eq(ImageCacheVO_.imageUuid, vm.imageUuid) + .list() + assert caches.find { it.installUrl.contains(NEW_ROOT_POOL_NAME) } != null + assert caches.size() >= 2 + + List queriedCaches = queryImageCache { + conditions = asList("primaryStorageUuid=${primaryStorage.uuid}".toString(), "imageUuid=${vm.imageUuid}".toString()) + } + assert queriedCaches.size() >= 2 + + destroyVmInstance { + uuid = imageCachePoolVm.uuid + } + expungeVmInstance { + uuid = imageCachePoolVm.uuid + } + } finally { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + } + } + + void testDefaultImageCachePoolStrategyUsesDefaultPool() { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + String defaultImageCachePoolName = CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL.getTokenByResourceUuid(primaryStorage.uuid, CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL_TOKEN) + + List existingCaches = Q.New(ImageCacheVO.class) + .eq(ImageCacheVO_.primaryStorageUuid, primaryStorage.uuid) + .eq(ImageCacheVO_.imageUuid, vm.imageUuid) + .list() + assert !existingCaches.isEmpty() + + CephPrimaryStorageBase.CloneCmd cloneCmd = null + env.preSimulator(CephPrimaryStorageBase.CLONE_PATH) { HttpEntity e -> + cloneCmd = json(e.body, CephPrimaryStorageBase.CloneCmd.class) + } + + VmInstanceInventory defaultStrategyVm = createVmInstance { + name = "default-image-cache-pool-vm" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = vm.imageUuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + rootVolumeSystemTags = [CephSystemTags.USE_CEPH_ROOT_POOL.instantiateTag([(CephSystemTags.USE_CEPH_ROOT_POOL_TOKEN): NEW_ROOT_POOL_NAME])] + } as VmInstanceInventory + + assert cloneCmd != null + assert existingCaches.find { it.installUrl == cloneCmd.srcPath } != null + assert cloneCmd.srcPath.contains(defaultImageCachePoolName) + assert cloneCmd.dstPath.contains(NEW_ROOT_POOL_NAME) + + destroyVmInstance { + uuid = defaultStrategyVm.uuid + } + expungeVmInstance { + uuid = defaultStrategyVm.uuid + } + } + + void testDefaultStrategyCopiesCephBackupStorageImageToDefaultPool() { + ImageInventory image5 = env.inventoryByName("image5") as ImageInventory + String defaultImageCachePoolName = CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL.getTokenByResourceUuid(primaryStorage.uuid, CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL_TOKEN) + String backupStorageInstallPath = image5.backupStorageRefs[0].installPath + CephPrimaryStorageBase.CpCmd cpCmd = null + CephPrimaryStorageBase.CloneCmd cloneCmd = null + VmInstanceInventory defaultStrategyVm = null + + env.hijackSimulator(CephPrimaryStorageBase.CP_PATH) { rsp, HttpEntity e -> + cpCmd = json(e.body, CephPrimaryStorageBase.CpCmd.class) + return rsp + } + env.preSimulator(CephPrimaryStorageBase.CLONE_PATH) { HttpEntity e -> + cloneCmd = json(e.body, CephPrimaryStorageBase.CloneCmd.class) + } + + try { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + defaultStrategyVm = createVmInstance { + name = "default-strategy-copy-ceph-bs-image-vm" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = image5.uuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + rootVolumeSystemTags = [CephSystemTags.USE_CEPH_ROOT_POOL.instantiateTag([(CephSystemTags.USE_CEPH_ROOT_POOL_TOKEN): NEW_ROOT_POOL_NAME])] + } as VmInstanceInventory + + assert cpCmd != null + assert cpCmd.srcPath == backupStorageInstallPath + assert cpCmd.dstPath.contains(defaultImageCachePoolName) + assert cloneCmd != null + assert cloneCmd.srcPath.contains(defaultImageCachePoolName) + assert cloneCmd.dstPath.contains(NEW_ROOT_POOL_NAME) + } finally { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + if (defaultStrategyVm != null) { + destroyVmInstance { + uuid = defaultStrategyVm.uuid + } + expungeVmInstance { + uuid = defaultStrategyVm.uuid + } + } + } + } + + void testPreferVolumePoolFallbackToDefaultPool() { + String defaultImageCachePoolName = CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL.getTokenByResourceUuid(primaryStorage.uuid, CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL_TOKEN) + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.PreferVolumePool.toString()) + + CephPrimaryStorageBase.CloneCmd cloneCmd = null + env.preSimulator(CephPrimaryStorageBase.CLONE_PATH) { HttpEntity e -> + cloneCmd = json(e.body, CephPrimaryStorageBase.CloneCmd.class) + } + + try { + VmInstanceInventory fallbackVm = createVmInstance { + name = "prefer-volume-pool-fallback-vm" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = vm.imageUuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + rootVolumeSystemTags = [CephSystemTags.USE_CEPH_ROOT_POOL.instantiateTag([(CephSystemTags.USE_CEPH_ROOT_POOL_TOKEN): ROOT_ONLY_POOL_NAME])] + } as VmInstanceInventory + + assert cloneCmd != null + assert cloneCmd.srcPath.contains(defaultImageCachePoolName) + assert cloneCmd.dstPath.contains(ROOT_ONLY_POOL_NAME) + + destroyVmInstance { + uuid = fallbackVm.uuid + } + expungeVmInstance { + uuid = fallbackVm.uuid + } + } finally { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + } + } + + void testPreferExistingCacheStrategyPrefersDefaultPoolCache() { + ensureNewRootPoolIsImageCachePool() + + String defaultImageCachePoolName = CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL.getTokenByResourceUuid(primaryStorage.uuid, CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL_TOKEN) + VmInstanceInventory defaultCacheVm = null + CephPrimaryStorageBase.CloneCmd cloneCmd = null + + try { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.PreferVolumePool.toString()) + defaultCacheVm = createVmInstance { + name = "default-cache-vm" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = vm.imageUuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + } as VmInstanceInventory + + ImageCacheVO defaultCache = Q.New(ImageCacheVO.class) + .eq(ImageCacheVO_.primaryStorageUuid, primaryStorage.uuid) + .eq(ImageCacheVO_.imageUuid, vm.imageUuid) + .like(ImageCacheVO_.installUrl, String.format("ceph://%s/%%", defaultImageCachePoolName)) + .find() + assert defaultCache != null + + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.PreferExistingCache.toString()) + env.preSimulator(CephPrimaryStorageBase.CLONE_PATH) { HttpEntity e -> + cloneCmd = json(e.body, CephPrimaryStorageBase.CloneCmd.class) + } + + VmInstanceInventory imageCachePoolVm = createVmInstance { + name = "prefer-existing-cache-vm" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = vm.imageUuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + rootVolumeSystemTags = [CephSystemTags.USE_CEPH_ROOT_POOL.instantiateTag([(CephSystemTags.USE_CEPH_ROOT_POOL_TOKEN): NEW_ROOT_POOL_NAME])] + } as VmInstanceInventory + + assert cloneCmd != null + assert cloneCmd.srcPath.contains(defaultImageCachePoolName) + + destroyVmInstance { + uuid = imageCachePoolVm.uuid + } + expungeVmInstance { + uuid = imageCachePoolVm.uuid + } + } finally { + if (defaultCacheVm != null) { + destroyVmInstance { + uuid = defaultCacheVm.uuid + } + expungeVmInstance { + uuid = defaultCacheVm.uuid + } + } + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + } + } + + void testPreferExistingCacheStrategyUsesNonDefaultPoolCache() { + ensureNewRootPoolIsImageCachePool() + ImageInventory image2 = env.inventoryByName("image2") as ImageInventory + String defaultRootPoolName = CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_ROOT_VOLUME_POOL.getTokenByResourceUuid(primaryStorage.uuid, CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_ROOT_VOLUME_POOL_TOKEN) + VmInstanceInventory nonDefaultCacheVm = null + VmInstanceInventory preferExistingVm = null + + try { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.PreferVolumePool.toString()) + nonDefaultCacheVm = createVmInstance { + name = "non-default-cache-vm" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = image2.uuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + rootVolumeSystemTags = [CephSystemTags.USE_CEPH_ROOT_POOL.instantiateTag([(CephSystemTags.USE_CEPH_ROOT_POOL_TOKEN): NEW_ROOT_POOL_NAME])] + } as VmInstanceInventory + + assert Q.New(ImageCacheVO.class) + .eq(ImageCacheVO_.primaryStorageUuid, primaryStorage.uuid) + .eq(ImageCacheVO_.imageUuid, image2.uuid) + .like(ImageCacheVO_.installUrl, String.format("ceph://%s/%%", NEW_ROOT_POOL_NAME)) + .isExists() + + destroyVmInstance { + uuid = nonDefaultCacheVm.uuid + } + expungeVmInstance { + uuid = nonDefaultCacheVm.uuid + } + nonDefaultCacheVm = null + + CephPrimaryStorageBase.CloneCmd cloneCmd = null + env.preSimulator(CephPrimaryStorageBase.CLONE_PATH) { HttpEntity e -> + cloneCmd = json(e.body, CephPrimaryStorageBase.CloneCmd.class) + } + + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.PreferExistingCache.toString()) + preferExistingVm = createVmInstance { + name = "prefer-existing-non-default-cache-vm" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = image2.uuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + } as VmInstanceInventory + + assert cloneCmd != null + assert cloneCmd.srcPath.contains(NEW_ROOT_POOL_NAME) + assert cloneCmd.dstPath.contains(defaultRootPoolName) + } finally { + if (preferExistingVm != null) { + destroyVmInstance { + uuid = preferExistingVm.uuid + } + expungeVmInstance { + uuid = preferExistingVm.uuid + } + } + if (nonDefaultCacheVm != null) { + destroyVmInstance { + uuid = nonDefaultCacheVm.uuid + } + expungeVmInstance { + uuid = nonDefaultCacheVm.uuid + } + } + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + } + } + + void testPreferExistingCacheStrategySkipsStaleDefaultPoolCache() { + ensureNewRootPoolIsImageCachePool() + ImageInventory image2 = env.inventoryByName("image2") as ImageInventory + String defaultImageCachePoolName = CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL.getTokenByResourceUuid(primaryStorage.uuid, CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL_TOKEN) + String defaultRootPoolName = CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_ROOT_VOLUME_POOL.getTokenByResourceUuid(primaryStorage.uuid, CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_ROOT_VOLUME_POOL_TOKEN) + VmInstanceInventory defaultCacheVm = null + VmInstanceInventory nonDefaultCacheVm = null + VmInstanceInventory skipStaleDefaultVm = null + ImageCacheVO staleDefaultCache = null + ImageCacheVO nonDefaultCache = null + ImageCacheVolumeRefVO staleRef = null + CephPrimaryStorageBase.CloneCmd cloneCmd = null + + try { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + defaultCacheVm = createVmInstance { + name = "prefer-existing-stale-default-source-vm" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = image2.uuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + } as VmInstanceInventory + destroyVmInstance { + uuid = defaultCacheVm.uuid + } + expungeVmInstance { + uuid = defaultCacheVm.uuid + } + defaultCacheVm = null + + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.PreferVolumePool.toString()) + nonDefaultCacheVm = createVmInstance { + name = "prefer-existing-non-default-source-vm" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = image2.uuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + rootVolumeSystemTags = [CephSystemTags.USE_CEPH_ROOT_POOL.instantiateTag([(CephSystemTags.USE_CEPH_ROOT_POOL_TOKEN): NEW_ROOT_POOL_NAME])] + } as VmInstanceInventory + destroyVmInstance { + uuid = nonDefaultCacheVm.uuid + } + expungeVmInstance { + uuid = nonDefaultCacheVm.uuid + } + nonDefaultCacheVm = null + + staleDefaultCache = Q.New(ImageCacheVO.class) + .eq(ImageCacheVO_.primaryStorageUuid, primaryStorage.uuid) + .eq(ImageCacheVO_.imageUuid, image2.uuid) + .like(ImageCacheVO_.installUrl, String.format("ceph://%s/%%", defaultImageCachePoolName)) + .find() + nonDefaultCache = Q.New(ImageCacheVO.class) + .eq(ImageCacheVO_.primaryStorageUuid, primaryStorage.uuid) + .eq(ImageCacheVO_.imageUuid, image2.uuid) + .like(ImageCacheVO_.installUrl, String.format("ceph://%s/%%", NEW_ROOT_POOL_NAME)) + .find() + assert nonDefaultCache != null + + if (staleDefaultCache == null) { + staleDefaultCache = new ImageCacheVO() + staleDefaultCache.setPrimaryStorageUuid(primaryStorage.uuid) + staleDefaultCache.setImageUuid(image2.uuid) + staleDefaultCache.setInstallUrl(nonDefaultCache.installUrl.replaceFirst(String.format("ceph://%s/", NEW_ROOT_POOL_NAME), String.format("ceph://%s/", defaultImageCachePoolName))) + staleDefaultCache.setMediaType(nonDefaultCache.mediaType) + staleDefaultCache.setSize(nonDefaultCache.size) + staleDefaultCache.setMd5sum(nonDefaultCache.md5sum) + staleDefaultCache = bean(DatabaseFacade.class).persistAndRefresh(staleDefaultCache) + } + + staleRef = new ImageCacheVolumeRefVO() + staleRef.setImageCacheId(staleDefaultCache.id) + staleRef.setPrimaryStorageUuid(primaryStorage.uuid) + staleRef.setVolumeUuid(vm.rootVolumeUuid) + bean(DatabaseFacade.class).persist(staleRef) + + env.simulator(CephPrimaryStorageBase.CHECK_BITS_PATH) { HttpEntity e -> + CephPrimaryStorageBase.CheckIsBitsExistingCmd cmd = json(e.body, CephPrimaryStorageBase.CheckIsBitsExistingCmd.class) + CephPrimaryStorageBase.CheckIsBitsExistingRsp rsp = new CephPrimaryStorageBase.CheckIsBitsExistingRsp() + rsp.setExisting(!cmd.installPath.contains(defaultImageCachePoolName)) + return rsp + } + env.afterSimulator(CephPrimaryStorageBase.CHECK_BITS_PATH) { rsp, HttpEntity e -> + return rsp + } + env.preSimulator(CephPrimaryStorageBase.CLONE_PATH) { HttpEntity e -> + cloneCmd = json(e.body, CephPrimaryStorageBase.CloneCmd.class) + } + + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.PreferExistingCache.toString()) + skipStaleDefaultVm = createVmInstance { + name = "prefer-existing-skip-stale-default-cache-vm" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = image2.uuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + } as VmInstanceInventory + + assert dbFindById(staleDefaultCache.id, ImageCacheVO.class) == null + assert dbFindById(staleRef.id, ImageCacheVolumeRefVO.class) == null + assert dbFindById(nonDefaultCache.id, ImageCacheVO.class) != null + assert cloneCmd != null + assert cloneCmd.srcPath.contains(NEW_ROOT_POOL_NAME) + assert cloneCmd.dstPath.contains(defaultRootPoolName) + } finally { + if (skipStaleDefaultVm != null) { + destroyVmInstance { + uuid = skipStaleDefaultVm.uuid + } + expungeVmInstance { + uuid = skipStaleDefaultVm.uuid + } + } + if (nonDefaultCacheVm != null) { + destroyVmInstance { + uuid = nonDefaultCacheVm.uuid + } + expungeVmInstance { + uuid = nonDefaultCacheVm.uuid + } + } + if (defaultCacheVm != null) { + destroyVmInstance { + uuid = defaultCacheVm.uuid + } + expungeVmInstance { + uuid = defaultCacheVm.uuid + } + } + restoreCheckBitsSimulator() + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + } + } + + void testMissingSelectedCacheBitsCopiesFromOtherPool() { + ensureNewRootPoolIsImageCachePool() + ImageInventory image3 = env.inventoryByName("image3") as ImageInventory + String defaultImageCachePoolName = CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL.getTokenByResourceUuid(primaryStorage.uuid, CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL_TOKEN) + VFS vfs = CephPrimaryStorageSpec.vfs1("7ff218d9-f525-435f-8a40-3618d1772a64", env) + + ImageCacheVO sourceCache = new ImageCacheVO() + sourceCache.setPrimaryStorageUuid(primaryStorage.uuid) + sourceCache.setImageUuid(image3.uuid) + sourceCache.setInstallUrl(String.format("ceph://%s/%s-source@%s-source", defaultImageCachePoolName, image3.uuid, image3.uuid)) + sourceCache.setMediaType(ImageMediaType.RootVolumeTemplate) + sourceCache.setSize(SizeUnit.GIGABYTE.toByte(1)) + sourceCache.setMd5sum("not calculated") + sourceCache = bean(DatabaseFacade.class).persistAndRefresh(sourceCache) + + ImageCacheVO staleCache = new ImageCacheVO() + staleCache.setPrimaryStorageUuid(primaryStorage.uuid) + staleCache.setImageUuid(image3.uuid) + staleCache.setInstallUrl(String.format("ceph://%s/%s-stale@%s-stale", NEW_ROOT_POOL_NAME, image3.uuid, image3.uuid)) + staleCache.setMediaType(ImageMediaType.RootVolumeTemplate) + staleCache.setSize(SizeUnit.GIGABYTE.toByte(1)) + staleCache.setMd5sum("not calculated") + staleCache = bean(DatabaseFacade.class).persistAndRefresh(staleCache) + + String sourceCacheVolumePath = CephPrimaryStorageSpec.cephPathToVFSPath(sourceCache.installUrl.split("@")[0]) + String sourceCachePath = CephPrimaryStorageSpec.cephPathToVFSPath(sourceCache.installUrl) + vfs.createDirectories(String.format("/%s", defaultImageCachePoolName)) + if (!vfs.exists(sourceCacheVolumePath)) { + vfs.createCephRaw(sourceCacheVolumePath, 0L) + } + if (!vfs.exists(sourceCachePath)) { + vfs.createCephRaw(sourceCachePath, 0L) + } + String staleCachePath = CephPrimaryStorageSpec.cephPathToVFSPath(staleCache.installUrl) + String staleCacheVolumePath = CephPrimaryStorageSpec.cephPathToVFSPath(staleCache.installUrl.split("@")[0]) + if (vfs.exists(staleCachePath)) { + vfs.delete(staleCachePath) + } + if (vfs.exists(staleCacheVolumePath)) { + vfs.delete(staleCacheVolumePath) + } + + CephPrimaryStorageBase.CpCmd cpCmd = null + CephPrimaryStorageBase.CloneCmd cloneCmd = null + env.simulator(CephPrimaryStorageBase.CHECK_BITS_PATH) { HttpEntity e -> + CephPrimaryStorageBase.CheckIsBitsExistingCmd cmd = json(e.body, CephPrimaryStorageBase.CheckIsBitsExistingCmd.class) + CephPrimaryStorageBase.CheckIsBitsExistingRsp rsp = new CephPrimaryStorageBase.CheckIsBitsExistingRsp() + rsp.setExisting(!cmd.installPath.contains(NEW_ROOT_POOL_NAME)) + return rsp + } + env.afterSimulator(CephPrimaryStorageBase.CHECK_BITS_PATH) { rsp, HttpEntity e -> + return rsp + } + env.hijackSimulator(CephPrimaryStorageBase.CP_PATH) { rsp, HttpEntity e -> + cpCmd = json(e.body, CephPrimaryStorageBase.CpCmd.class) + return rsp + } + env.preSimulator(CephPrimaryStorageBase.CLONE_PATH) { HttpEntity e -> + cloneCmd = json(e.body, CephPrimaryStorageBase.CloneCmd.class) + } + + try { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.PreferVolumePool.toString()) + VmInstanceInventory staleCacheVm = createVmInstance { + name = "missing-selected-cache-bits-vm" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = image3.uuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + rootVolumeSystemTags = [CephSystemTags.USE_CEPH_ROOT_POOL.instantiateTag([(CephSystemTags.USE_CEPH_ROOT_POOL_TOKEN): NEW_ROOT_POOL_NAME])] + } as VmInstanceInventory + + assert dbFindById(staleCache.id, ImageCacheVO.class) == null + assert cpCmd != null + assert cpCmd.srcPath.contains(defaultImageCachePoolName) + assert cpCmd.dstPath.contains(NEW_ROOT_POOL_NAME) + assert cloneCmd != null + assert cloneCmd.srcPath.contains(NEW_ROOT_POOL_NAME) + + destroyVmInstance { + uuid = staleCacheVm.uuid + } + expungeVmInstance { + uuid = staleCacheVm.uuid + } + } finally { + restoreCheckBitsSimulator() + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + } + } + + void testCleanupImageCacheKeepsReferencedCacheOnly() { + restoreCheckBitsSimulator() + String cleanupImageUuid = "cleanup-image-cache-image" + String reuseImageUuid = "cleanup-image-cache-reuse-image" + String reuseTreeUuid = null + String defaultImageCachePoolName = CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL.getTokenByResourceUuid(primaryStorage.uuid, CephSystemTags.DEFAULT_CEPH_PRIMARY_STORAGE_IMAGE_CACHE_POOL_TOKEN) + + try { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + + ImageCacheVO protectedCache = new ImageCacheVO() + protectedCache.setPrimaryStorageUuid(primaryStorage.uuid) + protectedCache.setImageUuid(cleanupImageUuid) + protectedCache.setInstallUrl(String.format("ceph://%s/%s-protected@%s-protected", NEW_ROOT_POOL_NAME, cleanupImageUuid, cleanupImageUuid)) + protectedCache.setMediaType(ImageMediaType.RootVolumeTemplate) + protectedCache.setSize(SizeUnit.GIGABYTE.toByte(1)) + protectedCache.setMd5sum("not calculated") + protectedCache = bean(DatabaseFacade.class).persistAndRefresh(protectedCache) + + ImageCacheVO deletedCache = new ImageCacheVO() + deletedCache.setPrimaryStorageUuid(primaryStorage.uuid) + deletedCache.setImageUuid(cleanupImageUuid) + deletedCache.setInstallUrl(String.format("ceph://%s/%s-deleted@%s-deleted", defaultImageCachePoolName, cleanupImageUuid, cleanupImageUuid)) + deletedCache.setMediaType(ImageMediaType.RootVolumeTemplate) + deletedCache.setSize(SizeUnit.GIGABYTE.toByte(1)) + deletedCache.setMd5sum("not calculated") + deletedCache = bean(DatabaseFacade.class).persistAndRefresh(deletedCache) + + ImageCacheVO reuseCache = new ImageCacheVO() + reuseCache.setPrimaryStorageUuid(primaryStorage.uuid) + reuseCache.setImageUuid(reuseImageUuid) + reuseCache.setInstallUrl(String.format("volumeSnapshotReuse://%s", reuseImageUuid)) + reuseCache.setMediaType(ImageMediaType.RootVolumeTemplate) + reuseCache.setSize(SizeUnit.GIGABYTE.toByte(1)) + reuseCache.setMd5sum("not calculated") + reuseCache = bean(DatabaseFacade.class).persistAndRefresh(reuseCache) + + VolumeSnapshotReferenceTreeVO reuseTree = new VolumeSnapshotReferenceTreeVO() + reuseTree.setUuid(Platform.getUuid()) + reuseTree.setRootImageUuid(reuseImageUuid) + reuseTree.setRootInstallUrl(reuseCache.installUrl) + reuseTree.setPrimaryStorageUuid(primaryStorage.uuid) + bean(DatabaseFacade.class).persist(reuseTree) + reuseTreeUuid = reuseTree.uuid + + VFS vfs = CephPrimaryStorageSpec.vfs1("7ff218d9-f525-435f-8a40-3618d1772a64", env) + vfs.createDirectories(String.format("/%s", NEW_ROOT_POOL_NAME)) + vfs.createDirectories(String.format("/%s", defaultImageCachePoolName)) + [protectedCache, deletedCache].each { ImageCacheVO cache -> + String imagePath = CephPrimaryStorageSpec.cephPathToVFSPath(cache.installUrl.split("@")[0]) + String snapshotPath = CephPrimaryStorageSpec.cephPathToVFSPath(cache.installUrl) + if (!vfs.exists(imagePath)) { + vfs.createCephRaw(imagePath, 0L) + } + if (!vfs.exists(snapshotPath)) { + vfs.createCephRaw(snapshotPath, 0L) + } + } + + ImageCacheVolumeRefVO ref = new ImageCacheVolumeRefVO() + ref.setImageCacheId(protectedCache.id) + ref.setPrimaryStorageUuid(primaryStorage.uuid) + ref.setVolumeUuid(vm.rootVolumeUuid) + bean(DatabaseFacade.class).persist(ref) + + List deleteCmds = [] + env.preSimulator(CephPrimaryStorageBase.DELETE_IMAGE_CACHE) { HttpEntity e -> + deleteCmds.add(json(e.body, CephPrimaryStorageBase.DeleteImageCacheCmd.class)) + } + + cleanUpImageCacheOnPrimaryStorage { + uuid = primaryStorage.uuid + force = true + } + + retryInSecs { + assert dbFindById(protectedCache.id, ImageCacheVO.class) != null + assert dbFindById(deletedCache.id, ImageCacheVO.class) == null + assert dbFindById(reuseCache.id, ImageCacheVO.class) != null + assert !Q.New(ImageCacheShadowVO.class).eq(ImageCacheShadowVO_.installUrl, deletedCache.installUrl).isExists() + assert !vfs.exists(CephPrimaryStorageSpec.cephPathToVFSPath(deletedCache.installUrl.split("@")[0])) + assert !vfs.exists(CephPrimaryStorageSpec.cephPathToVFSPath(deletedCache.installUrl)) + assert deleteCmds.find { it.snapshotPath == deletedCache.installUrl } != null + assert deleteCmds.find { it.snapshotPath == protectedCache.installUrl } == null + assert deleteCmds.find { it.snapshotPath == reuseCache.installUrl } == null + } + } finally { + if (reuseTreeUuid != null) { + bean(DatabaseFacade.class).removeByPrimaryKey(reuseTreeUuid, VolumeSnapshotReferenceTreeVO.class) + } + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + } } void testAddCephPoolWithChinese(){ @@ -330,6 +1025,14 @@ class CephPrimaryStorageVolumePoolsCase extends SubCase { } void testReimageVmAndAllocatePool() { + ensureNewRootPoolIsImageCachePool() + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.PreferVolumePool.toString()) + + CephPrimaryStorageBase.CloneCmd cloneCmd = null + env.preSimulator(CephPrimaryStorageBase.CLONE_PATH) { HttpEntity e -> + cloneCmd = json(e.body, CephPrimaryStorageBase.CloneCmd.class) + } + L3NetworkSpec l3Spec = env.specByName("l3") as L3NetworkSpec VmInstanceInventory new_root_pool_vm = createVmInstance { name = "new_root_pool_vm" @@ -345,12 +1048,77 @@ class CephPrimaryStorageVolumePoolsCase extends SubCase { uuid = new_root_pool_vm.uuid } - reimageVmInstance { - vmInstanceUuid = new_root_pool_vm.uuid + cloneCmd = null + try { + reimageVmInstance { + vmInstanceUuid = new_root_pool_vm.uuid + } + } finally { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) } String VolumeInstallPath = Q.New(VolumeVO.class).select(VolumeVO_.installPath).eq(VolumeVO_.uuid, new_root_pool_vm.rootVolumeUuid).findValue() assert VolumeInstallPath.contains(NEW_ROOT_POOL_NAME) + assert cloneCmd != null + assert cloneCmd.srcPath.contains(NEW_ROOT_POOL_NAME) + } + + void testReimageVmCopiesImageToSelectedPoolAfterImageDeleted() { + ensureNewRootPoolIsImageCachePool() + ImageInventory image4 = env.inventoryByName("image4") as ImageInventory + VmInstanceInventory reimageVm = null + CephPrimaryStorageBase.CpCmd cpCmd = null + CephPrimaryStorageBase.CloneCmd cloneCmd = null + env.hijackSimulator(CephPrimaryStorageBase.CP_PATH) { rsp, HttpEntity e -> + cpCmd = json(e.body, CephPrimaryStorageBase.CpCmd.class) + return rsp + } + env.preSimulator(CephPrimaryStorageBase.CLONE_PATH) { HttpEntity e -> + cloneCmd = json(e.body, CephPrimaryStorageBase.CloneCmd.class) + } + + try { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + reimageVm = createVmInstance { + name = "reimage-vm-after-image-deleted" + instanceOfferingUuid = vm.instanceOfferingUuid + imageUuid = image4.uuid + l3NetworkUuids = asList(l3.uuid) + sessionId = adminSession() + rootVolumeSystemTags = [CephSystemTags.USE_CEPH_ROOT_POOL.instantiateTag([(CephSystemTags.USE_CEPH_ROOT_POOL_TOKEN): NEW_ROOT_POOL_NAME])] + } as VmInstanceInventory + + stopVmInstance { + uuid = reimageVm.uuid + } + + deleteImage { + uuid = image4.uuid + } + + cpCmd = null + cloneCmd = null + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.PreferVolumePool.toString()) + reimageVmInstance { + vmInstanceUuid = reimageVm.uuid + } + + assert cpCmd != null + assert cpCmd.dstPath.contains(NEW_ROOT_POOL_NAME) + assert cloneCmd != null + assert cloneCmd.srcPath.contains(NEW_ROOT_POOL_NAME) + } finally { + CephGlobalConfig.IMAGE_CACHE_POOL_STRATEGY.updateValue(CephImageCachePoolStrategy.DefaultImageCachePool.toString()) + + if (reimageVm != null) { + destroyVmInstance { + uuid = reimageVm.uuid + } + expungeVmInstance { + uuid = reimageVm.uuid + } + } + } } void testCreateVmInstanceWithCustomDiskOffering() { @@ -433,8 +1201,18 @@ class CephPrimaryStorageVolumePoolsCase extends SubCase { testAddPoolWithCheckExistenceFailure() testQueryPool() testAddSameCephPool() + testPreferVolumePoolImageCacheStrategy() + testDefaultImageCachePoolStrategyUsesDefaultPool() + testDefaultStrategyCopiesCephBackupStorageImageToDefaultPool() + testPreferVolumePoolFallbackToDefaultPool() + testPreferExistingCacheStrategyPrefersDefaultPoolCache() + testPreferExistingCacheStrategyUsesNonDefaultPoolCache() + testPreferExistingCacheStrategySkipsStaleDefaultPoolCache() + testMissingSelectedCacheBitsCopiesFromOtherPool() + testCleanupImageCacheKeepsReferencedCacheOnly() testAddCephPoolWithChinese() testReimageVmAndAllocatePool() + testReimageVmCopiesImageToSelectedPoolAfterImageDeleted() testCreateVmInstanceWithCustomDiskOffering() } } diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/capacity/CephOpenSourcePoolCapacityCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/capacity/CephOpenSourcePoolCapacityCase.groovy index 09c74368ad3..edf752b09b1 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/capacity/CephOpenSourcePoolCapacityCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/capacity/CephOpenSourcePoolCapacityCase.groovy @@ -212,4 +212,4 @@ class CephOpenSourcePoolCapacityCase extends SubCase { assert bsCapacity.totalCapacity == 106300440576 // 99G env.cleanSimulatorAndMessageHandlers() } -} \ No newline at end of file +} diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/sandstone/capacity/CephSandStonePoolCapacityCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/sandstone/capacity/CephSandStonePoolCapacityCase.groovy index df1fc1d7d0e..0eb92a7cfd7 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/sandstone/capacity/CephSandStonePoolCapacityCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/sandstone/capacity/CephSandStonePoolCapacityCase.groovy @@ -163,4 +163,5 @@ class CephSandStonePoolCapacityCase extends SubCase { assert afterBs.availableCapacity == bs.availableCapacity + addSize assert afterBs.totalCapacity == bs.totalCapacity + addSize } + } diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/xsky/capacity/CephXskyPoolCapacityCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/xsky/capacity/CephXskyPoolCapacityCase.groovy index 356599909f6..f23ed4a3c3a 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/xsky/capacity/CephXskyPoolCapacityCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/ceph/xsky/capacity/CephXskyPoolCapacityCase.groovy @@ -205,4 +205,5 @@ class CephXskyPoolCapacityCase extends SubCase { SQL.New(VolumeSnapshotVO.class).eq(VolumeSnapshotVO_.uuid, rootSnapshot.uuid) .set(VolumeSnapshotVO_.primaryStorageInstallPath, volumeSnapshotInstallPath).update() } + } diff --git a/testlib/src/main/java/org/zstack/testlib/CephPrimaryStoragePoolSpec.groovy b/testlib/src/main/java/org/zstack/testlib/CephPrimaryStoragePoolSpec.groovy index c0cfec41607..12d6eb293b4 100755 --- a/testlib/src/main/java/org/zstack/testlib/CephPrimaryStoragePoolSpec.groovy +++ b/testlib/src/main/java/org/zstack/testlib/CephPrimaryStoragePoolSpec.groovy @@ -13,6 +13,8 @@ class CephPrimaryStoragePoolSpec extends Spec { String description @SpecParam(required = true) String type + @SpecParam + boolean isCreate = true CephPrimaryStoragePoolInventory inventory @@ -40,7 +42,7 @@ class CephPrimaryStoragePoolSpec extends Spec { delegate.resourceUuid = uuid delegate.sessionId = sessionId delegate.type = type - delegate.isCreate = true + delegate.isCreate = isCreate } return id(poolName, inventory.uuid) diff --git a/testlib/src/main/java/org/zstack/testlib/CephPrimaryStorageSpec.groovy b/testlib/src/main/java/org/zstack/testlib/CephPrimaryStorageSpec.groovy index e8f90f1d110..9521d8dec87 100755 --- a/testlib/src/main/java/org/zstack/testlib/CephPrimaryStorageSpec.groovy +++ b/testlib/src/main/java/org/zstack/testlib/CephPrimaryStorageSpec.groovy @@ -37,8 +37,10 @@ class CephPrimaryStorageSpec extends PrimaryStorageSpec { String rootVolumePoolName = "pri-v-r-" + Platform.getUuid() @SpecParam String dataVolumePoolName = "pri-v-d-" + Platform.getUuid() - @SpecParam - String imageCachePoolName = "pri-c-" + Platform.getUuid() + @SpecParam + String imageCachePoolName = "pri-c-" + Platform.getUuid() + @SpecParam + String extraImageCachePoolName CephPrimaryStorageSpec(EnvSpec envSpec) { super(envSpec) @@ -78,9 +80,10 @@ class CephPrimaryStorageSpec extends PrimaryStorageSpec { } class CephPrimaryStorageStruct { - String rootVolumePoolName - String dataVolumePoolName - String imageCachePoolName + String rootVolumePoolName + String dataVolumePoolName + String imageCachePoolName + String extraImageCachePoolName } static class Simulators implements Simulator { @@ -496,18 +499,19 @@ class CephPrimaryStorageSpec extends PrimaryStorageSpec { return new CephPrimaryStorageBase.AgentResponse() } - simulator(CephPrimaryStorageBase.ADD_POOL_PATH) { HttpEntity entity, EnvSpec spec -> - def cmd = JSONObjectUtil.toObject(entity.body, CephPrimaryStorageBase.AddPoolCmd.class) + simulator(CephPrimaryStorageBase.ADD_POOL_PATH) { HttpEntity entity, EnvSpec spec -> + def cmd = JSONObjectUtil.toObject(entity.body, CephPrimaryStorageBase.AddPoolCmd.class) CephPrimaryStorageSpec cspec = spec.specByUuid(cmd.uuid) CephPrimaryStorageBase.AddPoolRsp rsp = new CephPrimaryStorageBase.AddPoolRsp() - rsp.totalCapacity = cspec.totalCapacity - rsp.availableCapacity = cspec.availableCapacity - long rootSize = cspec.availableCapacity / 3 - long dataSize = cspec.availableCapacity / 3 - long cacheSize = cspec.totalCapacity - rootSize - dataSize - rsp.setAvailableCapacity(SizeUnit.GIGABYTE.toByte(100)) - rsp.setTotalCapacity(SizeUnit.GIGABYTE.toByte(100)) + def newPoolCapacity = SizeUnit.GIGABYTE.toByte(100) + rsp.totalCapacity = newPoolCapacity + rsp.availableCapacity = newPoolCapacity + long rootSize = cspec.availableCapacity / 3 + long dataSize = cspec.availableCapacity / 3 + long cacheSize = cspec.totalCapacity - rootSize - dataSize + rsp.setAvailableCapacity(newPoolCapacity) + rsp.setTotalCapacity(newPoolCapacity) List poolCapacities = [ new CephPoolCapacity( name: cspec.rootVolumePoolName, @@ -539,17 +543,17 @@ class CephPrimaryStorageSpec extends PrimaryStorageSpec { diskUtilization: 0.33, relatedOsds: 'osd.3' ), - new CephPoolCapacity( - name: cmd.poolName, - availableCapacity: SizeUnit.GIGABYTE.toByte(100), - usedCapacity: 0, - totalCapacity: SizeUnit.GIGABYTE.toByte(100), - securityPolicy: DataSecurityPolicy.Copy.toString(), - replicatedSize: 3, - diskUtilization: 0.33, - relatedOsds: "osd.4" - ) - ] + new CephPoolCapacity( + name: cmd.poolName, + availableCapacity: newPoolCapacity, + usedCapacity: 0, + totalCapacity: newPoolCapacity, + securityPolicy: DataSecurityPolicy.Copy.toString(), + replicatedSize: 3, + diskUtilization: 0.33, + relatedOsds: 'osd.4' + ) + ] rsp.setPoolCapacities(poolCapacities) return rsp @@ -568,7 +572,7 @@ class CephPrimaryStorageSpec extends PrimaryStorageSpec { return } - if (f.parent?.setVolumeChainInstallPaths() == snapshotPath) { + if (f.parent?.pathString() == snapshotPath) { children.add(f.pathString()) } } @@ -820,14 +824,24 @@ class CephPrimaryStorageSpec extends PrimaryStorageSpec { delegate.imageCachePoolName = imageCachePoolName } as PrimaryStorageInventory - postCreate { - inventory = queryCephPrimaryStorage { - conditions=["uuid=${inventory.uuid}".toString()] - }[0] - } - - return id(name, inventory.uuid) - } + postCreate { + inventory = queryCephPrimaryStorage { + conditions=["uuid=${inventory.uuid}".toString()] + }[0] + } + + if (extraImageCachePoolName != null) { + addCephPrimaryStoragePool { + delegate.primaryStorageUuid = inventory.uuid + delegate.poolName = extraImageCachePoolName + delegate.type = CephPrimaryStoragePoolType.ImageCache.toString() + delegate.isCreate = true + delegate.sessionId = sessionId + } + } + + return id(name, inventory.uuid) + } CephPrimaryStoragePoolSpec pool(@DelegatesTo(strategy = Closure.DELEGATE_FIRST, value = CephPrimaryStoragePoolSpec.class) Closure c) { def spec = new CephPrimaryStoragePoolSpec(envSpec)