CSTACKEX-18: Cloudstack snapshot workflow for NFS3 protocol

plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/driver/OntapPrimaryDatastoreDriver.java

@@ -27,6 +27,11 @@
 import com.cloud.storage.Storage;
 import com.cloud.storage.StoragePool;
 import com.cloud.storage.Volume;
+import com.cloud.storage.VolumeVO;
+import com.cloud.storage.dao.SnapshotDetailsDao;
+import com.cloud.storage.dao.SnapshotDetailsVO;
+import com.cloud.storage.dao.VolumeDao;
+import com.cloud.storage.dao.VolumeDetailsDao;
 import com.cloud.utils.Pair;
 import com.cloud.utils.exception.CloudRuntimeException;
 import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo;
@@ -41,14 +46,18 @@
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
 import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
 import org.apache.cloudstack.storage.command.CommandResult;
+import org.apache.cloudstack.storage.command.CreateObjectAnswer;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
+import org.apache.cloudstack.storage.feign.model.FileInfo;
 import org.apache.cloudstack.storage.service.StorageStrategy;
 import org.apache.cloudstack.storage.service.model.CloudStackVolume;
 import org.apache.cloudstack.storage.service.model.ProtocolType;
+import org.apache.cloudstack.storage.to.SnapshotObjectTO;
 import org.apache.cloudstack.storage.utils.Constants;
 import org.apache.cloudstack.storage.utils.Utility;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -62,6 +71,9 @@ public class OntapPrimaryDatastoreDriver implements PrimaryDataStoreDriver {
 
     @Inject private StoragePoolDetailsDao storagePoolDetailsDao;
     @Inject private PrimaryDataStoreDao storagePoolDao;
+    @Inject private VolumeDao volumeDao;
+    @Inject private VolumeDetailsDao volumeDetailsDao;
+    @Inject private SnapshotDetailsDao snapshotDetailsDao;
 
     @Override
     public Map<String, String> getCapabilities() {
@@ -227,6 +239,79 @@ public long getUsedIops(StoragePool storagePool) {
     @Override
     public void takeSnapshot(SnapshotInfo snapshot, AsyncCompletionCallback<CreateCmdResult> callback) {
 
+        CreateCmdResult result;
+
+        try {
+            VolumeInfo volumeInfo = snapshot.getBaseVolume();
+
+            VolumeVO volumeVO = volumeDao.findById(volumeInfo.getId());
+            if(volumeVO == null) {
+                throw new CloudRuntimeException("takeSnapshot: VolumeVO not found for id: " + volumeInfo.getId());
+            }
+
+            /** we are keeping file path at volumeVO.getPath() */
+
+            StoragePoolVO storagePool = storagePoolDao.findById(volumeVO.getPoolId());
+            if(storagePool == null) {
+                s_logger.error("takeSnapshot : Storage Pool not found for id: " + volumeVO.getPoolId());
+                throw new CloudRuntimeException("takeSnapshot : Storage Pool not found for id: " + volumeVO.getPoolId());
+            }
+            Map<String, String> poolDetails = storagePoolDetailsDao.listDetailsKeyPairs(volumeVO.getPoolId());
+            StorageStrategy storageStrategy = Utility.getStrategyByStoragePoolDetails(poolDetails);
+
+            CloudStackVolume cloudStackVolumeRequest = getCloudStackVolumeRequestByProtocol(poolDetails, volumeVO.getPath());
+            CloudStackVolume cloudStackVolume = storageStrategy.getCloudStackVolume(cloudStackVolumeRequest);
+            if (cloudStackVolume == null || cloudStackVolume.getFile() == null) {
+                throw new CloudRuntimeException("takeSnapshot: Failed to get source file to take snapshot");
+            }
+            long capacityBytes = storagePool.getCapacityBytes();
+
+            long usedBytes = getUsedBytes(storagePool);
+            long fileSize = cloudStackVolume.getFile().getSize();
+
+            usedBytes += fileSize;
+
+            if (usedBytes > capacityBytes) {
+                throw new CloudRuntimeException("Insufficient space remains in this primary storage to take a snapshot");
+            }
+
+            storagePool.setUsedBytes(usedBytes);
+
+            SnapshotObjectTO snapshotObjectTo = (SnapshotObjectTO)snapshot.getTO();
+
+            String fileSnapshotName = volumeInfo.getName() + "-" + snapshot.getUuid();
+
+            int maxSnapshotNameLength = 64;
+            int trimRequired = fileSnapshotName.length() - maxSnapshotNameLength;
+
+            if (trimRequired > 0) {
+                fileSnapshotName = StringUtils.left(volumeInfo.getName(), (volumeInfo.getName().length() - trimRequired)) + "-" + snapshot.getUuid();
+            }
+
+            CloudStackVolume snapCloudStackVolumeRequest = snapshotCloudStackVolumeRequestByProtocol(poolDetails, volumeVO.getPath(), fileSnapshotName);
+            CloudStackVolume cloneCloudStackVolume = storageStrategy.snapshotCloudStackVolume(snapCloudStackVolumeRequest);
+
+            updateSnapshotDetails(snapshot.getId(), volumeInfo.getId(), poolDetails.get(Constants.VOLUME_UUID), cloneCloudStackVolume.getFile().getPath(), volumeVO.getPoolId(), fileSize);
+
+            snapshotObjectTo.setPath(Constants.ONTAP_SNAP_ID +"="+cloneCloudStackVolume.getFile().getPath());
+
+            /** Update size for the storage-pool including snapshot size */
+            storagePoolDao.update(volumeVO.getPoolId(), storagePool);
+
+            CreateObjectAnswer createObjectAnswer = new CreateObjectAnswer(snapshotObjectTo);
+
+            result = new CreateCmdResult(null, createObjectAnswer);
+
+            result.setResult(null);
+        }
+        catch (Exception ex) {
+            s_logger.error("takeSnapshot: Failed due to ", ex);
+            result = new CreateCmdResult(null, new CreateObjectAnswer(ex.toString()));
+
+            result.setResult(ex.toString());
+        }
+
+        callback.complete(result);
     }
 
     @Override
@@ -293,4 +378,87 @@ public boolean isStorageSupportHA(Storage.StoragePoolType type) {
     public void detachVolumeFromAllStorageNodes(Volume volume) {
 
     }
+
+
+    private CloudStackVolume getCloudStackVolumeRequestByProtocol(Map<String, String> details, String filePath) {
+        CloudStackVolume cloudStackVolumeRequest = null;
+        ProtocolType protocolType = null;
+        String protocol = null;
+
+        try {
+            protocol = details.get(Constants.PROTOCOL);
+            protocolType = ProtocolType.valueOf(protocol);
+        } catch (IllegalArgumentException e) {
+            throw new CloudRuntimeException("getCloudStackVolumeRequestByProtocol: Protocol: "+ protocol +" is not valid");
+        }
+        switch (protocolType) {
+            case NFS3:
+                cloudStackVolumeRequest = new CloudStackVolume();
+                FileInfo fileInfo = new FileInfo();
+                fileInfo.setPath(filePath);
+                cloudStackVolumeRequest.setFile(fileInfo);
+                String volumeUuid = details.get(Constants.VOLUME_UUID);
+                cloudStackVolumeRequest.setFlexVolumeUuid(volumeUuid);
+                break;
+            default:
+                throw new CloudRuntimeException("createCloudStackVolumeRequestByProtocol: Unsupported protocol " + protocol);
+        }
+        return cloudStackVolumeRequest;
+    }
+
+private CloudStackVolume snapshotCloudStackVolumeRequestByProtocol(Map<String, String> details,
+                                                                   String sourcePath,
+                                                                   String destinationPath) {
+    CloudStackVolume cloudStackVolumeRequest = null;
+    ProtocolType protocolType = null;
+    String protocol = null;
+
+    try {
+        protocol = details.get(Constants.PROTOCOL);
+        protocolType = ProtocolType.valueOf(protocol);
+    } catch (IllegalArgumentException e) {
+        throw new CloudRuntimeException("getCloudStackVolumeRequestByProtocol: Protocol: "+ protocol +" is not valid");
+    }
+    switch (protocolType) {
+        case NFS3:
+            cloudStackVolumeRequest = new CloudStackVolume();
+            FileInfo fileInfo = new FileInfo();
+            fileInfo.setPath(sourcePath);
+            cloudStackVolumeRequest.setFile(fileInfo);
+            String volumeUuid = details.get(Constants.VOLUME_UUID);
+            cloudStackVolumeRequest.setFlexVolumeUuid(volumeUuid);
+            cloudStackVolumeRequest.setDestinationPath(destinationPath);
+            break;
+        default:
+            throw new CloudRuntimeException("createCloudStackVolumeRequestByProtocol: Unsupported protocol " + protocol);
+
+    }
+    return cloudStackVolumeRequest;
+}
+
+    /**
+     *
+     * @param csSnapshotId: generated snapshot id from cloudstack
+     * @param csVolumeId: Source CS volume id
+     * @param ontapVolumeUuid: storage flexvolume id
+     * @param ontapNewSnapshot: generated snapshot id from ONTAP
+     * @param storagePoolId: primary storage pool id
+     * @param ontapSnapSize: Size of snapshot CS volume(LUN/file)
+     */
+    private void updateSnapshotDetails(long csSnapshotId, long csVolumeId, String ontapVolumeUuid, String ontapNewSnapshot, long storagePoolId, long ontapSnapSize) {
+    SnapshotDetailsVO snapshotDetail = new SnapshotDetailsVO(csSnapshotId, Constants.SRC_CS_VOLUME_ID,  String.valueOf(csVolumeId), false);
+    snapshotDetailsDao.persist(snapshotDetail);
+
+    snapshotDetail = new SnapshotDetailsVO(csSnapshotId, Constants.BASE_ONTAP_FV_ID, String.valueOf(ontapVolumeUuid), false);
+    snapshotDetailsDao.persist(snapshotDetail);
+
+    snapshotDetail = new SnapshotDetailsVO(csSnapshotId, Constants.ONTAP_SNAP_ID, String.valueOf(ontapNewSnapshot), false);
+    snapshotDetailsDao.persist(snapshotDetail);
+
+    snapshotDetail = new SnapshotDetailsVO(csSnapshotId, Constants.PRIMARY_POOL_ID, String.valueOf(storagePoolId), false);
+    snapshotDetailsDao.persist(snapshotDetail);
+
+    snapshotDetail = new SnapshotDetailsVO(csSnapshotId, Constants.ONTAP_SNAP_SIZE, String.valueOf(ontapSnapSize), false);
+    snapshotDetailsDao.persist(snapshotDetail);
+}
 }

plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/client/NASFeignClient.java

@@ -21,7 +21,9 @@
 
 import feign.QueryMap;
 import org.apache.cloudstack.storage.feign.model.ExportPolicy;
+import org.apache.cloudstack.storage.feign.model.FileClone;
 import org.apache.cloudstack.storage.feign.model.FileInfo;
+import org.apache.cloudstack.storage.feign.model.response.JobResponse;
 import org.apache.cloudstack.storage.feign.model.response.OntapResponse;
 import feign.Headers;
 import feign.Param;
@@ -58,6 +60,11 @@ void createFile(@Param("authHeader") String authHeader,
                     @Param("path") String filePath,
                     FileInfo file);
 
+    @RequestLine("POST /api/storage/volumes/{volumeUuid}/files/{path}")
+    @Headers({"Authorization: {authHeader}"})
+    JobResponse cloneFile(@Param("authHeader") String authHeader,
+                          FileClone fileClone);
+
     // Export Policy Operations
     @RequestLine("POST /api/protocols/nfs/export-policies")
     @Headers({"Authorization: {authHeader}"})

plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/FileClone.java

@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.storage.feign.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class FileClone {
+    @JsonProperty("source_path")
+    private String sourcePath;
+    @JsonProperty("destination_path")
+    private String destinationPath;
+    @JsonProperty("volume")
+    private VolumeConcise volume;
+    public VolumeConcise getVolume() {
+        return volume;
+    }
+    public void setVolume(VolumeConcise volume) {
+        this.volume = volume;
+    }
+    public String getSourcePath() {
+        return sourcePath;
+    }
+    public void setSourcePath(String sourcePath) {
+        this.sourcePath = sourcePath;
+    }
+    public String getDestinationPath() {
+        return destinationPath;
+    }
+    public void setDestinationPath(String destinationPath) {}
+}

plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/feign/model/VolumeConcise.java

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.storage.feign.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class VolumeConcise {
+    @JsonProperty("uuid")
+    private String uuid;
+    @JsonProperty("name")
+    private String name;
+    public String getUuid() {
+        return uuid;
+    }
+    public void setUuid(String uuid) {
+        this.uuid = uuid;
+    }
+    public String getName() {
+        return name;
+    }
+    public void setName(String name) {}
+}

plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/StorageStrategy.java

@@ -393,8 +393,6 @@ public String getStoragePath() {
         return targetIqn;
     }
 
-
-
     /**
      * Get the network ip interface
      *
@@ -484,7 +482,19 @@ public String getNetworkInterface() {
      * @param cloudstackVolume the CloudStack volume to retrieve
      * @return the retrieved CloudStackVolume object
      */
-    abstract CloudStackVolume getCloudStackVolume(CloudStackVolume cloudstackVolume);
+    public abstract CloudStackVolume getCloudStackVolume(CloudStackVolume cloudstackVolume);
+
+    /**
+     * Method encapsulates the behavior based on the opted protocol in subclasses.
+     * it is going to mimic
+     * snapshotLun       for iSCSI, FC protocols
+     * snapshotFile      for NFS3.0 and NFS4.1 protocols
+     *
+     *
+     * @param cloudstackVolume the source CloudStack volume
+     * @return the retrieved snapshot CloudStackVolume object
+     */
+    public abstract CloudStackVolume snapshotCloudStackVolume(CloudStackVolume cloudstackVolume);
 
     /**
      * Method encapsulates the behavior based on the opted protocol in subclasses
@@ -547,7 +557,7 @@ public String getNetworkInterface() {
      */
     abstract void disableLogicalAccess(Map<String, String> values);
 
-    private Boolean jobPollForSuccess(String jobUUID) {
+    public Boolean jobPollForSuccess(String jobUUID) {
         //Create URI for GET Job API
         int jobRetryCount = 0;
         Job jobResp = null;