Linstor add support for incremental nasbackup

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -636,6 +636,23 @@ public class AgentProperties{
      */
     public static final Property<String> KVM_SCRIPTS_DIR = new Property<>("kvm.scripts.dir", "scripts/vm/hypervisor/kvm");
 
+    /**
+     * Host-local parent directory for the NAS backup pull-mode scratch area: the NBD socket and
+     * the libvirt-managed fleecing (copy-before-write) images used while backing up a running VM
+     * on raw block-device storage such as LINSTOR/DRBD.
+     * <p>
+     * Must be on a real on-disk filesystem with room for the guest's write churn during a backup,
+     * and must NOT be on the NAS share: copy-before-write runs inline with guest writes, so a slow
+     * or remote location stalls the VM. Keep the path short — the resulting socket path is subject
+     * to the kernel's 108-byte UNIX socket limit.
+     * <p>
+     * Data type: String.<br>
+     * Default value: <code>/var/tmp</code>
+     *
+     * @since 4.22.0
+     */
+    public static final Property<String> NAS_BACKUP_PULL_SCRATCH_DIR = new Property<>("nas.backup.pull.scratch.dir", "/var/tmp");
+
     /**
      * Specifies start MAC address for private IP range.<br>
      * Data type: String.<br>

plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupChainKeys.java

@@ -48,6 +48,20 @@ public final class NASBackupChainKeys {
     // "legacy-full" mode token (which sets make_checkpoint=0).
     public static final String TYPE_LEGACY_FULL = "legacy-full";
 
+    /**
+     * Backup taken through the content-based (libvirt pull mode) path, used for raw
+     * block-device storage such as LINSTOR/DRBD which cannot carry QEMU persistent dirty
+     * bitmaps. A full standalone qcow2, written sparse in a single pass.
+     */
+    public static final String TYPE_CONTENT_FULL = "content-full";
+
+    /**
+     * Delta backup produced by content comparison rather than dirty bitmaps: a qcow2 overlay
+     * on the disk's NBD export, safe-rebased onto the parent so it holds only the clusters
+     * that differ. Same chain shape as {@link #TYPE_INCREMENTAL}.
+     */
+    public static final String TYPE_CONTENT_INCREMENTAL = "content-incremental";
+
     /**
      * VM-scoped detail (stored in {@code vm_instance_details}) holding the QEMU dirty-bitmap
      * name that currently exists on the running VM and is therefore the only valid parent

plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java

@@ -224,15 +224,24 @@ static final class ChainDecision {
         final List<String> parentPaths;
         final String chainId;         // chain identifier this backup belongs to
         final int chainPosition;      // 0 for full, N for the Nth incremental in the chain
+        // Content-based chains have no bitmap to look the parent up by, so the parent backup's
+        // uuid is resolved when the decision is made. null for bitmap-based decisions.
+        final String parentBackupUuid;
 
         private ChainDecision(String mode, String bitmapNew, String bitmapParent, List<String> parentPaths,
                               String chainId, int chainPosition) {
+            this(mode, bitmapNew, bitmapParent, parentPaths, chainId, chainPosition, null);
+        }
+
+        private ChainDecision(String mode, String bitmapNew, String bitmapParent, List<String> parentPaths,
+                              String chainId, int chainPosition, String parentBackupUuid) {
             this.mode = mode;
             this.bitmapNew = bitmapNew;
             this.bitmapParent = bitmapParent;
             this.parentPaths = parentPaths;
             this.chainId = chainId;
             this.chainPosition = chainPosition;
+            this.parentBackupUuid = parentBackupUuid;
         }
 
         static ChainDecision fullStart(String bitmapName) {
@@ -255,8 +264,30 @@ static ChainDecision incremental(String bitmapNew, String bitmapParent, List<Str
                     parentPaths, chainId, chainPosition);
         }
 
+        /**
+         * Full backup on the content-based (pull mode) path: no bitmap, no checkpoint, but it
+         * does anchor a chain so later content-incrementals can hang off it.
+         */
+        static ChainDecision contentFull() {
+            return new ChainDecision(NASBackupChainKeys.TYPE_CONTENT_FULL, null, null, null,
+                    UUID.randomUUID().toString(), 0, null);
+        }
+
+        static ChainDecision contentIncremental(List<String> parentPaths, String chainId,
+                                                int chainPosition, String parentBackupUuid) {
+            return new ChainDecision(NASBackupChainKeys.TYPE_CONTENT_INCREMENTAL, null, null,
+                    parentPaths, chainId, chainPosition, parentBackupUuid);
+        }
+
         boolean isIncremental() {
-            return NASBackupChainKeys.TYPE_INCREMENTAL.equals(mode);
+            return NASBackupChainKeys.TYPE_INCREMENTAL.equals(mode)
+                    || NASBackupChainKeys.TYPE_CONTENT_INCREMENTAL.equals(mode);
+        }
+
+        /** True for the pull-mode content-diff path (no dirty bitmaps involved). */
+        boolean isContentBased() {
+            return NASBackupChainKeys.TYPE_CONTENT_FULL.equals(mode)
+                    || NASBackupChainKeys.TYPE_CONTENT_INCREMENTAL.equals(mode);
         }
 
         boolean isLegacyFull() {
@@ -281,14 +312,25 @@ protected ChainDecision decideChain(VirtualMachine vm) {
         // behaves exactly like the pre-incremental full-only path: no bitmap is generated and no
         // chain/checkpoint metadata is created, sent to the agent, or persisted (legacy-full).
         Boolean incrementalEnabled = NASBackupIncrementalEnabled.valueIn(vm.getDataCenterId());
-        if (incrementalEnabled == null || !incrementalEnabled) {
+        final boolean incrementalOn = incrementalEnabled != null && incrementalEnabled;
+
+        // Raw block-device storage (LINSTOR/DRBD) cannot carry QEMU persistent dirty bitmaps, so
+        // libvirt checkpoints — and with them the push-mode incremental path — are unavailable.
+        // Those VMs take the content-based pull-mode path instead. It is chosen regardless of the
+        // master switch, because even a full backup benefits: pull mode writes a sparse qcow2 in a
+        // single pass rather than a fully allocated one needing a second re-convert pass.
+        if (allVolumesOnContentDiffCapableStorage(vm)) {
+            return decideContentChain(vm, incrementalOn);
+        }
+
+        if (!incrementalOn) {
             return ChainDecision.legacyFull();
         }
 
         // Incremental backups rely on QEMU dirty bitmaps / libvirt checkpoints, which only exist
-        // on file-based qcow2 storage. Storage such as Ceph-RBD and Linstor cannot carry per-disk
-        // checkpoints, so a VM with any volume on such a pool must stay on the full-only (legacy)
-        // path — otherwise an incremental attempt would fail or regress those storages.
+        // on file-based qcow2 storage. Storage such as Ceph-RBD cannot carry per-disk checkpoints
+        // and has no content-diff path either, so such a VM stays on the full-only (legacy) path —
+        // otherwise an incremental attempt would fail or regress those storages.
         if (!allVolumesOnCheckpointCapableStorage(vm)) {
             return ChainDecision.legacyFull();
         }
@@ -352,6 +394,78 @@ protected ChainDecision decideChain(VirtualMachine vm) {
                 parentChainId, parentChainPosition + 1);
     }
 
+    /**
+     * Chain decision for content-based (pull mode) storage such as LINSTOR/DRBD.
+     *
+     * <p>Unlike the bitmap path there is no host-side state to anchor on. The delta is derived by
+     * comparing the disk's point-in-time NBD export against the parent backup, so the chain cannot
+     * be invalidated by a VM restart, live migration or restore — the comparison is stateless. The
+     * parent is therefore simply the most recent BackedUp backup of the same chain, and the agent
+     * independently verifies the parent files still exist on the NAS, degrading to a full if they
+     * do not (INCREMENTAL_FALLBACK).</p>
+     */
+    protected ChainDecision decideContentChain(VirtualMachine vm, boolean incrementalEnabled) {
+        if (!incrementalEnabled) {
+            return ChainDecision.contentFull();
+        }
+
+        // Stopped VMs are backed up straight from the disk with qemu-img convert; that path has no
+        // point-in-time export to diff against a parent, so it is always a full.
+        if (VirtualMachine.State.Stopped.equals(vm.getState())) {
+            return ChainDecision.contentFull();
+        }
+
+        Integer fullEvery = NASBackupFullEvery.valueIn(vm.getDataCenterId());
+        if (fullEvery == null || fullEvery <= 1) {
+            return ChainDecision.contentFull();
+        }
+
+        Backup parent = findLatestBackedUpBackup(vm.getId());
+        if (parent == null) {
+            return ChainDecision.contentFull();
+        }
+
+        String parentChainId = readDetail(parent, NASBackupChainKeys.CHAIN_ID);
+        int parentChainPosition = chainPosition(parent);
+        if (parentChainId == null || parentChainPosition == Integer.MAX_VALUE) {
+            return ChainDecision.contentFull();
+        }
+
+        // Force a fresh full when the chain has reached the configured length.
+        if (parentChainPosition + 1 >= fullEvery) {
+            return ChainDecision.contentFull();
+        }
+
+        List<String> parentPaths = composeParentBackupPaths(parent, vm.getId());
+        if (parentPaths == null) {
+            LOG.debug("VM {} parent backup {} volume layout no longer matches current VM — forcing full",
+                    vm.getInstanceName(), parent.getUuid());
+            return ChainDecision.contentFull();
+        }
+        return ChainDecision.contentIncremental(parentPaths, parentChainId, parentChainPosition + 1,
+                parent.getUuid());
+    }
+
+    /**
+     * True when EVERY volume of the VM sits on storage whose backups must be derived by content
+     * comparison instead of dirty bitmaps — currently LINSTOR, whose volumes are raw DRBD block
+     * devices and so cannot persist a bitmap (persistence is a qcow2-only feature). A VM with a
+     * mix of such volumes and others is not eligible, since one backup run uses a single mode.
+     */
+    protected boolean allVolumesOnContentDiffCapableStorage(VirtualMachine vm) {
+        List<VolumeVO> volumes = volumeDao.findByInstance(vm.getId());
+        if (CollectionUtils.isEmpty(volumes)) {
+            return false;
+        }
+        for (VolumeVO volume : volumes) {
+            StoragePoolVO pool = primaryDataStoreDao.findById(volume.getPoolId());
+            if (pool == null || !Storage.StoragePoolType.Linstor.equals(pool.getPoolType())) {
+                return false;
+            }
+        }
+        return true;
+    }
+
     /**
      * Incremental backups require QEMU dirty bitmaps / libvirt checkpoints, which are only
      * possible on file-based qcow2 storage. Returns {@code true} only when EVERY volume of the
@@ -489,7 +603,10 @@ private void persistChainMetadata(Backup backup, ChainDecision decision, String
         // source of truth. Not duplicated into backup_details.
         if (decision.isIncremental()) {
             // Resolve the parent backup's UUID so restore can walk the chain by id, not by path.
-            String parentUuid = lookupParentBackupUuid(backup.getVmId(), decision.bitmapParent);
+            // Content-based decisions carry the parent uuid directly (no bitmap to look it up by).
+            String parentUuid = decision.parentBackupUuid != null
+                    ? decision.parentBackupUuid
+                    : lookupParentBackupUuid(backup.getVmId(), decision.bitmapParent);
             if (parentUuid != null) {
                 backupDetailsDao.persist(new BackupDetailVO(backup.getId(), NASBackupChainKeys.PARENT_BACKUP_ID, parentUuid, true));
             }
@@ -614,7 +731,9 @@ public Pair<Boolean, Backup> takeBackup(final VirtualMachine vm, Boolean quiesce
             // backup as a full and start a new chain.
             ChainDecision effective = decision;
             if (answer.getIncrementalFallback()) {
-                effective = ChainDecision.fullStart(decision.bitmapNew);
+                effective = decision.isContentBased()
+                        ? ChainDecision.contentFull()
+                        : ChainDecision.fullStart(decision.bitmapNew);
                 backupVO.setType("FULL");
             }
             List<Volume> volumes = new ArrayList<>(volumeDao.findByInstance(vm.getId()));
@@ -628,11 +747,15 @@ public Pair<Boolean, Backup> takeBackup(final VirtualMachine vm, Boolean quiesce
                     // created — the only valid parent for the next incremental (see decideChain).
                     // If the agent reports no bitmap (bitmapCreated=null), clear any stale detail
                     // so the next backup starts a fresh full.
-                    String confirmedBitmap = answer.getBitmapCreated();
-                    if (confirmedBitmap != null) {
-                        upsertVmActiveCheckpoint(vm.getId(), confirmedBitmap);
-                    } else {
-                        clearVmActiveCheckpoint(vm.getId());
+                    // The content-based path creates no bitmaps, so there is no active
+                    // checkpoint to track — its chain is anchored purely on backup history.
+                    if (!decision.isContentBased()) {
+                        String confirmedBitmap = answer.getBitmapCreated();
+                        if (confirmedBitmap != null) {
+                            upsertVmActiveCheckpoint(vm.getId(), confirmedBitmap);
+                        } else {
+                            clearVmActiveCheckpoint(vm.getId());
+                        }
                     }
                 }
                 return new Pair<>(true, backupVO);