Merge branch '4.22' into fix/oauth2-google-thread-safety-4.22

Author: rhenar0

api/src/main/java/com/cloud/resource/ResourceService.java

@@ -91,7 +91,7 @@ public interface ResourceService {
 
     DataCenter getZone(Long zoneId);
 
-    List<HypervisorType> getSupportedHypervisorTypes(long zoneId, boolean forVirtualRouter, Long podId);
+    List<HypervisorType> getSupportedHypervisorTypes(long zoneId, boolean forSystemVm, Long podId);
 
     boolean releaseHostReservation(Long hostId);
 

api/src/main/java/org/apache/cloudstack/backup/Backup.java

@@ -31,7 +31,11 @@
 public interface Backup extends ControlledEntity, InternalIdentity, Identity {
 
     enum Status {
-        Allocated, Queued, BackingUp, BackedUp, Error, Failed, Restoring, Removed, Expunged
+        Allocated, Queued, BackingUp, BackedUp, Error, Failed, Restoring, Removed, Expunged,
+        // Hidden: a chain backup kept as a tombstone after the user deleted it while it still has
+        // live descendants (incremental chains). Excluded from listBackups and from all backup
+        // operations (which require BackedUp); swept from the DB once its last descendant is gone.
+        Hidden
     }
 
     class Metric {

api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java

@@ -87,6 +87,18 @@ public interface BackupProvider {
      */
     boolean deleteBackup(Backup backup, boolean forced);
 
+    /**
+     * Whether {@link #deleteBackup(Backup, boolean)} owns DB-row removal and resource-count /
+     * usage accounting for every backup it physically removes. Providers that manage incremental
+     * chains (e.g. NAS) delete several backups per call — the leaf plus swept delete-pending
+     * ancestors — and decrement once per removed backup themselves, so the manager must NOT
+     * decrement or remove the row again. Defaults to {@code false}: the manager does the
+     * single-backup accounting (the historical behaviour for non-chain providers).
+     */
+    default boolean handlesChainDeleteResourceAccounting() {
+        return false;
+    }
+
     Pair<Boolean, String> restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid);
 
     /**

core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java

@@ -29,6 +29,12 @@ public class BackupAnswer extends Answer {
     private Long virtualSize;
     private Map<String, String> volumes;
     Boolean needsCleanup;
+    // Set by the NAS backup provider after a checkpoint/bitmap was created during this backup.
+    // The provider persists it in backup_details under NASBackupChainKeys.BITMAP_NAME.
+    private String bitmapCreated;
+    // Set when an incremental was requested but the agent had to fall back to a full
+    // (e.g. VM was stopped). Provider should record this backup as type=full.
+    private Boolean incrementalFallback;
 
     public BackupAnswer(final Command command, final boolean success, final String details) {
         super(command, success, details);
@@ -68,4 +74,21 @@ public Boolean getNeedsCleanup() {
     public void setNeedsCleanup(Boolean needsCleanup) {
         this.needsCleanup = needsCleanup;
     }
+
+    public String getBitmapCreated() {
+        return bitmapCreated;
+    }
+
+    public void setBitmapCreated(String bitmapCreated) {
+        this.bitmapCreated = bitmapCreated;
+    }
+
+    public Boolean getIncrementalFallback() {
+        return incrementalFallback != null && incrementalFallback;
+    }
+
+    public void setIncrementalFallback(Boolean incrementalFallback) {
+        this.incrementalFallback = incrementalFallback;
+    }
+
 }

core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java

@@ -36,6 +36,17 @@ public class TakeBackupCommand extends Command {
     @LogLevel(LogLevel.Log4jLevel.Off)
     private String mountOptions;
 
+    // Incremental backup fields (NAS provider; null/empty for legacy full-only callers).
+    private String mode;          // "full" or "incremental"; null => legacy behaviour (script default)
+    private String bitmapNew;     // Checkpoint/bitmap name to create with this backup (timestamp-based)
+    private String bitmapParent;  // Incremental: parent bitmap to read changes since
+
+    // Per-volume parent backup file paths (one per VM volume, ordered by deviceId — same
+    // order as volumePaths). The script rebases each new qcow2 onto the matching parent.
+    // Backup file UUIDs differ across volumes, so a single parentPath would have rebased
+    // every data disk onto the root file. New callers MUST populate parentPaths.
+    private List<String> parentPaths;
+
     public TakeBackupCommand(String vmName, String backupPath) {
         super();
         this.vmName = vmName;
@@ -106,6 +117,38 @@ public void setQuiesce(Boolean quiesce) {
         this.quiesce = quiesce;
     }
 
+    public String getMode() {
+        return mode;
+    }
+
+    public void setMode(String mode) {
+        this.mode = mode;
+    }
+
+    public String getBitmapNew() {
+        return bitmapNew;
+    }
+
+    public void setBitmapNew(String bitmapNew) {
+        this.bitmapNew = bitmapNew;
+    }
+
+    public String getBitmapParent() {
+        return bitmapParent;
+    }
+
+    public void setBitmapParent(String bitmapParent) {
+        this.bitmapParent = bitmapParent;
+    }
+
+    public List<String> getParentPaths() {
+        return parentPaths;
+    }
+
+    public void setParentPaths(List<String> parentPaths) {
+        this.parentPaths = parentPaths;
+    }
+
     @Override
     public boolean executeInSequence() {
         return true;

engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java

@@ -66,7 +66,7 @@ public class BackupVO implements Backup {
     private String externalId;
 
     @Column(name = "type")
-    private String backupType;
+    private String type;
 
     @Column(name = "date")
     @Temporal(value = TemporalType.DATE)
@@ -113,7 +113,7 @@ public BackupVO() {
     @Override
     public String toString() {
         return String.format("Backup %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(
-                this, "id", "uuid", "vmId", "backupType", "externalId"));
+                this, "id", "uuid", "vmId", "type", "externalId"));
     }
 
     @Override
@@ -145,11 +145,11 @@ public void setExternalId(String externalId) {
     }
 
     public String getType() {
-        return backupType;
+        return type;
     }
 
     public void setType(String type) {
-        this.backupType = type;
+        this.type = type;
     }
 
     @Override

engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java

@@ -472,7 +472,15 @@ private boolean canBypassSecondaryStorage(DataObject srcData, DataObject destDat
                 return true;
             }
 
+            if (Hypervisor.HypervisorType.XenServer.equals(((VolumeInfo) srcData).getHypervisorType())) {
+                return false;
+            }
+
             if (destData instanceof VolumeInfo) {
+                if (Hypervisor.HypervisorType.XenServer.equals(((VolumeInfo) destData).getHypervisorType())) {
+                    return false;
+                }
+
                 Scope srcDataStoreScope = srcData.getDataStore().getScope();
                 Scope destDataStoreScope = destData.getDataStore().getScope();
                 logger.info("srcDataStoreScope: {}, srcData pool type: {}; destDataStoreScope: {}, destData pool type: {}",

engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategyTest.java

@@ -123,6 +123,7 @@ public void testCanBypassSecondaryStorageForDirectDownload() throws NoSuchMethod
     @Test
     public void testCanBypassSecondaryStorageForUnsupportedDataObject() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
         VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject());
+        Mockito.doReturn(HypervisorType.KVM).when(srcVolumeInfo).getHypervisorType();
 
         TemplateObject destTemplateInfo = Mockito.spy(new TemplateObject());
 
@@ -136,12 +137,14 @@ public void testCanBypassSecondaryStorageForUnsupportedDataObject() throws NoSuc
     @Test
     public void testCanBypassSecondaryStorageForUnsupportedSrcPoolType() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
         VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject());
+        Mockito.doReturn(HypervisorType.KVM).when(srcVolumeInfo).getHypervisorType();
         DataStore srcDataStore = Mockito.mock(DataStore.class);
         Mockito.doReturn(new ZoneScope(1L)).when(srcDataStore).getScope();
         Mockito.doReturn(srcDataStore).when(srcVolumeInfo).getDataStore();
         Mockito.doReturn(Storage.StoragePoolType.PowerFlex).when(srcVolumeInfo).getStoragePoolType();
 
         VolumeObject destVolumeInfo = Mockito.spy(new VolumeObject());
+        Mockito.doReturn(HypervisorType.KVM).when(destVolumeInfo).getHypervisorType();
         DataStore destDataStore = Mockito.mock(DataStore.class);
         Mockito.doReturn(new ZoneScope(1L)).when(destDataStore).getScope();
         Mockito.doReturn(destDataStore).when(destVolumeInfo).getDataStore();
@@ -157,12 +160,14 @@ public void testCanBypassSecondaryStorageForUnsupportedSrcPoolType() throws NoSu
     @Test
     public void testCanBypassSecondaryStorageForUnsupportedDestPoolType() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
         VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject());
+        Mockito.doReturn(HypervisorType.KVM).when(srcVolumeInfo).getHypervisorType();
         DataStore srcDataStore = Mockito.mock(DataStore.class);
         Mockito.doReturn(new ZoneScope(1L)).when(srcDataStore).getScope();
         Mockito.doReturn(srcDataStore).when(srcVolumeInfo).getDataStore();
         Mockito.doReturn(Storage.StoragePoolType.NetworkFilesystem).when(srcVolumeInfo).getStoragePoolType();
 
         VolumeObject destVolumeInfo = Mockito.spy(new VolumeObject());
+        Mockito.doReturn(HypervisorType.KVM).when(destVolumeInfo).getHypervisorType();
         DataStore destDataStore = Mockito.mock(DataStore.class);
         Mockito.doReturn(new ZoneScope(1L)).when(destDataStore).getScope();
         Mockito.doReturn(destDataStore).when(destVolumeInfo).getDataStore();
@@ -178,12 +183,14 @@ public void testCanBypassSecondaryStorageForUnsupportedDestPoolType() throws NoS
     @Test
     public void testCanBypassSecondaryStorageWithZoneWideNFSPoolsInSameZone() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
         VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject());
+        Mockito.doReturn(HypervisorType.KVM).when(srcVolumeInfo).getHypervisorType();
         DataStore srcDataStore = Mockito.mock(DataStore.class);
         Mockito.doReturn(new ZoneScope(1L)).when(srcDataStore).getScope();
         Mockito.doReturn(srcDataStore).when(srcVolumeInfo).getDataStore();
         Mockito.doReturn(Storage.StoragePoolType.NetworkFilesystem).when(srcVolumeInfo).getStoragePoolType();
 
         VolumeObject destVolumeInfo = Mockito.spy(new VolumeObject());
+        Mockito.doReturn(HypervisorType.KVM).when(destVolumeInfo).getHypervisorType();
         DataStore destDataStore = Mockito.mock(DataStore.class);
         Mockito.doReturn(new ZoneScope(1L)).when(destDataStore).getScope();
         Mockito.doReturn(destDataStore).when(destVolumeInfo).getDataStore();
@@ -199,12 +206,14 @@ public void testCanBypassSecondaryStorageWithZoneWideNFSPoolsInSameZone() throws
     @Test
     public void testCanBypassSecondaryStorageWithClusterWideNFSPoolsInSameCluster() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
         VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject());
+        Mockito.doReturn(HypervisorType.KVM).when(srcVolumeInfo).getHypervisorType();
         DataStore srcDataStore = Mockito.mock(DataStore.class);
         Mockito.doReturn(new ClusterScope(5L, 2L, 1L)).when(srcDataStore).getScope();
         Mockito.doReturn(srcDataStore).when(srcVolumeInfo).getDataStore();
         Mockito.doReturn(Storage.StoragePoolType.NetworkFilesystem).when(srcVolumeInfo).getStoragePoolType();
 
         VolumeObject destVolumeInfo = Mockito.spy(new VolumeObject());
+        Mockito.doReturn(HypervisorType.KVM).when(destVolumeInfo).getHypervisorType();
         DataStore destDataStore = Mockito.mock(DataStore.class);
         Mockito.doReturn(new ClusterScope(5L, 2L, 1L)).when(destDataStore).getScope();
         Mockito.doReturn(destDataStore).when(destVolumeInfo).getDataStore();
@@ -220,12 +229,14 @@ public void testCanBypassSecondaryStorageWithClusterWideNFSPoolsInSameCluster()
     @Test
     public void testCanBypassSecondaryStorageWithLocalAndClusterWideNFSPoolsInSameCluster() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
         VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject());
+        Mockito.doReturn(HypervisorType.KVM).when(srcVolumeInfo).getHypervisorType();
         DataStore srcDataStore = Mockito.mock(DataStore.class);
         Mockito.doReturn(new HostScope(1L, 1L, 1L)).when(srcDataStore).getScope();
         Mockito.doReturn(srcDataStore).when(srcVolumeInfo).getDataStore();
         Mockito.doReturn(Storage.StoragePoolType.Filesystem).when(srcVolumeInfo).getStoragePoolType();
 
         VolumeObject destVolumeInfo = Mockito.spy(new VolumeObject());
+        Mockito.doReturn(HypervisorType.KVM).when(destVolumeInfo).getHypervisorType();
         DataStore destDataStore = Mockito.mock(DataStore.class);
         Mockito.doReturn(new ClusterScope(1L, 1L, 1L)).when(destDataStore).getScope();
         Mockito.doReturn(destDataStore).when(destVolumeInfo).getDataStore();
@@ -244,12 +255,14 @@ public void testCanBypassSecondaryStorageWithLocalAndClusterWideNFSPoolsInSameCl
     @Test
     public void testCanBypassSecondaryStorageWithLocalAndZoneWideNFSPoolsInSameZone() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
         VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject());
+        Mockito.doReturn(HypervisorType.KVM).when(srcVolumeInfo).getHypervisorType();
         DataStore srcDataStore = Mockito.mock(DataStore.class);
         Mockito.doReturn(new HostScope(1L, 1L, 1L)).when(srcDataStore).getScope();
         Mockito.doReturn(srcDataStore).when(srcVolumeInfo).getDataStore();
         Mockito.doReturn(Storage.StoragePoolType.Filesystem).when(srcVolumeInfo).getStoragePoolType();
 
         VolumeObject destVolumeInfo = Mockito.spy(new VolumeObject());
+        Mockito.doReturn(HypervisorType.KVM).when(destVolumeInfo).getHypervisorType();
         DataStore destDataStore = Mockito.mock(DataStore.class);
         Mockito.doReturn(new ZoneScope(1L)).when(destDataStore).getScope();
         Mockito.doReturn(destDataStore).when(destVolumeInfo).getDataStore();
@@ -268,12 +281,14 @@ public void testCanBypassSecondaryStorageWithLocalAndZoneWideNFSPoolsInSameZone(
     @Test
     public void testCanBypassSecondaryStorageWithClusterWideNFSAndZoneWideNFSPoolsInSameZone() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
         VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject());
+        Mockito.doReturn(HypervisorType.KVM).when(srcVolumeInfo).getHypervisorType();
         DataStore srcDataStore = Mockito.mock(DataStore.class);
         Mockito.doReturn(new ClusterScope(5L, 2L, 1L)).when(srcDataStore).getScope();
         Mockito.doReturn(srcDataStore).when(srcVolumeInfo).getDataStore();
         Mockito.doReturn(Storage.StoragePoolType.NetworkFilesystem).when(srcVolumeInfo).getStoragePoolType();
 
         VolumeObject destVolumeInfo = Mockito.spy(new VolumeObject());
+        Mockito.doReturn(HypervisorType.KVM).when(destVolumeInfo).getHypervisorType();
         DataStore destDataStore = Mockito.mock(DataStore.class);
         Mockito.doReturn(new ZoneScope(1L)).when(destDataStore).getScope();
         Mockito.doReturn(destDataStore).when(destVolumeInfo).getDataStore();

plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupChainKeys.java

@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.backup;
+
+/**
+ * Keys used by the NAS backup provider when storing incremental-chain metadata
+ * in the existing {@code backup_details} key/value table. Stored here (not on
+ * the {@code backups} table) so other providers do not need a schema change to
+ * support their own incremental implementations.
+ */
+public final class NASBackupChainKeys {
+
+    /** UUID of the parent backup (full or previous incremental). Empty for full backups. */
+    public static final String PARENT_BACKUP_ID = "nas.parent_backup_id";
+
+    /** QEMU dirty-bitmap name created by this backup, used as the {@code <incremental>} reference for the next one. */
+    public static final String BITMAP_NAME = "nas.bitmap_name";
+
+    /** Identifier shared by every backup in the same chain (the full anchors a chain; its incrementals inherit the id). */
+    public static final String CHAIN_ID = "nas.chain_id";
+
+    /** Position within the chain: 0 for the full, 1 for the first incremental, and so on. */
+    public static final String CHAIN_POSITION = "nas.chain_position";
+
+    /**
+     * In-memory chain-mode sentinels used by {@code ChainDecision.mode}. The persisted
+     * full-vs-incremental backup type lives on the {@code backup.type} column (set in
+     * {@code takeBackup}) — single source of truth. Not duplicated into backup_details.
+     */
+    public static final String TYPE_FULL = "full";
+    public static final String TYPE_INCREMENTAL = "incremental";
+    // Feature disabled: behave exactly like the pre-incremental full-only backup — no QEMU
+    // bitmap/checkpoint is created and no chain metadata is persisted. Matches nasbackup.sh's
+    // "legacy-full" mode token (which sets make_checkpoint=0).
+    public static final String TYPE_LEGACY_FULL = "legacy-full";
+
+    /**
+     * VM-scoped detail (stored in {@code vm_instance_details}) holding the QEMU dirty-bitmap
+     * name that currently exists on the running VM and is therefore the only valid parent
+     * for the next incremental backup. Written by {@link #BITMAP_NAME} on each successful
+     * backup; cleared on restore (the restored disk image has no bitmap, so the next backup
+     * must be a fresh full). When the VM has no detail, {@code decideChain} forces full.
+     */
+    public static final String VM_ACTIVE_CHECKPOINT_ID = "nas.active_checkpoint_id";
+
+    private NASBackupChainKeys() {
+    }
+}