Merge branch 'main' of https://github.com/apache/cloudstack into clvm-enhancements

Author: Pearl Dsilva

.asf.yaml

@@ -51,7 +51,7 @@ github:
 
   collaborators:
     - ingox
-    - gpordeus
+    - gp-santos
     - erikbocks
     - Imvedansh
     - Damans227

.github/workflows/merge-conflict-checker.yml

@@ -17,16 +17,14 @@
 
 name: "PR Merge Conflict Check"
 on:
-  push:
-  pull_request:
-    types: [opened, synchronize, reopened]
+  schedule:
+    - cron: '*/10 * * * *'
+  workflow_dispatch:
 
-permissions:  # added using https://github.com/step-security/secure-workflows
-  contents: read
+permissions: {}
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: "gh-aw-${{ github.workflow }}"
 
 jobs:
   triage:

api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java

@@ -51,6 +51,7 @@ public class VirtualMachineTO {
 
     private long minRam;
     private long maxRam;
+    private long requestedRam;
     private String hostName;
     private String arch;
     private String os;
@@ -207,15 +208,20 @@ public long getMinRam() {
         return minRam;
     }
 
-    public void setRam(long minRam, long maxRam) {
+    public void setRam(long minRam, long maxRam, long requestedRam) {
         this.minRam = minRam;
         this.maxRam = maxRam;
+        this.requestedRam = requestedRam;
     }
 
     public long getMaxRam() {
         return maxRam;
     }
 
+    public long getRequestedRam() {
+        return requestedRam;
+    }
+
     public String getHostName() {
         return hostName;
     }

api/src/main/java/org/apache/cloudstack/api/command/admin/ca/ProvisionCertificateCmd.java

@@ -63,6 +63,12 @@ public class ProvisionCertificateCmd extends BaseAsyncCmd {
             description = "Name of the CA service provider, otherwise the default configured provider plugin will be used")
     private String provider;
 
+    @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN,
+            description = "When true, uses SSH to re-provision the agent's certificate, bypassing the NIO agent connection. " +
+            "Use this when agents are disconnected due to a CA change. Supported for KVM hosts and SystemVMs. Default is false",
+            since = "4.23.0")
+    private Boolean forced;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -79,6 +85,10 @@ public String getProvider() {
         return provider;
     }
 
+    public boolean isForced() {
+        return forced != null && forced;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////
@@ -90,7 +100,7 @@ public void execute() {
             throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find host by ID: " + getHostId());
         }
 
-        boolean result = caManager.provisionCertificate(host, getReconnect(), getProvider());
+        boolean result = caManager.provisionCertificate(host, getReconnect(), getProvider(), isForced());
         SuccessResponse response = new SuccessResponse(getCommandName());
         response.setSuccess(result);
         setResponseObject(response);

api/src/main/java/org/apache/cloudstack/api/command/admin/user/GetUserCmd.java

@@ -22,7 +22,7 @@
 import org.apache.cloudstack.api.BaseCmd;
 import org.apache.cloudstack.api.Parameter;
 import org.apache.cloudstack.api.response.UserResponse;
-
+import org.apache.cloudstack.api.ApiArgValidator;
 import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.user.UserAccount;
 
@@ -35,7 +35,7 @@ public class GetUserCmd extends BaseCmd {
     //////////////// API parameters /////////////////////
     /////////////////////////////////////////////////////
 
-    @Parameter(name = ApiConstants.USER_API_KEY, type = CommandType.STRING, required = true, description = "API key of the user")
+    @Parameter(name = ApiConstants.USER_API_KEY, type = CommandType.STRING, required = true, description = "API key of the user", validations = {ApiArgValidator.NotNullOrEmpty})
     private String apiKey;
 
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/ca/CAManager.java

@@ -23,6 +23,8 @@
 import java.util.List;
 import java.util.Map;
 
+import com.trilead.ssh2.Connection;
+
 import org.apache.cloudstack.framework.ca.CAProvider;
 import org.apache.cloudstack.framework.ca.CAService;
 import org.apache.cloudstack.framework.ca.Certificate;
@@ -39,7 +41,10 @@ public interface CAManager extends CAService, Configurable, PluggableService {
     ConfigKey<String> CAProviderPlugin = new ConfigKey<>("Advanced", String.class,
             "ca.framework.provider.plugin",
             "root",
-            "The CA provider plugin that is used for secure CloudStack management server-agent communication for encryption and authentication. Restart management server(s) when changed.", true);
+            "The CA provider plugin used for CloudStack internal certificate management (MS-agent encryption and authentication). " +
+            "The default 'root' provider auto-generates a CA on first startup, but also supports user-provided custom CA material " +
+            "via the ca.plugin.root.private.key, ca.plugin.root.public.key, and ca.plugin.root.ca.certificate settings. " +
+            "Restart management server(s) when changed.", false);
 
     ConfigKey<Integer> CertKeySize = new ConfigKey<>("Advanced", Integer.class,
                                     "ca.framework.cert.keysize",
@@ -85,6 +90,12 @@ public interface CAManager extends CAService, Configurable, PluggableService {
                     "The actual implementation will depend on the configured CA provider.",
             false);
 
+    ConfigKey<Boolean> CaInjectDefaultTruststore = new ConfigKey<>("Advanced", Boolean.class,
+            "ca.framework.inject.default.truststore", "true",
+            "When true, injects the CA provider's certificate into the JVM default truststore on management server startup. " +
+            "This allows outgoing HTTPS connections from the management server to trust servers with certificates signed by the configured CA. " +
+            "Restart management server(s) when changed.", false);
+
     /**
      * Returns a list of available CA provider plugins
      * @return returns list of CAProvider
@@ -130,12 +141,26 @@ public interface CAManager extends CAService, Configurable, PluggableService {
     boolean revokeCertificate(final BigInteger certSerial, final String certCn, final String provider);
 
     /**
-     * Provisions certificate for given active and connected agent host
+     * Provisions certificate for given agent host.
+     * When forced=true, uses SSH to re-provision bypassing the NIO agent connection (for disconnected agents).
      * @param host
+     * @param reconnect
      * @param provider
+     * @param forced when true, provisions via SSH instead of NIO; supports KVM hosts and SystemVMs
      * @return returns success/failure as boolean
      */
-    boolean provisionCertificate(final Host host, final Boolean reconnect, final String provider);
+    boolean provisionCertificate(final Host host, final Boolean reconnect, final String provider, final boolean forced);
+
+    /**
+     * Provisions certificate for a KVM host using an existing SSH connection.
+     * Runs keystore-setup to generate a CSR, issues a certificate, then runs keystore-cert-import.
+     * Used during host discovery and for forced re-provisioning when the NIO agent is unreachable.
+     * @param sshConnection active SSH connection to the KVM host
+     * @param agentIp IP address of the KVM host agent
+     * @param agentHostname hostname of the KVM host agent
+     * @param caProvider optional CA provider plugin name (null uses default)
+     */
+    void provisionCertificateViaSsh(Connection sshConnection, String agentIp, String agentHostname, String caProvider);
 
     /**
      * Setups up a new keystore and generates CSR for a host

core/src/main/java/com/cloud/agent/api/ScaleVmCommand.java

@@ -30,6 +30,7 @@ public class ScaleVmCommand extends Command {
     Integer maxSpeed;
     long minRam;
     long maxRam;
+    private boolean limitCpuUseChange;
 
     public VirtualMachineTO getVm() {
         return vm;
@@ -43,7 +44,7 @@ public int getCpus() {
         return cpus;
     }
 
-    public ScaleVmCommand(String vmName, int cpus, Integer minSpeed, Integer maxSpeed, long minRam, long maxRam, boolean limitCpuUse) {
+    public ScaleVmCommand(String vmName, int cpus, Integer minSpeed, Integer maxSpeed, long minRam, long maxRam, boolean limitCpuUse, Double cpuQuotaPercentage, boolean limitCpuUseChange) {
         super();
         this.vmName = vmName;
         this.cpus = cpus;
@@ -52,6 +53,8 @@ public ScaleVmCommand(String vmName, int cpus, Integer minSpeed, Integer maxSpee
         this.minRam = minRam;
         this.maxRam = maxRam;
         this.vm = new VirtualMachineTO(1L, vmName, null, cpus, minSpeed, maxSpeed, minRam, maxRam, null, null, false, limitCpuUse, null);
+        this.vm.setCpuQuotaPercentage(cpuQuotaPercentage);
+        this.limitCpuUseChange = limitCpuUseChange;
     }
 
     public void setCpus(int cpus) {
@@ -102,6 +105,10 @@ public VirtualMachineTO getVirtualMachine() {
         return vm;
     }
 
+    public boolean getLimitCpuUseChange() {
+        return limitCpuUseChange;
+    }
+
     @Override
     public boolean executeInSequence() {
         return true;

engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java

@@ -133,6 +133,20 @@ public interface CapacityManager {
             "capacity.calculate.workers", "1",
             "Number of worker threads to be used for capacities calculation", true);
 
+    ConfigKey<Integer> KvmMemoryDynamicScalingCapacity = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED,
+            Integer.class, "kvm.memory.dynamic.scaling.capacity", "0",
+            "Defines the maximum memory capacity in MiB for which VMs can be dynamically scaled to with KVM. " +
+                    "The 'kvm.memory.dynamic.scaling.capacity' setting's value will be used to define the value of the " +
+                    "'<maxMemory />' element of domain XMLs. If it is set to a value less than or equal to '0', then the host's memory capacity will be considered.",
+            true, ConfigKey.Scope.Cluster);
+
+    ConfigKey<Integer> KvmCpuDynamicScalingCapacity = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED,
+            Integer.class, "kvm.cpu.dynamic.scaling.capacity", "0",
+            "Defines the maximum vCPU capacity for which VMs can be dynamically scaled to with KVM. " +
+                    "The 'kvm.cpu.dynamic.scaling.capacity' setting's value will be used to define the value of the " +
+                    "'<vcpu />' element of domain XMLs. If it is set to a value less than or equal to '0', then the host's CPU cores capacity will be considered.",
+            true, ConfigKey.Scope.Cluster);
+
     public boolean releaseVmCapacity(VirtualMachine vm, boolean moveFromReserved, boolean moveToReservered, Long hostId);
 
     void allocateVmCapacity(VirtualMachine vm, boolean fromLastHost);

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -52,6 +52,7 @@
 
 import com.cloud.agent.api.PostMigrationCommand;
 import com.cloud.storage.clvm.ClvmPoolManager;
+import com.cloud.hypervisor.KVMGuru;
 import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao;
 import org.apache.cloudstack.annotation.AnnotationService;
 import org.apache.cloudstack.annotation.dao.AnnotationDao;
@@ -5228,7 +5229,7 @@ public VMInstanceVO reConfigureVm(final String vmUuid, final ServiceOffering old
             try {
                 result = retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome);
             } catch (Exception ex) {
-                throw new RuntimeException("Unhandled exception", ex);
+                throw new RuntimeException("Unable to reconfigure VM.", ex);
             }
 
             if (result != null) {
@@ -5241,22 +5242,29 @@ public VMInstanceVO reConfigureVm(final String vmUuid, final ServiceOffering old
 
     private VMInstanceVO orchestrateReConfigureVm(String vmUuid, ServiceOffering oldServiceOffering, ServiceOffering newServiceOffering,
                                                   boolean reconfiguringOnExistingHost) throws ResourceUnavailableException, ConcurrentOperationException {
-        final VMInstanceVO vm = _vmDao.findByUuid(vmUuid);
+        VMInstanceVO vm = _vmDao.findByUuid(vmUuid);
 
         HostVO hostVo = _hostDao.findById(vm.getHostId());
 
-        Long clustedId = hostVo.getClusterId();
-        Float memoryOvercommitRatio = CapacityManager.MemOverprovisioningFactor.valueIn(clustedId);
-        Float cpuOvercommitRatio = CapacityManager.CpuOverprovisioningFactor.valueIn(clustedId);
-        boolean divideMemoryByOverprovisioning = HypervisorGuruBase.VmMinMemoryEqualsMemoryDividedByMemOverprovisioningFactor.valueIn(clustedId);
-        boolean divideCpuByOverprovisioning = HypervisorGuruBase.VmMinCpuSpeedEqualsCpuSpeedDividedByCpuOverprovisioningFactor.valueIn(clustedId);
+        Long clusterId = hostVo.getClusterId();
+        Float memoryOvercommitRatio = CapacityManager.MemOverprovisioningFactor.valueIn(clusterId);
+        Float cpuOvercommitRatio = CapacityManager.CpuOverprovisioningFactor.valueIn(clusterId);
+        boolean divideMemoryByOverprovisioning = HypervisorGuruBase.VmMinMemoryEqualsMemoryDividedByMemOverprovisioningFactor.valueIn(clusterId);
+        boolean divideCpuByOverprovisioning = HypervisorGuruBase.VmMinCpuSpeedEqualsCpuSpeedDividedByCpuOverprovisioningFactor.valueIn(clusterId);
 
         int minMemory = (int)(newServiceOffering.getRamSize() / (divideMemoryByOverprovisioning ? memoryOvercommitRatio : 1));
         int minSpeed = (int)(newServiceOffering.getSpeed() / (divideCpuByOverprovisioning ? cpuOvercommitRatio : 1));
 
-        ScaleVmCommand scaleVmCommand =
-                new ScaleVmCommand(vm.getInstanceName(), newServiceOffering.getCpu(), minSpeed,
-                        newServiceOffering.getSpeed(), minMemory * 1024L * 1024L, newServiceOffering.getRamSize() * 1024L * 1024L, newServiceOffering.getLimitCpuUse());
+        Double cpuQuotaPercentage = null;
+        if (newServiceOffering.getLimitCpuUse() && vm.getHypervisorType().equals(HypervisorType.KVM)) {
+            KVMGuru kvmGuru = (KVMGuru) _hvGuruMgr.getGuru(vm.getHypervisorType());
+            cpuQuotaPercentage = kvmGuru.getCpuQuotaPercentage(minSpeed, hostVo.getSpeed());
+        }
+
+        boolean limitCpuUseChange = oldServiceOffering.getLimitCpuUse() != newServiceOffering.getLimitCpuUse();
+        ScaleVmCommand scaleVmCommand = new ScaleVmCommand(vm.getInstanceName(), newServiceOffering.getCpu(), minSpeed, newServiceOffering.getSpeed(),
+                minMemory * 1024L * 1024L, newServiceOffering.getRamSize() * 1024L * 1024L,
+                newServiceOffering.getLimitCpuUse(), cpuQuotaPercentage, limitCpuUseChange);
 
         scaleVmCommand.getVirtualMachine().setId(vm.getId());
         scaleVmCommand.getVirtualMachine().setUuid(vm.getUuid());
@@ -5285,16 +5293,20 @@ private VMInstanceVO orchestrateReConfigureVm(String vmUuid, ServiceOffering old
                 throw new CloudRuntimeException("Unable to scale vm due to " + (reconfigureAnswer == null ? "" : reconfigureAnswer.getDetails()));
             }
 
-            upgradeVmDb(vm.getId(), newServiceOffering, oldServiceOffering);
+            if (reconfiguringOnExistingHost) {
+                _capacityMgr.releaseVmCapacity(vm, false, false, vm.getHostId());
+            }
+
+            boolean vmUpgraded = upgradeVmDb(vm.getId(), newServiceOffering, oldServiceOffering);
+            if (vmUpgraded) {
+                vm = _vmDao.findById(vm.getId());
+            }
 
             if (vm.getType().equals(VirtualMachine.Type.User)) {
                 _userVmMgr.generateUsageEvent(vm, vm.isDisplayVm(), EventTypes.EVENT_VM_DYNAMIC_SCALE);
             }
 
             if (reconfiguringOnExistingHost) {
-                vm.setServiceOfferingId(oldServiceOffering.getId());
-                _capacityMgr.releaseVmCapacity(vm, false, false, vm.getHostId());
-                vm.setServiceOfferingId(newServiceOffering.getId());
                 _capacityMgr.allocateVmCapacity(vm, false);
             }
 

engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java

@@ -1073,7 +1073,7 @@ protected void updateRegisteredTemplateDetails(Long templateId, MetadataTemplate
         }
         Hypervisor.HypervisorType hypervisorType = templateDetails.getHypervisorType();
         updateSystemVMEntries(templateId, hypervisorType);
-        updateConfigurationParams(hypervisorType, templateDetails.getName(), zoneId);
+        updateConfigurationParams(hypervisorType, templateVO.getName(), zoneId);
     }
 
     protected void updateTemplateUrlChecksumAndGuestOsId(VMTemplateVO templateVO,