Add ROOT CAs to the trust store

api/src/main/java/org/apache/cloudstack/api/command/admin/ca/ProvisionCertificateCmd.java

@@ -63,6 +63,12 @@ public class ProvisionCertificateCmd extends BaseAsyncCmd {
             description = "Name of the CA service provider, otherwise the default configured provider plugin will be used")
     private String provider;
 
+    @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN,
+            description = "When true, uses SSH to re-provision the agent's certificate, bypassing the NIO agent connection. " +
+            "Use this when agents are disconnected due to a CA change. Supported for KVM hosts and SystemVMs. Default is false",
+            since = "4.23.0")
+    private Boolean forced;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -79,6 +85,10 @@ public String getProvider() {
         return provider;
     }
 
+    public boolean isForced() {
+        return forced != null && forced;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////
@@ -90,7 +100,7 @@ public void execute() {
             throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find host by ID: " + getHostId());
         }
 
-        boolean result = caManager.provisionCertificate(host, getReconnect(), getProvider());
+        boolean result = caManager.provisionCertificate(host, getReconnect(), getProvider(), isForced());
         SuccessResponse response = new SuccessResponse(getCommandName());
         response.setSuccess(result);
         setResponseObject(response);

api/src/main/java/org/apache/cloudstack/ca/CAManager.java

@@ -23,6 +23,8 @@
 import java.util.List;
 import java.util.Map;
 
+import com.trilead.ssh2.Connection;
+
 import org.apache.cloudstack.framework.ca.CAProvider;
 import org.apache.cloudstack.framework.ca.CAService;
 import org.apache.cloudstack.framework.ca.Certificate;
@@ -39,7 +41,10 @@ public interface CAManager extends CAService, Configurable, PluggableService {
     ConfigKey<String> CAProviderPlugin = new ConfigKey<>("Advanced", String.class,
             "ca.framework.provider.plugin",
             "root",
-            "The CA provider plugin that is used for secure CloudStack management server-agent communication for encryption and authentication. Restart management server(s) when changed.", true);
+            "The CA provider plugin used for CloudStack internal certificate management (MS-agent encryption and authentication). " +
+            "The default 'root' provider auto-generates a CA on first startup, but also supports user-provided custom CA material " +
+            "via the ca.plugin.root.private.key, ca.plugin.root.public.key, and ca.plugin.root.ca.certificate settings. " +
+            "Restart management server(s) when changed.", true);
 
     ConfigKey<Integer> CertKeySize = new ConfigKey<>("Advanced", Integer.class,
                                     "ca.framework.cert.keysize",
@@ -85,6 +90,12 @@ public interface CAManager extends CAService, Configurable, PluggableService {
                     "The actual implementation will depend on the configured CA provider.",
             false);
 
+    ConfigKey<Boolean> CaInjectDefaultTruststore = new ConfigKey<>("Advanced", Boolean.class,
+            "ca.framework.inject.default.truststore", "true",
+            "When true, injects the CA provider's certificate into the JVM default truststore on management server startup. " +
+            "This allows outgoing HTTPS connections from the management server to trust servers with certificates signed by the configured CA. " +
+            "Restart management server(s) when changed.", true);
+
     /**
      * Returns a list of available CA provider plugins
      * @return returns list of CAProvider
@@ -130,12 +141,26 @@ public interface CAManager extends CAService, Configurable, PluggableService {
     boolean revokeCertificate(final BigInteger certSerial, final String certCn, final String provider);
 
     /**
-     * Provisions certificate for given active and connected agent host
+     * Provisions certificate for given agent host.
+     * When forced=true, uses SSH to re-provision bypassing the NIO agent connection (for disconnected agents).
      * @param host
+     * @param reconnect
      * @param provider
+     * @param forced when true, provisions via SSH instead of NIO; supports KVM hosts and SystemVMs
      * @return returns success/failure as boolean
      */
-    boolean provisionCertificate(final Host host, final Boolean reconnect, final String provider);
+    boolean provisionCertificate(final Host host, final Boolean reconnect, final String provider, final boolean forced);
+
+    /**
+     * Provisions certificate for a KVM host using an existing SSH connection.
+     * Runs keystore-setup to generate a CSR, issues a certificate, then runs keystore-cert-import.
+     * Used during host discovery and for forced re-provisioning when the NIO agent is unreachable.
+     * @param sshConnection active SSH connection to the KVM host
+     * @param agentIp IP address of the KVM host agent
+     * @param agentHostname hostname of the KVM host agent
+     * @param caProvider optional CA provider plugin name (null uses default)
+     */
+    void provisionCertificateViaSsh(Connection sshConnection, String agentIp, String agentHostname, String caProvider);
 
     /**
      * Setups up a new keystore and generates CSR for a host

plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManager.java

@@ -40,17 +40,17 @@ public final class RootCACustomTrustManager implements X509TrustManager {
     private boolean authStrictness = true;
     private boolean allowExpiredCertificate = true;
     private CrlDao crlDao;
-    private X509Certificate caCertificate;
+    private List<X509Certificate> caCertificates;
     private Map<String, X509Certificate> activeCertMap;
 
-    public RootCACustomTrustManager(final String clientAddress, final boolean authStrictness, final boolean allowExpiredCertificate, final Map<String, X509Certificate> activeCertMap, final X509Certificate caCertificate, final CrlDao crlDao) {
+    public RootCACustomTrustManager(final String clientAddress, final boolean authStrictness, final boolean allowExpiredCertificate, final Map<String, X509Certificate> activeCertMap, final List<X509Certificate> caCertificates, final CrlDao crlDao) {
         if (StringUtils.isNotEmpty(clientAddress)) {
             this.clientAddress = clientAddress.replace("/", "").split(":")[0];
         }
         this.authStrictness = authStrictness;
         this.allowExpiredCertificate = allowExpiredCertificate;
         this.activeCertMap = activeCertMap;
-        this.caCertificate = caCertificate;
+        this.caCertificates = caCertificates;
         this.crlDao = crlDao;
     }
 
@@ -151,6 +151,6 @@ public void checkServerTrusted(X509Certificate[] x509Certificates, String s) thr
 
     @Override
     public X509Certificate[] getAcceptedIssuers() {
-        return new X509Certificate[]{caCertificate};
+        return caCertificates.toArray(new X509Certificate[0]);
     }
 }

plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java

@@ -40,7 +40,6 @@
 import java.security.spec.InvalidKeySpecException;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.Enumeration;
 import java.util.HashSet;
 import java.util.List;
@@ -92,6 +91,7 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con
 
     private static KeyPair caKeyPair = null;
     private static X509Certificate caCertificate = null;
+    private static List<X509Certificate> caCertificates = null;
     private static KeyStore managementKeyStore = null;
 
     @Inject
@@ -106,17 +106,22 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con
     private static ConfigKey<String> rootCAPrivateKey = new ConfigKey<>("Hidden", String.class,
             "ca.plugin.root.private.key",
             null,
-            "The ROOT CA private key.", true);
+            "The ROOT CA private key in PEM format. " +
+            "When set along with the public key and certificate, CloudStack uses this custom CA instead of auto-generating one. " +
+            "All three ca.plugin.root.* keys must be set together. Restart management server(s) when changed.", true);
 
     private static ConfigKey<String> rootCAPublicKey = new ConfigKey<>("Hidden", String.class,
             "ca.plugin.root.public.key",
             null,
-            "The ROOT CA public key.", true);
+            "The ROOT CA public key in PEM format (X.509/SPKI: must start with '-----BEGIN PUBLIC KEY-----'). " +
+            "Required when providing a custom CA. Restart management server(s) when changed.", true);
 
     private static ConfigKey<String> rootCACertificate = new ConfigKey<>("Hidden", String.class,
             "ca.plugin.root.ca.certificate",
             null,
-            "The ROOT CA certificate.", true);
+            "The CA certificate(s) in PEM format (must start with '-----BEGIN CERTIFICATE-----'). " +
+            "For intermediate CAs, concatenate the signing cert first, followed by intermediate(s) and root. " +
+            "Required when providing a custom CA. Restart management server(s) when changed.", true);
 
     private static ConfigKey<String> rootCAIssuerDN = new ConfigKey<>("Advanced", String.class,
             "ca.plugin.root.issuer.dn",
@@ -151,7 +156,7 @@ private Certificate generateCertificate(final List<String> domainNames, final Li
                 caCertificate, caKeyPair, keyPair.getPublic(),
                 subject, CAManager.CertSignatureAlgorithm.value(),
                 validityDays, domainNames, ipAddresses);
-        return new Certificate(clientCertificate, keyPair.getPrivate(), Collections.singletonList(caCertificate));
+        return new Certificate(clientCertificate, keyPair.getPrivate(), caCertificates);
     }
 
     private Certificate generateCertificateUsingCsr(final String csr, final List<String> names, final List<String> ips, final int validityDays) throws NoSuchAlgorithmException, InvalidKeyException, NoSuchProviderException, CertificateException, SignatureException, IOException, OperatorCreationException {
@@ -205,7 +210,7 @@ private Certificate generateCertificateUsingCsr(final String csr, final List<Str
                 caCertificate, caKeyPair, request.getPublicKey(),
                 subject, CAManager.CertSignatureAlgorithm.value(),
                 validityDays, dnsNames, ipAddresses);
-        return new Certificate(clientCertificate, null, Collections.singletonList(caCertificate));
+        return new Certificate(clientCertificate, null, caCertificates);
     }
 
     ////////////////////////////////////////////////////////
@@ -219,7 +224,7 @@ public boolean canProvisionCertificates() {
 
     @Override
     public List<X509Certificate> getCaCertificate() {
-        return Collections.singletonList(caCertificate);
+        return caCertificates;
     }
 
     @Override
@@ -254,8 +259,8 @@ public boolean revokeCertificate(final BigInteger certSerial, final String certC
     private KeyStore getCaKeyStore() throws CertificateException, NoSuchAlgorithmException, IOException, KeyStoreException {
         final KeyStore ks = KeyStore.getInstance("JKS");
         ks.load(null, null);
-        if (caKeyPair != null && caCertificate != null) {
-            ks.setKeyEntry(caAlias, caKeyPair.getPrivate(), getKeyStorePassphrase(), new X509Certificate[]{caCertificate});
+        if (caKeyPair != null && CollectionUtils.isNotEmpty(caCertificates)) {
+            ks.setKeyEntry(caAlias, caKeyPair.getPrivate(), getKeyStorePassphrase(), caCertificates.toArray(new X509Certificate[0]));
         } else {
             return null;
         }
@@ -274,7 +279,7 @@ public SSLEngine createSSLEngine(final SSLContext sslContext, final String remot
         final boolean authStrictness = rootCAAuthStrictness.value();
         final boolean allowExpiredCertificate = rootCAAllowExpiredCert.value();
 
-        TrustManager[] tms = new TrustManager[]{new RootCACustomTrustManager(remoteAddress, authStrictness, allowExpiredCertificate, certMap, caCertificate, crlDao)};
+        TrustManager[] tms = new TrustManager[]{new RootCACustomTrustManager(remoteAddress, authStrictness, allowExpiredCertificate, certMap, caCertificates, crlDao)};
 
         sslContext.init(kmf.getKeyManagers(), tms, new SecureRandom());
         final SSLEngine sslEngine = sslContext.createSSLEngine();
@@ -360,9 +365,23 @@ private boolean loadRootCACertificate() {
             return false;
         }
         try {
-            caCertificate = CertUtils.pemToX509Certificate(rootCACertificate.value());
-            caCertificate.verify(caKeyPair.getPublic());
-        } catch (final IOException | CertificateException | NoSuchAlgorithmException | InvalidKeyException | SignatureException | NoSuchProviderException e) {
+            caCertificates = CertUtils.pemToX509Certificates(rootCACertificate.value());
+            if (CollectionUtils.isEmpty(caCertificates)) {
+                logger.error("No certificates found in ca.plugin.root.ca.certificate");
+                return false;
+            }
+            caCertificate = caCertificates.get(0);
+
+            // Verify key ownership without enforcing self-signature
+            if (!caCertificate.getPublicKey().equals(caKeyPair.getPublic())) {
+                logger.error("The public key in the CA certificate does not match the configured CA public key");
+                return false;
+            }
+
+            if (caCertificates.size() > 1) {
+                logger.info("Loaded CA certificate chain with {} certificate(s)", caCertificates.size());
+            }
+        } catch (final IOException | CertificateException e) {
             logger.error("Failed to load saved RootCA certificate due to exception:", e);
             return false;
         }
@@ -389,9 +408,15 @@ private boolean loadManagementKeyStore() {
         try {
             managementKeyStore = KeyStore.getInstance("JKS");
             managementKeyStore.load(null, null);
-            managementKeyStore.setCertificateEntry(caAlias, caCertificate);
+            int caIndex = 0;
+            for (final X509Certificate cert : caCertificates) {
+                managementKeyStore.setCertificateEntry(caAlias + "-" + caIndex++, cert);
+            }
+            final List<X509Certificate> fullChain = new ArrayList<>();
+            fullChain.add(serverCertificate.getClientCertificate());
+            fullChain.addAll(caCertificates);
             managementKeyStore.setKeyEntry(managementAlias, serverCertificate.getPrivateKey(), getKeyStorePassphrase(),
-                    new X509Certificate[]{serverCertificate.getClientCertificate(), caCertificate});
+                    fullChain.toArray(new X509Certificate[0]));
         } catch (final CertificateException | NoSuchAlgorithmException | KeyStoreException | IOException  e) {
             logger.error("Failed to load root CA management-server keystore due to exception: ", e);
             return false;
@@ -422,13 +447,28 @@ protected void addConfiguredManagementIp(List<String> ipList) {
 
 
     private boolean setupCA() {
-        if (!loadRootCAKeyPair() && !saveNewRootCAKeypair()) {
-            logger.error("Failed to save and load root CA keypair");
-            return false;
+        if (!loadRootCAKeyPair()) {
+            if (hasUserProvidedCAKeys()) {
+                logger.error("Failed to load user-provided CA keys from configuration. " +
+                    "Check that ca.plugin.root.private.key, ca.plugin.root.public.key, and " +
+                    "ca.plugin.root.ca.certificate are all set and in the correct PEM format. " +
+                    "Overwriting with auto-generated keys.");
+            }
+            if (!saveNewRootCAKeypair()) {
+                logger.error("Failed to save and load root CA keypair");
+                return false;
+            }
         }
-        if (!loadRootCACertificate() && !saveNewRootCACertificate()) {
-            logger.error("Failed to save and load root CA certificate");
-            return false;
+        if (!loadRootCACertificate()) {
+            if (hasUserProvidedCAKeys()) {
+                logger.error("Failed to load user-provided CA certificate. " +
+                    "Check that ca.plugin.root.ca.certificate is set and in PEM format. " +
+                    "Overwriting with auto-generated certificate.");
+            }
+            if (!saveNewRootCACertificate()) {
+                logger.error("Failed to save and load root CA certificate");
+                return false;
+            }
         }
         if (!loadManagementKeyStore()) {
             logger.error("Failed to check and configure management server keystore");
@@ -437,10 +477,16 @@ private boolean setupCA() {
         return true;
     }
 
+    private boolean hasUserProvidedCAKeys() {
+        return StringUtils.isNotEmpty(rootCAPublicKey.value())
+            || StringUtils.isNotEmpty(rootCAPrivateKey.value())
+            || StringUtils.isNotEmpty(rootCACertificate.value());
+    }
+
     @Override
     public boolean start() {
         managementCertificateCustomSAN = CAManager.CertManagementCustomSubjectAlternativeName.value();
-        return loadRootCAKeyPair() && loadRootCAKeyPair() && loadManagementKeyStore();
+        return loadRootCAKeyPair() && loadRootCACertificate() && loadManagementKeyStore();
     }
 
     @Override