kvm: reload VNC TLS certificate on running VMs after cert renewal

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -518,11 +518,11 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
 
     public static final String CGROUP_V2 = "cgroup2fs";
 
-    protected long getHypervisorLibvirtVersion() {
+    public long getHypervisorLibvirtVersion() {
         return hypervisorLibvirtVersion;
     }
 
-    protected long getHypervisorQemuVersion() {
+    public long getHypervisorQemuVersion() {
         return hypervisorQemuVersion;
     }
 

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapper.java

@@ -19,6 +19,9 @@
 
 import org.apache.cloudstack.ca.PostCertificateRenewalCommand;
 import org.apache.cloudstack.ca.SetupCertificateAnswer;
+import org.libvirt.Connect;
+import org.libvirt.Domain;
+import org.libvirt.LibvirtException;
 
 import com.cloud.agent.api.Answer;
 import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
@@ -29,18 +32,82 @@
 @ResourceWrapper(handles =  PostCertificateRenewalCommand.class)
 public final class LibvirtPostCertificateRenewalCommandWrapper extends CommandWrapper<PostCertificateRenewalCommand, Answer, LibvirtComputingResource> {
 
+    /**
+     * QMP {@code display-reload} command asking QEMU to reload the VNC display's TLS credentials off disk.
+     * {@code tls-certs: true} is required, otherwise QEMU reloads the display without touching the certificates.
+     * Added in QEMU 6.0 by commit 9cc07651655ee86eca41059f5ead8c4e5607c734 ("qmp: add new qmp display-reload",
+     * merged 2021-03-23) - https://github.com/qemu/qemu/commit/9cc07651655ee86eca41059f5ead8c4e5607c734
+     */
+    private static final String QEMU_MONITOR_DISPLAY_RELOAD_VNC_TLS_CERTS_COMMAND =
+            "{\"execute\":\"display-reload\",\"arguments\":{\"type\":\"vnc\",\"tls-certs\":true}}";
+
+    /** Minimum QEMU version supporting {@link #QEMU_MONITOR_DISPLAY_RELOAD_VNC_TLS_CERTS_COMMAND}. */
+    private static final long MIN_QEMU_VERSION_FOR_VNC_TLS_CERT_RELOAD = 6000000L;
 
     @Override
     public Answer execute(final PostCertificateRenewalCommand command, final LibvirtComputingResource serverResource) {
         logger.info("Restarting libvirt after certificate provisioning/renewal");
         if (command != null) {
-            final int timeout = 30000;
-            Script script = new Script(true, "service", timeout, logger);
-            script.add("libvirtd");
-            script.add("restart");
-            script.execute();
+            pushRenewedVncCertificateToRunningVms(serverResource);
+            restartLibvirtd();
             return new SetupCertificateAnswer(true);
        }
         return new SetupCertificateAnswer(false);
     }
+
+    private void restartLibvirtd() {
+        final int timeout = 30000;
+        Script script = new Script(true, "service", timeout, logger);
+        script.add("libvirtd");
+        script.add("restart");
+        script.execute();
+    }
+
+    /**
+     * The VNC TLS certificate on KVM is the host's agent certificate, applied host-wide via libvirtd's
+     * {@code vnc_tls_x509_cert_dir} setting. Restarting libvirtd does not affect VMs already running, since QEMU
+     * only loads that certificate once, at VM start - so reload it live on every running VM here, instead of
+     * leaving them on the previous (possibly expired) certificate until stopped/started or migrated.
+     */
+    private void pushRenewedVncCertificateToRunningVms(final LibvirtComputingResource serverResource) {
+        final long qemuVersion = serverResource.getHypervisorQemuVersion();
+        if (qemuVersion < MIN_QEMU_VERSION_FOR_VNC_TLS_CERT_RELOAD) {
+            logger.warn("QEMU {} on this host does not support reloading the VNC TLS certificate of a running VM (QEMU >= {} required), " +
+                    "running VMs will keep using the previous certificate until they are stopped/started or migrated",
+                    qemuVersion, MIN_QEMU_VERSION_FOR_VNC_TLS_CERT_RELOAD);
+            return;
+        }
+
+        final Connect conn;
+        final int[] domainIds;
+        try {
+            conn = serverResource.getLibvirtUtilitiesHelper().getConnection();
+            domainIds = conn.listDomains();
+        } catch (final LibvirtException e) {
+            logger.warn("Unable to list running VMs to reload their renewed VNC certificate", e);
+            return;
+        }
+
+        for (final int domainId : domainIds) {
+            Domain vm = null;
+            String vmName = null;
+            try {
+                vm = conn.domainLookupByID(domainId);
+                vmName = vm.getName();
+                vm.qemuMonitorCommand(QEMU_MONITOR_DISPLAY_RELOAD_VNC_TLS_CERTS_COMMAND, 0);
+                logger.debug("Reloaded VNC TLS certificate for VM [{}]", vmName);
+            } catch (final Exception e) {
+                logger.warn("Failed to reload the renewed VNC certificate for VM [{}], it will keep using the previous " +
+                        "certificate until it is stopped/started or migrated", vmName, e);
+            } finally {
+                if (vm != null) {
+                    try {
+                        vm.free();
+                    } catch (final LibvirtException e) {
+                        logger.trace("Ignoring libvirt error.", e);
+                    }
+                }
+            }
+        }
+    }
 }

plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapperTest.java

@@ -0,0 +1,147 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.hypervisor.kvm.resource.wrapper;
+
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
+import org.apache.cloudstack.ca.PostCertificateRenewalCommand;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.libvirt.Connect;
+import org.libvirt.Domain;
+import org.libvirt.LibvirtException;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockedConstruction;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import com.cloud.agent.api.Answer;
+import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
+import com.cloud.utils.script.Script;
+
+@RunWith(MockitoJUnitRunner.class)
+public class LibvirtPostCertificateRenewalCommandWrapperTest {
+
+    private static final long SUPPORTED_QEMU_VERSION = 6000000L;
+
+    @Mock
+    private LibvirtComputingResource libvirtComputingResource;
+    @Mock
+    private LibvirtUtilitiesHelper libvirtUtilitiesHelper;
+    @Mock
+    private Connect connect;
+
+    private final LibvirtPostCertificateRenewalCommandWrapper wrapper = new LibvirtPostCertificateRenewalCommandWrapper();
+
+    @Before
+    public void setUp() {
+        when(libvirtComputingResource.getHypervisorQemuVersion()).thenReturn(SUPPORTED_QEMU_VERSION);
+    }
+
+    private Answer executeWithScriptMocked() {
+        try (MockedConstruction<Script> ignored = Mockito.mockConstruction(Script.class)) {
+            return wrapper.execute(new PostCertificateRenewalCommand(), libvirtComputingResource);
+        }
+    }
+
+    @Test
+    public void testExecuteReloadsVncTlsCertificateForEachRunningVm() throws Exception {
+        final Domain vm1 = mock(Domain.class);
+        when(vm1.getName()).thenReturn("i-2-3-VM");
+        final Domain vm2 = mock(Domain.class);
+        when(vm2.getName()).thenReturn("i-4-5-VM");
+
+        when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper);
+        when(libvirtUtilitiesHelper.getConnection()).thenReturn(connect);
+        when(connect.listDomains()).thenReturn(new int[]{1, 2});
+        when(connect.domainLookupByID(1)).thenReturn(vm1);
+        when(connect.domainLookupByID(2)).thenReturn(vm2);
+
+        final Answer answer = executeWithScriptMocked();
+
+        assertTrue(answer.getResult());
+        final ArgumentCaptor<String> monitorCommandCaptor = ArgumentCaptor.forClass(String.class);
+        verify(vm1, times(1)).qemuMonitorCommand(monitorCommandCaptor.capture(), Mockito.eq(0));
+        verify(vm2, times(1)).qemuMonitorCommand(Mockito.anyString(), Mockito.eq(0));
+        final String capturedCommand = monitorCommandCaptor.getValue();
+        assertTrue(capturedCommand.contains("display-reload"));
+        assertTrue(capturedCommand.contains("\"type\":\"vnc\""));
+        assertTrue(capturedCommand.contains("\"tls-certs\":true"));
+        verify(vm1, times(1)).free();
+        verify(vm2, times(1)).free();
+    }
+
+    @Test
+    public void testExecuteContinuesWithOtherVmsWhenOneReloadFails() throws Exception {
+        final Domain failingVm = mock(Domain.class);
+        when(failingVm.getName()).thenReturn("i-2-3-VM");
+        when(failingVm.qemuMonitorCommand(Mockito.anyString(), Mockito.eq(0))).thenThrow(mock(LibvirtException.class));
+        final Domain workingVm = mock(Domain.class);
+        when(workingVm.getName()).thenReturn("i-4-5-VM");
+
+        when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper);
+        when(libvirtUtilitiesHelper.getConnection()).thenReturn(connect);
+        when(connect.listDomains()).thenReturn(new int[]{1, 2});
+        when(connect.domainLookupByID(1)).thenReturn(failingVm);
+        when(connect.domainLookupByID(2)).thenReturn(workingVm);
+
+        final Answer answer = executeWithScriptMocked();
+
+        assertTrue(answer.getResult());
+        verify(workingVm, times(1)).qemuMonitorCommand(Mockito.anyString(), Mockito.eq(0));
+        verify(failingVm, times(1)).free();
+        verify(workingVm, times(1)).free();
+    }
+
+    @Test
+    public void testExecuteContinuesWhenNoRunningVms() throws Exception {
+        when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper);
+        when(libvirtUtilitiesHelper.getConnection()).thenReturn(connect);
+        when(connect.listDomains()).thenReturn(new int[]{});
+
+        final Answer answer = executeWithScriptMocked();
+
+        assertTrue(answer.getResult());
+    }
+
+    @Test
+    public void testExecuteHandlesUnableToListRunningVms() throws Exception {
+        when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper);
+        when(libvirtUtilitiesHelper.getConnection()).thenThrow(mock(LibvirtException.class));
+
+        final Answer answer = executeWithScriptMocked();
+
+        assertTrue(answer.getResult());
+    }
+
+    @Test
+    public void testExecuteSkipsVncCertReloadWhenQemuVersionTooOld() throws Exception {
+        when(libvirtComputingResource.getHypervisorQemuVersion()).thenReturn(5002000L);
+
+        final Answer answer = executeWithScriptMocked();
+
+        assertTrue(answer.getResult());
+        verifyNoInteractions(libvirtUtilitiesHelper);
+    }
+}