From 908815243cefa9490c26000058b31e91fbfd239f Mon Sep 17 00:00:00 2001 From: Viktor Pashaiev Date: Thu, 22 Oct 2026 10:56:42 +0200 Subject: [PATCH] fix(ec2): wait for candidate NICs on fast-booting instances via Amazon PCI detection On fast-booting EC2 instances (both Graviton arm64 and AMD/Intel amd64, e.g. hpc7a, m8g, r8g, t4g, x2gd, m7a), cloud-init-local can execute before the kernel modular ena driver finishes probing PCI devices. When that happens, find_candidate_nics() finds no interfaces and DataSourceEc2 fails early with 'The instance must have at least one eligible NIC'. Previously, PR #7065 added an allowlist for hpc7a.96xlarge, and PR #7138 initially extended it with Graviton product prefixes. However, benchmark data and reports from production fleets show that this boot race affects multiple instance families across both arm64 and amd64, making instance allowlists an unsustainable maintenance burden. Instead of matching DMI product names or architectures, detect in-flight kernel probes directly via the PCI bus: - If find_candidate_nics() finds an interface, proceed immediately with 0ms overhead. - If no interfaces are found yet, check /sys/bus/pci/devices/ for an Amazon network controller (Vendor ID 0x1d0f, class 0x02*). - If present, wait up to 60s for candidate NICs to appear. - If no Amazon network device is on the PCI bus, do not wait, avoiding unnecessary stalls on isolated/network-less instances. Fixes GH-6697 LP: #2151279 --- cloudinit/sources/DataSourceEc2.py | 79 +++++++++++--- tests/unittests/sources/test_ec2.py | 156 ++++++++++++++++++++++++++-- 2 files changed, 208 insertions(+), 27 deletions(-) diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 4b1ba1deef0..70dd688dbcb 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -68,14 +68,59 @@ def skip_404_tag_errors(exception): # https://github.com/amzn/amzn-drivers/ ELASTIC_DRIVERS = ["ena", "efa"] -# Instance types known to exhibit a NIC initialization race condition -# where kernel network drivers are not ready when cloud-init-local runs. -# Polling for NICs is restricted to these product names (from DMI -# system-product-name) to avoid unnecessary boot delays on other instances. -# Addresses GH-6697. -NETWORK_POLLING_PRODUCT_ALLOW_LIST = [ - "hpc7a.96xlarge", -] +# Hardware identifiers for Amazon network controllers (ENA, EFA) +# used to detect in-flight kernel probe during early boot. Addresses GH-6697. +AMAZON_PCI_VENDOR_ID = "0x1d0f" +PCI_NETWORK_CLASS_PREFIX = "0x02" +SYS_BUS_PCI_DEVICES = "/sys/bus/pci/devices" +FALLBACK_NIC_TIMEOUT = 60 + + +def _has_amazon_network_device( + pci_devices_path: str = SYS_BUS_PCI_DEVICES, +) -> bool: + """Return True if an Amazon network controller is on the PCI bus. + + When cloud-init-local executes early during fast boots (e.g. on Graviton + or fast AMD/Intel instances with modular ena drivers), candidate network + interfaces may not yet have appeared in /sys/class/net because kernel + probe is still in-flight. Presence of an Amazon network controller + (vendor 0x1d0f, class 0x02*) indicates candidate NICs may still appear. + """ + try: + devices = os.listdir(pci_devices_path) + except OSError: + return False + + for dev in devices: + try: + dev_dir = os.path.join(pci_devices_path, dev) + vendor_file = os.path.join(dev_dir, "vendor") + class_file = os.path.join(dev_dir, "class") + + with open(vendor_file, "r", encoding="utf-8") as f: + vendor = f.read().strip().lower() + if not vendor.startswith("0x"): + vendor = f"0x{vendor}" + if vendor != AMAZON_PCI_VENDOR_ID: + continue + + with open(class_file, "r", encoding="utf-8") as f: + pci_class = f.read().strip().lower() + if not pci_class.startswith("0x"): + pci_class = f"0x{pci_class}" + if not pci_class.startswith(PCI_NETWORK_CLASS_PREFIX): + continue + + LOG.debug( + "Amazon network controller %s detected on PCI bus", + dev, + ) + return True + except OSError: + continue + + return False class DataSourceEc2(sources.DataSource): @@ -168,17 +213,17 @@ def _get_data(self): if util.is_FreeBSD(): LOG.debug("FreeBSD doesn't support running dhclient with -sf") return False - product_name = dmi.read_dmi_data("system-product-name") - wait_for_nics = ( - product_name is not None - and product_name.lower() in NETWORK_POLLING_PRODUCT_ALLOW_LIST - ) - if wait_for_nics: + candidate_nics = net.find_candidate_nics() + if not candidate_nics and _has_amazon_network_device(): + LOG.info( + "Amazon network controller detected on PCI bus but no " + "candidate NICs found; waiting up to %s seconds for " + "NIC driver probe", + FALLBACK_NIC_TIMEOUT, + ) candidate_nics = net.wait_for_candidate_nics( - timeout=60, sleep_interval=1 + timeout=FALLBACK_NIC_TIMEOUT, sleep_interval=1 ) - else: - candidate_nics = net.find_candidate_nics() LOG.debug("Looking for the primary NIC in: %s", candidate_nics) # Abort if NICs are still unavailable # (whether retry was attempted or skipped). diff --git a/tests/unittests/sources/test_ec2.py b/tests/unittests/sources/test_ec2.py index a6490e7c356..6a01a1688d4 100644 --- a/tests/unittests/sources/test_ec2.py +++ b/tests/unittests/sources/test_ec2.py @@ -916,18 +916,102 @@ def test_ec2_local_returns_false_on_bsd( "FreeBSD doesn't support running dhclient with -sf" in caplog.text ) + @pytest.mark.parametrize( + "vendor,pci_class,expected", + [ + ("0x1d0f", "0x020000", True), + ("0x1d0f", "0x028000", True), + ("0x1d0f", "0x020028", True), + ("0x1d0f", "0x010802", False), + ("0x8086", "0x020000", False), + ("0x10ec", "0x020000", False), + ("", "", False), + ], + ) + def test_has_amazon_network_device( + self, tmpdir, vendor, pci_class, expected + ): + pci_dir = tmpdir.mkdir("pci_devices") + dev_dir = pci_dir.mkdir("0000:00:05.0") + if vendor: + dev_dir.join("vendor").write(vendor) + if pci_class: + dev_dir.join("class").write(pci_class) + + res = ec2._has_amazon_network_device(str(pci_dir)) + assert res is expected + + def test_has_amazon_network_device_no_pci_dir(self, tmpdir): + missing_dir = str(tmpdir.join("nonexistent")) + assert ec2._has_amazon_network_device(missing_dir) is False + + def test_has_amazon_network_device_multiple_devices(self, tmp_path): + pci_dir = tmp_path / "pci_devices" + pci_dir.mkdir() + # Non-matching device (non-Amazon NIC) + intel_dev = pci_dir / "0000:00:04.0" + intel_dev.mkdir() + (intel_dev / "vendor").write_text("0x8086", encoding="utf-8") + (intel_dev / "class").write_text("0x020000", encoding="utf-8") + # Non-matching Amazon storage controller (NVMe) + nvme_dev = pci_dir / "0000:00:05.0" + nvme_dev.mkdir() + (nvme_dev / "vendor").write_text("0x1d0f", encoding="utf-8") + (nvme_dev / "class").write_text("0x010802", encoding="utf-8") + + # Negative case: non-Amazon NIC and Amazon NVMe present, but no ENA NIC + assert ec2._has_amazon_network_device(str(pci_dir)) is False + + # Positive case: add matching Amazon network controller (ENA) + ena_dev = pci_dir / "0000:00:06.0" + ena_dev.mkdir() + (ena_dev / "vendor").write_text("0x1d0f", encoding="utf-8") + (ena_dev / "class").write_text("0x020000", encoding="utf-8") + + assert ec2._has_amazon_network_device(str(pci_dir)) is True + + def test_has_amazon_network_device_partial_entry(self, tmp_path): + pci_dir = tmp_path / "pci_devices" + pci_dir.mkdir() + # Malformed or incomplete device entry without class attribute + broken_dev = pci_dir / "0000:00:04.0" + broken_dev.mkdir() + (broken_dev / "vendor").write_text("0x1d0f", encoding="utf-8") + + assert ec2._has_amazon_network_device(str(pci_dir)) is False + + def test_has_amazon_network_device_unreadable_entry(self, tmp_path): + pci_dir = tmp_path / "pci_devices" + pci_dir.mkdir() + dev = pci_dir / "0000:00:04.0" + dev.mkdir() + (dev / "vendor").write_text("0x1d0f", encoding="utf-8") + (dev / "class").write_text("0x020000", encoding="utf-8") + + with mock.patch( + "cloudinit.sources.DataSourceEc2.open", + create=True, + side_effect=PermissionError, + ) as m_open: + assert ec2._has_amazon_network_device(str(pci_dir)) is False + assert m_open.called + + @mock.patch( + "cloudinit.sources.DataSourceEc2._has_amazon_network_device", + return_value=True, + ) @mock.patch("cloudinit.sources.DataSourceEc2.net.wait_for_candidate_nics") @mock.patch("cloudinit.sources.DataSourceEc2.net.find_candidate_nics") - @mock.patch("cloudinit.sources.DataSourceEc2.dmi.read_dmi_data") @mock.patch("cloudinit.sources.DataSourceEc2.util.is_FreeBSD") - def test_ec2_local_waits_for_allowlisted_product( + def test_ec2_local_waits_when_amazon_pci_device_present( self, m_is_freebsd, - m_read_dmi, m_find_candidate_nics, m_wait_for_candidate_nics, + m_has_pci, mocker, tmpdir, + caplog, ): self.datasource = ec2.DataSourceEc2Local ds = self._setup_ds( @@ -938,26 +1022,42 @@ def test_ec2_local_waits_for_allowlisted_product( tmpdir=tmpdir, ) m_is_freebsd.return_value = False - m_read_dmi.return_value = "hpc7a.96xlarge" m_find_candidate_nics.return_value = [] m_wait_for_candidate_nics.return_value = [] + caplog.set_level( + logging.INFO, logger="cloudinit.sources.DataSourceEc2" + ) assert ds.get_data() is False + m_find_candidate_nics.assert_called_once_with() + m_has_pci.assert_called_once_with() m_wait_for_candidate_nics.assert_called_once_with( - timeout=60, sleep_interval=1 + timeout=ec2.FALLBACK_NIC_TIMEOUT, sleep_interval=1 + ) + expected_msg = ( + "Amazon network controller detected on PCI bus but no " + f"candidate NICs found; waiting up to {ec2.FALLBACK_NIC_TIMEOUT} " + "seconds for NIC driver probe" ) - m_find_candidate_nics.assert_not_called() + assert ( + "cloudinit.sources.DataSourceEc2", + logging.INFO, + expected_msg, + ) in caplog.record_tuples + @mock.patch( + "cloudinit.sources.DataSourceEc2._has_amazon_network_device", + return_value=False, + ) @mock.patch("cloudinit.sources.DataSourceEc2.net.wait_for_candidate_nics") @mock.patch("cloudinit.sources.DataSourceEc2.net.find_candidate_nics") - @mock.patch("cloudinit.sources.DataSourceEc2.dmi.read_dmi_data") @mock.patch("cloudinit.sources.DataSourceEc2.util.is_FreeBSD") - def test_ec2_local_does_not_wait_for_non_allowlisted_product( + def test_ec2_local_does_not_wait_when_no_amazon_pci_device( self, m_is_freebsd, - m_read_dmi, m_find_candidate_nics, m_wait_for_candidate_nics, + m_has_pci, mocker, tmpdir, ): @@ -970,12 +1070,48 @@ def test_ec2_local_does_not_wait_for_non_allowlisted_product( tmpdir=tmpdir, ) m_is_freebsd.return_value = False - m_read_dmi.return_value = "m7i.48xlarge" m_find_candidate_nics.return_value = [] assert ds.get_data() is False + m_find_candidate_nics.assert_called_once_with() + m_has_pci.assert_called_once_with() m_wait_for_candidate_nics.assert_not_called() + + @mock.patch("cloudinit.sources.DataSourceEc2._has_amazon_network_device") + @mock.patch("cloudinit.sources.DataSourceEc2.net.wait_for_candidate_nics") + @mock.patch( + "cloudinit.sources.DataSourceEc2.net.find_candidate_nics", + return_value=["eth0"], + ) + @mock.patch("cloudinit.sources.DataSourceEc2.util.is_FreeBSD") + def test_ec2_local_does_not_wait_when_nic_already_present( + self, + m_is_freebsd, + m_find_candidate_nics, + m_wait_for_candidate_nics, + m_has_pci, + mocker, + tmpdir, + ): + self.datasource = ec2.DataSourceEc2Local + ds = self._setup_ds( + platform_data=self.valid_platform_data, + sys_cfg={"datasource": {"Ec2": {"strict_id": False}}}, + md=None, + mocker=mocker, + tmpdir=tmpdir, + ) + m_is_freebsd.return_value = False + + mocker.patch.object( + ds, "crawl_metadata", return_value={"instance-id": "i-123"} + ) + mocker.patch("cloudinit.sources.DataSourceEc2.EphemeralIPNetwork") + + assert ds._get_data() is True m_find_candidate_nics.assert_called_once_with() + m_has_pci.assert_not_called() + m_wait_for_candidate_nics.assert_not_called() @responses.activate @pytest.mark.usefixtures("disable_netdev_info")