fix(ec2): wait for candidate NICs on fast-booting instances via Amazon PCI detection

cloudinit/sources/DataSourceEc2.py

@@ -68,14 +68,59 @@ def skip_404_tag_errors(exception):
 # https://github.com/amzn/amzn-drivers/
 ELASTIC_DRIVERS = ["ena", "efa"]
 
-# Instance types known to exhibit a NIC initialization race condition
-# where kernel network drivers are not ready when cloud-init-local runs.
-# Polling for NICs is restricted to these product names (from DMI
-# system-product-name) to avoid unnecessary boot delays on other instances.
-# Addresses GH-6697.
-NETWORK_POLLING_PRODUCT_ALLOW_LIST = [
-    "hpc7a.96xlarge",
-]
+# Hardware identifiers for Amazon network controllers (ENA, EFA)
+# used to detect in-flight kernel probe during early boot. Addresses GH-6697.
+AMAZON_PCI_VENDOR_ID = "0x1d0f"
+PCI_NETWORK_CLASS_PREFIX = "0x02"
+SYS_BUS_PCI_DEVICES = "/sys/bus/pci/devices"
+FALLBACK_NIC_TIMEOUT = 60
+
+
+def _has_amazon_network_device(
+    pci_devices_path: str = SYS_BUS_PCI_DEVICES,
+) -> bool:
+    """Return True if an Amazon network controller is on the PCI bus.
+
+    When cloud-init-local executes early during fast boots (e.g. on Graviton
+    or fast AMD/Intel instances with modular ena drivers), candidate network
+    interfaces may not yet have appeared in /sys/class/net because kernel
+    probe is still in-flight. Presence of an Amazon network controller
+    (vendor 0x1d0f, class 0x02*) indicates candidate NICs may still appear.
+    """
+    try:
+        devices = os.listdir(pci_devices_path)
+    except OSError:
+        return False
+
+    for dev in devices:
+        try:
+            dev_dir = os.path.join(pci_devices_path, dev)
+            vendor_file = os.path.join(dev_dir, "vendor")
+            class_file = os.path.join(dev_dir, "class")
+
+            with open(vendor_file, "r", encoding="utf-8") as f:
+                vendor = f.read().strip().lower()
+            if not vendor.startswith("0x"):
+                vendor = f"0x{vendor}"
+            if vendor != AMAZON_PCI_VENDOR_ID:
+                continue
+
+            with open(class_file, "r", encoding="utf-8") as f:
+                pci_class = f.read().strip().lower()
+            if not pci_class.startswith("0x"):
+                pci_class = f"0x{pci_class}"
+            if not pci_class.startswith(PCI_NETWORK_CLASS_PREFIX):
+                continue
+
+            LOG.debug(
+                "Amazon network controller %s detected on PCI bus",
+                dev,
+            )
+            return True
+        except OSError:
+            continue
+
+    return False
 
 
 class DataSourceEc2(sources.DataSource):
@@ -168,17 +213,17 @@ def _get_data(self):
             if util.is_FreeBSD():
                 LOG.debug("FreeBSD doesn't support running dhclient with -sf")
                 return False
-            product_name = dmi.read_dmi_data("system-product-name")
-            wait_for_nics = (
-                product_name is not None
-                and product_name.lower() in NETWORK_POLLING_PRODUCT_ALLOW_LIST
-            )
-            if wait_for_nics:
+            candidate_nics = net.find_candidate_nics()
+            if not candidate_nics and _has_amazon_network_device():
+                LOG.info(
+                    "Amazon network controller detected on PCI bus but no "
+                    "candidate NICs found; waiting up to %s seconds for "
+                    "NIC driver probe",
+                    FALLBACK_NIC_TIMEOUT,
+                )
                 candidate_nics = net.wait_for_candidate_nics(
-                    timeout=60, sleep_interval=1
+                    timeout=FALLBACK_NIC_TIMEOUT, sleep_interval=1
                 )
-            else:
-                candidate_nics = net.find_candidate_nics()
             LOG.debug("Looking for the primary NIC in: %s", candidate_nics)
             # Abort if NICs are still unavailable
             # (whether retry was attempted or skipped).

tests/unittests/sources/test_ec2.py

@@ -916,18 +916,102 @@ def test_ec2_local_returns_false_on_bsd(
             "FreeBSD doesn't support running dhclient with -sf" in caplog.text
         )
 
+    @pytest.mark.parametrize(
+        "vendor,pci_class,expected",
+        [
+            ("0x1d0f", "0x020000", True),
+            ("0x1d0f", "0x028000", True),
+            ("0x1d0f", "0x020028", True),
+            ("0x1d0f", "0x010802", False),
+            ("0x8086", "0x020000", False),
+            ("0x10ec", "0x020000", False),
+            ("", "", False),
+        ],
+    )
+    def test_has_amazon_network_device(
+        self, tmpdir, vendor, pci_class, expected
+    ):
+        pci_dir = tmpdir.mkdir("pci_devices")
+        dev_dir = pci_dir.mkdir("0000:00:05.0")
+        if vendor:
+            dev_dir.join("vendor").write(vendor)
+        if pci_class:
+            dev_dir.join("class").write(pci_class)
+
+        res = ec2._has_amazon_network_device(str(pci_dir))
+        assert res is expected
+
+    def test_has_amazon_network_device_no_pci_dir(self, tmpdir):
+        missing_dir = str(tmpdir.join("nonexistent"))
+        assert ec2._has_amazon_network_device(missing_dir) is False
+
+    def test_has_amazon_network_device_multiple_devices(self, tmp_path):
+        pci_dir = tmp_path / "pci_devices"
+        pci_dir.mkdir()
+        # Non-matching device (non-Amazon NIC)
+        intel_dev = pci_dir / "0000:00:04.0"
+        intel_dev.mkdir()
+        (intel_dev / "vendor").write_text("0x8086", encoding="utf-8")
+        (intel_dev / "class").write_text("0x020000", encoding="utf-8")
+        # Non-matching Amazon storage controller (NVMe)
+        nvme_dev = pci_dir / "0000:00:05.0"
+        nvme_dev.mkdir()
+        (nvme_dev / "vendor").write_text("0x1d0f", encoding="utf-8")
+        (nvme_dev / "class").write_text("0x010802", encoding="utf-8")
+
+        # Negative case: non-Amazon NIC and Amazon NVMe present, but no ENA NIC
+        assert ec2._has_amazon_network_device(str(pci_dir)) is False
+
+        # Positive case: add matching Amazon network controller (ENA)
+        ena_dev = pci_dir / "0000:00:06.0"
+        ena_dev.mkdir()
+        (ena_dev / "vendor").write_text("0x1d0f", encoding="utf-8")
+        (ena_dev / "class").write_text("0x020000", encoding="utf-8")
+
+        assert ec2._has_amazon_network_device(str(pci_dir)) is True
+
+    def test_has_amazon_network_device_partial_entry(self, tmp_path):
+        pci_dir = tmp_path / "pci_devices"
+        pci_dir.mkdir()
+        # Malformed or incomplete device entry without class attribute
+        broken_dev = pci_dir / "0000:00:04.0"
+        broken_dev.mkdir()
+        (broken_dev / "vendor").write_text("0x1d0f", encoding="utf-8")
+
+        assert ec2._has_amazon_network_device(str(pci_dir)) is False
+
+    def test_has_amazon_network_device_unreadable_entry(self, tmp_path):
+        pci_dir = tmp_path / "pci_devices"
+        pci_dir.mkdir()
+        dev = pci_dir / "0000:00:04.0"
+        dev.mkdir()
+        (dev / "vendor").write_text("0x1d0f", encoding="utf-8")
+        (dev / "class").write_text("0x020000", encoding="utf-8")
+
+        with mock.patch(
+            "cloudinit.sources.DataSourceEc2.open",
+            create=True,
+            side_effect=PermissionError,
+        ) as m_open:
+            assert ec2._has_amazon_network_device(str(pci_dir)) is False
+            assert m_open.called
+
+    @mock.patch(
+        "cloudinit.sources.DataSourceEc2._has_amazon_network_device",
+        return_value=True,
+    )
     @mock.patch("cloudinit.sources.DataSourceEc2.net.wait_for_candidate_nics")
     @mock.patch("cloudinit.sources.DataSourceEc2.net.find_candidate_nics")
-    @mock.patch("cloudinit.sources.DataSourceEc2.dmi.read_dmi_data")
     @mock.patch("cloudinit.sources.DataSourceEc2.util.is_FreeBSD")
-    def test_ec2_local_waits_for_allowlisted_product(
+    def test_ec2_local_waits_when_amazon_pci_device_present(
         self,
         m_is_freebsd,
-        m_read_dmi,
         m_find_candidate_nics,
         m_wait_for_candidate_nics,
+        m_has_pci,
         mocker,
         tmpdir,
+        caplog,
     ):
         self.datasource = ec2.DataSourceEc2Local
         ds = self._setup_ds(
@@ -938,26 +1022,42 @@ def test_ec2_local_waits_for_allowlisted_product(
             tmpdir=tmpdir,
         )
         m_is_freebsd.return_value = False
-        m_read_dmi.return_value = "hpc7a.96xlarge"
         m_find_candidate_nics.return_value = []
         m_wait_for_candidate_nics.return_value = []
 
+        caplog.set_level(
+            logging.INFO, logger="cloudinit.sources.DataSourceEc2"
+        )
         assert ds.get_data() is False
+        m_find_candidate_nics.assert_called_once_with()
+        m_has_pci.assert_called_once_with()
         m_wait_for_candidate_nics.assert_called_once_with(
-            timeout=60, sleep_interval=1
+            timeout=ec2.FALLBACK_NIC_TIMEOUT, sleep_interval=1
+        )
+        expected_msg = (
+            "Amazon network controller detected on PCI bus but no "
+            f"candidate NICs found; waiting up to {ec2.FALLBACK_NIC_TIMEOUT} "
+            "seconds for NIC driver probe"
         )
-        m_find_candidate_nics.assert_not_called()
+        assert (
+            "cloudinit.sources.DataSourceEc2",
+            logging.INFO,
+            expected_msg,
+        ) in caplog.record_tuples
 
+    @mock.patch(
+        "cloudinit.sources.DataSourceEc2._has_amazon_network_device",
+        return_value=False,
+    )
     @mock.patch("cloudinit.sources.DataSourceEc2.net.wait_for_candidate_nics")
     @mock.patch("cloudinit.sources.DataSourceEc2.net.find_candidate_nics")
-    @mock.patch("cloudinit.sources.DataSourceEc2.dmi.read_dmi_data")
     @mock.patch("cloudinit.sources.DataSourceEc2.util.is_FreeBSD")
-    def test_ec2_local_does_not_wait_for_non_allowlisted_product(
+    def test_ec2_local_does_not_wait_when_no_amazon_pci_device(
         self,
         m_is_freebsd,
-        m_read_dmi,
         m_find_candidate_nics,
         m_wait_for_candidate_nics,
+        m_has_pci,
         mocker,
         tmpdir,
     ):
@@ -970,12 +1070,48 @@ def test_ec2_local_does_not_wait_for_non_allowlisted_product(
             tmpdir=tmpdir,
         )
         m_is_freebsd.return_value = False
-        m_read_dmi.return_value = "m7i.48xlarge"
         m_find_candidate_nics.return_value = []
 
         assert ds.get_data() is False
+        m_find_candidate_nics.assert_called_once_with()
+        m_has_pci.assert_called_once_with()
         m_wait_for_candidate_nics.assert_not_called()
+
+    @mock.patch("cloudinit.sources.DataSourceEc2._has_amazon_network_device")
+    @mock.patch("cloudinit.sources.DataSourceEc2.net.wait_for_candidate_nics")
+    @mock.patch(
+        "cloudinit.sources.DataSourceEc2.net.find_candidate_nics",
+        return_value=["eth0"],
+    )
+    @mock.patch("cloudinit.sources.DataSourceEc2.util.is_FreeBSD")
+    def test_ec2_local_does_not_wait_when_nic_already_present(
+        self,
+        m_is_freebsd,
+        m_find_candidate_nics,
+        m_wait_for_candidate_nics,
+        m_has_pci,
+        mocker,
+        tmpdir,
+    ):
+        self.datasource = ec2.DataSourceEc2Local
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={"datasource": {"Ec2": {"strict_id": False}}},
+            md=None,
+            mocker=mocker,
+            tmpdir=tmpdir,
+        )
+        m_is_freebsd.return_value = False
+
+        mocker.patch.object(
+            ds, "crawl_metadata", return_value={"instance-id": "i-123"}
+        )
+        mocker.patch("cloudinit.sources.DataSourceEc2.EphemeralIPNetwork")
+
+        assert ds._get_data() is True
         m_find_candidate_nics.assert_called_once_with()
+        m_has_pci.assert_not_called()
+        m_wait_for_candidate_nics.assert_not_called()
 
     @responses.activate
     @pytest.mark.usefixtures("disable_netdev_info")