discovery: account for future gossip memory

discovery/gossiper.go

@@ -61,10 +61,25 @@ const (
 	// updates that we'll hold onto.
 	maxPrematureUpdates = 100
 
-	// maxFutureMessages tracks the max amount of future messages that
-	// we'll hold onto.
+	// maxFutureMessages tracks the max amount of ordinary future messages
+	// that we'll hold onto.
 	maxFutureMessages = 1000
 
+	// futureMsgMinSize is the minimum cache charge for a future message.
+	// Using the maximum wire body size preserves the existing count limit
+	// while allowing larger decoded objects to receive a larger charge.
+	futureMsgMinSize = lnwire.MaxMsgBody
+
+	// maxFutureMsgCacheSize is the retained-memory budget for future
+	// messages. This is equivalent to the previous limit of 1,000 maximum
+	// sized wire messages.
+	maxFutureMsgCacheSize = maxFutureMessages * futureMsgMinSize
+
+	// futureFeatureEntrySize conservatively accounts for the map bucket,
+	// key, top-hash, and overflow storage retained by each decoded feature
+	// bit.
+	futureFeatureEntrySize = 16
+
 	// DefaultSubBatchDelay is the default delay we'll use when
 	// broadcasting the next announcement batch.
 	DefaultSubBatchDelay = 5 * time.Second
@@ -591,7 +606,7 @@ func New(cfg Config, selfKeyDesc *keychain.KeyDescriptor) *AuthenticatedGossiper
 		selfKeyLoc:        selfKeyDesc.KeyLocator,
 		cfg:               &cfg,
 		networkMsgs:       make(chan *networkMsg),
-		futureMsgs:        newFutureMsgCache(maxFutureMessages),
+		futureMsgs:        newFutureMsgCache(maxFutureMsgCacheSize),
 		quit:              make(chan struct{}),
 		chanPolicyUpdates: make(chan *chanPolicyUpdateRequest),
 		prematureChannelUpdates: lru.NewCache[uint64, *cachedNetworkMsg]( //nolint: ll
@@ -788,12 +803,32 @@ type cachedFutureMsg struct {
 
 	// height is the block height.
 	height uint32
+
+	// size is the conservative amount of retained memory charged to the
+	// cache.
+	size uint64
 }
 
 // Size returns the size of the message.
 func (c *cachedFutureMsg) Size() (uint64, error) {
-	// Return a constant 1.
-	return 1, nil
+	return c.size, nil
+}
+
+// futureMsgSize returns a conservative retained-memory charge for a future
+// message. The minimum charge preserves the cache's previous 1,000-message
+// bound. ChannelAnnouncement1 feature maps receive an additional charge for
+// each decoded map entry because their retained representation can be much
+// larger than their wire encoding.
+func futureMsgSize(msg lnwire.Message) uint64 {
+	size := uint64(futureMsgMinSize)
+
+	ann, ok := msg.(*lnwire.ChannelAnnouncement1)
+	if !ok || ann.Features == nil {
+		return size
+	}
+
+	return size + uint64(ann.Features.NumFeatures())*
+		futureFeatureEntrySize
 }
 
 // resendFutureMessages takes a block height, resends all the future messages
@@ -2245,9 +2280,10 @@ func (d *AuthenticatedGossiper) isPremature(chanID lnwire.ShortChannelID,
 	cachedMsg := &cachedFutureMsg{
 		msg:    copied,
 		height: msgHeight,
+		size:   futureMsgSize(copied.msg),
 	}
 
-	// Increment the msg ID and add it to the cache.
+	// Increment the message ID and add it to the cache.
 	nextMsgID := d.futureMsgs.nextMsgID()
 	_, err := d.futureMsgs.Put(nextMsgID, cachedMsg)
 	if err != nil {

discovery/gossiper_test.go

@@ -702,6 +702,7 @@ const (
 type fundingTxOpts struct {
 	extraBytes    []byte
 	fundingTxPrep fundingTxPrepType
+	features      *lnwire.RawFeatureVector
 }
 
 type fundingTxOption func(*fundingTxOpts)
@@ -718,6 +719,12 @@ func withFundingTxPrep(prep fundingTxPrepType) fundingTxOption {
 	}
 }
 
+func withFeatures(features *lnwire.RawFeatureVector) fundingTxOption {
+	return func(opts *fundingTxOpts) {
+		opts.features = features
+	}
+}
+
 func (ctx *testCtx) createAnnouncementWithoutProof(blockHeight uint32,
 	key1, key2 *btcec.PublicKey,
 	options ...fundingTxOption) *lnwire.ChannelAnnouncement1 {
@@ -789,6 +796,9 @@ func (ctx *testCtx) createAnnouncementWithoutProof(blockHeight uint32,
 		},
 		Features: testFeatures,
 	}
+	if opts.features != nil {
+		a.Features = opts.features
+	}
 	copy(a.NodeID1[:], key1.SerializeCompressed())
 	copy(a.NodeID2[:], key2.SerializeCompressed())
 	copy(a.BitcoinKey1[:], bitcoinKeyPub1.SerializeCompressed())
@@ -4442,13 +4452,19 @@ func TestFutureMsgCacheEviction(t *testing.T) {
 	//
 	// Put the first item.
 	id := c.nextMsgID()
-	evicted, err := c.Put(id, &cachedFutureMsg{height: uint32(id)})
+	evicted, err := c.Put(id, &cachedFutureMsg{
+		height: 1,
+		size:   1,
+	})
 	require.NoError(t, err)
 	require.False(t, evicted, "should not be evicted")
 
 	// Put the second item.
 	id = c.nextMsgID()
-	evicted, err = c.Put(id, &cachedFutureMsg{height: uint32(id)})
+	evicted, err = c.Put(id, &cachedFutureMsg{
+		height: 2,
+		size:   1,
+	})
 	require.NoError(t, err)
 	require.True(t, evicted, "should be evicted")
 
@@ -4464,6 +4480,88 @@ func TestFutureMsgCacheEviction(t *testing.T) {
 	require.EqualValues(t, 2, item.height, "should be the second item")
 }
 
+// denseFeatureVector returns a maximally populated feature vector. This is
+// the largest decoded feature map representable by FeatureBit.
+func denseFeatureVector() *lnwire.RawFeatureVector {
+	features := lnwire.NewRawFeatureVector()
+	for i := 0; i < 1<<16; i++ {
+		features.Set(lnwire.FeatureBit(i))
+	}
+
+	return features
+}
+
+// TestFutureChanAnnCacheBounds verifies that separately allocated but
+// content-identical dense announcements are each charged according to their
+// decoded representation and remain within the retained-memory budget.
+func TestFutureChanAnnCacheBounds(t *testing.T) {
+	t.Parallel()
+
+	const (
+		startHeight  = uint32(100)
+		futureHeight = uint32(200)
+	)
+
+	tCtx, err := createTestCtx(t, startHeight, false)
+	require.NoError(t, err)
+
+	ann, err := tCtx.createRemoteChannelAnnouncement(
+		futureHeight, withFundingTxPrep(fundingTxPrepTypeNone),
+		withFeatures(denseFeatureVector()),
+	)
+	require.NoError(t, err)
+
+	entrySize := futureMsgSize(ann)
+	require.Equal(
+		t, uint64(futureMsgMinSize+(1<<16)*futureFeatureEntrySize),
+		entrySize,
+	)
+	tCtx.gossiper.futureMsgs = newFutureMsgCache(2 * entrySize)
+
+	nodePeer := &mockPeer{
+		pk: remoteKeyPriv1.PubKey(),
+	}
+	err = mustProcess(t, tCtx.gossiper.ProcessRemoteAnnouncement(
+		t.Context(), ann, nodePeer,
+	))
+	require.NoError(t, err)
+	require.Equal(t, 1, tCtx.gossiper.futureMsgs.Len())
+	require.Equal(t, entrySize, tCtx.gossiper.futureMsgs.Size())
+
+	// Model decoding the identical wire message again. The object and its
+	// feature map are distinct allocations, while the canonical wire content
+	// is the same.
+	annCopy := *ann
+	annCopy.Features = ann.Features.Clone()
+	require.NotSame(t, ann, &annCopy)
+	require.NotSame(t, ann.Features, annCopy.Features)
+
+	err = mustProcess(t, tCtx.gossiper.ProcessRemoteAnnouncement(
+		t.Context(), &annCopy, nodePeer,
+	))
+	require.NoError(t, err)
+	require.Equal(t, 2, tCtx.gossiper.futureMsgs.Len())
+	require.Equal(t, 2*entrySize, tCtx.gossiper.futureMsgs.Size())
+
+	// A third dense announcement evicts the oldest entry. The cache never
+	// exceeds its configured retained-memory capacity.
+	nextAnn, err := tCtx.createRemoteChannelAnnouncement(
+		futureHeight+1, withFundingTxPrep(fundingTxPrepTypeNone),
+		withFeatures(denseFeatureVector()),
+	)
+	require.NoError(t, err)
+
+	err = mustProcess(t, tCtx.gossiper.ProcessRemoteAnnouncement(
+		t.Context(), nextAnn, nodePeer,
+	))
+	require.NoError(t, err)
+
+	require.Equal(t, 2, tCtx.gossiper.futureMsgs.Len())
+	require.Equal(t, 2*entrySize, tCtx.gossiper.futureMsgs.Size())
+	_, err = tCtx.gossiper.futureMsgs.Get(1)
+	require.ErrorIs(t, err, cache.ErrElementNotFound)
+}
+
 // TestChanAnnBanningNonChanPeer asserts that non-channel peers who send bogus
 // channel announcements are banned properly.
 func TestChanAnnBanningNonChanPeer(t *testing.T) {

docs/release-notes/release-notes-0.22.0.md

@@ -47,6 +47,10 @@
   the reported network statistics such as total network capacity, channel
   count and max out degree.
 
+* The future gossip cache now [accounts for decoded feature-map
+  memory](https://github.com/lightningnetwork/lnd/pull/11158), preserving its
+  ordinary message-count limit while bounding dense announcement retention.
+
 # New Features
 
 ## Functional Enhancements
@@ -166,4 +170,5 @@
 * Boris Nagaev
 * Erick Cestari
 * Jared Tobin
+* moscowchill
 * Nishant Bansal