From 3aa29dde17c87b5c9565e1191d8bb16bb5bceb1b Mon Sep 17 00:00:00 2001 From: Abhimanyu Saharan Date: Sat, 31 Oct 2026 18:08:09 +0530 Subject: [PATCH 1/2] Fence search projection caches without flushing unrelated public data --- .env.example | 2 + apps/api/src/cache.py | 9 +- apps/api/src/search.py | 4 +- compose.yaml | 3 + dev/search-cache-epochs.md | 61 +++++++++ .../versions/0041_search_cache_publication.py | 28 ++++ packages/core/OBSERVABILITY.md | 5 +- packages/core/src/cache.py | 65 ++++++++-- packages/core/src/config.py | 1 + packages/core/src/models.py | 1 + packages/core/src/search_index.py | 28 +++- packages/core/src/version.py | 2 +- tests/http/test_cache_policy_contracts.py | 3 +- .../test_article_refresh_migrations.py | 3 +- tests/infrastructure/test_cache.py | 100 +++++++++++++- .../infrastructure/test_cache_integration.py | 122 ++++++++++++++++++ tests/infrastructure/test_migration_paths.py | 4 +- .../test_search_cache_migration.py | 48 +++++++ tests/partners/catalog/test_migration.py | 4 +- tests/search/test_search_outbox_unit.py | 9 +- 20 files changed, 468 insertions(+), 34 deletions(-) create mode 100644 dev/search-cache-epochs.md create mode 100644 migrations/versions/0041_search_cache_publication.py create mode 100644 tests/infrastructure/test_search_cache_migration.py diff --git a/.env.example b/.env.example index b9f5b3824..469b11953 100644 --- a/.env.example +++ b/.env.example @@ -247,6 +247,8 @@ DEVFEED_SEARCH_QUERY_KEY= DEVFEED_SEARCH_HTTP_MAX_CONNECTIONS=32 DEVFEED_SEARCH_HTTP_POOL_TIMEOUT_SECONDS=2 DEVFEED_SEARCH_INDEX_BATCH_SIZE=200 +# Enable only after all public APIs and indexers support the index cache epoch. +DEVFEED_SEARCH_CACHE_ISOLATION_ENABLED=false # Search-indexer retries are bounded and apply only to transient 503/network/import failures. # Write timeout adapts upward from this value using observed successful import duration. # DEVFEED_SEARCH_INDEX_RETRY_ATTEMPTS=3 diff --git a/apps/api/src/cache.py b/apps/api/src/cache.py index 19c2a7787..ff896b24c 100644 --- a/apps/api/src/cache.py +++ b/apps/api/src/cache.py @@ -28,6 +28,7 @@ class CachePolicy(BaseModel): namespace: str = Field(min_length=1, max_length=64, pattern=r"^[a-z][a-z0-9_-]*$") revision: int = Field(ge=1) ttl: Literal["content", "metadata"] + index_generation: bool = False def openapi_extra(self) -> dict[str, JsonValue]: return {CACHE_POLICY_EXTENSION: self.model_dump()} @@ -129,7 +130,13 @@ async def render() -> Response: try: while True: lookup = await run_in_threadpool( - lambda: cache.lookup(key, "public", observe=False) + lambda: ( + cache.lookup( + key, "public", observe=False, dependencies=("search-index",) + ) + if policy.index_generation + else cache.lookup(key, "public", observe=False) + ) ) if lookup.body is not None: return tagged( diff --git a/apps/api/src/search.py b/apps/api/src/search.py index bc7e47692..f8f916761 100644 --- a/apps/api/src/search.py +++ b/apps/api/src/search.py @@ -170,7 +170,9 @@ def persist_metadata(session: Session) -> str: @router.get( "", response_model=SearchResponse, - openapi_extra=CachePolicy(namespace="search", revision=1, ttl="content").openapi_extra(), + openapi_extra=CachePolicy( + namespace="search", revision=2, ttl="content", index_generation=True + ).openapi_extra(), ) def search( session: DB, diff --git a/compose.yaml b/compose.yaml index 0f9740713..5b7b1d829 100644 --- a/compose.yaml +++ b/compose.yaml @@ -343,6 +343,7 @@ services: <<: *backend environment: <<: [*core-environment, *clickhouse-public] + DEVFEED_SEARCH_CACHE_ISOLATION_ENABLED: ${DEVFEED_SEARCH_CACHE_ISOLATION_ENABLED:-false} DEVFEED_SEARCH_ENABLED: ${DEVFEED_SEARCH_ENABLED:-false} DEVFEED_SEARCH_URL: ${DEVFEED_SEARCH_URL:-http://typesense:8108} DEVFEED_SEARCH_QUERY_KEY: ${DEVFEED_SEARCH_QUERY_KEY:-} @@ -432,6 +433,7 @@ services: DEVFEED_SEARCH_QUERY_KEY: ${DEVFEED_SEARCH_QUERY_KEY:-} DEVFEED_SEARCH_COLLECTION_PREFIX: ${DEVFEED_SEARCH_COLLECTION_PREFIX:-devfeed} DEVFEED_SEARCH_INDEX_BATCH_SIZE: ${DEVFEED_SEARCH_INDEX_BATCH_SIZE:-200} + DEVFEED_SEARCH_CACHE_ISOLATION_ENABLED: ${DEVFEED_SEARCH_CACHE_ISOLATION_ENABLED:-false} depends_on: <<: *ready typesense: @@ -450,6 +452,7 @@ services: DEVFEED_SEARCH_ADMIN_KEY: ${DEVFEED_SEARCH_ADMIN_KEY:-} DEVFEED_SEARCH_COLLECTION_PREFIX: ${DEVFEED_SEARCH_COLLECTION_PREFIX:-devfeed} DEVFEED_SEARCH_INDEX_BATCH_SIZE: ${DEVFEED_SEARCH_INDEX_BATCH_SIZE:-200} + DEVFEED_SEARCH_CACHE_ISOLATION_ENABLED: ${DEVFEED_SEARCH_CACHE_ISOLATION_ENABLED:-false} depends_on: <<: *ready search-setup: diff --git a/dev/search-cache-epochs.md b/dev/search-cache-epochs.md new file mode 100644 index 000000000..ffe258a2d --- /dev/null +++ b/dev/search-cache-epochs.md @@ -0,0 +1,61 @@ +# Search projection cache epochs + +Search-index publication and public database data have separate Redis epochs. +Cached Typesense search results depend on both: a completed projection must not +leave a previously cached empty search behind, and content withdrawal must not +leave old visible search results behind. Article/feed/catalog responses depend +only on the public-data epoch. Lua checks every dependency and loader token in +one atomic publish so an old renderer cannot repopulate an invalidated epoch. +Suggestions and PostgreSQL full-text feed filtering keep their own data semantics; +they do not inherit the Typesense publication epoch. + +## Conservative source provenance + +Migration `0041` adds `search_events.public_change` with a constant true default. +Every retained event, database-triggered business change and unknown/manual event +therefore retains the old public-data invalidation fallback. This matters for +raw psycopg and direct SQL writers that do not run application commit hooks. + +Indexer-created projection/fanout continuation events explicitly carry false. +A completed nonempty batch always invalidates the search-index epoch, and +invalidates the public epoch when any selected event may represent a real data +change. Normal application commits still invalidate public data immediately; +raw SQL source events retain the existing eventual indexer fallback before their +remaining fanout pages drain. Backfill explicitly fences public data once after +enqueue commit and marks its pages projection-only, preserving repair semantics +without flushing unrelated content on every page. + +This does not promise to remove every observed indexer invalidation: source +business events remain conservative. It does prevent index maintenance, duplicate +projection pages and fanout continuation from unconditionally expelling unrelated +public data. Direct SQL that bypasses or disables triggers still requires the +explicit backfill/repair fence, as before. + +## Rollout + +1. Apply the single additive migration `0041` before upgrading the applications. + Its constant default retains unknown events as public changes without rewriting + their tuples; a populated 10,000-event test checks unchanged IDs/xmin. +2. Upgrade every public API and search indexer replica. Leave + `DEVFEED_SEARCH_CACHE_ISOLATION_ENABLED=false` during mixed versions. Updated + APIs always understand both epochs; the default-off indexer gate continues + invalidating public data for every completed batch, matching old APIs. +3. Enable the gate only after all public APIs and indexers are compatible. + Verify cached article/feed reads survive projection-only pages while real + source/article changes, raw SQL, withdrawal and rejected sources still fence + old readers. Monitor cache hit rate, API CPU, SQL work, index lag and freshness. +4. Roll back by disabling the gate first. The old invalidation policy resumes; + keep the additive column until the compatible image rollback is complete. + +No TTL, replica count, search delay or production resource limit changes here. +The gate is intentionally off until a compatible rollout is explicitly enabled. + +## Validation + +Real Redis/PostgreSQL tests cover atomic dependency fences, projection-only pages, +raw-SQL source edits, explicit repair backfills and populated schema round trips. +Twenty projection-only pages produce twenty unrelated content-cache misses and +SQL reads with the gate off; with isolation enabled the same reads stay cached +and perform no SQL. This is a controlled demonstration of avoided work, not an +estimate that all production cache misses or CPU would disappear. Native ARM64 +workload measurement and hosted CI must pass before promoting the draft. diff --git a/migrations/versions/0041_search_cache_publication.py b/migrations/versions/0041_search_cache_publication.py new file mode 100644 index 000000000..cfe9679ba --- /dev/null +++ b/migrations/versions/0041_search_cache_publication.py @@ -0,0 +1,28 @@ +"""Keep source-change invalidation distinct from index-only projection work. + +Revision ID: 0041 +Revises: 0040 +""" + +import sqlalchemy as sa +from alembic import op + +revision = "0041" +down_revision = "0040" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute("SET LOCAL lock_timeout = '5s'") + # PostgreSQL's constant default preserves every retained/unknown event as a + # conservative public-data invalidation without rewriting the outbox. + op.add_column( + "search_events", + sa.Column("public_change", sa.Boolean(), nullable=False, server_default=sa.true()), + ) + + +def downgrade(): + op.execute("SET LOCAL lock_timeout = '5s'") + op.drop_column("search_events", "public_change") diff --git a/packages/core/OBSERVABILITY.md b/packages/core/OBSERVABILITY.md index 6ad79b82d..a106ac006 100644 --- a/packages/core/OBSERVABILITY.md +++ b/packages/core/OBSERVABILITY.md @@ -100,9 +100,10 @@ pressure. A cache hit does not bypass authentication or visibility checks. Keep the existing generation boundaries and TTLs until measurements identify a specific cause. `devfeed.cache.invalidations` records one attempt with -`cache.domain=public|operations` and +`cache.domain=public|operations|search-index` and `cache.outcome=success|cache_unavailable|disabled`. Only `success` means a generation -was replaced. Failures and the Redis circuit breaker are counted without failing +was replaced. The separate `search-index` domain fences Typesense projection +completion; search responses also depend on the public-data domain. Failures and the Redis circuit breaker are counted without failing the committed database write; disabled caching does not contact Redis. `devfeed.cache.invalidation.causes` adds `cache.invalidation_reason`. The bounded diff --git a/packages/core/src/cache.py b/packages/core/src/cache.py index b001503cc..a62b62be1 100644 --- a/packages/core/src/cache.py +++ b/packages/core/src/cache.py @@ -103,7 +103,9 @@ def record_cache_read( def record_cache_invalidation(domain: str, outcome: str, reasons: Collection[str]) -> None: if runtime := current(): attributes = { - "cache.domain": domain if domain in {"public", "operations"} else "other", + "cache.domain": domain + if domain in {"public", "operations", "search-index"} + else "other", "cache.outcome": ( outcome if outcome in {"success", "cache_unavailable", "disabled"} else "other" ), @@ -121,13 +123,17 @@ def record_cache_invalidation(domain: str, outcome: str, reasons: Collection[str # A late user may not repopulate a generation invalidated by a committed write, # or replace another loader's response after losing its lock. PUBLISH = """ -if redis.call('GET', KEYS[1]) == ARGV[1] - and redis.call('GET', KEYS[2]) == ARGV[2] then - redis.call('SET', KEYS[3], ARGV[3], 'EX', ARGV[4]) +local generations = #KEYS - 2 +for i = 1, generations do + if redis.call('GET', KEYS[i]) ~= ARGV[i] then return 0 end +end +if redis.call('GET', KEYS[generations + 1]) == ARGV[generations + 1] then + redis.call('SET', KEYS[generations + 2], ARGV[generations + 2], 'EX', ARGV[generations + 3]) return 1 end return 0 """ + RELEASE = """ if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) @@ -148,6 +154,7 @@ class CacheLookup: lock_key: str token: str | None = None body: bytes | None = None + dependencies: tuple[tuple[str, bytes], ...] = () T = TypeVar("T") @@ -176,9 +183,19 @@ def _run(self, operation: Callable[[], T]) -> T: logger.warning("cache_unavailable", extra={"error_type": type(exc).__name__}) raise CacheUnavailable from None - def lookup(self, identity: str, domain: str, *, observe: bool = True) -> CacheLookup: - def operation() -> CacheLookup: - generation_key = f"{self.namespace}:{domain}:generation" + def lookup( + self, + identity: str, + domain: str, + *, + observe: bool = True, + dependencies: tuple[str, ...] = (), + ) -> CacheLookup: + if any(dependency != "search-index" for dependency in dependencies): + raise ValueError("Unknown cache dependency") + + def epoch(name: str) -> tuple[str, bytes]: + generation_key = f"{self.namespace}:{name}:generation" generation = self.redis.get(generation_key) if generation is None: self.redis.set(generation_key, uuid.uuid4().hex, nx=True) @@ -190,7 +207,19 @@ def operation() -> CacheLookup: ): # Never fall back to a fixed epoch after eviction/corruption. raise CacheUnavailable - digest = hashlib.sha256(identity.encode()).hexdigest() + return generation_key, generation + + def operation() -> CacheLookup: + generation_key, generation = epoch(domain) + extra = tuple(epoch(name) for name in dependencies) + # Include all dependency epochs in the response identity and fence + # them again atomically with publication below. + cache_identity = ( + json.dumps([identity, [value.decode("ascii") for _, value in extra]]) + if extra + else identity + ) + digest = hashlib.sha256(cache_identity.encode()).hexdigest() key = f"{self.namespace}:{domain}:{generation.decode('ascii')}:{digest}" lock_key = f"{key}:lock" body = self.redis.get(key) @@ -207,7 +236,7 @@ def operation() -> CacheLookup: candidate = uuid.uuid4().hex if self.redis.set(lock_key, candidate, nx=True, ex=LOCK_SECONDS): token = candidate - return CacheLookup(generation_key, generation, key, lock_key, token, body) + return CacheLookup(generation_key, generation, key, lock_key, token, body, extra) name = { "public": "public_response", @@ -232,11 +261,13 @@ def publish(self, lookup: CacheLookup, body: bytes, ttl: int) -> bool: self._run( lambda: self.redis.eval( PUBLISH, - 3, + 3 + len(lookup.dependencies), lookup.generation_key, + *(key for key, _ in lookup.dependencies), lookup.lock_key, lookup.key, lookup.generation, + *(generation for _, generation in lookup.dependencies), token, body, ttl, @@ -252,7 +283,7 @@ def release(self, lookup: CacheLookup) -> None: def invalidate( self, domain: str = "public", *, reasons: Collection[str] = ("unspecified",) ) -> None: - if domain not in {"public", "operations"}: + if domain not in {"public", "operations", "search-index"}: raise ValueError("Unknown cache domain") try: self._run( @@ -382,3 +413,15 @@ def invalidate_public_cache(*, reasons: Collection[str] = ("unspecified",)) -> N get_cache().invalidate(reasons=reasons) else: record_cache_invalidation("public", "disabled", reasons) + + +def invalidate_search_cache(*, public_change: bool) -> None: + """Fence index completion separately, preserving unknown/raw SQL changes.""" + settings = get_settings() + if public_change or not settings.search_cache_isolation_enabled: + invalidate_public_cache(reasons=("search_index",)) + if settings.cache_enabled: + with suppress(CacheUnavailable): + get_cache().invalidate("search-index", reasons=("search_index",)) + else: + record_cache_invalidation("search-index", "disabled", ("search_index",)) diff --git a/packages/core/src/config.py b/packages/core/src/config.py index bbf8cdea5..9fd4e6323 100644 --- a/packages/core/src/config.py +++ b/packages/core/src/config.py @@ -203,6 +203,7 @@ def validate_partner_caller_key(cls, value: SecretStr | None) -> SecretStr | Non return value search_enabled: bool = False + search_cache_isolation_enabled: bool = False search_analytics_enabled: bool = False search_url: str | None = None search_query_key: SecretStr | None = None diff --git a/packages/core/src/models.py b/packages/core/src/models.py index 718da22eb..741b7f2bf 100644 --- a/packages/core/src/models.py +++ b/packages/core/src/models.py @@ -1519,6 +1519,7 @@ class SearchEvent(Base): kind: Mapped[str] = mapped_column(String(20)) entity_id: Mapped[uuid.UUID] = mapped_column(Uuid) fanout: Mapped[bool] = mapped_column(Boolean, server_default="false") + public_change: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true") after_id: Mapped[uuid.UUID | None] = mapped_column(Uuid) created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), server_default=text("now()") diff --git a/packages/core/src/search_index.py b/packages/core/src/search_index.py index 512cdae57..8740b11ce 100644 --- a/packages/core/src/search_index.py +++ b/packages/core/src/search_index.py @@ -9,7 +9,7 @@ from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.sql import Select -from devfeed_core.cache import invalidate_public_cache +from devfeed_core.cache import invalidate_public_cache, invalidate_search_cache from devfeed_core.catalog_cache import snapshot from devfeed_core.config import get_settings from devfeed_core.models import ( @@ -153,11 +153,17 @@ def backfill(factory: sessionmaker[Session]) -> int: for kind, model in MODELS.items(): queued = ( insert(SearchEvent) - .from_select(["kind", "entity_id"], select(literal(kind), model.id)) + .from_select( + ["kind", "entity_id", "public_change"], + select(literal(kind), model.id, literal(False)), + ) .returning(SearchEvent.id) .cte("queued") ) count += session.scalar(select(func.count()).select_from(queued)) or 0 + # Backfill may repair direct SQL writes whose triggers were disabled. Fence + # public data once after the enqueue commit, rather than once per page. + invalidate_public_cache(reasons=("search_index",)) return count @@ -170,6 +176,7 @@ def sync_batch(factory: sessionmaker[Session], engine: Typesense | None = None) events = session.scalars(select(SearchEvent).order_by(SearchEvent.id).limit(batch)).all() if not events: return 0 + public_change = any(event.public_change is not False for event in events) for kind in KINDS: ids = {event.entity_id for event in events if event.kind == kind} if ids: @@ -182,8 +189,8 @@ def sync_batch(factory: sessionmaker[Session], engine: Typesense | None = None) for kind, (link, foreign) in LINKS.items(): session.execute( insert(SearchEvent).from_select( - ["kind", "entity_id"], - select(literal(kind), foreign) + ["kind", "entity_id", "public_change"], + select(literal(kind), foreign, literal(False)) .where(link.article_id.in_(article_ids)) .distinct(), ) @@ -207,12 +214,19 @@ def sync_batch(factory: sessionmaker[Session], engine: Typesense | None = None) if related_ids: session.execute( insert(SearchEvent), - [{"kind": "articles", "entity_id": value} for value in related_ids], + [ + {"kind": "articles", "entity_id": value, "public_change": False} + for value in related_ids + ], ) if len(related_ids) == batch: session.add( SearchEvent( - kind=kind, entity_id=identifier, fanout=True, after_id=related_ids[-1] + kind=kind, + entity_id=identifier, + fanout=True, + after_id=related_ids[-1], + public_change=False, ) ) # Only acknowledge this batch. Changes committed during an HTTP import @@ -220,5 +234,5 @@ def sync_batch(factory: sessionmaker[Session], engine: Typesense | None = None) session.execute( delete(SearchEvent).where(SearchEvent.id.in_([event.id for event in events])) ) - invalidate_public_cache(reasons=("search_index",)) + invalidate_search_cache(public_change=public_change) return len(events) diff --git a/packages/core/src/version.py b/packages/core/src/version.py index a5b693495..78bd616da 100644 --- a/packages/core/src/version.py +++ b/packages/core/src/version.py @@ -6,7 +6,7 @@ # Schema revisions are not application versions. Advance this when adding a # migration required by this application; never change an existing migration. -SCHEMA_REVISION = "0040" +SCHEMA_REVISION = "0041" # Retention maintenance requires the durable checkpoints added after the scheduler # clock and indexed topic decision facts in 0035 and 0036. diff --git a/tests/http/test_cache_policy_contracts.py b/tests/http/test_cache_policy_contracts.py index df8ebd5ea..e80f4bc42 100644 --- a/tests/http/test_cache_policy_contracts.py +++ b/tests/http/test_cache_policy_contracts.py @@ -14,7 +14,7 @@ "/v1/feed": ("feed", 1, "content"), "/v1/feed/options": ("feed-options", 1, "content"), "/v1/articles/{article_id}": ("article", 1, "content"), - "/v1/search": ("search", 1, "content"), + "/v1/search": ("search", 2, "content"), "/v1/search/suggestions": ("search-suggestions", 1, "content"), "/v1/sources": ("sources", 1, "metadata"), "/v1/sources/{source_id}": ("source", 1, "metadata"), @@ -41,6 +41,7 @@ def test_public_cache_inventory_marks_only_the_twelve_shared_read_operations(): policy = CachePolicy.model_validate(extra[CACHE_POLICY_EXTENSION]) assert policy.namespace not in namespaces namespaces.add(policy.namespace) + assert policy.index_generation == (context.path == "/v1/search") actual[context.path] = (policy.namespace, policy.revision, policy.ttl) exported = app.openapi()["paths"][context.path]["get"][CACHE_POLICY_EXTENSION] assert exported == policy.model_dump() diff --git a/tests/infrastructure/test_article_refresh_migrations.py b/tests/infrastructure/test_article_refresh_migrations.py index 44725cc54..179943d07 100644 --- a/tests/infrastructure/test_article_refresh_migrations.py +++ b/tests/infrastructure/test_article_refresh_migrations.py @@ -20,6 +20,7 @@ from devfeed_core.analytics_cutover import LegacyAnalyticsCutover from devfeed_core.config import get_settings from devfeed_core.db import get_engine +from devfeed_core.version import SCHEMA_REVISION from psycopg.rows import dict_row from sqlalchemy import event, text from sqlalchemy.exc import DBAPIError, IntegrityError @@ -489,7 +490,7 @@ def test_refresh_history_refuses_downgrade_and_owner_deletion_but_article_cascad assert index_state(database) == indexes assert ordinary_fingerprints(database) == before with database() as session: - assert session.scalar(text("SELECT version_num FROM alembic_version")) == "0040" + assert session.scalar(text("SELECT version_num FROM alembic_version")) == SCHEMA_REVISION assert session.scalar(text("SELECT count(*) FROM article_refreshes")) == 1 assert ( tuple( diff --git a/tests/infrastructure/test_cache.py b/tests/infrastructure/test_cache.py index d12013b2b..2c7dd5763 100644 --- a/tests/infrastructure/test_cache.py +++ b/tests/infrastructure/test_cache.py @@ -52,9 +52,14 @@ def eval(self, script, count, *args): self.values.pop(key, None) return 1 elif script == cache.PUBLISH: - epoch_key, lock_key, key, epoch, token, body, ttl = args - if self.get(epoch_key) == epoch and self.get(lock_key) == token.encode(): - self.set(key, body, ex=ttl) + keys, values = args[:count], args[count:] + generations = count - 2 + token, body, ttl = values[generations:] + if ( + all(self.get(keys[i]) == values[i] for i in range(generations)) + and self.get(keys[-2]) == token.encode() + ): + self.set(keys[-1], body, ex=ttl) return 1 else: pytest.fail("Unexpected Redis script") @@ -742,3 +747,92 @@ def test_loaded_tag_reassigned_its_current_value_does_not_evict_public_responses cache_events.track_objects(session, None, None) assert not session.info.get(cache_events.DIRTY) assert not session.info.get(cache_events.REASONS) + + +@pytest.mark.parametrize("changed", ["public", "search-index"]) +def test_search_cache_publisher_fences_both_epochs(response_cache, changed): + lookup = response_cache.lookup("search", "public", dependencies=("search-index",)) + assert lookup.token + response_cache.invalidate(changed) + assert not response_cache.publish(lookup, b'{"items":[]}', 60) + response_cache.release(lookup) + fresh = response_cache.lookup("search", "public", dependencies=("search-index",)) + assert fresh.key != lookup.key and fresh.body is None + + +def test_projection_epoch_preserves_content_but_changes_search_results(response_cache): + content = response_cache.lookup("article", "public") + search = response_cache.lookup("search", "public", dependencies=("search-index",)) + assert response_cache.publish(content, b'{"title":"Approved"}', 60) + assert response_cache.publish(search, b'{"items":[]}', 60) + response_cache.invalidate("search-index") + assert response_cache.lookup("article", "public").body == b'{"title":"Approved"}' + assert response_cache.lookup("search", "public", dependencies=("search-index",)).body is None + + +@pytest.mark.parametrize("enabled", [False, True]) +@pytest.mark.parametrize("public_change", [False, True]) +def test_projection_cache_isolation_retains_conservative_fallback( + response_cache, monkeypatch, enabled, public_change +): + monkeypatch.setattr(get_settings(), "search_cache_isolation_enabled", enabled) + content = response_cache.lookup("article", "public") + search = response_cache.lookup("search", "public", dependencies=("search-index",)) + cache.invalidate_search_cache(public_change=public_change) + current = response_cache.lookup("article", "public") + assert (current.key != content.key) == (public_change or not enabled) + assert ( + response_cache.lookup("search", "public", dependencies=("search-index",)).key != search.key + ) + + +def test_search_route_uses_both_epochs_without_flushing_unrelated_reads(response_cache): + from devfeed_api.cache import CachedReadRoute, CachePolicy + from fastapi import APIRouter, FastAPI + from fastapi.testclient import TestClient + + app, router = FastAPI(), APIRouter(route_class=CachedReadRoute) + counts = {"content": 0, "search": 0} + for kind in counts: + + def endpoint(current=kind): + counts[current] += 1 + return {"kind": current} + + router.add_api_route( + f"/{kind}", + endpoint, + methods=["GET"], + openapi_extra=CachePolicy( + namespace=kind, revision=1, ttl="content", index_generation=kind == "search" + ).openapi_extra(), + ) + app.include_router(router) + with TestClient(app) as client: + for path in ("/content", "/search"): + assert client.get(path).headers["x-cache"] == "MISS" + assert client.get(path).headers["x-cache"] == "HIT" + response_cache.invalidate("search-index") + assert client.get("/content").headers["x-cache"] == "HIT" + assert client.get("/search").headers["x-cache"] == "MISS" + response_cache.invalidate("public") + assert client.get("/content").headers["x-cache"] == "MISS" + assert client.get("/search").headers["x-cache"] == "MISS" + assert counts == {"content": 2, "search": 3} + + +def test_unknown_cache_dependency_is_rejected_without_reading_redis(response_cache): + with pytest.raises(ValueError, match="Unknown cache dependency"): + response_cache.lookup("record", "public", dependencies=("arbitrary",)) + + +def test_disabled_search_cache_records_domains_without_contacting_redis(monkeypatch): + monkeypatch.setattr(get_settings(), "cache_enabled", False) + recorder = [] + monkeypatch.setattr(cache, "record_cache_invalidation", lambda *args: recorder.append(args)) + monkeypatch.setattr(cache, "get_cache", lambda: pytest.fail("Cache disabled")) + cache.invalidate_search_cache(public_change=True) + assert recorder == [ + ("public", "disabled", ("search_index",)), + ("search-index", "disabled", ("search_index",)), + ] diff --git a/tests/infrastructure/test_cache_integration.py b/tests/infrastructure/test_cache_integration.py index 2713fae33..a637adcf5 100644 --- a/tests/infrastructure/test_cache_integration.py +++ b/tests/infrastructure/test_cache_integration.py @@ -348,4 +348,126 @@ def test_search_index_cause_requires_a_completed_nonempty_batch(database, observ (point.attributes["cache.outcome"], point.value) for point in points if point.attributes["cache.invalidation_reason"] == "search_index" + and point.attributes["cache.domain"] == "public" ] == [("success", 1)] + + +def test_projection_only_indexing_keeps_content_cache_but_fences_search( + database, client, monkeypatch +): + from unittest.mock import Mock + + from devfeed_core.models import SearchEvent + + monkeypatch.setattr(get_settings(), "search_cache_isolation_enabled", True) + identifier = approved_source(database) + engine = Mock() + assert sync_batch(database, engine) > 0 + path = f"/v1/sources/{identifier}" + assert client.get(path).status_code == 200 + assert client.get(path).headers["x-cache"] == "HIT" + store = cache.get_cache() + search = store.lookup("search-result", "public", dependencies=("search-index",)) + content = store.lookup("unrelated-publisher", "public") + with database.begin() as session: + session.add(SearchEvent(kind="sources", entity_id=identifier, public_change=False)) + assert sync_batch(database, engine) == 1 + assert client.get(path).headers["x-cache"] == "HIT" + assert not store.publish(search, b'{"items":[]}', 30) + assert store.publish(content, b'{"title":"Approved"}', 30) + + +@pytest.mark.parametrize("reject", [False, True]) +def test_raw_sql_changes_keep_the_conservative_indexer_invalidation( + database, client, monkeypatch, reject +): + from unittest.mock import Mock + + from devfeed_core.models import SearchEvent + from sqlalchemy import text + + monkeypatch.setattr(get_settings(), "search_cache_isolation_enabled", True) + identifier = approved_source(database) + engine = Mock() + assert sync_batch(database, engine) > 0 + path = f"/v1/sources/{identifier}" + assert client.get(path).json()["name"] == "Example" + assert client.get(path).headers["x-cache"] == "HIT" + old = cache.get_cache().lookup("late-source-reader", "public") + with get_engine().begin() as connection: + connection.execute( + text("UPDATE sources SET approval_status='rejected' WHERE id=:id") + if reject + else text("UPDATE sources SET name='Raw SQL' WHERE id=:id"), + {"id": identifier}, + ) + with database() as session: + assert session.scalar(select(SearchEvent.public_change)) is True + assert client.get(path).headers["x-cache"] == "HIT" + assert sync_batch(database, engine) == 1 + result = client.get(path) + if reject: + assert result.status_code == 404 + else: + assert result.headers["x-cache"] == "MISS" and result.json()["name"] == "Raw SQL" + assert not cache.get_cache().publish(old, b'{"name":"Old"}', 30) + + +def test_backfill_fences_unknown_data_once_and_marks_projection_work(database, client, monkeypatch): + from unittest.mock import Mock + + from devfeed_core.models import SearchEvent + from devfeed_core.search_index import backfill + from sqlalchemy import text + + monkeypatch.setattr(get_settings(), "search_cache_isolation_enabled", True) + identifier = approved_source(database) + assert sync_batch(database, Mock()) > 0 + path = f"/v1/sources/{identifier}" + assert client.get(path).status_code == 200 + with get_engine().begin() as connection: + # Represent an explicit repair after direct maintenance; the backfill + # itself must provide the conservative public fence before page drains. + connection.execute( + text("UPDATE sources SET description='Repair' WHERE id=:id"), {"id": identifier} + ) + connection.execute(text("DELETE FROM search_events")) + assert client.get(path).headers["x-cache"] == "HIT" + assert backfill(database) > 0 + assert client.get(path).headers["x-cache"] == "MISS" + with database() as session: + assert all(value is False for value in session.scalars(select(SearchEvent.public_change))) + assert sync_batch(database, Mock()) > 0 + assert client.get(path).headers["x-cache"] == "HIT" + + +@pytest.mark.parametrize("isolation", [False, True]) +def test_projection_pages_do_not_repeat_unrelated_api_queries( + database, client, monkeypatch, isolation +): + from unittest.mock import Mock + + from devfeed_core.models import SearchEvent + + monkeypatch.setattr(get_settings(), "search_cache_isolation_enabled", isolation) + identifier = approved_source(database) + engine = Mock() + assert sync_batch(database, engine) > 0 + path = f"/v1/sources/{identifier}" + assert client.get(path).status_code == 200 + statuses, queries = [], [] + + def observed(_connection, _cursor, statement, _parameters, _context, _many): + queries.append(statement) + + for _ in range(20): + with database.begin() as session: + session.add(SearchEvent(kind="sources", entity_id=identifier, public_change=False)) + assert sync_batch(database, engine) == 1 + event.listen(get_engine(), "before_cursor_execute", observed) + try: + statuses.append(client.get(path).headers["x-cache"]) + finally: + event.remove(get_engine(), "before_cursor_execute", observed) + assert statuses == ["HIT" if isolation else "MISS"] * 20 + assert len(queries) == (0 if isolation else 20) diff --git a/tests/infrastructure/test_migration_paths.py b/tests/infrastructure/test_migration_paths.py index 5954e38bd..fbf636b07 100644 --- a/tests/infrastructure/test_migration_paths.py +++ b/tests/infrastructure/test_migration_paths.py @@ -31,9 +31,9 @@ def test_migration_history_loads_from_unrelated_working_directory(tmp_path): check=False, ) assert result.returncode == 0, result.stderr - assert f"0039 -> {SCHEMA_REVISION} (head)" in result.stdout + assert f"0040 -> {SCHEMA_REVISION} (head)" in result.stdout scripts = ScriptDirectory.from_config(Config(str(REPO_ROOT / "alembic.ini"))) - assert scripts.get_revision(SCHEMA_REVISION).down_revision == "0039" + assert scripts.get_revision(SCHEMA_REVISION).down_revision == "0040" @pytest.mark.integration diff --git a/tests/infrastructure/test_search_cache_migration.py b/tests/infrastructure/test_search_cache_migration.py new file mode 100644 index 000000000..d9f3661e3 --- /dev/null +++ b/tests/infrastructure/test_search_cache_migration.py @@ -0,0 +1,48 @@ +"""Search cache provenance preserves retained and raw-SQL event authority.""" + +import pytest +from alembic import command +from alembic.config import Config +from sqlalchemy import text + +from tests.support.legacy_migrations import database_at_revision +from tests.support.paths import REPO_ROOT + +pytestmark = pytest.mark.integration + + +def test_search_cache_upgrade_keeps_retained_unknown_events_conservative( + integration_environment, monkeypatch +): + with database_at_revision(monkeypatch, "0040") as factory: + with factory.begin() as session: + session.execute( + text( + "INSERT INTO search_events(kind,entity_id) SELECT 'articles'," + "md5(g::text)::uuid FROM generate_series(1,10000) g" + ) + ) + before = session.scalar( + text( + "SELECT md5(string_agg(id::text||':'||xmin::text,',' ORDER BY id)) " + "FROM search_events" + ) + ) + command.upgrade(Config(str(REPO_ROOT / "alembic.ini")), "0041") + with factory() as session: + assert ( + session.scalar(text("SELECT count(*) FROM search_events WHERE public_change")) + == 10000 + ) + assert ( + session.scalar( + text( + "SELECT md5(string_agg(id::text||':'||xmin::text,',' ORDER BY id)) " + "FROM search_events" + ) + ) + == before + ) + command.downgrade(Config(str(REPO_ROOT / "alembic.ini")), "0040") + with factory() as session: + assert session.scalar(text("SELECT count(*) FROM search_events")) == 10000 diff --git a/tests/partners/catalog/test_migration.py b/tests/partners/catalog/test_migration.py index cc7d81d6f..98b874c39 100644 --- a/tests/partners/catalog/test_migration.py +++ b/tests/partners/catalog/test_migration.py @@ -36,7 +36,8 @@ def test_partnership_revision_follows_master_schema(): scripts = ScriptDirectory.from_config(Config(str(ROOT / "alembic.ini"))) - assert scripts.get_heads() == [SCHEMA_REVISION] == ["0040"] + assert scripts.get_heads() == [SCHEMA_REVISION] == ["0041"] + assert scripts.get_revision("0041").down_revision == "0040" assert scripts.get_revision("0040").down_revision == "0039" assert scripts.get_revision("0039").down_revision == "0038" assert scripts.get_revision("0038").down_revision == "0037" @@ -56,6 +57,7 @@ def test_partnership_revision_follows_master_schema(): assert scripts.get_revision("0024").down_revision == "0023" assert scripts.get_revision("0023").down_revision == "0022" assert [revision.revision for revision in scripts.walk_revisions("0022", "head")] == [ + "0041", "0040", "0039", "0038", diff --git a/tests/search/test_search_outbox_unit.py b/tests/search/test_search_outbox_unit.py index 720059a7d..20398d773 100644 --- a/tests/search/test_search_outbox_unit.py +++ b/tests/search/test_search_outbox_unit.py @@ -52,7 +52,8 @@ def runtime(monkeypatch): monkeypatch.setattr(index, "get_settings", lambda: configuration(search_index_batch_size=10)) project, invalidate = Mock(), Mock() monkeypatch.setattr(index, "documents", project) - monkeypatch.setattr(index, "invalidate_public_cache", invalidate) + monkeypatch.setattr(index, "invalidate_search_cache", invalidate) + monkeypatch.setattr(index, "invalidate_public_cache", Mock()) return SimpleNamespace( session=session, engine=engine, factory=factory, project=project, invalidate=invalidate ) @@ -88,7 +89,7 @@ def acknowledge(statement, *args): def invalidated(**kwargs): assert not runtime.factory.open - assert kwargs == {"reasons": ("search_index",)} + assert kwargs == {"public_change": True} runtime.invalidate.side_effect = invalidated @@ -137,7 +138,9 @@ def test_catalog_fanout_queues_selected_article_ids_and_only_full_pages_continue ] assert len(writes) == int(bool(ids)) if ids: - assert writes[0].args[1] == [{"kind": "articles", "entity_id": value} for value in ids] + assert writes[0].args[1] == [ + {"kind": "articles", "entity_id": value, "public_change": False} for value in ids + ] if size == 10: continuation = runtime.session.add.call_args.args[0] assert continuation.kind == "topics" and continuation.entity_id == identifier From e68cfb4c00d1c06ded60eecff830655cc4d85781 Mon Sep 17 00:00:00 2001 From: Abhimanyu Saharan Date: Sat, 31 Oct 2026 19:29:25 +0530 Subject: [PATCH 2/2] Verify PostgreSQL work and cache provenance migration costs --- dev/search-cache-epochs.md | 55 +++++++++++++++++++ .../infrastructure/test_cache_integration.py | 23 +++++++- .../test_search_cache_migration.py | 20 +++++++ 3 files changed, 95 insertions(+), 3 deletions(-) diff --git a/dev/search-cache-epochs.md b/dev/search-cache-epochs.md index ffe258a2d..7923bbc85 100644 --- a/dev/search-cache-epochs.md +++ b/dev/search-cache-epochs.md @@ -59,3 +59,58 @@ SQL reads with the gate off; with isolation enabled the same reads stay cached and perform no SQL. This is a controlled demonstration of avoided work, not an estimate that all production cache misses or CPU would disappear. Native ARM64 workload measurement and hosted CI must pass before promoting the draft. + +## PostgreSQL impact + +Local x86-64 PostgreSQL 18, session-mode PgBouncer and Redis; three samples per +isolation mode, alternating order. Each sample includes the producer insertion, +indexer processing/acknowledgement and public API read for 20 projection-only pages. +The baseline is isolation off, which retains the old broad invalidation behavior. +A separate observer role captures PostgreSQL utility/nested statements, driver +configuration, transaction commands and pooler resets. Warm-up is excluded. All +samples use the same hydrated response and preserve the raw-writer/publication fences. + +| Metric | Before | After | Measurement basis | +| --- | ---: | ---: | --- | +| Public API cache outcomes | 20 MISS | 20 HIT | 20 projection-only pages per mode, three samples. | +| API data SELECTs / all recorded PG commands | 20 / 360 | 0 / 260 | Entire producer + indexer + API stage; no extra indexer query replaces the saved reads. | +| Rows returned/affected / shared buffer hits | 100 / 720 | 80 / 660 | Includes inserted/acknowledged outbox rows. | +| PG planning + execution elapsed | 28.449 ms | 24.192 ms | Medians; elapsed time is not database CPU. | +| PostgreSQL-container CPU | 152.606 ms | 131.571 ms | Median cgroup delta, includes logging, background work and probes. | +| Read transaction rollbacks / pooler resets | 20 / 60 | 0 / 40 | Producer/indexer retain 40 BEGIN and 40 COMMIT in both modes. | +| Real Redis outage: commands / rows / buffer hits | 360 / 100 / 720 | 360 / 100 / 720 | Owned Redis paused; 20 API bypasses in each mode, three samples. | +| Real Redis outage: PG-container CPU range | 150.297–172.076 ms | 152.390–164.025 ms | Overlapping ranges with identical statement/row/buffer work. | +| Public churn: commands / rows / buffer hits | 360 / 100 / 720 | 360 / 100 / 720 | Public epoch invalidated before every read; 20 misses in both modes, three samples. | +| Public churn: PG-container CPU range | 149.617–163.599 ms | 149.616–166.850 ms | Overlapping ranges; no fallback SQL added by isolation. | +| Total workload WAL | 4,440–4,539 B | 4,440–4,539 B | Observed range in all three scenarios on the new schema, including outbox writes and pruning. | +| Backend startups / remaining locks / idle transactions | 0 / 0 / 0 | 0 / 0 / 0 | Warmed session pool; same in-transaction advisory/table lock footprint sampled during remote sync. | + +These gate-off/on results isolate runtime invalidation behavior on the new schema. +They do not hide the additive schema cost below, establish an across-the-board +PostgreSQL CPU reduction under failure, or predict whole-production container savings. +The degraded cases retain equal SQL work; they are not an outage capacity guarantee. + +### Migration and ongoing outbox storage + +A fresh disposable database at revision 0040 held 103,000 events before the actual +Alembic upgrade. Before/after checks matched every ID/xmin checksum, relation filenode +and the 7,888,896-byte heap. All old rows read `public_change=true`. No heap rewrite +or row-by-row backfill occurred. A deliberately held reader made the requested +AccessExclusiveLock wait and fail at the migration's five-second lock deadline; +a subsequent uncontended upgrade succeeded. + +| Metric | Before | After | Measurement basis | +| --- | ---: | ---: | --- | +| Retained event heap / IDs / xmin | 7,888,896 B / 103,000 unchanged | Same | One actual 0040→0041 upgrade; filenode and row checksum identical. | +| Migration commands / WAL | Not applicable | 8 / 2,153 B | Actual Alembic upgrade; ALTER TABLE metadata used 2,020 B and version-row update 133 B. | +| Migration lock | Reader holds AccessShareLock | AccessExclusiveLock requested; blocked attempt timed out at 5.038 s | Deliberate contention in the disposable database; this is not a production lock-duration estimate. | +| INSERT commands / rows | 1 / 1,000 | 1 / 1,000 | Three 1,000-event samples before/after, raw SQL, no new lookup or backfill. | +| INSERT WAL | 172,250 B median | 173,250 B median | Approximately one extra WAL byte per new event; both true and false provenance measured. B-tree/sequence page boundaries also affect individual samples. | +| INSERT planning + execution elapsed | 4.676 ms | 4.709 ms true / 4.743 ms false | Three samples per form; small timing differences are not proof of zero overhead. | +| INSERT PG-container CPU | 24.045 ms | 23.528 ms true / 23.890 ms false | Medians, probe/background-inclusive; no whole-production CPU inference. | + +The boolean provenance has a concrete storage/WAL cost, even while the fixed runtime +workload avoids API SQL. Keep the PR in draft and isolation disabled until migration +contention, native ARM64 and a representative production canary have been reviewed. +Do not weaken raw-SQL or source-withdrawal fences to remove this byte. Rollout still +requires every public API and indexer to support both epochs before enabling isolation. diff --git a/tests/infrastructure/test_cache_integration.py b/tests/infrastructure/test_cache_integration.py index a637adcf5..4fcbe56a6 100644 --- a/tests/infrastructure/test_cache_integration.py +++ b/tests/infrastructure/test_cache_integration.py @@ -442,8 +442,9 @@ def test_backfill_fences_unknown_data_once_and_marks_projection_work(database, c @pytest.mark.parametrize("isolation", [False, True]) +@pytest.mark.parametrize("scenario", ["available", "cache_unavailable", "public_churn"]) def test_projection_pages_do_not_repeat_unrelated_api_queries( - database, client, monkeypatch, isolation + database, client, monkeypatch, isolation, scenario ): from unittest.mock import Mock @@ -456,6 +457,13 @@ def test_projection_pages_do_not_repeat_unrelated_api_queries( path = f"/v1/sources/{identifier}" assert client.get(path).status_code == 200 statuses, queries = [], [] + store = cache.get_cache() + if scenario == "cache_unavailable": + + def unavailable(*_): + raise cache.CacheUnavailable + + monkeypatch.setattr(store, "_run", unavailable) def observed(_connection, _cursor, statement, _parameters, _context, _many): queries.append(statement) @@ -464,10 +472,19 @@ def observed(_connection, _cursor, statement, _parameters, _context, _many): with database.begin() as session: session.add(SearchEvent(kind="sources", entity_id=identifier, public_change=False)) assert sync_batch(database, engine) == 1 + if scenario == "public_churn": + store.invalidate("public") event.listen(get_engine(), "before_cursor_execute", observed) try: statuses.append(client.get(path).headers["x-cache"]) finally: event.remove(get_engine(), "before_cursor_execute", observed) - assert statuses == ["HIT" if isolation else "MISS"] * 20 - assert len(queries) == (0 if isolation else 20) + expected = ( + "BYPASS" + if scenario == "cache_unavailable" + else "HIT" + if isolation and scenario == "available" + else "MISS" + ) + assert statuses == [expected] * 20 + assert len(queries) == (0 if isolation and scenario == "available" else 20) diff --git a/tests/infrastructure/test_search_cache_migration.py b/tests/infrastructure/test_search_cache_migration.py index d9f3661e3..8b627404a 100644 --- a/tests/infrastructure/test_search_cache_migration.py +++ b/tests/infrastructure/test_search_cache_migration.py @@ -28,8 +28,28 @@ def test_search_cache_upgrade_keeps_retained_unknown_events_conservative( "FROM search_events" ) ) + heap_before = session.execute( + text( + "SELECT pg_relation_filenode('search_events'),pg_relation_size('search_events')" + ) + ).one() command.upgrade(Config(str(REPO_ROOT / "alembic.ini")), "0041") with factory() as session: + assert ( + session.execute( + text( + "SELECT pg_relation_filenode('search_events')," + "pg_relation_size('search_events')" + ) + ).one() + == heap_before + ) + assert session.scalar( + text( + "SELECT atthasmissing FROM pg_attribute " + "WHERE attrelid='search_events'::regclass AND attname='public_change'" + ) + ) assert ( session.scalar(text("SELECT count(*) FROM search_events WHERE public_change")) == 10000