Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,8 @@ DEVFEED_SEARCH_QUERY_KEY=
DEVFEED_SEARCH_HTTP_MAX_CONNECTIONS=32
DEVFEED_SEARCH_HTTP_POOL_TIMEOUT_SECONDS=2
DEVFEED_SEARCH_INDEX_BATCH_SIZE=200
# Enable only after all public APIs and indexers support the index cache epoch.
DEVFEED_SEARCH_CACHE_ISOLATION_ENABLED=false
# Search-indexer retries are bounded and apply only to transient 503/network/import failures.
# Write timeout adapts upward from this value using observed successful import duration.
# DEVFEED_SEARCH_INDEX_RETRY_ATTEMPTS=3
Expand Down
9 changes: 8 additions & 1 deletion apps/api/src/cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ class CachePolicy(BaseModel):
namespace: str = Field(min_length=1, max_length=64, pattern=r"^[a-z][a-z0-9_-]*$")
revision: int = Field(ge=1)
ttl: Literal["content", "metadata"]
index_generation: bool = False

def openapi_extra(self) -> dict[str, JsonValue]:
return {CACHE_POLICY_EXTENSION: self.model_dump()}
Expand Down Expand Up @@ -129,7 +130,13 @@ async def render() -> Response:
try:
while True:
lookup = await run_in_threadpool(
lambda: cache.lookup(key, "public", observe=False)
lambda: (
cache.lookup(
key, "public", observe=False, dependencies=("search-index",)
)
if policy.index_generation
else cache.lookup(key, "public", observe=False)
)
)
if lookup.body is not None:
return tagged(
Expand Down
4 changes: 3 additions & 1 deletion apps/api/src/search.py
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,9 @@ def persist_metadata(session: Session) -> str:
@router.get(
"",
response_model=SearchResponse,
openapi_extra=CachePolicy(namespace="search", revision=1, ttl="content").openapi_extra(),
openapi_extra=CachePolicy(
namespace="search", revision=2, ttl="content", index_generation=True
).openapi_extra(),
)
def search(
session: DB,
Expand Down
3 changes: 3 additions & 0 deletions compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,7 @@ services:
<<: *backend
environment:
<<: [*core-environment, *clickhouse-public]
DEVFEED_SEARCH_CACHE_ISOLATION_ENABLED: ${DEVFEED_SEARCH_CACHE_ISOLATION_ENABLED:-false}
DEVFEED_SEARCH_ENABLED: ${DEVFEED_SEARCH_ENABLED:-false}
DEVFEED_SEARCH_URL: ${DEVFEED_SEARCH_URL:-http://typesense:8108}
DEVFEED_SEARCH_QUERY_KEY: ${DEVFEED_SEARCH_QUERY_KEY:-}
Expand Down Expand Up @@ -432,6 +433,7 @@ services:
DEVFEED_SEARCH_QUERY_KEY: ${DEVFEED_SEARCH_QUERY_KEY:-}
DEVFEED_SEARCH_COLLECTION_PREFIX: ${DEVFEED_SEARCH_COLLECTION_PREFIX:-devfeed}
DEVFEED_SEARCH_INDEX_BATCH_SIZE: ${DEVFEED_SEARCH_INDEX_BATCH_SIZE:-200}
DEVFEED_SEARCH_CACHE_ISOLATION_ENABLED: ${DEVFEED_SEARCH_CACHE_ISOLATION_ENABLED:-false}
depends_on:
<<: *ready
typesense:
Expand All @@ -450,6 +452,7 @@ services:
DEVFEED_SEARCH_ADMIN_KEY: ${DEVFEED_SEARCH_ADMIN_KEY:-}
DEVFEED_SEARCH_COLLECTION_PREFIX: ${DEVFEED_SEARCH_COLLECTION_PREFIX:-devfeed}
DEVFEED_SEARCH_INDEX_BATCH_SIZE: ${DEVFEED_SEARCH_INDEX_BATCH_SIZE:-200}
DEVFEED_SEARCH_CACHE_ISOLATION_ENABLED: ${DEVFEED_SEARCH_CACHE_ISOLATION_ENABLED:-false}
depends_on:
<<: *ready
search-setup:
Expand Down
116 changes: 116 additions & 0 deletions dev/search-cache-epochs.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
# Search projection cache epochs

Search-index publication and public database data have separate Redis epochs.
Cached Typesense search results depend on both: a completed projection must not
leave a previously cached empty search behind, and content withdrawal must not
leave old visible search results behind. Article/feed/catalog responses depend
only on the public-data epoch. Lua checks every dependency and loader token in
one atomic publish so an old renderer cannot repopulate an invalidated epoch.
Suggestions and PostgreSQL full-text feed filtering keep their own data semantics;
they do not inherit the Typesense publication epoch.

## Conservative source provenance

Migration `0041` adds `search_events.public_change` with a constant true default.
Every retained event, database-triggered business change and unknown/manual event
therefore retains the old public-data invalidation fallback. This matters for
raw psycopg and direct SQL writers that do not run application commit hooks.

Indexer-created projection/fanout continuation events explicitly carry false.
A completed nonempty batch always invalidates the search-index epoch, and
invalidates the public epoch when any selected event may represent a real data
change. Normal application commits still invalidate public data immediately;
raw SQL source events retain the existing eventual indexer fallback before their
remaining fanout pages drain. Backfill explicitly fences public data once after
enqueue commit and marks its pages projection-only, preserving repair semantics
without flushing unrelated content on every page.

This does not promise to remove every observed indexer invalidation: source
business events remain conservative. It does prevent index maintenance, duplicate
projection pages and fanout continuation from unconditionally expelling unrelated
public data. Direct SQL that bypasses or disables triggers still requires the
explicit backfill/repair fence, as before.

## Rollout

1. Apply the single additive migration `0041` before upgrading the applications.
Its constant default retains unknown events as public changes without rewriting
their tuples; a populated 10,000-event test checks unchanged IDs/xmin.
2. Upgrade every public API and search indexer replica. Leave
`DEVFEED_SEARCH_CACHE_ISOLATION_ENABLED=false` during mixed versions. Updated
APIs always understand both epochs; the default-off indexer gate continues
invalidating public data for every completed batch, matching old APIs.
3. Enable the gate only after all public APIs and indexers are compatible.
Verify cached article/feed reads survive projection-only pages while real
source/article changes, raw SQL, withdrawal and rejected sources still fence
old readers. Monitor cache hit rate, API CPU, SQL work, index lag and freshness.
4. Roll back by disabling the gate first. The old invalidation policy resumes;
keep the additive column until the compatible image rollback is complete.

No TTL, replica count, search delay or production resource limit changes here.
The gate is intentionally off until a compatible rollout is explicitly enabled.

## Validation

Real Redis/PostgreSQL tests cover atomic dependency fences, projection-only pages,
raw-SQL source edits, explicit repair backfills and populated schema round trips.
Twenty projection-only pages produce twenty unrelated content-cache misses and
SQL reads with the gate off; with isolation enabled the same reads stay cached
and perform no SQL. This is a controlled demonstration of avoided work, not an
estimate that all production cache misses or CPU would disappear. Native ARM64
workload measurement and hosted CI must pass before promoting the draft.

## PostgreSQL impact

Local x86-64 PostgreSQL 18, session-mode PgBouncer and Redis; three samples per
isolation mode, alternating order. Each sample includes the producer insertion,
indexer processing/acknowledgement and public API read for 20 projection-only pages.
The baseline is isolation off, which retains the old broad invalidation behavior.
A separate observer role captures PostgreSQL utility/nested statements, driver
configuration, transaction commands and pooler resets. Warm-up is excluded. All
samples use the same hydrated response and preserve the raw-writer/publication fences.

| Metric | Before | After | Measurement basis |
| --- | ---: | ---: | --- |
| Public API cache outcomes | 20 MISS | 20 HIT | 20 projection-only pages per mode, three samples. |
| API data SELECTs / all recorded PG commands | 20 / 360 | 0 / 260 | Entire producer + indexer + API stage; no extra indexer query replaces the saved reads. |
| Rows returned/affected / shared buffer hits | 100 / 720 | 80 / 660 | Includes inserted/acknowledged outbox rows. |
| PG planning + execution elapsed | 28.449 ms | 24.192 ms | Medians; elapsed time is not database CPU. |
| PostgreSQL-container CPU | 152.606 ms | 131.571 ms | Median cgroup delta, includes logging, background work and probes. |
| Read transaction rollbacks / pooler resets | 20 / 60 | 0 / 40 | Producer/indexer retain 40 BEGIN and 40 COMMIT in both modes. |
| Real Redis outage: commands / rows / buffer hits | 360 / 100 / 720 | 360 / 100 / 720 | Owned Redis paused; 20 API bypasses in each mode, three samples. |
| Real Redis outage: PG-container CPU range | 150.297–172.076 ms | 152.390–164.025 ms | Overlapping ranges with identical statement/row/buffer work. |
| Public churn: commands / rows / buffer hits | 360 / 100 / 720 | 360 / 100 / 720 | Public epoch invalidated before every read; 20 misses in both modes, three samples. |
| Public churn: PG-container CPU range | 149.617–163.599 ms | 149.616–166.850 ms | Overlapping ranges; no fallback SQL added by isolation. |
| Total workload WAL | 4,440–4,539 B | 4,440–4,539 B | Observed range in all three scenarios on the new schema, including outbox writes and pruning. |
| Backend startups / remaining locks / idle transactions | 0 / 0 / 0 | 0 / 0 / 0 | Warmed session pool; same in-transaction advisory/table lock footprint sampled during remote sync. |

These gate-off/on results isolate runtime invalidation behavior on the new schema.
They do not hide the additive schema cost below, establish an across-the-board
PostgreSQL CPU reduction under failure, or predict whole-production container savings.
The degraded cases retain equal SQL work; they are not an outage capacity guarantee.

### Migration and ongoing outbox storage

A fresh disposable database at revision 0040 held 103,000 events before the actual
Alembic upgrade. Before/after checks matched every ID/xmin checksum, relation filenode
and the 7,888,896-byte heap. All old rows read `public_change=true`. No heap rewrite
or row-by-row backfill occurred. A deliberately held reader made the requested
AccessExclusiveLock wait and fail at the migration's five-second lock deadline;
a subsequent uncontended upgrade succeeded.

| Metric | Before | After | Measurement basis |
| --- | ---: | ---: | --- |
| Retained event heap / IDs / xmin | 7,888,896 B / 103,000 unchanged | Same | One actual 0040→0041 upgrade; filenode and row checksum identical. |
| Migration commands / WAL | Not applicable | 8 / 2,153 B | Actual Alembic upgrade; ALTER TABLE metadata used 2,020 B and version-row update 133 B. |
| Migration lock | Reader holds AccessShareLock | AccessExclusiveLock requested; blocked attempt timed out at 5.038 s | Deliberate contention in the disposable database; this is not a production lock-duration estimate. |
| INSERT commands / rows | 1 / 1,000 | 1 / 1,000 | Three 1,000-event samples before/after, raw SQL, no new lookup or backfill. |
| INSERT WAL | 172,250 B median | 173,250 B median | Approximately one extra WAL byte per new event; both true and false provenance measured. B-tree/sequence page boundaries also affect individual samples. |
| INSERT planning + execution elapsed | 4.676 ms | 4.709 ms true / 4.743 ms false | Three samples per form; small timing differences are not proof of zero overhead. |
| INSERT PG-container CPU | 24.045 ms | 23.528 ms true / 23.890 ms false | Medians, probe/background-inclusive; no whole-production CPU inference. |

The boolean provenance has a concrete storage/WAL cost, even while the fixed runtime
workload avoids API SQL. Keep the PR in draft and isolation disabled until migration
contention, native ARM64 and a representative production canary have been reviewed.
Do not weaken raw-SQL or source-withdrawal fences to remove this byte. Rollout still
requires every public API and indexer to support both epochs before enabling isolation.
28 changes: 28 additions & 0 deletions migrations/versions/0041_search_cache_publication.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
"""Keep source-change invalidation distinct from index-only projection work.

Revision ID: 0041
Revises: 0040
"""

import sqlalchemy as sa
from alembic import op

revision = "0041"
down_revision = "0040"
branch_labels = None
depends_on = None


def upgrade():
op.execute("SET LOCAL lock_timeout = '5s'")
# PostgreSQL's constant default preserves every retained/unknown event as a
# conservative public-data invalidation without rewriting the outbox.
op.add_column(
"search_events",
sa.Column("public_change", sa.Boolean(), nullable=False, server_default=sa.true()),
)


def downgrade():
op.execute("SET LOCAL lock_timeout = '5s'")
op.drop_column("search_events", "public_change")
5 changes: 3 additions & 2 deletions packages/core/OBSERVABILITY.md
Original file line number Diff line number Diff line change
Expand Up @@ -100,9 +100,10 @@ pressure. A cache hit does not bypass authentication or visibility checks.

Keep the existing generation boundaries and TTLs until measurements identify a
specific cause. `devfeed.cache.invalidations` records one attempt with
`cache.domain=public|operations` and
`cache.domain=public|operations|search-index` and
`cache.outcome=success|cache_unavailable|disabled`. Only `success` means a generation
was replaced. Failures and the Redis circuit breaker are counted without failing
was replaced. The separate `search-index` domain fences Typesense projection
completion; search responses also depend on the public-data domain. Failures and the Redis circuit breaker are counted without failing
the committed database write; disabled caching does not contact Redis.

`devfeed.cache.invalidation.causes` adds `cache.invalidation_reason`. The bounded
Expand Down
65 changes: 54 additions & 11 deletions packages/core/src/cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,9 @@ def record_cache_read(
def record_cache_invalidation(domain: str, outcome: str, reasons: Collection[str]) -> None:
if runtime := current():
attributes = {
"cache.domain": domain if domain in {"public", "operations"} else "other",
"cache.domain": domain
if domain in {"public", "operations", "search-index"}
else "other",
"cache.outcome": (
outcome if outcome in {"success", "cache_unavailable", "disabled"} else "other"
),
Expand All @@ -121,13 +123,17 @@ def record_cache_invalidation(domain: str, outcome: str, reasons: Collection[str
# A late user may not repopulate a generation invalidated by a committed write,
# or replace another loader's response after losing its lock.
PUBLISH = """
if redis.call('GET', KEYS[1]) == ARGV[1]
and redis.call('GET', KEYS[2]) == ARGV[2] then
redis.call('SET', KEYS[3], ARGV[3], 'EX', ARGV[4])
local generations = #KEYS - 2
for i = 1, generations do
if redis.call('GET', KEYS[i]) ~= ARGV[i] then return 0 end
end
if redis.call('GET', KEYS[generations + 1]) == ARGV[generations + 1] then
redis.call('SET', KEYS[generations + 2], ARGV[generations + 2], 'EX', ARGV[generations + 3])
return 1
end
return 0
"""

RELEASE = """
if redis.call('GET', KEYS[1]) == ARGV[1] then
return redis.call('DEL', KEYS[1])
Expand All @@ -148,6 +154,7 @@ class CacheLookup:
lock_key: str
token: str | None = None
body: bytes | None = None
dependencies: tuple[tuple[str, bytes], ...] = ()


T = TypeVar("T")
Expand Down Expand Up @@ -176,9 +183,19 @@ def _run(self, operation: Callable[[], T]) -> T:
logger.warning("cache_unavailable", extra={"error_type": type(exc).__name__})
raise CacheUnavailable from None

def lookup(self, identity: str, domain: str, *, observe: bool = True) -> CacheLookup:
def operation() -> CacheLookup:
generation_key = f"{self.namespace}:{domain}:generation"
def lookup(
self,
identity: str,
domain: str,
*,
observe: bool = True,
dependencies: tuple[str, ...] = (),
) -> CacheLookup:
if any(dependency != "search-index" for dependency in dependencies):
raise ValueError("Unknown cache dependency")

def epoch(name: str) -> tuple[str, bytes]:
generation_key = f"{self.namespace}:{name}:generation"
generation = self.redis.get(generation_key)
if generation is None:
self.redis.set(generation_key, uuid.uuid4().hex, nx=True)
Expand All @@ -190,7 +207,19 @@ def operation() -> CacheLookup:
):
# Never fall back to a fixed epoch after eviction/corruption.
raise CacheUnavailable
digest = hashlib.sha256(identity.encode()).hexdigest()
return generation_key, generation

def operation() -> CacheLookup:
generation_key, generation = epoch(domain)
extra = tuple(epoch(name) for name in dependencies)
# Include all dependency epochs in the response identity and fence
# them again atomically with publication below.
cache_identity = (
json.dumps([identity, [value.decode("ascii") for _, value in extra]])
if extra
else identity
)
digest = hashlib.sha256(cache_identity.encode()).hexdigest()
key = f"{self.namespace}:{domain}:{generation.decode('ascii')}:{digest}"
lock_key = f"{key}:lock"
body = self.redis.get(key)
Expand All @@ -207,7 +236,7 @@ def operation() -> CacheLookup:
candidate = uuid.uuid4().hex
if self.redis.set(lock_key, candidate, nx=True, ex=LOCK_SECONDS):
token = candidate
return CacheLookup(generation_key, generation, key, lock_key, token, body)
return CacheLookup(generation_key, generation, key, lock_key, token, body, extra)

name = {
"public": "public_response",
Expand All @@ -232,11 +261,13 @@ def publish(self, lookup: CacheLookup, body: bytes, ttl: int) -> bool:
self._run(
lambda: self.redis.eval(
PUBLISH,
3,
3 + len(lookup.dependencies),
lookup.generation_key,
*(key for key, _ in lookup.dependencies),
lookup.lock_key,
lookup.key,
lookup.generation,
*(generation for _, generation in lookup.dependencies),
token,
body,
ttl,
Expand All @@ -252,7 +283,7 @@ def release(self, lookup: CacheLookup) -> None:
def invalidate(
self, domain: str = "public", *, reasons: Collection[str] = ("unspecified",)
) -> None:
if domain not in {"public", "operations"}:
if domain not in {"public", "operations", "search-index"}:
raise ValueError("Unknown cache domain")
try:
self._run(
Expand Down Expand Up @@ -382,3 +413,15 @@ def invalidate_public_cache(*, reasons: Collection[str] = ("unspecified",)) -> N
get_cache().invalidate(reasons=reasons)
else:
record_cache_invalidation("public", "disabled", reasons)


def invalidate_search_cache(*, public_change: bool) -> None:
"""Fence index completion separately, preserving unknown/raw SQL changes."""
settings = get_settings()
if public_change or not settings.search_cache_isolation_enabled:
invalidate_public_cache(reasons=("search_index",))
if settings.cache_enabled:
with suppress(CacheUnavailable):
get_cache().invalidate("search-index", reasons=("search_index",))
else:
record_cache_invalidation("search-index", "disabled", ("search_index",))
1 change: 1 addition & 0 deletions packages/core/src/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,7 @@ def validate_partner_caller_key(cls, value: SecretStr | None) -> SecretStr | Non
return value

search_enabled: bool = False
search_cache_isolation_enabled: bool = False
search_analytics_enabled: bool = False
search_url: str | None = None
search_query_key: SecretStr | None = None
Expand Down
1 change: 1 addition & 0 deletions packages/core/src/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -1519,6 +1519,7 @@ class SearchEvent(Base):
kind: Mapped[str] = mapped_column(String(20))
entity_id: Mapped[uuid.UUID] = mapped_column(Uuid)
fanout: Mapped[bool] = mapped_column(Boolean, server_default="false")
public_change: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
after_id: Mapped[uuid.UUID | None] = mapped_column(Uuid)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=text("now()")
Expand Down
Loading
Loading