Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,8 @@ DEVFEED_SOLVER_SERVICES=[]

# Server-side GET caching uses DEVFEED_REDIS_URL; no additional connection URL.
DEVFEED_CACHE_ENABLED=true
# Opt in after checking source-tag snapshot hits, edit frequency and worker memory.
DEVFEED_SOURCE_TAG_SHARED_CACHE_ENABLED=false
DEVFEED_CACHE_TTL_SECONDS=300
DEVFEED_CACHE_METADATA_TTL_SECONDS=600
DEVFEED_CACHE_PUBLIC_PROFILE_TTL_SECONDS=21600
Expand Down
17 changes: 15 additions & 2 deletions apps/aggregator/src/tasks.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import logging
import time
import uuid
from contextlib import nullcontext
from datetime import datetime
from typing import TypedDict

Expand All @@ -25,13 +26,19 @@
utcnow,
)
from devfeed_core.polling import observe, schedule
from devfeed_core.source_tags import attach_source_tags, resolve_source_tags, source_tag_names
from devfeed_core.source_tags import (
attach_source_tags,
prefetch_source_tags,
resolve_source_tags,
source_tag_names,
)
from devfeed_core.source_types import SourceType
from devfeed_core.taxonomy import classify, classify_tags, detect_content_type
from devfeed_core.telemetry import current
from devfeed_core.urls import fingerprint
from sqlalchemy import case, func, select, update
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.engine import Connection
from sqlalchemy.orm import Session, sessionmaker

from devfeed_aggregator.languages import detect_feed_languages
Expand Down Expand Up @@ -246,7 +253,13 @@ def _ingest_claimed(
if result.status == 200
else None
)
with factory.begin() as session:
labels = (tag for entry in parsed.entries for tag in entry.tags) if parsed else ()
prefetch = (
nullcontext()
if isinstance(getattr(factory, "kw", {}).get("bind"), Connection)
else prefetch_source_tags(labels)
)
with prefetch, factory.begin() as session:
job = owned_job(session, IngestionJob, identifier, lease_token)
if job is None:
logger.warning("ingestion_lease_lost")
Expand Down
Loading
Loading