From f2eb29cd963a839599a9bce5fcd93e3749059608 Mon Sep 17 00:00:00 2001 From: Abhimanyu Saharan Date: Sat, 31 Oct 2026 17:55:28 +0530 Subject: [PATCH 1/2] Load ingestion heuristic catalogs only when classification needs them --- apps/aggregator/src/tasks.py | 12 ++++- benchmarks/ingestion-heuristic-catalogs.md | 37 +++++++++++++++ tests/sources/test_ingestion.py | 54 ++++++++++++++++++++++ 3 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 benchmarks/ingestion-heuristic-catalogs.md diff --git a/apps/aggregator/src/tasks.py b/apps/aggregator/src/tasks.py index e77e8b28e..3490cb62c 100644 --- a/apps/aggregator/src/tasks.py +++ b/apps/aggregator/src/tasks.py @@ -1,6 +1,7 @@ import logging import time import uuid +from collections.abc import Sequence from datetime import datetime from typing import TypedDict @@ -58,11 +59,10 @@ def store_entries( *, activity: list[datetime | None] | None = None, ) -> int: - topics = session.scalars(select(Topic).where(Topic.status == "active")).all() + heuristic_catalog: tuple[Sequence[Topic], Sequence[Tag]] | None = None supplied_tags, _ = resolve_source_tags( session, (tag for entry in parsed.entries for tag in entry.tags) ) - tags = session.scalars(select(Tag)).all() created = 0 # Stable lock order across workers importing the same URLs from different feeds. for entry in sorted(parsed.entries, key=lambda item: item.canonical_url): @@ -168,6 +168,14 @@ def store_entries( ) ) in {"ai", "manual"}: continue + if heuristic_catalog is None: + # Explicit feed labels are independent of heuristic classification. + # AI, known-origin and human-owned entries need no ORM catalog load. + heuristic_catalog = ( + session.scalars(select(Topic).where(Topic.status == "active")).all(), + session.scalars(select(Tag)).all(), + ) + topics, tags = heuristic_catalog for topic_id in sorted(classify(entry.title, entry.summary, entry.tags, topics)): session.execute( insert(ArticleTopic) diff --git a/benchmarks/ingestion-heuristic-catalogs.md b/benchmarks/ingestion-heuristic-catalogs.md new file mode 100644 index 000000000..bbd034579 --- /dev/null +++ b/benchmarks/ingestion-heuristic-catalogs.md @@ -0,0 +1,37 @@ +# Ingestion heuristic catalog hydration + +AI ingestion imported explicit publisher labels and then skipped heuristic +classification, but still hydrated every active Topic and every Tag as ORM objects +before checking any origins. Repeated HTTP 200 feeds paid the same cost even when +all entries were already known. + +Catalog loading now occurs only when an entry reaches heuristic classification: +AI is disabled, the origin is new, and the retained article has no manual or AI +classification. Both catalogs are shared by the eligible entries in that feed +transaction. Publisher-supplied tags still resolve and attach on repeated imports. + +A disposable PostgreSQL integration case seeds 10,000 active topics and 10,000 +unrelated tags, imports two entries, and observes actual ORM load events: + +| Mode | First import catalog ORM objects | Repeated import catalog ORM objects | +| --- | ---: | ---: | +| AI enabled | 0 | 0 | +| Heuristic classifier | 20,000 | 0 | + +The test also confirms that both first-pass entries are inserted and the repeated +pass creates none. Existing suites cover heuristic classification, source tag +restoration, AI/manual classification protection, multilingual articles, source +changes, lost leases, and rollback/retry behavior. + +```sh +uv run pytest -q tests/sources/test_ingestion.py \ + tests/sources/test_source_tags_integration.py \ + tests/taxonomy/test_languages_integration.py +``` + +Use disposable `*_test` PostgreSQL and Redis database 15 as described in +[the test guide](../tests/README.md). Tests truncate the configured databases. +The local validation ran on x86-64, not native ARM64 or production. Object-load +counts prove removed work; they are not a claim about total worker CPU savings. +The independent lightweight source-tag identity projection still runs when feed +labels need resolution. The changed path does not add a cache or migration. diff --git a/tests/sources/test_ingestion.py b/tests/sources/test_ingestion.py index 533be2844..198aec442 100644 --- a/tests/sources/test_ingestion.py +++ b/tests/sources/test_ingestion.py @@ -1,5 +1,6 @@ import uuid from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace from datetime import timedelta import pytest @@ -378,3 +379,56 @@ def test_publisher_metadata_wins_over_discovery_without_losing_provenance( assert origins["publisher"]["author"] == "Example Author" assert origins["aggregator"]["submitter"] == "Example Author" assert origins["aggregator"]["tags"] == ["python"] + + +@pytest.mark.parametrize("ai_enabled", [False, True]) +def test_ingestion_loads_heuristic_catalog_only_for_entries_that_need_it( + database, rss_bytes, monkeypatch, ai_enabled +): + from devfeed_core.config import get_settings + from devfeed_core.feeds.parser import parse_feed + from devfeed_core.models import Tag + from sqlalchemy import event, text + from sqlalchemy.orm import Session + + monkeypatch.setattr(get_settings(), "ai_enabled", ai_enabled) + source_id, _ = make_job(database) + # A realistic unrelated catalog should not become one ORM graph per poll. + with database.begin() as session: + session.execute( + text( + "INSERT INTO topics(id,name,slug,kind,status,keywords,aliases," + "facts,created_at,updated_at) " + "SELECT md5('ingestion-topic-'||g::text)::uuid,'Topic '||g,'topic-'||g," + "'discipline','active',ARRAY['unrelated-'||g],ARRAY[]::varchar[],'[]',now(),now() " + "FROM generate_series(1,10000) g" + ) + ) + session.execute( + text( + "INSERT INTO tags(id,name,slug,aliases) " + "SELECT md5('ingestion-tag-'||g::text)::uuid,'Tag '||g,'tag-'||g," + "ARRAY[]::varchar[] FROM generate_series(1,10000) g" + ) + ) + parsed = parse_feed(rss_bytes, "https://example.com/rss", utcnow(), source_type="publisher") + parsed = replace(parsed, entries=[replace(entry, tags=[]) for entry in parsed.entries]) + hydrated = [] + + def loaded(_session, instance): + if isinstance(instance, (Topic, Tag)): + hydrated.append(type(instance).__name__) + + event.listen(Session, "loaded_as_persistent", loaded) + try: + with database.begin() as session: + assert tasks.store_entries(session, source_id, parsed) == 2 + first = len(hydrated) + # With AI disabled, one lazy catalog is shared by both new entries. + assert first == (0 if ai_enabled else 20000) + hydrated.clear() + with database.begin() as session: + assert tasks.store_entries(session, source_id, parsed) == 0 + assert hydrated == [] + finally: + event.remove(Session, "loaded_as_persistent", loaded) From f7669f301083e9ce1e5cd6e761b94afb66601477 Mon Sep 17 00:00:00 2001 From: Abhimanyu Saharan Date: Sat, 31 Oct 2026 19:28:05 +0530 Subject: [PATCH 2/2] Record PostgreSQL work removed by lazy ingestion catalogs --- ...gestion-heuristic-catalogs-postgresql.json | 1170 +++++++++++++++++ benchmarks/ingestion-heuristic-catalogs.md | 34 + tests/sources/test_ingestion.py | 14 +- 3 files changed, 1217 insertions(+), 1 deletion(-) create mode 100644 benchmarks/ingestion-heuristic-catalogs-postgresql.json diff --git a/benchmarks/ingestion-heuristic-catalogs-postgresql.json b/benchmarks/ingestion-heuristic-catalogs-postgresql.json new file mode 100644 index 000000000..bfb5ae812 --- /dev/null +++ b/benchmarks/ingestion-heuristic-catalogs-postgresql.json @@ -0,0 +1,1170 @@ +{ + "schema": 1, + "baseline_ref": "6f12ed1d892bb70d30249df1a08537996e907673", + "after_implementation_ref": "f2eb29cd963a839599a9bce5fcd93e3749059608", + "environment": "local x86-64, PostgreSQL18 Docker, pg_stat_statements track=all/track_planning=on, Redis DB15, disposable *_test database", + "fixture": { + "active_topics": 4450, + "tags": 55000, + "aliases_per_tag": 1, + "feed_entries": 2 + }, + "method": "Same production store_entries function from baseline/current git source; fresh fixture per variant, ANALYZE before measurement, pg_stat_statements_reset immediately before workload. SQLAlchemy events capture app calls, checkout and transaction intervals. PostgreSQL top-level totals include transaction control; nested trigger work is included in enclosing statement cost. Container CPU includes fixed cgroup-read probes/background maintenance and is one observation, not isolated backend CPU.", + "samples_per_scenario": 1, + "records": [ + { + "checkouts": 1, + "connect_hold_ms": 1365.6437050012755, + "for_update": 2, + "label": "catalogs/ai_new/before", + "max_tx_ms": 1363.8369889995374, + "postgres": { + "calls": 20.0, + "rows": 59460.0, + "shared_blks_dirtied": 51.0, + "shared_blks_hit": 3449.0, + "shared_blks_read": 43.0, + "shared_blks_written": 29.0, + "temp_blks_read": 0.0, + "temp_blks_written": 0.0, + "total_exec_time": 106.104945, + "total_plan_time": 9.60558, + "wal_bytes": 13259.0 + }, + "postgres_container_cpu_ms": 147.462, + "queries": [ + { + "calls": 1, + "query": "BEGIN", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 0, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.000972, + "total_plan_time": 0.0, + "wal_bytes": "0" + }, + { + "calls": 1, + "query": "COMMIT", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 0, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.001559, + "total_plan_time": 0.0, + "wal_bytes": "0" + }, + { + "calls": 2, + "query": "INSERT INTO article_enrichment_jobs (id, article_id, refresh_id, http_status, outcome, changed_fields, result, error, requires_solver, status, attempts, available_at, dispatched_at, lease_until, lease_token, created_at, finished_at) VALUES ($1::UUID, $2::UUID, $3::UUID, $4::INTEGER, $5::VARCHAR, $6::VARCHAR(30)[], $7::JSONB, $8::VARCHAR, $9, $10::VARCHAR, $11::INTEGER, $12::TIMESTAMP WITH TIME ZONE, $13::TIMESTAMP WITH TIME ZONE, $14::TIMESTAMP WITH TIME ZONE, $15::UUID, $16::TIMESTAMP WITH TIME ZONE, $17::TIMESTAMP WITH TIME ZONE)", + "rows": 2, + "shared_blks_dirtied": 13, + "shared_blks_hit": 45, + "shared_blks_read": 0, + "shared_blks_written": 7, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 1.981442, + "total_plan_time": 0.07014100000000001, + "wal_bytes": "2384" + }, + { + "calls": 2, + "query": "INSERT INTO article_origins (id, article_id, source_id, entry_key, original_url, source_metadata) VALUES ($1::UUID, $2::UUID, $3::UUID, $4::VARCHAR, $5::VARCHAR, $6::JSONB) ON CONFLICT (source_id, entry_key) DO NOTHING", + "rows": 2, + "shared_blks_dirtied": 14, + "shared_blks_hit": 517, + "shared_blks_read": 2, + "shared_blks_written": 8, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 8.593985, + "total_plan_time": 0.074809, + "wal_bytes": "4288" + }, + { + "calls": 2, + "query": "INSERT INTO articles (id, canonical_url, url_hash, title, summary, review_status, publication_status, editorial_revision, classification_provenance, sponsor_names, author, image_url, managed_image, language, content_type, content_format, published_at, feed_at, metadata_source_type, discovered_at, automation_next_check_at) VALUES ($1::UUID, $2::VARCHAR, $3::VARCHAR, $4::VARCHAR, $5::VARCHAR, $6::VARCHAR, $7::VARCHAR, $8::INTEGER, $9::JSONB, $10::JSONB, $11::VARCHAR, $12::VARCHAR, $13::JSONB, $14::VARCHAR, $15::VARCHAR, $16::VARCHAR, $17::TIMESTAMP WITH TIME ZONE, $18::TIMESTAMP WITH TIME ZONE, $19::VARCHAR, $20::TIMESTAMP WITH TIME ZONE, $21::TIMESTAMP WITH TIME ZONE) ON CONFLICT (url_hash) DO NOTHING RETURNING articles.id", + "rows": 2, + "shared_blks_dirtied": 24, + "shared_blks_hit": 1190, + "shared_blks_read": 27, + "shared_blks_written": 14, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 18.71182, + "total_plan_time": 1.306209, + "wal_bytes": "6479" + }, + { + "calls": 2, + "query": "SELECT article_enrichment_jobs.id \nFROM article_enrichment_jobs \nWHERE article_enrichment_jobs.article_id = $1::UUID AND article_enrichment_jobs.refresh_id IS NOT DISTINCT FROM NULL \n LIMIT $2::INTEGER", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 15, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.03931, + "total_plan_time": 0.211708, + "wal_bytes": "0" + }, + { + "calls": 2, + "query": "SELECT article_enrichment_jobs.id, article_enrichment_jobs.article_id, article_enrichment_jobs.refresh_id, article_enrichment_jobs.http_status, article_enrichment_jobs.outcome, article_enrichment_jobs.changed_fields, article_enrichment_jobs.result, article_enrichment_jobs.error, article_enrichment_jobs.requires_solver, article_enrichment_jobs.status, article_enrichment_jobs.attempts, article_enrichment_jobs.available_at, article_enrichment_jobs.dispatched_at, article_enrichment_jobs.lease_until, article_enrichment_jobs.lease_token, article_enrichment_jobs.created_at, article_enrichment_jobs.finished_at \nFROM article_enrichment_jobs \nWHERE article_enrichment_jobs.article_id = $1::UUID AND article_enrichment_jobs.status IN ($2 /*, ... */)", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 216, + "shared_blks_read": 9, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.035019, + "total_plan_time": 3.220949, + "wal_bytes": "0" + }, + { + "calls": 2, + "query": "SELECT article_origins.article_id \nFROM article_origins \nWHERE article_origins.source_id = $1::UUID AND article_origins.entry_key = $2::VARCHAR", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 100, + "shared_blks_read": 5, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.051457, + "total_plan_time": 0.962344, + "wal_bytes": "0" + }, + { + "calls": 2, + "query": "SELECT articles.id, articles.canonical_url, articles.url_hash, articles.title, articles.slug, articles.summary, articles.matching_tokens, articles.ai_title, articles.ai_summary, articles.ai_description, articles.review_status, articles.publication_status, articles.published_to_feed_at, articles.editorial_revision, articles.classification_provenance, articles.sponsor_names, articles.author, articles.image_url, articles.managed_image, articles.language, articles.content_type, articles.content_format, articles.published_at, articles.feed_at, articles.metadata_source_type, articles.discovered_at, articles.automation_started_at, articles.automation_next_check_at \nFROM articles \nWHERE articles.id = $1::UUID FOR UPDATE OF articles", + "rows": 2, + "shared_blks_dirtied": 0, + "shared_blks_hit": 8, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.088964, + "total_plan_time": 0.204655, + "wal_bytes": "108" + }, + { + "calls": 2, + "query": "SELECT sources.id \nFROM sources \nWHERE sources.id IN (SELECT article_origins.source_id \nFROM article_origins \nWHERE article_origins.article_id = $1::UUID) AND sources.approval_status = $2::VARCHAR ORDER BY sources.id", + "rows": 2, + "shared_blks_dirtied": 0, + "shared_blks_hit": 36, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.13158599999999998, + "total_plan_time": 0.5215000000000001, + "wal_bytes": "0" + }, + { + "calls": 1, + "query": "SELECT tags.id, tags.name, tags.slug, tags.aliases, tags.topic_id, tags.auto_link_topic, tags.topic_match_revision, tags.topic_match_status, tags.topic_match_checked_at \nFROM tags", + "rows": 55000, + "shared_blks_dirtied": 0, + "shared_blks_hit": 970, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 64.948503, + "total_plan_time": 0.617586, + "wal_bytes": "0" + }, + { + "calls": 1, + "query": "SELECT topics.id, topics.name, topics.slug, topics.kind, topics.aliases, topics.identity_keys, topics.matching_tokens, topics.keywords, topics.status, topics.description, topics.ai_description, topics.website_url, topics.logo_url, topics.managed_logo, topics.facts, topics.created_at, topics.updated_at \nFROM topics \nWHERE topics.status = $1::VARCHAR", + "rows": 4450, + "shared_blks_dirtied": 0, + "shared_blks_hit": 352, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 11.520328, + "total_plan_time": 2.415679, + "wal_bytes": "0" + } + ], + "sql_calls": 18, + "sql_ms": 141.12502799616777, + "tx_count": 1, + "tx_ms": 1363.8369889995374, + "wall_ms": 1382.488408999052 + }, + { + "checkouts": 1, + "connect_hold_ms": 74.74333999925875, + "for_update": 2, + "label": "catalogs/ai_new/after", + "max_tx_ms": 72.6065449998714, + "postgres": { + "calls": 18.0, + "rows": 10.0, + "shared_blks_dirtied": 51.0, + "shared_blks_hit": 2186.0, + "shared_blks_read": 30.0, + "shared_blks_written": 29.0, + "temp_blks_read": 0.0, + "temp_blks_written": 0.0, + "total_exec_time": 29.675924, + "total_plan_time": 6.436269, + "wal_bytes": 12918.0 + }, + "postgres_container_cpu_ms": 76.272, + "queries": [ + { + "calls": 1, + "query": "BEGIN", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 0, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.001115, + "total_plan_time": 0.0, + "wal_bytes": "0" + }, + { + "calls": 1, + "query": "COMMIT", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 0, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.001186, + "total_plan_time": 0.0, + "wal_bytes": "0" + }, + { + "calls": 2, + "query": "INSERT INTO article_enrichment_jobs (id, article_id, refresh_id, http_status, outcome, changed_fields, result, error, requires_solver, status, attempts, available_at, dispatched_at, lease_until, lease_token, created_at, finished_at) VALUES ($1::UUID, $2::UUID, $3::UUID, $4::INTEGER, $5::VARCHAR, $6::VARCHAR(30)[], $7::JSONB, $8::VARCHAR, $9, $10::VARCHAR, $11::INTEGER, $12::TIMESTAMP WITH TIME ZONE, $13::TIMESTAMP WITH TIME ZONE, $14::TIMESTAMP WITH TIME ZONE, $15::UUID, $16::TIMESTAMP WITH TIME ZONE, $17::TIMESTAMP WITH TIME ZONE)", + "rows": 2, + "shared_blks_dirtied": 13, + "shared_blks_hit": 45, + "shared_blks_read": 0, + "shared_blks_written": 7, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 2.5247780000000004, + "total_plan_time": 0.07108600000000001, + "wal_bytes": "2384" + }, + { + "calls": 2, + "query": "INSERT INTO article_origins (id, article_id, source_id, entry_key, original_url, source_metadata) VALUES ($1::UUID, $2::UUID, $3::UUID, $4::VARCHAR, $5::VARCHAR, $6::JSONB) ON CONFLICT (source_id, entry_key) DO NOTHING", + "rows": 2, + "shared_blks_dirtied": 14, + "shared_blks_hit": 504, + "shared_blks_read": 2, + "shared_blks_written": 8, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 7.1668140000000005, + "total_plan_time": 0.055371, + "wal_bytes": "4288" + }, + { + "calls": 2, + "query": "INSERT INTO articles (id, canonical_url, url_hash, title, summary, review_status, publication_status, editorial_revision, classification_provenance, sponsor_names, author, image_url, managed_image, language, content_type, content_format, published_at, feed_at, metadata_source_type, discovered_at, automation_next_check_at) VALUES ($1::UUID, $2::VARCHAR, $3::VARCHAR, $4::VARCHAR, $5::VARCHAR, $6::VARCHAR, $7::VARCHAR, $8::INTEGER, $9::JSONB, $10::JSONB, $11::VARCHAR, $12::VARCHAR, $13::JSONB, $14::VARCHAR, $15::VARCHAR, $16::VARCHAR, $17::TIMESTAMP WITH TIME ZONE, $18::TIMESTAMP WITH TIME ZONE, $19::VARCHAR, $20::TIMESTAMP WITH TIME ZONE, $21::TIMESTAMP WITH TIME ZONE) ON CONFLICT (url_hash) DO NOTHING RETURNING articles.id", + "rows": 2, + "shared_blks_dirtied": 24, + "shared_blks_hit": 1213, + "shared_blks_read": 14, + "shared_blks_written": 14, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 19.631241, + "total_plan_time": 1.344577, + "wal_bytes": "6138" + }, + { + "calls": 2, + "query": "SELECT article_enrichment_jobs.id \nFROM article_enrichment_jobs \nWHERE article_enrichment_jobs.article_id = $1::UUID AND article_enrichment_jobs.refresh_id IS NOT DISTINCT FROM NULL \n LIMIT $2::INTEGER", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 15, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.040848, + "total_plan_time": 0.349957, + "wal_bytes": "0" + }, + { + "calls": 2, + "query": "SELECT article_enrichment_jobs.id, article_enrichment_jobs.article_id, article_enrichment_jobs.refresh_id, article_enrichment_jobs.http_status, article_enrichment_jobs.outcome, article_enrichment_jobs.changed_fields, article_enrichment_jobs.result, article_enrichment_jobs.error, article_enrichment_jobs.requires_solver, article_enrichment_jobs.status, article_enrichment_jobs.attempts, article_enrichment_jobs.available_at, article_enrichment_jobs.dispatched_at, article_enrichment_jobs.lease_until, article_enrichment_jobs.lease_token, article_enrichment_jobs.created_at, article_enrichment_jobs.finished_at \nFROM article_enrichment_jobs \nWHERE article_enrichment_jobs.article_id = $1::UUID AND article_enrichment_jobs.status IN ($2 /*, ... */)", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 217, + "shared_blks_read": 9, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.030177000000000002, + "total_plan_time": 2.260806, + "wal_bytes": "0" + }, + { + "calls": 2, + "query": "SELECT article_origins.article_id \nFROM article_origins \nWHERE article_origins.source_id = $1::UUID AND article_origins.entry_key = $2::VARCHAR", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 148, + "shared_blks_read": 5, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.065359, + "total_plan_time": 1.558411, + "wal_bytes": "0" + }, + { + "calls": 2, + "query": "SELECT articles.id, articles.canonical_url, articles.url_hash, articles.title, articles.slug, articles.summary, articles.matching_tokens, articles.ai_title, articles.ai_summary, articles.ai_description, articles.review_status, articles.publication_status, articles.published_to_feed_at, articles.editorial_revision, articles.classification_provenance, articles.sponsor_names, articles.author, articles.image_url, articles.managed_image, articles.language, articles.content_type, articles.content_format, articles.published_at, articles.feed_at, articles.metadata_source_type, articles.discovered_at, articles.automation_started_at, articles.automation_next_check_at \nFROM articles \nWHERE articles.id = $1::UUID FOR UPDATE OF articles", + "rows": 2, + "shared_blks_dirtied": 0, + "shared_blks_hit": 8, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.091988, + "total_plan_time": 0.243577, + "wal_bytes": "108" + }, + { + "calls": 2, + "query": "SELECT sources.id \nFROM sources \nWHERE sources.id IN (SELECT article_origins.source_id \nFROM article_origins \nWHERE article_origins.article_id = $1::UUID) AND sources.approval_status = $2::VARCHAR ORDER BY sources.id", + "rows": 2, + "shared_blks_dirtied": 0, + "shared_blks_hit": 36, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.122418, + "total_plan_time": 0.552484, + "wal_bytes": "0" + } + ], + "sql_calls": 16, + "sql_ms": 60.14477900316706, + "tx_count": 1, + "tx_ms": 72.6065449998714, + "wall_ms": 89.59783400132437 + }, + { + "checkouts": 1, + "connect_hold_ms": 1333.3139969981858, + "for_update": 0, + "label": "catalogs/ai_repeat/before", + "max_tx_ms": 1332.326440999168, + "postgres": { + "calls": 6.0, + "rows": 59452.0, + "shared_blks_dirtied": 0.0, + "shared_blks_hit": 1412.0, + "shared_blks_read": 0.0, + "shared_blks_written": 0.0, + "temp_blks_read": 0.0, + "temp_blks_written": 0.0, + "total_exec_time": 70.353538, + "total_plan_time": 3.238525, + "wal_bytes": 182.0 + }, + "postgres_container_cpu_ms": 384.648, + "queries": [ + { + "calls": 1, + "query": "BEGIN", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 0, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.001097, + "total_plan_time": 0.0, + "wal_bytes": "0" + }, + { + "calls": 1, + "query": "COMMIT", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 0, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.001754, + "total_plan_time": 0.0, + "wal_bytes": "0" + }, + { + "calls": 2, + "query": "SELECT article_origins.article_id \nFROM article_origins \nWHERE article_origins.source_id = $1::UUID AND article_origins.entry_key = $2::VARCHAR", + "rows": 2, + "shared_blks_dirtied": 0, + "shared_blks_hit": 97, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.050695, + "total_plan_time": 0.820844, + "wal_bytes": "0" + }, + { + "calls": 1, + "query": "SELECT tags.id, tags.name, tags.slug, tags.aliases, tags.topic_id, tags.auto_link_topic, tags.topic_match_revision, tags.topic_match_status, tags.topic_match_checked_at \nFROM tags", + "rows": 55000, + "shared_blks_dirtied": 0, + "shared_blks_hit": 970, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 58.867994, + "total_plan_time": 0.598875, + "wal_bytes": "0" + }, + { + "calls": 1, + "query": "SELECT topics.id, topics.name, topics.slug, topics.kind, topics.aliases, topics.identity_keys, topics.matching_tokens, topics.keywords, topics.status, topics.description, topics.ai_description, topics.website_url, topics.logo_url, topics.managed_logo, topics.facts, topics.created_at, topics.updated_at \nFROM topics \nWHERE topics.status = $1::VARCHAR", + "rows": 4450, + "shared_blks_dirtied": 0, + "shared_blks_hit": 345, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 11.431998, + "total_plan_time": 1.818806, + "wal_bytes": "182" + } + ], + "sql_calls": 4, + "sql_ms": 81.84107599663548, + "tx_count": 1, + "tx_ms": 1332.326440999168, + "wall_ms": 1343.7818129968946 + }, + { + "checkouts": 1, + "connect_hold_ms": 7.158757998695364, + "for_update": 0, + "label": "catalogs/ai_repeat/after", + "max_tx_ms": 6.591973000467988, + "postgres": { + "calls": 4.0, + "rows": 2.0, + "shared_blks_dirtied": 0.0, + "shared_blks_hit": 145.0, + "shared_blks_read": 0.0, + "shared_blks_written": 0.0, + "temp_blks_read": 0.0, + "temp_blks_written": 0.0, + "total_exec_time": 0.054914000000000004, + "total_plan_time": 1.216651, + "wal_bytes": 0.0 + }, + "postgres_container_cpu_ms": 32.204, + "queries": [ + { + "calls": 1, + "query": "BEGIN", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 0, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.000846, + "total_plan_time": 0.0, + "wal_bytes": "0" + }, + { + "calls": 1, + "query": "COMMIT", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 0, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.000946, + "total_plan_time": 0.0, + "wal_bytes": "0" + }, + { + "calls": 2, + "query": "SELECT article_origins.article_id \nFROM article_origins \nWHERE article_origins.source_id = $1::UUID AND article_origins.entry_key = $2::VARCHAR", + "rows": 2, + "shared_blks_dirtied": 0, + "shared_blks_hit": 145, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.053122, + "total_plan_time": 1.216651, + "wal_bytes": "0" + } + ], + "sql_calls": 2, + "sql_ms": 5.698920002032537, + "tx_count": 1, + "tx_ms": 6.591973000467988, + "wall_ms": 18.64181799828657 + }, + { + "checkouts": 1, + "connect_hold_ms": 13058.474309000303, + "for_update": 2, + "label": "catalogs/heuristic_new/before", + "max_tx_ms": 13056.97449299987, + "postgres": { + "calls": 20.0, + "rows": 59460.0, + "shared_blks_dirtied": 63.0, + "shared_blks_hit": 3710.0, + "shared_blks_read": 38.0, + "shared_blks_written": 35.0, + "temp_blks_read": 0.0, + "temp_blks_written": 0.0, + "total_exec_time": 95.35521, + "total_plan_time": 9.256267, + "wal_bytes": 14835.0 + }, + "postgres_container_cpu_ms": 224.452, + "queries": [ + { + "calls": 1, + "query": "BEGIN", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 0, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.000428, + "total_plan_time": 0.0, + "wal_bytes": "0" + }, + { + "calls": 1, + "query": "COMMIT", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 0, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.000602, + "total_plan_time": 0.0, + "wal_bytes": "0" + }, + { + "calls": 2, + "query": "INSERT INTO article_image_jobs (requires_solver, operation, storage, id, article_id, refresh_id, topic_id, source_id, product_id, account_id, subject_revision, storage_version, http_status, outcome, image_url, method, error, status, attempts, available_at, dispatched_at, lease_until, lease_token, created_at, finished_at) VALUES ($1, $2::VARCHAR, $3::JSONB, $4::UUID, $5::UUID, $6::UUID, $7::UUID, $8::UUID, $9::UUID, $10::UUID, $11::INTEGER, $12::VARCHAR, $13::INTEGER, $14::VARCHAR, $15::VARCHAR, $16::VARCHAR, $17::VARCHAR, $18::VARCHAR, $19::INTEGER, $20::TIMESTAMP WITH TIME ZONE, $21::TIMESTAMP WITH TIME ZONE, $22::TIMESTAMP WITH TIME ZONE, $23::UUID, $24::TIMESTAMP WITH TIME ZONE, $25::TIMESTAMP WITH TIME ZONE)", + "rows": 2, + "shared_blks_dirtied": 25, + "shared_blks_hit": 153, + "shared_blks_read": 0, + "shared_blks_written": 13, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 2.878608, + "total_plan_time": 0.072043, + "wal_bytes": "3594" + }, + { + "calls": 2, + "query": "INSERT INTO article_origins (id, article_id, source_id, entry_key, original_url, source_metadata) VALUES ($1::UUID, $2::UUID, $3::UUID, $4::VARCHAR, $5::VARCHAR, $6::JSONB) ON CONFLICT (source_id, entry_key) DO NOTHING", + "rows": 2, + "shared_blks_dirtied": 14, + "shared_blks_hit": 501, + "shared_blks_read": 2, + "shared_blks_written": 8, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 7.158745, + "total_plan_time": 0.044625, + "wal_bytes": "4288" + }, + { + "calls": 2, + "query": "INSERT INTO articles (id, canonical_url, url_hash, title, summary, review_status, publication_status, editorial_revision, classification_provenance, sponsor_names, author, image_url, managed_image, language, content_type, content_format, published_at, feed_at, metadata_source_type, discovered_at, automation_next_check_at) VALUES ($1::UUID, $2::VARCHAR, $3::VARCHAR, $4::VARCHAR, $5::VARCHAR, $6::VARCHAR, $7::VARCHAR, $8::INTEGER, $9::JSONB, $10::JSONB, $11::VARCHAR, $12::VARCHAR, $13::JSONB, $14::VARCHAR, $15::VARCHAR, $16::VARCHAR, $17::TIMESTAMP WITH TIME ZONE, $18::TIMESTAMP WITH TIME ZONE, $19::VARCHAR, $20::TIMESTAMP WITH TIME ZONE, $21::TIMESTAMP WITH TIME ZONE) ON CONFLICT (url_hash) DO NOTHING RETURNING articles.id", + "rows": 2, + "shared_blks_dirtied": 24, + "shared_blks_hit": 1196, + "shared_blks_read": 14, + "shared_blks_written": 14, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 18.156761, + "total_plan_time": 1.301302, + "wal_bytes": "6147" + }, + { + "calls": 2, + "query": "SELECT CAST(articles.classification_provenance ->> $1::TEXT AS VARCHAR) AS anon_1 \nFROM articles \nWHERE articles.id = $2::UUID", + "rows": 2, + "shared_blks_dirtied": 0, + "shared_blks_hit": 6, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.060907, + "total_plan_time": 0.161362, + "wal_bytes": "0" + }, + { + "calls": 2, + "query": "SELECT article_image_jobs.id \nFROM article_image_jobs \nWHERE article_image_jobs.article_id = $1::UUID AND article_image_jobs.operation = $2::VARCHAR AND CAST((article_image_jobs.storage ->> $3::TEXT) AS VARCHAR) = $4::VARCHAR \n LIMIT $5::INTEGER", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 25, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.035907, + "total_plan_time": 0.343032, + "wal_bytes": "0" + }, + { + "calls": 2, + "query": "SELECT article_image_jobs.requires_solver, article_image_jobs.operation, article_image_jobs.storage, article_image_jobs.id, article_image_jobs.article_id, article_image_jobs.refresh_id, article_image_jobs.topic_id, article_image_jobs.source_id, article_image_jobs.product_id, article_image_jobs.account_id, article_image_jobs.subject_revision, article_image_jobs.storage_version, article_image_jobs.http_status, article_image_jobs.outcome, article_image_jobs.image_url, article_image_jobs.method, article_image_jobs.error, article_image_jobs.status, article_image_jobs.attempts, article_image_jobs.available_at, article_image_jobs.dispatched_at, article_image_jobs.lease_until, article_image_jobs.lease_token, article_image_jobs.created_at, article_image_jobs.finished_at \nFROM article_image_jobs \nWHERE article_image_jobs.article_id = $1::UUID AND article_image_jobs.status IN ($2 /*, ... */)", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 385, + "shared_blks_read": 17, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.062225, + "total_plan_time": 3.771095, + "wal_bytes": "110" + }, + { + "calls": 2, + "query": "SELECT article_origins.article_id \nFROM article_origins \nWHERE article_origins.source_id = $1::UUID AND article_origins.entry_key = $2::VARCHAR", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 105, + "shared_blks_read": 5, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.052048000000000004, + "total_plan_time": 0.953132, + "wal_bytes": "100" + }, + { + "calls": 2, + "query": "SELECT articles.id, articles.canonical_url, articles.url_hash, articles.title, articles.slug, articles.summary, articles.matching_tokens, articles.ai_title, articles.ai_summary, articles.ai_description, articles.review_status, articles.publication_status, articles.published_to_feed_at, articles.editorial_revision, articles.classification_provenance, articles.sponsor_names, articles.author, articles.image_url, articles.managed_image, articles.language, articles.content_type, articles.content_format, articles.published_at, articles.feed_at, articles.metadata_source_type, articles.discovered_at, articles.automation_started_at, articles.automation_next_check_at \nFROM articles \nWHERE articles.id = $1::UUID FOR UPDATE OF articles", + "rows": 2, + "shared_blks_dirtied": 0, + "shared_blks_hit": 8, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.078303, + "total_plan_time": 0.173789, + "wal_bytes": "108" + }, + { + "calls": 1, + "query": "SELECT tags.id, tags.name, tags.slug, tags.aliases, tags.topic_id, tags.auto_link_topic, tags.topic_match_revision, tags.topic_match_status, tags.topic_match_checked_at \nFROM tags", + "rows": 55000, + "shared_blks_dirtied": 0, + "shared_blks_hit": 978, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 53.760191999999996, + "total_plan_time": 0.62939, + "wal_bytes": "0" + }, + { + "calls": 1, + "query": "SELECT topics.id, topics.name, topics.slug, topics.kind, topics.aliases, topics.identity_keys, topics.matching_tokens, topics.keywords, topics.status, topics.description, topics.ai_description, topics.website_url, topics.logo_url, topics.managed_logo, topics.facts, topics.created_at, topics.updated_at \nFROM topics \nWHERE topics.status = $1::VARCHAR", + "rows": 4450, + "shared_blks_dirtied": 0, + "shared_blks_hit": 353, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 13.110484, + "total_plan_time": 1.806497, + "wal_bytes": "488" + } + ], + "sql_calls": 18, + "sql_ms": 129.29026699930546, + "tx_count": 1, + "tx_ms": 13056.97449299987, + "wall_ms": 13069.794438997633 + }, + { + "checkouts": 1, + "connect_hold_ms": 14354.318439000053, + "for_update": 2, + "label": "catalogs/heuristic_new/after", + "max_tx_ms": 14352.47905399956, + "postgres": { + "calls": 20.0, + "rows": 59460.0, + "shared_blks_dirtied": 63.0, + "shared_blks_hit": 3736.0, + "shared_blks_read": 38.0, + "shared_blks_written": 35.0, + "temp_blks_read": 0.0, + "temp_blks_written": 0.0, + "total_exec_time": 114.59714299999999, + "total_plan_time": 10.405931, + "wal_bytes": 14812.0 + }, + "postgres_container_cpu_ms": 185.755, + "queries": [ + { + "calls": 1, + "query": "BEGIN", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 0, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.001766, + "total_plan_time": 0.0, + "wal_bytes": "0" + }, + { + "calls": 1, + "query": "COMMIT", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 0, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.001799, + "total_plan_time": 0.0, + "wal_bytes": "0" + }, + { + "calls": 2, + "query": "INSERT INTO article_image_jobs (requires_solver, operation, storage, id, article_id, refresh_id, topic_id, source_id, product_id, account_id, subject_revision, storage_version, http_status, outcome, image_url, method, error, status, attempts, available_at, dispatched_at, lease_until, lease_token, created_at, finished_at) VALUES ($1, $2::VARCHAR, $3::JSONB, $4::UUID, $5::UUID, $6::UUID, $7::UUID, $8::UUID, $9::UUID, $10::UUID, $11::INTEGER, $12::VARCHAR, $13::INTEGER, $14::VARCHAR, $15::VARCHAR, $16::VARCHAR, $17::VARCHAR, $18::VARCHAR, $19::INTEGER, $20::TIMESTAMP WITH TIME ZONE, $21::TIMESTAMP WITH TIME ZONE, $22::TIMESTAMP WITH TIME ZONE, $23::UUID, $24::TIMESTAMP WITH TIME ZONE, $25::TIMESTAMP WITH TIME ZONE)", + "rows": 2, + "shared_blks_dirtied": 25, + "shared_blks_hit": 184, + "shared_blks_read": 0, + "shared_blks_written": 13, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 3.5318799999999997, + "total_plan_time": 0.082994, + "wal_bytes": "3594" + }, + { + "calls": 2, + "query": "INSERT INTO article_origins (id, article_id, source_id, entry_key, original_url, source_metadata) VALUES ($1::UUID, $2::UUID, $3::UUID, $4::VARCHAR, $5::VARCHAR, $6::JSONB) ON CONFLICT (source_id, entry_key) DO NOTHING", + "rows": 2, + "shared_blks_dirtied": 14, + "shared_blks_hit": 501, + "shared_blks_read": 2, + "shared_blks_written": 8, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 8.510597, + "total_plan_time": 0.06548100000000001, + "wal_bytes": "4288" + }, + { + "calls": 2, + "query": "INSERT INTO articles (id, canonical_url, url_hash, title, summary, review_status, publication_status, editorial_revision, classification_provenance, sponsor_names, author, image_url, managed_image, language, content_type, content_format, published_at, feed_at, metadata_source_type, discovered_at, automation_next_check_at) VALUES ($1::UUID, $2::VARCHAR, $3::VARCHAR, $4::VARCHAR, $5::VARCHAR, $6::VARCHAR, $7::VARCHAR, $8::INTEGER, $9::JSONB, $10::JSONB, $11::VARCHAR, $12::VARCHAR, $13::JSONB, $14::VARCHAR, $15::VARCHAR, $16::VARCHAR, $17::TIMESTAMP WITH TIME ZONE, $18::TIMESTAMP WITH TIME ZONE, $19::VARCHAR, $20::TIMESTAMP WITH TIME ZONE, $21::TIMESTAMP WITH TIME ZONE) ON CONFLICT (url_hash) DO NOTHING RETURNING articles.id", + "rows": 2, + "shared_blks_dirtied": 24, + "shared_blks_hit": 1218, + "shared_blks_read": 14, + "shared_blks_written": 14, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 23.265009999999997, + "total_plan_time": 1.613901, + "wal_bytes": "6300" + }, + { + "calls": 2, + "query": "SELECT CAST(articles.classification_provenance ->> $1::TEXT AS VARCHAR) AS anon_1 \nFROM articles \nWHERE articles.id = $2::UUID", + "rows": 2, + "shared_blks_dirtied": 0, + "shared_blks_hit": 6, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.059261999999999995, + "total_plan_time": 0.146601, + "wal_bytes": "0" + }, + { + "calls": 2, + "query": "SELECT article_image_jobs.id \nFROM article_image_jobs \nWHERE article_image_jobs.article_id = $1::UUID AND article_image_jobs.operation = $2::VARCHAR AND CAST((article_image_jobs.storage ->> $3::TEXT) AS VARCHAR) = $4::VARCHAR \n LIMIT $5::INTEGER", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 25, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.030220999999999998, + "total_plan_time": 0.325041, + "wal_bytes": "0" + }, + { + "calls": 2, + "query": "SELECT article_image_jobs.requires_solver, article_image_jobs.operation, article_image_jobs.storage, article_image_jobs.id, article_image_jobs.article_id, article_image_jobs.refresh_id, article_image_jobs.topic_id, article_image_jobs.source_id, article_image_jobs.product_id, article_image_jobs.account_id, article_image_jobs.subject_revision, article_image_jobs.storage_version, article_image_jobs.http_status, article_image_jobs.outcome, article_image_jobs.image_url, article_image_jobs.method, article_image_jobs.error, article_image_jobs.status, article_image_jobs.attempts, article_image_jobs.available_at, article_image_jobs.dispatched_at, article_image_jobs.lease_until, article_image_jobs.lease_token, article_image_jobs.created_at, article_image_jobs.finished_at \nFROM article_image_jobs \nWHERE article_image_jobs.article_id = $1::UUID AND article_image_jobs.status IN ($2 /*, ... */)", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 390, + "shared_blks_read": 17, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.027635, + "total_plan_time": 3.998647, + "wal_bytes": "0" + }, + { + "calls": 2, + "query": "SELECT article_origins.article_id \nFROM article_origins \nWHERE article_origins.source_id = $1::UUID AND article_origins.entry_key = $2::VARCHAR", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 153, + "shared_blks_read": 5, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.11085499999999998, + "total_plan_time": 1.646075, + "wal_bytes": "0" + }, + { + "calls": 2, + "query": "SELECT articles.id, articles.canonical_url, articles.url_hash, articles.title, articles.slug, articles.summary, articles.matching_tokens, articles.ai_title, articles.ai_summary, articles.ai_description, articles.review_status, articles.publication_status, articles.published_to_feed_at, articles.editorial_revision, articles.classification_provenance, articles.sponsor_names, articles.author, articles.image_url, articles.managed_image, articles.language, articles.content_type, articles.content_format, articles.published_at, articles.feed_at, articles.metadata_source_type, articles.discovered_at, articles.automation_started_at, articles.automation_next_check_at \nFROM articles \nWHERE articles.id = $1::UUID FOR UPDATE OF articles", + "rows": 2, + "shared_blks_dirtied": 0, + "shared_blks_hit": 8, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.103659, + "total_plan_time": 0.322946, + "wal_bytes": "108" + }, + { + "calls": 1, + "query": "SELECT tags.id, tags.name, tags.slug, tags.aliases, tags.topic_id, tags.auto_link_topic, tags.topic_match_revision, tags.topic_match_status, tags.topic_match_checked_at \nFROM tags", + "rows": 55000, + "shared_blks_dirtied": 0, + "shared_blks_hit": 969, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 67.33341399999999, + "total_plan_time": 0.817085, + "wal_bytes": "64" + }, + { + "calls": 1, + "query": "SELECT topics.id, topics.name, topics.slug, topics.kind, topics.aliases, topics.identity_keys, topics.matching_tokens, topics.keywords, topics.status, topics.description, topics.ai_description, topics.website_url, topics.logo_url, topics.managed_logo, topics.facts, topics.created_at, topics.updated_at \nFROM topics \nWHERE topics.status = $1::VARCHAR", + "rows": 4450, + "shared_blks_dirtied": 0, + "shared_blks_hit": 282, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 11.621045, + "total_plan_time": 1.38716, + "wal_bytes": "458" + } + ], + "sql_calls": 18, + "sql_ms": 149.32273600061308, + "tx_count": 1, + "tx_ms": 14352.47905399956, + "wall_ms": 14367.184949001967 + }, + { + "checkouts": 1, + "connect_hold_ms": 1415.8782289996452, + "for_update": 0, + "label": "catalogs/heuristic_repeat/before", + "max_tx_ms": 1414.9540639991756, + "postgres": { + "calls": 6.0, + "rows": 59452.0, + "shared_blks_dirtied": 0.0, + "shared_blks_hit": 1412.0, + "shared_blks_read": 0.0, + "shared_blks_written": 0.0, + "temp_blks_read": 0.0, + "temp_blks_written": 0.0, + "total_exec_time": 63.731201000000006, + "total_plan_time": 3.339691, + "wal_bytes": 376.0 + }, + "postgres_container_cpu_ms": 103.136, + "queries": [ + { + "calls": 1, + "query": "BEGIN", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 0, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.000877, + "total_plan_time": 0.0, + "wal_bytes": "0" + }, + { + "calls": 1, + "query": "COMMIT", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 0, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.001027, + "total_plan_time": 0.0, + "wal_bytes": "0" + }, + { + "calls": 2, + "query": "SELECT article_origins.article_id \nFROM article_origins \nWHERE article_origins.source_id = $1::UUID AND article_origins.entry_key = $2::VARCHAR", + "rows": 2, + "shared_blks_dirtied": 0, + "shared_blks_hit": 97, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.078285, + "total_plan_time": 0.8457279999999999, + "wal_bytes": "0" + }, + { + "calls": 1, + "query": "SELECT tags.id, tags.name, tags.slug, tags.aliases, tags.topic_id, tags.auto_link_topic, tags.topic_match_revision, tags.topic_match_status, tags.topic_match_checked_at \nFROM tags", + "rows": 55000, + "shared_blks_dirtied": 0, + "shared_blks_hit": 970, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 51.857245000000006, + "total_plan_time": 0.669927, + "wal_bytes": "0" + }, + { + "calls": 1, + "query": "SELECT topics.id, topics.name, topics.slug, topics.kind, topics.aliases, topics.identity_keys, topics.matching_tokens, topics.keywords, topics.status, topics.description, topics.ai_description, topics.website_url, topics.logo_url, topics.managed_logo, topics.facts, topics.created_at, topics.updated_at \nFROM topics \nWHERE topics.status = $1::VARCHAR", + "rows": 4450, + "shared_blks_dirtied": 0, + "shared_blks_hit": 345, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 11.793767, + "total_plan_time": 1.824036, + "wal_bytes": "376" + } + ], + "sql_calls": 4, + "sql_ms": 74.81568900402635, + "tx_count": 1, + "tx_ms": 1414.9540639991756, + "wall_ms": 1428.9215870012413 + }, + { + "checkouts": 1, + "connect_hold_ms": 7.0077169984870125, + "for_update": 0, + "label": "catalogs/heuristic_repeat/after", + "max_tx_ms": 6.574854000064079, + "postgres": { + "calls": 4.0, + "rows": 2.0, + "shared_blks_dirtied": 0.0, + "shared_blks_hit": 145.0, + "shared_blks_read": 0.0, + "shared_blks_written": 0.0, + "temp_blks_read": 0.0, + "temp_blks_written": 0.0, + "total_exec_time": 0.06840900000000001, + "total_plan_time": 1.2672919999999999, + "wal_bytes": 0.0 + }, + "postgres_container_cpu_ms": 31.672, + "queries": [ + { + "calls": 1, + "query": "BEGIN", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 0, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.001292, + "total_plan_time": 0.0, + "wal_bytes": "0" + }, + { + "calls": 1, + "query": "COMMIT", + "rows": 0, + "shared_blks_dirtied": 0, + "shared_blks_hit": 0, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.001004, + "total_plan_time": 0.0, + "wal_bytes": "0" + }, + { + "calls": 2, + "query": "SELECT article_origins.article_id \nFROM article_origins \nWHERE article_origins.source_id = $1::UUID AND article_origins.entry_key = $2::VARCHAR", + "rows": 2, + "shared_blks_dirtied": 0, + "shared_blks_hit": 145, + "shared_blks_read": 0, + "shared_blks_written": 0, + "temp_blks_read": 0, + "temp_blks_written": 0, + "toplevel": true, + "total_exec_time": 0.066113, + "total_plan_time": 1.2672919999999999, + "wal_bytes": "0" + } + ], + "sql_calls": 2, + "sql_ms": 5.572623998887138, + "tx_count": 1, + "tx_ms": 6.574854000064079, + "wall_ms": 17.685999999230262 + } + ] +} diff --git a/benchmarks/ingestion-heuristic-catalogs.md b/benchmarks/ingestion-heuristic-catalogs.md index bbd034579..99bc92d3e 100644 --- a/benchmarks/ingestion-heuristic-catalogs.md +++ b/benchmarks/ingestion-heuristic-catalogs.md @@ -35,3 +35,37 @@ The local validation ran on x86-64, not native ARM64 or production. Object-load counts prove removed work; they are not a claim about total worker CPU savings. The independent lightweight source-tag identity projection still runs when feed labels need resolution. The changed path does not add a cache or migration. + +## PostgreSQL work, connection and lock audit + +A separate controlled local x86-64 PostgreSQL 18 run used 4,450 active topics, +55,000 tags with one alias each, and the same two-entry feed. It executed the +actual baseline `store_entries` from `6f12ed1d` and this implementation against +fresh disposable fixtures. `ANALYZE` preceded each sample; +`pg_stat_statements` was reset immediately before the workload. Application +statement counts exclude driver transaction commands; returned rows, shared +buffers and SQL execution time come from PostgreSQL's top-level counters. +Nested trigger costs belong to their enclosing statement. See the +[recorded statements and counters](ingestion-heuristic-catalogs-postgresql.json). + +| Workload | App SQL before → after | SQL rows before → after | Shared hit/read blocks before → after | SQL execution ms before → after | Open transaction ms before → after | +| --- | ---: | ---: | ---: | ---: | ---: | +| AI, new entries | 18 → 16 | 59,460 → 10 | 3,492 → 2,216 | 106.10 → 29.68 | 1,363.8 → 72.6 | +| AI, retained entries | 4 → 2 | 59,452 → 2 | 1,412 → 145 | 70.35 → 0.05 | 1,332.3 → 6.6 | +| Heuristic, new entries | 18 → 18 | 59,460 → 59,460 | 3,748 → 3,774 | 95.36 → 114.60 | 13,057.0 → 14,352.5 | +| Heuristic, retained entries | 4 → 2 | 59,452 → 2 | 1,412 → 145 | 63.73 → 0.07 | 1,415.0 → 6.6 | + +Each call used one transaction and checkout in both versions. New-entry cases +used two existing article row-locking statements in both; retained cases used +none. The removed catalog scans add no Redis dependency, invalidation query, +background transaction or replacement database calculation. The native regression +now checks actual catalog query row counts as well as ORM load events. + +These are one paired observation per workload with warm analyzed fixture data, +not a cold-storage, concurrent production or native ARM64 sizing result. The +heuristic path performs identical SQL work; its elapsed time was higher in this +sample and it receives no database speedup claim. PostgreSQL-container CPU +observations are preserved in the JSON, including probe/background cost; they do +not establish isolated backend CPU or a production resource limit. Cache outage +and cache churn do not change this PR's path because it adds no cache. Sustained +concurrency, PostgreSQL CPU and slot occupancy remain rollout measurements. diff --git a/tests/sources/test_ingestion.py b/tests/sources/test_ingestion.py index 198aec442..c4bb28aac 100644 --- a/tests/sources/test_ingestion.py +++ b/tests/sources/test_ingestion.py @@ -413,22 +413,34 @@ def test_ingestion_loads_heuristic_catalog_only_for_entries_that_need_it( ) parsed = parse_feed(rss_bytes, "https://example.com/rss", utcnow(), source_type="publisher") parsed = replace(parsed, entries=[replace(entry, tags=[]) for entry in parsed.entries]) - hydrated = [] + hydrated, catalog_rows = [], [] + + def executed(_connection, cursor, statement, *_): + if statement.startswith("SELECT") and ( + "FROM topics" in statement or "FROM tags" in statement + ): + catalog_rows.append(cursor.rowcount) def loaded(_session, instance): if isinstance(instance, (Topic, Tag)): hydrated.append(type(instance).__name__) + engine = database.kw["bind"] event.listen(Session, "loaded_as_persistent", loaded) + event.listen(engine, "after_cursor_execute", executed) try: with database.begin() as session: assert tasks.store_entries(session, source_id, parsed) == 2 first = len(hydrated) # With AI disabled, one lazy catalog is shared by both new entries. assert first == (0 if ai_enabled else 20000) + assert catalog_rows == ([] if ai_enabled else [10000, 10000]) hydrated.clear() + catalog_rows.clear() with database.begin() as session: assert tasks.store_entries(session, source_id, parsed) == 0 assert hydrated == [] + assert catalog_rows == [] # No server-side catalog scan or returned rows either. finally: event.remove(Session, "loaded_as_persistent", loaded) + event.remove(engine, "after_cursor_execute", executed)