Check the feed hash before parsing

gtfsdb/conditional_import_test.go

@@ -325,3 +325,33 @@ func TestClearAllGTFSData(t *testing.T) {
 	require.NoError(t, err, "Import metadata should still exist after clear")
 	assert.NotEmpty(t, metadata.FileHash, "Import metadata should not be cleared")
 }
+
+func TestSkipIfImported(t *testing.T) {
+	client, err := NewClient(Config{DBPath: ":memory:", Env: appconf.Test})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = client.Close() })
+
+	imported, err := client.SkipIfImported(t.Context(), "hash", "source")
+	require.NoError(t, err)
+	assert.False(t, imported, "an empty database holds no feed")
+
+	raba, err := os.ReadFile("../testdata/raba.zip")
+	require.NoError(t, err)
+	parsed, err := ParseGtfsData(raba, "source")
+	require.NoError(t, err)
+	_, err = client.StoreGtfsData(t.Context(), parsed)
+	require.NoError(t, err)
+
+	for _, tt := range []struct {
+		hash, source string
+		want         bool
+	}{
+		{parsed.Hash, "source", true},
+		{parsed.Hash, "other-source", false},
+		{"other-hash", "source", false},
+	} {
+		imported, err := client.SkipIfImported(t.Context(), tt.hash, tt.source)
+		require.NoError(t, err)
+		assert.Equal(t, tt.want, imported, "%s from %s", tt.hash, tt.source)
+	}
+}

gtfsdb/helpers.go

@@ -7,6 +7,7 @@ import (
 	"database/sql"
 	_ "embed"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"iter"
 	"log/slog"
@@ -23,9 +24,9 @@ import (
 //go:embed schema.sql
 var ddl string
 
-// GtfsData bundles parsed static GTFS data with the sha256 hash of the
-// bytes it was parsed from. The hash is used to skip reimport when the
-// source data is unchanged.
+// GtfsData bundles parsed static GTFS data with the hash that identifies the
+// feed in import_metadata. The hash is used to skip reimport when the feed is
+// unchanged.
 type GtfsData struct {
 	Static *gtfs.Static
 	Hash   string
@@ -36,8 +37,12 @@ type GtfsData struct {
 // The given source is stored on the returned struct for later use by StoreGtfsData.
 func ParseGtfsData(b []byte, source string) (*GtfsData, error) {
 	hash := sha256.Sum256(b)
-	hashStr := hex.EncodeToString(hash[:])
+	return ParseGtfsDataWithHash(b, hex.EncodeToString(hash[:]), source)
+}
 
+// ParseGtfsDataWithHash parses and structurally validates GTFS zip bytes whose
+// identifying hash the caller has already computed.
+func ParseGtfsDataWithHash(b []byte, hash, source string) (*GtfsData, error) {
 	logger := slog.Default().With(slog.String("component", "gtfs_parser"))
 	logging.LogOperation(logger, "parsing_gtfs_data", slog.Int("bytes", len(b)))
 	staticData, err := gtfs.ParseStatic(b, gtfs.ParseStaticOptions{})
@@ -49,7 +54,7 @@ func ParseGtfsData(b []byte, source string) (*GtfsData, error) {
 		return nil, fmt.Errorf("GTFS validation failed: %w", err)
 	}
 
-	return &GtfsData{Static: staticData, Hash: hashStr, Source: source}, nil
+	return &GtfsData{Static: staticData, Hash: hash, Source: source}, nil
 }
 
 // metricsWrapper wraps *sql.DB for metric reporting purposes
@@ -203,6 +208,49 @@ func (c *Client) withTransaction(ctx context.Context, tx *sql.Tx, label string,
 	return nil
 }
 
+// SkipIfImported reports whether the database already holds the feed with
+// this hash from this source, so a caller can skip parsing and storing it.
+// When it does, it refreshes the feed's derived metadata, as StoreGtfsData
+// does for an unchanged feed.
+func (c *Client) SkipIfImported(ctx context.Context, hash, source string) (bool, error) {
+	metadata, err := c.Queries.GetImportMetadata(ctx)
+	if errors.Is(err, sql.ErrNoRows) {
+		return false, nil
+	}
+	if err != nil {
+		return false, fmt.Errorf("error checking import metadata: %w", err)
+	}
+	return c.skipUnchangedFeed(ctx, metadata, hash, source)
+}
+
+// skipUnchangedFeed is the one rule for a feed the database already holds.
+// When import_metadata describes the feed with this hash from this source, it
+// refreshes the derived metadata and reports true; otherwise it does nothing.
+func (c *Client) skipUnchangedFeed(ctx context.Context, metadata ImportMetadatum, hash, source string) (bool, error) {
+	if metadata.FileHash != hash || metadata.FileSource != source {
+		return false, nil
+	}
+	if err := c.refreshFeedExpiry(ctx); err != nil {
+		return false, err
+	}
+	logger := slog.Default().With(slog.String("component", "gtfs_importer"))
+	logging.LogOperation(logger, "gtfs_data_unchanged_skipping_import", slog.String("hash", hash[:8]))
+	return true, nil
+}
+
+// refreshFeedExpiry recomputes feed_expires_at from the stored calendar. It
+// runs when a feed is found unchanged, so existing databases pick up
+// corrections to the expiry calculation on upgrade without a re-import.
+func (c *Client) refreshFeedExpiry(ctx context.Context) error {
+	err := c.withTransaction(ctx, nil, "refresh_feed_expiry", func(tx *sql.Tx) error {
+		return updateFeedExpiresAtFromCalendar(ctx, c.Queries.WithTx(tx))
+	})
+	if err != nil {
+		return fmt.Errorf("failed to refresh feed expiry: %w", err)
+	}
+	return nil
+}
+
 func (c *Client) StoreGtfsData(ctx context.Context, data *GtfsData) (bool, error) {
 	logger := slog.Default().With(slog.String("component", "gtfs_importer"))
 
@@ -222,17 +270,11 @@ func (c *Client) StoreGtfsData(ctx context.Context, data *GtfsData) (bool, error
 	existingMetadata, err := c.Queries.GetImportMetadata(ctx)
 	if err == nil {
 		hasExisting = true
-		// We have existing metadata, check if hash matches
-		if existingMetadata.FileHash == data.Hash && existingMetadata.FileSource == data.Source {
-			// Refresh derived metadata even when the feed is unchanged, so existing
-			// databases pick up corrections to the expiry calculation on upgrade.
-			if err := c.withTransaction(ctx, nil, "refresh_feed_expiry", func(tx *sql.Tx) error {
-				return updateFeedExpiresAtFromCalendar(ctx, c.Queries.WithTx(tx))
-			}); err != nil {
-				return false, fmt.Errorf("failed to refresh feed expiry: %w", err)
-			}
-			logging.LogOperation(logger, "gtfs_data_unchanged_skipping_import",
-				slog.String("hash", data.Hash[:8]))
+		skipped, err := c.skipUnchangedFeed(ctx, existingMetadata, data.Hash, data.Source)
+		if err != nil {
+			return false, err
+		}
+		if skipped {
 			return false, nil
 		}
 	} else if err != sql.ErrNoRows {