Add a parser for obacloud's bundle-inputs manifest

internal/appconf/json_config.go

@@ -302,24 +302,30 @@ func (j *JSONConfig) validateStaticFeeds() error {
 	if j.GtfsStaticFeeds != nil && len(j.GtfsStaticFeeds) == 0 {
 		return fmt.Errorf("gtfs-static-feeds cannot be empty")
 	}
+	return ValidateStaticFeeds(j.GtfsStaticFeeds, j.staticFeedsKey)
+}
 
+// ValidateStaticFeeds checks a static feed list wherever it came from:
+// unique ids, unique default agencies, and valid fields in each feed.
+// key names the list's source in errors.
+func ValidateStaticFeeds(feeds []GtfsStaticFeed, key string) error {
 	seenIDs := make(map[string]bool)
 	seenDefaultAgencyIDs := make(map[string]bool)
-	for _, feed := range j.GtfsStaticFeeds {
+	for _, feed := range feeds {
 		if seenIDs[feed.ID] {
-			return fmt.Errorf("duplicate %s id: %q", j.staticFeedsKey, feed.ID)
+			return fmt.Errorf("duplicate %s id: %q", key, feed.ID)
 		}
 		seenIDs[feed.ID] = true
 
 		if feed.DefaultAgencyID != "" {
 			if seenDefaultAgencyIDs[feed.DefaultAgencyID] {
-				return fmt.Errorf("duplicate %s default-agency-id: %q", j.staticFeedsKey, feed.DefaultAgencyID)
+				return fmt.Errorf("duplicate %s default-agency-id: %q", key, feed.DefaultAgencyID)
 			}
 			seenDefaultAgencyIDs[feed.DefaultAgencyID] = true
 		}
 
 		if err := feed.validate(); err != nil {
-			return fmt.Errorf("%s %q: %w", j.staticFeedsKey, feed.ID, err)
+			return fmt.Errorf("%s %q: %w", key, feed.ID, err)
 		}
 	}
 	return nil
@@ -355,13 +361,18 @@ func (feed GtfsStaticFeed) validate() error {
 	if strings.HasPrefix(strings.ToLower(feed.URL), "file://") {
 		return fmt.Errorf("file:// URLs are not allowed in url for security reasons")
 	}
-	isHTTPURL := strings.HasPrefix(feed.URL, "http://") || strings.HasPrefix(feed.URL, "https://")
-	if isHTTPURL {
+	if IsHTTPURL(feed.URL) {
 		return nil
 	}
 	return validatePath(feed.URL, "url")
 }
 
+// IsHTTPURL reports whether source is an http:// or https:// URL rather
+// than a local path.
+func IsHTTPURL(source string) bool {
+	return strings.HasPrefix(source, "http://") || strings.HasPrefix(source, "https://")
+}
+
 // validatePath checks a file path for security issues
 func validatePath(path, fieldName string) error {
 	if path == "" {

internal/appconf/static_feeds_test.go

@@ -396,3 +396,22 @@ func TestToGtfsConfigData_WarnsAboutIgnoredStaticFeeds(t *testing.T) {
 		})
 	}
 }
+
+func TestIsHTTPURL(t *testing.T) {
+	tests := []struct {
+		source string
+		want   bool
+	}{
+		{source: "https://example.com/a.zip", want: true},
+		{source: "http://example.com/a.zip", want: true},
+		{source: "./a.zip", want: false},
+		{source: "/data/a.zip", want: false},
+		{source: "file:///data/a.zip", want: false},
+		{source: "HTTPS://example.com/a.zip", want: false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.source, func(t *testing.T) {
+			assert.Equal(t, tt.want, IsHTTPURL(tt.source))
+		})
+	}
+}

internal/bundleinputs/bundleinputs.go

@@ -0,0 +1,117 @@
+// Package bundleinputs reads the bundle-inputs.json manifest that obacloud
+// publishes to list the static GTFS feeds a server should load.
+package bundleinputs
+
+import (
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+
+	"maglev.onebusaway.org/internal/appconf"
+)
+
+// SupportedVersion is the only manifest version Parse accepts.
+const SupportedVersion = 1
+
+// sha256HexLength is the length of a SHA-256 digest written as hex.
+const sha256HexLength = 64
+
+// Manifest is a parsed bundle-inputs.json file.
+type Manifest struct {
+	Version int    `json:"version"`
+	Feeds   []Feed `json:"feeds"`
+	// StopConsolidationURL is optional and empty when the manifest has none.
+	StopConsolidationURL string `json:"stopConsolidationUrl"`
+}
+
+// Feed is one static feed in a manifest. Feed order is load order.
+type Feed struct {
+	ID              string `json:"id"`
+	Name            string `json:"name"`
+	DefaultAgencyID string `json:"defaultAgencyId"`
+	URL             string `json:"url"`
+	// ByteSize and SHA256 describe the published zip, for checking the
+	// download. Nothing checks them yet.
+	ByteSize int64  `json:"byteSize"`
+	SHA256   string `json:"sha256"`
+}
+
+// Parse decodes and validates a manifest. Unknown keys are ignored.
+func Parse(r io.Reader) (*Manifest, error) {
+	var manifest Manifest
+	decoder := json.NewDecoder(r)
+	if err := decoder.Decode(&manifest); err != nil {
+		return nil, fmt.Errorf("invalid bundle inputs JSON: %w", err)
+	}
+	if _, err := decoder.Token(); err != io.EOF {
+		return nil, fmt.Errorf("invalid bundle inputs JSON: unexpected data after the manifest")
+	}
+	if err := manifest.validate(); err != nil {
+		return nil, err
+	}
+	return &manifest, nil
+}
+
+// StaticFeeds returns the manifest feeds as static feed config, in order.
+func (m *Manifest) StaticFeeds() []appconf.GtfsStaticFeed {
+	feeds := make([]appconf.GtfsStaticFeed, 0, len(m.Feeds))
+	for _, feed := range m.Feeds {
+		feeds = append(feeds, appconf.GtfsStaticFeed{
+			ID:              feed.ID,
+			URL:             feed.URL,
+			DefaultAgencyID: feed.DefaultAgencyID,
+		})
+	}
+	return feeds
+}
+
+// manifestFeedsKey names the manifest feed list in shared validation errors.
+const manifestFeedsKey = "bundle inputs feeds"
+
+// validate applies the manifest format rules, then the checks every static
+// feed list gets.
+func (m *Manifest) validate() error {
+	if m.Version != SupportedVersion {
+		return fmt.Errorf("unsupported bundle inputs version %d, want %d", m.Version, SupportedVersion)
+	}
+	if len(m.Feeds) == 0 {
+		return fmt.Errorf("bundle inputs feeds cannot be empty")
+	}
+	if m.StopConsolidationURL != "" && !appconf.IsHTTPURL(m.StopConsolidationURL) {
+		return fmt.Errorf("stopConsolidationUrl %q must be an http:// or https:// URL", m.StopConsolidationURL)
+	}
+	for i, feed := range m.Feeds {
+		if err := feed.validate(); err != nil {
+			return fmt.Errorf("bundle inputs feeds[%d]: %w", i, err)
+		}
+	}
+	return appconf.ValidateStaticFeeds(m.StaticFeeds(), manifestFeedsKey)
+}
+
+// validate applies the manifest format rules to one feed. The url must be
+// http(s): a manifest may come from a remote server, so it must not name
+// paths on this machine.
+func (feed Feed) validate() error {
+	if feed.ID == "" {
+		return fmt.Errorf("id is required")
+	}
+	if feed.DefaultAgencyID == "" {
+		return fmt.Errorf("defaultAgencyId is required")
+	}
+	if !appconf.IsHTTPURL(feed.URL) {
+		return fmt.Errorf("url %q must be an http:// or https:// URL", feed.URL)
+	}
+	if feed.SHA256 != "" && !isSHA256Hex(feed.SHA256) {
+		return fmt.Errorf("sha256 must be %d hex characters", sha256HexLength)
+	}
+	if feed.ByteSize < 0 {
+		return fmt.Errorf("byteSize cannot be negative")
+	}
+	return nil
+}
+
+func isSHA256Hex(value string) bool {
+	_, err := hex.DecodeString(value)
+	return err == nil && len(value) == sha256HexLength
+}

internal/bundleinputs/bundleinputs_test.go

@@ -0,0 +1,166 @@
+package bundleinputs
+
+import (
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"maglev.onebusaway.org/internal/appconf"
+)
+
+const goldenManifestPath = "testdata/bundle-inputs.json"
+
+func TestParse_GoldenManifest(t *testing.T) {
+	file, err := os.Open(goldenManifestPath)
+	require.NoError(t, err)
+	defer file.Close()
+
+	manifest, err := Parse(file)
+
+	require.NoError(t, err)
+	want := &Manifest{
+		Version: 1,
+		Feeds: []Feed{
+			{
+				ID:              "metro",
+				Name:            "King County Metro",
+				DefaultAgencyID: "1",
+				URL:             "https://example.com/bundles/metro.zip",
+				ByteSize:        1234567,
+				SHA256:          "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08",
+			},
+			{
+				ID:              "sound-transit",
+				Name:            "Sound Transit",
+				DefaultAgencyID: "40",
+				URL:             "https://example.com/bundles/sound-transit.zip",
+			},
+		},
+		StopConsolidationURL: "https://example.com/bundles/stop-consolidation.txt",
+	}
+	assert.Equal(t, want, manifest)
+
+	wantStaticFeeds := []appconf.GtfsStaticFeed{
+		{ID: "metro", URL: "https://example.com/bundles/metro.zip", DefaultAgencyID: "1"},
+		{ID: "sound-transit", URL: "https://example.com/bundles/sound-transit.zip", DefaultAgencyID: "40"},
+	}
+	assert.Equal(t, wantStaticFeeds, manifest.StaticFeeds())
+}
+
+func TestParse_Validation(t *testing.T) {
+	const validSHA256 = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"
+	feedA := `{"id": "a", "defaultAgencyId": "1", "url": "https://example.com/a.zip"}`
+
+	tests := []struct {
+		name     string
+		manifest string
+		wantErr  string
+	}{
+		{
+			name:     "minimal manifest without consolidation url",
+			manifest: `{"version": 1, "feeds": [` + feedA + `]}`,
+		},
+		{
+			name:     "uppercase sha256 is accepted",
+			manifest: `{"version": 1, "feeds": [{"id": "a", "defaultAgencyId": "1", "url": "https://example.com/a.zip", "sha256": "` + strings.ToUpper(validSHA256) + `"}]}`,
+		},
+		{
+			name:     "malformed JSON",
+			manifest: `{"version": 1,`,
+			wantErr:  "invalid bundle inputs JSON",
+		},
+		{
+			name:     "trailing data after the manifest",
+			manifest: `{"version": 1, "feeds": [` + feedA + `]} {}`,
+			wantErr:  "unexpected data after the manifest",
+		},
+		{
+			name:     "missing version",
+			manifest: `{"feeds": [` + feedA + `]}`,
+			wantErr:  "unsupported bundle inputs version 0",
+		},
+		{
+			name:     "future version",
+			manifest: `{"version": 2, "feeds": [` + feedA + `]}`,
+			wantErr:  "unsupported bundle inputs version 2",
+		},
+		{
+			name:     "no feeds",
+			manifest: `{"version": 1, "feeds": []}`,
+			wantErr:  "feeds cannot be empty",
+		},
+		{
+			name:     "missing id",
+			manifest: `{"version": 1, "feeds": [{"defaultAgencyId": "1", "url": "https://example.com/a.zip"}]}`,
+			wantErr:  "feeds[0]: id is required",
+		},
+		{
+			name:     "missing url",
+			manifest: `{"version": 1, "feeds": [{"id": "a", "defaultAgencyId": "1"}]}`,
+			wantErr:  `feeds[0]: url "" must be an http:// or https:// URL`,
+		},
+		{
+			name:     "local path url",
+			manifest: `{"version": 1, "feeds": [{"id": "a", "defaultAgencyId": "1", "url": "/etc/passwd"}]}`,
+			wantErr:  "must be an http:// or https:// URL",
+		},
+		{
+			name:     "missing defaultAgencyId",
+			manifest: `{"version": 1, "feeds": [{"id": "a", "url": "https://example.com/a.zip"}]}`,
+			wantErr:  "feeds[0]: defaultAgencyId is required",
+		},
+		{
+			name:     "defaultAgencyId with underscore",
+			manifest: `{"version": 1, "feeds": [{"id": "a", "defaultAgencyId": "1_2", "url": "https://example.com/a.zip"}]}`,
+			wantErr:  `bundle inputs feeds "a": default-agency-id "1_2" must not contain '_'`,
+		},
+		{
+			name:     "duplicate id",
+			manifest: `{"version": 1, "feeds": [` + feedA + `, {"id": "a", "defaultAgencyId": "2", "url": "https://example.com/b.zip"}]}`,
+			wantErr:  `duplicate bundle inputs feeds id: "a"`,
+		},
+		{
+			name:     "duplicate defaultAgencyId fails the shared static feed validator",
+			manifest: `{"version": 1, "feeds": [` + feedA + `, {"id": "b", "defaultAgencyId": "1", "url": "https://example.com/b.zip"}]}`,
+			wantErr:  `duplicate bundle inputs feeds default-agency-id: "1"`,
+		},
+		{
+			name:     "short sha256",
+			manifest: `{"version": 1, "feeds": [{"id": "a", "defaultAgencyId": "1", "url": "https://example.com/a.zip", "sha256": "abc123"}]}`,
+			wantErr:  "sha256 must be 64 hex characters",
+		},
+		{
+			name:     "non-hex sha256",
+			manifest: `{"version": 1, "feeds": [{"id": "a", "defaultAgencyId": "1", "url": "https://example.com/a.zip", "sha256": "` + strings.Repeat("z", 64) + `"}]}`,
+			wantErr:  "sha256 must be 64 hex characters",
+		},
+		{
+			name:     "negative byteSize",
+			manifest: `{"version": 1, "feeds": [{"id": "a", "defaultAgencyId": "1", "url": "https://example.com/a.zip", "byteSize": -1}]}`,
+			wantErr:  "byteSize cannot be negative",
+		},
+		{
+			name:     "local stopConsolidationUrl",
+			manifest: `{"version": 1, "feeds": [` + feedA + `], "stopConsolidationUrl": "stop-consolidation.txt"}`,
+			wantErr:  "stopConsolidationUrl",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			manifest, err := Parse(strings.NewReader(tt.manifest))
+
+			if tt.wantErr != "" {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), tt.wantErr)
+				assert.Nil(t, manifest)
+				return
+			}
+			require.NoError(t, err)
+			assert.NotEmpty(t, manifest.Feeds)
+		})
+	}
+}