diff --git a/internal/appconf/json_config.go b/internal/appconf/json_config.go index 502347388..6b01082b9 100644 --- a/internal/appconf/json_config.go +++ b/internal/appconf/json_config.go @@ -302,24 +302,30 @@ func (j *JSONConfig) validateStaticFeeds() error { if j.GtfsStaticFeeds != nil && len(j.GtfsStaticFeeds) == 0 { return fmt.Errorf("gtfs-static-feeds cannot be empty") } + return ValidateStaticFeeds(j.GtfsStaticFeeds, j.staticFeedsKey) +} +// ValidateStaticFeeds checks a static feed list wherever it came from: +// unique ids, unique default agencies, and valid fields in each feed. +// key names the list's source in errors. +func ValidateStaticFeeds(feeds []GtfsStaticFeed, key string) error { seenIDs := make(map[string]bool) seenDefaultAgencyIDs := make(map[string]bool) - for _, feed := range j.GtfsStaticFeeds { + for _, feed := range feeds { if seenIDs[feed.ID] { - return fmt.Errorf("duplicate %s id: %q", j.staticFeedsKey, feed.ID) + return fmt.Errorf("duplicate %s id: %q", key, feed.ID) } seenIDs[feed.ID] = true if feed.DefaultAgencyID != "" { if seenDefaultAgencyIDs[feed.DefaultAgencyID] { - return fmt.Errorf("duplicate %s default-agency-id: %q", j.staticFeedsKey, feed.DefaultAgencyID) + return fmt.Errorf("duplicate %s default-agency-id: %q", key, feed.DefaultAgencyID) } seenDefaultAgencyIDs[feed.DefaultAgencyID] = true } if err := feed.validate(); err != nil { - return fmt.Errorf("%s %q: %w", j.staticFeedsKey, feed.ID, err) + return fmt.Errorf("%s %q: %w", key, feed.ID, err) } } return nil @@ -355,13 +361,18 @@ func (feed GtfsStaticFeed) validate() error { if strings.HasPrefix(strings.ToLower(feed.URL), "file://") { return fmt.Errorf("file:// URLs are not allowed in url for security reasons") } - isHTTPURL := strings.HasPrefix(feed.URL, "http://") || strings.HasPrefix(feed.URL, "https://") - if isHTTPURL { + if IsHTTPURL(feed.URL) { return nil } return validatePath(feed.URL, "url") } +// IsHTTPURL reports whether source is an http:// or https:// URL rather +// than a local path. +func IsHTTPURL(source string) bool { + return strings.HasPrefix(source, "http://") || strings.HasPrefix(source, "https://") +} + // validatePath checks a file path for security issues func validatePath(path, fieldName string) error { if path == "" { diff --git a/internal/appconf/static_feeds_test.go b/internal/appconf/static_feeds_test.go index aee3e8881..900dd31b5 100644 --- a/internal/appconf/static_feeds_test.go +++ b/internal/appconf/static_feeds_test.go @@ -396,3 +396,22 @@ func TestToGtfsConfigData_WarnsAboutIgnoredStaticFeeds(t *testing.T) { }) } } + +func TestIsHTTPURL(t *testing.T) { + tests := []struct { + source string + want bool + }{ + {source: "https://example.com/a.zip", want: true}, + {source: "http://example.com/a.zip", want: true}, + {source: "./a.zip", want: false}, + {source: "/data/a.zip", want: false}, + {source: "file:///data/a.zip", want: false}, + {source: "HTTPS://example.com/a.zip", want: false}, + } + for _, tt := range tests { + t.Run(tt.source, func(t *testing.T) { + assert.Equal(t, tt.want, IsHTTPURL(tt.source)) + }) + } +} diff --git a/internal/bundleinputs/bundleinputs.go b/internal/bundleinputs/bundleinputs.go new file mode 100644 index 000000000..ab1d03506 --- /dev/null +++ b/internal/bundleinputs/bundleinputs.go @@ -0,0 +1,117 @@ +// Package bundleinputs reads the bundle-inputs.json manifest that obacloud +// publishes to list the static GTFS feeds a server should load. +package bundleinputs + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "io" + + "maglev.onebusaway.org/internal/appconf" +) + +// SupportedVersion is the only manifest version Parse accepts. +const SupportedVersion = 1 + +// sha256HexLength is the length of a SHA-256 digest written as hex. +const sha256HexLength = 64 + +// Manifest is a parsed bundle-inputs.json file. +type Manifest struct { + Version int `json:"version"` + Feeds []Feed `json:"feeds"` + // StopConsolidationURL is optional and empty when the manifest has none. + StopConsolidationURL string `json:"stopConsolidationUrl"` +} + +// Feed is one static feed in a manifest. Feed order is load order. +type Feed struct { + ID string `json:"id"` + Name string `json:"name"` + DefaultAgencyID string `json:"defaultAgencyId"` + URL string `json:"url"` + // ByteSize and SHA256 describe the published zip, for checking the + // download. Nothing checks them yet. + ByteSize int64 `json:"byteSize"` + SHA256 string `json:"sha256"` +} + +// Parse decodes and validates a manifest. Unknown keys are ignored. +func Parse(r io.Reader) (*Manifest, error) { + var manifest Manifest + decoder := json.NewDecoder(r) + if err := decoder.Decode(&manifest); err != nil { + return nil, fmt.Errorf("invalid bundle inputs JSON: %w", err) + } + if _, err := decoder.Token(); err != io.EOF { + return nil, fmt.Errorf("invalid bundle inputs JSON: unexpected data after the manifest") + } + if err := manifest.validate(); err != nil { + return nil, err + } + return &manifest, nil +} + +// StaticFeeds returns the manifest feeds as static feed config, in order. +func (m *Manifest) StaticFeeds() []appconf.GtfsStaticFeed { + feeds := make([]appconf.GtfsStaticFeed, 0, len(m.Feeds)) + for _, feed := range m.Feeds { + feeds = append(feeds, appconf.GtfsStaticFeed{ + ID: feed.ID, + URL: feed.URL, + DefaultAgencyID: feed.DefaultAgencyID, + }) + } + return feeds +} + +// manifestFeedsKey names the manifest feed list in shared validation errors. +const manifestFeedsKey = "bundle inputs feeds" + +// validate applies the manifest format rules, then the checks every static +// feed list gets. +func (m *Manifest) validate() error { + if m.Version != SupportedVersion { + return fmt.Errorf("unsupported bundle inputs version %d, want %d", m.Version, SupportedVersion) + } + if len(m.Feeds) == 0 { + return fmt.Errorf("bundle inputs feeds cannot be empty") + } + if m.StopConsolidationURL != "" && !appconf.IsHTTPURL(m.StopConsolidationURL) { + return fmt.Errorf("stopConsolidationUrl %q must be an http:// or https:// URL", m.StopConsolidationURL) + } + for i, feed := range m.Feeds { + if err := feed.validate(); err != nil { + return fmt.Errorf("bundle inputs feeds[%d]: %w", i, err) + } + } + return appconf.ValidateStaticFeeds(m.StaticFeeds(), manifestFeedsKey) +} + +// validate applies the manifest format rules to one feed. The url must be +// http(s): a manifest may come from a remote server, so it must not name +// paths on this machine. +func (feed Feed) validate() error { + if feed.ID == "" { + return fmt.Errorf("id is required") + } + if feed.DefaultAgencyID == "" { + return fmt.Errorf("defaultAgencyId is required") + } + if !appconf.IsHTTPURL(feed.URL) { + return fmt.Errorf("url %q must be an http:// or https:// URL", feed.URL) + } + if feed.SHA256 != "" && !isSHA256Hex(feed.SHA256) { + return fmt.Errorf("sha256 must be %d hex characters", sha256HexLength) + } + if feed.ByteSize < 0 { + return fmt.Errorf("byteSize cannot be negative") + } + return nil +} + +func isSHA256Hex(value string) bool { + _, err := hex.DecodeString(value) + return err == nil && len(value) == sha256HexLength +} diff --git a/internal/bundleinputs/bundleinputs_test.go b/internal/bundleinputs/bundleinputs_test.go new file mode 100644 index 000000000..c8e491b73 --- /dev/null +++ b/internal/bundleinputs/bundleinputs_test.go @@ -0,0 +1,166 @@ +package bundleinputs + +import ( + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "maglev.onebusaway.org/internal/appconf" +) + +const goldenManifestPath = "testdata/bundle-inputs.json" + +func TestParse_GoldenManifest(t *testing.T) { + file, err := os.Open(goldenManifestPath) + require.NoError(t, err) + defer file.Close() + + manifest, err := Parse(file) + + require.NoError(t, err) + want := &Manifest{ + Version: 1, + Feeds: []Feed{ + { + ID: "metro", + Name: "King County Metro", + DefaultAgencyID: "1", + URL: "https://example.com/bundles/metro.zip", + ByteSize: 1234567, + SHA256: "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", + }, + { + ID: "sound-transit", + Name: "Sound Transit", + DefaultAgencyID: "40", + URL: "https://example.com/bundles/sound-transit.zip", + }, + }, + StopConsolidationURL: "https://example.com/bundles/stop-consolidation.txt", + } + assert.Equal(t, want, manifest) + + wantStaticFeeds := []appconf.GtfsStaticFeed{ + {ID: "metro", URL: "https://example.com/bundles/metro.zip", DefaultAgencyID: "1"}, + {ID: "sound-transit", URL: "https://example.com/bundles/sound-transit.zip", DefaultAgencyID: "40"}, + } + assert.Equal(t, wantStaticFeeds, manifest.StaticFeeds()) +} + +func TestParse_Validation(t *testing.T) { + const validSHA256 = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + feedA := `{"id": "a", "defaultAgencyId": "1", "url": "https://example.com/a.zip"}` + + tests := []struct { + name string + manifest string + wantErr string + }{ + { + name: "minimal manifest without consolidation url", + manifest: `{"version": 1, "feeds": [` + feedA + `]}`, + }, + { + name: "uppercase sha256 is accepted", + manifest: `{"version": 1, "feeds": [{"id": "a", "defaultAgencyId": "1", "url": "https://example.com/a.zip", "sha256": "` + strings.ToUpper(validSHA256) + `"}]}`, + }, + { + name: "malformed JSON", + manifest: `{"version": 1,`, + wantErr: "invalid bundle inputs JSON", + }, + { + name: "trailing data after the manifest", + manifest: `{"version": 1, "feeds": [` + feedA + `]} {}`, + wantErr: "unexpected data after the manifest", + }, + { + name: "missing version", + manifest: `{"feeds": [` + feedA + `]}`, + wantErr: "unsupported bundle inputs version 0", + }, + { + name: "future version", + manifest: `{"version": 2, "feeds": [` + feedA + `]}`, + wantErr: "unsupported bundle inputs version 2", + }, + { + name: "no feeds", + manifest: `{"version": 1, "feeds": []}`, + wantErr: "feeds cannot be empty", + }, + { + name: "missing id", + manifest: `{"version": 1, "feeds": [{"defaultAgencyId": "1", "url": "https://example.com/a.zip"}]}`, + wantErr: "feeds[0]: id is required", + }, + { + name: "missing url", + manifest: `{"version": 1, "feeds": [{"id": "a", "defaultAgencyId": "1"}]}`, + wantErr: `feeds[0]: url "" must be an http:// or https:// URL`, + }, + { + name: "local path url", + manifest: `{"version": 1, "feeds": [{"id": "a", "defaultAgencyId": "1", "url": "/etc/passwd"}]}`, + wantErr: "must be an http:// or https:// URL", + }, + { + name: "missing defaultAgencyId", + manifest: `{"version": 1, "feeds": [{"id": "a", "url": "https://example.com/a.zip"}]}`, + wantErr: "feeds[0]: defaultAgencyId is required", + }, + { + name: "defaultAgencyId with underscore", + manifest: `{"version": 1, "feeds": [{"id": "a", "defaultAgencyId": "1_2", "url": "https://example.com/a.zip"}]}`, + wantErr: `bundle inputs feeds "a": default-agency-id "1_2" must not contain '_'`, + }, + { + name: "duplicate id", + manifest: `{"version": 1, "feeds": [` + feedA + `, {"id": "a", "defaultAgencyId": "2", "url": "https://example.com/b.zip"}]}`, + wantErr: `duplicate bundle inputs feeds id: "a"`, + }, + { + name: "duplicate defaultAgencyId fails the shared static feed validator", + manifest: `{"version": 1, "feeds": [` + feedA + `, {"id": "b", "defaultAgencyId": "1", "url": "https://example.com/b.zip"}]}`, + wantErr: `duplicate bundle inputs feeds default-agency-id: "1"`, + }, + { + name: "short sha256", + manifest: `{"version": 1, "feeds": [{"id": "a", "defaultAgencyId": "1", "url": "https://example.com/a.zip", "sha256": "abc123"}]}`, + wantErr: "sha256 must be 64 hex characters", + }, + { + name: "non-hex sha256", + manifest: `{"version": 1, "feeds": [{"id": "a", "defaultAgencyId": "1", "url": "https://example.com/a.zip", "sha256": "` + strings.Repeat("z", 64) + `"}]}`, + wantErr: "sha256 must be 64 hex characters", + }, + { + name: "negative byteSize", + manifest: `{"version": 1, "feeds": [{"id": "a", "defaultAgencyId": "1", "url": "https://example.com/a.zip", "byteSize": -1}]}`, + wantErr: "byteSize cannot be negative", + }, + { + name: "local stopConsolidationUrl", + manifest: `{"version": 1, "feeds": [` + feedA + `], "stopConsolidationUrl": "stop-consolidation.txt"}`, + wantErr: "stopConsolidationUrl", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + manifest, err := Parse(strings.NewReader(tt.manifest)) + + if tt.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + assert.Nil(t, manifest) + return + } + require.NoError(t, err) + assert.NotEmpty(t, manifest.Feeds) + }) + } +} diff --git a/internal/bundleinputs/fetch.go b/internal/bundleinputs/fetch.go new file mode 100644 index 000000000..a7d7364e4 --- /dev/null +++ b/internal/bundleinputs/fetch.go @@ -0,0 +1,95 @@ +package bundleinputs + +import ( + "bytes" + "context" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "time" + + "maglev.onebusaway.org/internal/appconf" + "maglev.onebusaway.org/internal/logging" +) + +const ( + // maxManifestBytes caps the manifest body. A real manifest lists a few + // dozen feeds and is a few kilobytes. + maxManifestBytes = 1 << 20 + fetchTimeout = 30 * time.Second +) + +// Fetch reads the manifest at source, an http(s) URL or a local path, and +// parses it. Errors name the source. +func Fetch(ctx context.Context, source string) (*Manifest, error) { + manifest, err := fetchAndParse(ctx, source) + if err != nil { + return nil, fmt.Errorf("bundle inputs %s: %w", source, err) + } + return manifest, nil +} + +func fetchAndParse(ctx context.Context, source string) (*Manifest, error) { + data, err := readSource(ctx, source) + if err != nil { + return nil, err + } + return Parse(bytes.NewReader(data)) +} + +// readSource returns the body at source, up to maxManifestBytes. +func readSource(ctx context.Context, source string) ([]byte, error) { + body, err := openSource(ctx, source) + if err != nil { + return nil, err + } + defer logging.SafeCloseWithLogging(body, + slog.Default().With(slog.String("component", "bundle_inputs")), + "bundle_inputs_body") + + data, err := io.ReadAll(io.LimitReader(body, maxManifestBytes+1)) + if err != nil { + return nil, err + } + if len(data) > maxManifestBytes { + return nil, fmt.Errorf("manifest exceeds size limit of %d bytes", maxManifestBytes) + } + return data, nil +} + +func openSource(ctx context.Context, source string) (io.ReadCloser, error) { + if !appconf.IsHTTPURL(source) { + return openLocalSource(source) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, source, nil) + if err != nil { + return nil, err + } + client := &http.Client{Timeout: fetchTimeout} + resp, err := client.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + _ = resp.Body.Close() + return nil, fmt.Errorf("received HTTP status %s", resp.Status) + } + return resp.Body, nil +} + +// openLocalSource opens a manifest on disk. Like the config file, it must +// be a regular file: a directory, device or FIFO is rejected instead of +// being read (a FIFO would block startup). +func openLocalSource(source string) (io.ReadCloser, error) { + info, err := os.Lstat(source) + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("manifest must be a regular file, not a %s", info.Mode().Type()) + } + return os.Open(source) +} diff --git a/internal/bundleinputs/fetch_test.go b/internal/bundleinputs/fetch_test.go new file mode 100644 index 000000000..136c1e5b3 --- /dev/null +++ b/internal/bundleinputs/fetch_test.go @@ -0,0 +1,94 @@ +package bundleinputs + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFetch(t *testing.T) { + golden, err := os.ReadFile(goldenManifestPath) + require.NoError(t, err) + + mux := http.NewServeMux() + mux.HandleFunc("/bundle-inputs.json", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(golden) + }) + mux.HandleFunc("/invalid.json", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"version": 2}`)) + }) + mux.HandleFunc("/too-large.json", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(strings.Repeat(" ", maxManifestBytes+1))) + }) + server := httptest.NewServer(mux) + defer server.Close() + + tests := []struct { + name string + source string + wantFeedIDs []string + wantErr string + }{ + { + name: "http URL", + source: server.URL + "/bundle-inputs.json", + wantFeedIDs: []string{"metro", "sound-transit"}, + }, + { + name: "local path", + source: goldenManifestPath, + wantFeedIDs: []string{"metro", "sound-transit"}, + }, + { + name: "HTTP error status", + source: server.URL + "/missing.json", + wantErr: "received HTTP status 404", + }, + { + name: "invalid manifest", + source: server.URL + "/invalid.json", + wantErr: "unsupported bundle inputs version 2", + }, + { + name: "body over the size limit", + source: server.URL + "/too-large.json", + wantErr: "exceeds size limit", + }, + { + name: "missing local file", + source: filepath.Join(t.TempDir(), "missing.json"), + wantErr: "no such file", + }, + { + name: "local directory", + source: t.TempDir(), + wantErr: "manifest must be a regular file", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + manifest, err := Fetch(context.Background(), tt.source) + + if tt.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + assert.Contains(t, err.Error(), tt.source, "the error names the manifest source") + return + } + require.NoError(t, err) + var gotFeedIDs []string + for _, feed := range manifest.Feeds { + gotFeedIDs = append(gotFeedIDs, feed.ID) + } + assert.Equal(t, tt.wantFeedIDs, gotFeedIDs) + }) + } +} diff --git a/internal/bundleinputs/testdata/bundle-inputs.json b/internal/bundleinputs/testdata/bundle-inputs.json new file mode 100644 index 000000000..55c711b4c --- /dev/null +++ b/internal/bundleinputs/testdata/bundle-inputs.json @@ -0,0 +1,21 @@ +{ + "version": 1, + "feeds": [ + { + "id": "metro", + "key": "kcm", + "name": "King County Metro", + "defaultAgencyId": "1", + "url": "https://example.com/bundles/metro.zip", + "byteSize": 1234567, + "sha256": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + }, + { + "id": "sound-transit", + "name": "Sound Transit", + "defaultAgencyId": "40", + "url": "https://example.com/bundles/sound-transit.zip" + } + ], + "stopConsolidationUrl": "https://example.com/bundles/stop-consolidation.txt" +} diff --git a/internal/gtfs/config.go b/internal/gtfs/config.go index 722711cf1..066ce8357 100644 --- a/internal/gtfs/config.go +++ b/internal/gtfs/config.go @@ -1,7 +1,6 @@ package gtfs import ( - "strings" "time" "maglev.onebusaway.org/internal/appconf" @@ -83,5 +82,5 @@ func (config Config) feedAgencyFilters() map[string]map[string]bool { } func (config Config) isLocalFile() bool { - return !strings.HasPrefix(config.GtfsURL, "http://") && !strings.HasPrefix(config.GtfsURL, "https://") + return !appconf.IsHTTPURL(config.GtfsURL) }