Cache complete workflow run metadata in logs (#59257)

Author: Copilot

docs/adr/59257-cache-complete-workflow-run-metadata-in-logs.md

@@ -0,0 +1,50 @@
+# ADR-59257: Cache complete workflow run metadata in logs
+
+**Date**: 2026-09-07
+**Status**: Draft
+**Deciders**: gh-aw maintainers
+
+---
+
+### Context
+
+`gh aw logs` previously omitted workflow-run metadata that downstream consumers had to retrieve separately with `gh api`. This PR updates the logs processing pipeline under `pkg/cli/` to fetch the full GitHub Actions run response, persist it as `run.json` beside each run's cached artifacts, and reuse that cache when possible. The diff also changes report-building logic so repository, actor, event, SHA, workflow path, and run attempt data come from GitHub metadata first and are overridden only by non-empty `aw_info.json` values. The repository needs an explicit decision for whether workflow run metadata should be treated as a first-class cached input of the logs subsystem rather than as an optional post-processing lookup.
+
+### Decision
+
+We will fetch the complete GitHub Actions workflow-run API response during `gh aw logs` processing, cache it per run as `run.json`, and use it as the baseline source for run metadata in logs reports. We will preserve and refresh that cache for both newly downloaded and previously cached runs, and we will exclude `run.json` from artifact inventories while retaining it during storage pruning. Non-empty `aw_info.json` fields will remain authoritative overrides so workflow-emitted metadata can correct or augment GitHub-provided values without discarding the GitHub baseline.
+
+### Alternatives Considered
+
+#### Alternative 1: Keep metadata uncached and require downstream consumers to call `gh api`
+
+This matches the previous behavior where logs output omitted some workflow-run metadata and consumers had to fetch it separately. It was considered because it keeps the logs pipeline simpler and avoids storing another API response on disk. It was not chosen because the PR description and diff both show repeated downstream need for repository, actor, event, attempt, commit, and workflow-path data, making separate follow-up API calls redundant and harder to reuse.
+
+#### Alternative 2: Cache only a reduced summary instead of the full workflow-run API response
+
+The project could translate the GitHub response directly into `run_summary.json` fields and avoid storing raw API output. This was considered because it would reduce stored data volume and keep the cache schema narrower. It was not chosen because the diff explicitly adds `run.json`, validates malformed or mismatched cached responses, and uses the raw response as a reusable source of truth that can backfill older summaries when metadata fields are later needed.
+
+#### Alternative 3: Treat `aw_info.json` as the sole source of metadata
+
+Another option would be to continue relying on workflow-emitted metadata files for repository, actor, SHA, event, and attempt data. This was considered because `aw_info.json` is already part of the logs ecosystem and can contain workflow-specific context. It was not chosen because `aw_info.json` may be absent or partially populated, and the diff intentionally changes precedence so GitHub API metadata provides a reliable baseline while `aw_info.json` overrides only non-empty fields.
+
+### Consequences
+
+#### Positive
+- Downstream logs consumers can read complete run metadata from cached output without making separate GitHub API calls.
+- Newly downloaded runs and previously cached runs share the same metadata source and healing behavior, improving consistency of `run_summary.json` and report output.
+- The logs report gains richer repository, organization, actor, attempt, SHA, and event fields even when `aw_info.json` is missing or incomplete.
+
+#### Negative
+- Each processed run now requires an additional GitHub Actions API request and an extra cached file, increasing API usage and local storage.
+- The logs subsystem must maintain parsing, cache validation, and merge logic for raw workflow-run responses in addition to existing summary and jobs caches.
+- Metadata precedence is more complex because the system must combine GitHub baseline fields with selective `aw_info.json` overrides.
+
+#### Neutral
+- `run.json` becomes a preserved internal cache artifact that is intentionally excluded from artifact listings shown to users.
+- Existing cached runs may be rewritten as metadata is backfilled or healed, even when no new workflow artifacts are downloaded.
+- Tests now need to cover both cache reuse behavior and metadata precedence between GitHub responses and `aw_info.json`.
+
+---
+
+*ADR created by [adr-writer agent]. Review and finalize before changing status from Draft to Accepted.*

pkg/cli/logs_download.go

@@ -385,7 +385,11 @@ func downloadArtifactsIndividually(ctx context.Context, opts downloadArtifactsOp
 	if err := downloadArtifactsByName(ctx, opts, downloadableNames); err != nil {
 		return err
 	}
-	if fileutil.IsDirEmpty(opts.outputDir) {
+	artifacts, err := listArtifacts(opts.outputDir)
+	if err != nil {
+		return err
+	}
+	if len(artifacts) == 0 {
 		// Downloads were attempted but none succeeded; treat as no artifacts.
 		return ErrNoArtifacts
 	}
@@ -429,9 +433,9 @@ func buildBulkDownloadArgs(opts downloadArtifactsOptions) []string {
 	ghArgs := []string{"run", "download", strconv.FormatInt(opts.runID, 10), "--dir", opts.outputDir}
 	if opts.owner != "" && opts.repo != "" {
 		if opts.hostname != "" && opts.hostname != "github.com" {
-			ghArgs = append(ghArgs, "-R", opts.hostname+"/"+opts.owner+"/"+opts.repo)
+			ghArgs = append(ghArgs, "-R", filepath.Join(opts.hostname, opts.owner, opts.repo))
 		} else {
-			ghArgs = append(ghArgs, "-R", opts.owner+"/"+opts.repo)
+			ghArgs = append(ghArgs, "-R", filepath.Join(opts.owner, opts.repo))
 		}
 	}
 	return ghArgs
@@ -493,7 +497,11 @@ func recoverBulkDownloadArtifacts(ctx context.Context, opts downloadArtifactsOpt
 		}
 	}
 
-	if skippedNonZipArtifacts && fileutil.IsDirEmpty(opts.outputDir) {
+	artifacts, err := listArtifacts(opts.outputDir)
+	if err != nil {
+		return err
+	}
+	if skippedNonZipArtifacts && len(artifacts) == 0 {
 		// All artifacts were non-zip (none could be extracted) so nothing was downloaded.
 		// Treat this the same as a run with no artifacts — the audit will rely solely on
 		// workflow logs rather than artifact content.

pkg/cli/logs_download_artifacts.go

@@ -42,8 +42,7 @@ func listArtifacts(outputDir string) ([]string, error) {
 			return err
 		}
 
-		// Skip directories and synthesized cache/summary files
-		if info.IsDir() || filepath.Base(path) == runSummaryFileName || filepath.Base(path) == jobsAPIResponseFileName {
+		if info.IsDir() {
 			return nil
 		}
 
@@ -52,6 +51,10 @@ func listArtifacts(outputDir string) ([]string, error) {
 		if err != nil {
 			return err
 		}
+		// Skip only root-level synthesized cache/summary files.
+		if relPath == runSummaryFileName || relPath == jobsAPIResponseFileName || relPath == runAPIResponseFileName {
+			return nil
+		}
 
 		artifacts = append(artifacts, relPath)
 		return nil

pkg/cli/logs_github_api.go

@@ -195,6 +195,146 @@ func writeSensitiveFile(path string, data []byte) (err error) {
 	return nil
 }
 
+type workflowRunAPIResponse struct {
+	ID           int64     `json:"id"`
+	RunNumber    int       `json:"run_number"`
+	RunAttempt   int       `json:"run_attempt"`
+	HTMLURL      string    `json:"html_url"`
+	Status       string    `json:"status"`
+	Conclusion   string    `json:"conclusion"`
+	Name         string    `json:"name"`
+	Path         string    `json:"path"`
+	CreatedAt    time.Time `json:"created_at"`
+	RunStartedAt time.Time `json:"run_started_at"`
+	UpdatedAt    time.Time `json:"updated_at"`
+	Event        string    `json:"event"`
+	HeadBranch   string    `json:"head_branch"`
+	HeadSHA      string    `json:"head_sha"`
+	DisplayTitle string    `json:"display_title"`
+	Actor        struct {
+		Login string `json:"login"`
+	} `json:"actor"`
+	Repository struct {
+		FullName string `json:"full_name"`
+	} `json:"repository"`
+}
+
+func parseWorkflowRunAPIResponse(data []byte) (WorkflowRun, error) {
+	var response workflowRunAPIResponse
+	if err := json.Unmarshal(data, &response); err != nil {
+		return WorkflowRun{}, fmt.Errorf("failed to parse workflow run API response: %w", err)
+	}
+	return WorkflowRun{
+		DatabaseID:   response.ID,
+		Number:       response.RunNumber,
+		URL:          response.HTMLURL,
+		Status:       response.Status,
+		Conclusion:   response.Conclusion,
+		WorkflowName: response.Name,
+		WorkflowPath: response.Path,
+		CreatedAt:    response.CreatedAt,
+		StartedAt:    response.RunStartedAt,
+		UpdatedAt:    response.UpdatedAt,
+		Event:        response.Event,
+		HeadBranch:   response.HeadBranch,
+		HeadSha:      response.HeadSHA,
+		DisplayTitle: response.DisplayTitle,
+		Attempt:      response.RunAttempt,
+		Repository:   response.Repository.FullName,
+		Actor:        response.Actor.Login,
+	}, nil
+}
+
+func applyWorkflowRunMetadata(run *WorkflowRun, metadata WorkflowRun) bool {
+	applied := applyNonZero(&run.DatabaseID, metadata.DatabaseID)
+	applied = applyNonZero(&run.Number, metadata.Number) || applied
+	applied = applyNonZero(&run.URL, metadata.URL) || applied
+	applied = applyNonZero(&run.Status, metadata.Status) || applied
+	applied = applyNonZero(&run.Conclusion, metadata.Conclusion) || applied
+	if !strings.HasPrefix(metadata.WorkflowName, constants.GithubDir) || run.WorkflowName == "" {
+		applied = applyNonZero(&run.WorkflowName, metadata.WorkflowName) || applied
+	}
+	applied = applyNonZero(&run.WorkflowPath, metadata.WorkflowPath) || applied
+	applied = applyNonZero(&run.CreatedAt, metadata.CreatedAt) || applied
+	applied = applyNonZero(&run.StartedAt, metadata.StartedAt) || applied
+	applied = applyNonZero(&run.UpdatedAt, metadata.UpdatedAt) || applied
+	applied = applyNonZero(&run.Event, metadata.Event) || applied
+	applied = applyNonZero(&run.HeadBranch, metadata.HeadBranch) || applied
+	applied = applyNonZero(&run.HeadSha, metadata.HeadSha) || applied
+	applied = applyNonZero(&run.DisplayTitle, metadata.DisplayTitle) || applied
+	applied = applyNonZero(&run.Attempt, metadata.Attempt) || applied
+	applied = applyNonZero(&run.Repository, metadata.Repository) || applied
+	applied = applyNonZero(&run.Actor, metadata.Actor) || applied
+	return applied
+}
+
+func applyNonZero[T comparable](target *T, value T) bool {
+	var zero T
+	if value == zero {
+		return false
+	}
+	changed := *target != value
+	*target = value
+	return changed
+}
+
+func fetchAndCacheWorkflowRunMetadata(ctx context.Context, currentRun WorkflowRun, outputDir, owner, repo, hostname string, verbose bool) (WorkflowRun, error) {
+	responsePath := filepath.Join(outputDir, runAPIResponseFileName)
+	if output, err := os.ReadFile(responsePath); err == nil {
+		if run, parseErr := parseWorkflowRunAPIResponse(output); parseErr == nil && cachedWorkflowRunMetadataIsCurrent(run, currentRun, owner, repo) {
+			return run, nil
+		}
+		logsGitHubAPILog.Printf("Ignoring invalid cached workflow run API response: path=%s", responsePath)
+	} else if !errors.Is(err, os.ErrNotExist) {
+		return WorkflowRun{}, fmt.Errorf("failed to read cached workflow run API response: %w", err)
+	}
+
+	args := buildWorkflowRunAPIArgs(currentRun.DatabaseID, owner, repo, hostname)
+	if verbose {
+		fmt.Fprintln(os.Stderr, console.FormatVerboseMessage("Executing: gh "+strings.Join(args, " ")))
+	}
+	output, err := workflow.RunGHCombinedContext(ctx, "Fetching run metadata...", args...)
+	if err != nil {
+		return WorkflowRun{}, classifyWorkflowRunMetadataError(currentRun.DatabaseID, err, output)
+	}
+	run, err := parseWorkflowRunAPIResponse(output)
+	if err != nil {
+		return WorkflowRun{}, err
+	}
+	if err := writeSensitiveFile(responsePath, output); err != nil {
+		return WorkflowRun{}, fmt.Errorf("failed to cache workflow run API response: %w", err)
+	}
+	return run, nil
+}
+
+func workflowRunMetadataCacheNeedsRefresh(outputDir string, currentRun WorkflowRun, owner, repo string) (bool, error) {
+	output, err := os.ReadFile(filepath.Join(outputDir, runAPIResponseFileName))
+	if errors.Is(err, os.ErrNotExist) {
+		return true, nil
+	}
+	if err != nil {
+		return false, fmt.Errorf("failed to read cached workflow run API response: %w", err)
+	}
+	cached, err := parseWorkflowRunAPIResponse(output)
+	return err != nil || !cachedWorkflowRunMetadataIsCurrent(cached, currentRun, owner, repo), nil
+}
+
+func cachedWorkflowRunMetadataIsCurrent(cached, current WorkflowRun, owner, repo string) bool {
+	if cached.DatabaseID != current.DatabaseID {
+		return false
+	}
+	if owner != "" && repo != "" && !strings.EqualFold(cached.Repository, filepath.Join(owner, repo)) {
+		return false
+	}
+	if current.Attempt > 0 && cached.Attempt != current.Attempt {
+		return false
+	}
+	if !current.UpdatedAt.IsZero() && cached.UpdatedAt.Before(current.UpdatedAt) {
+		return false
+	}
+	return cached.Status == "completed"
+}
+
 // fetchJobDetails gets detailed job information including durations for a workflow run.
 // Errors from the underlying API call are suppressed so that callers can continue
 // processing even when job data is unavailable (e.g. missing permissions).

pkg/cli/logs_github_api_test.go

@@ -42,6 +42,65 @@ func TestWorkflowRunUnmarshal(t *testing.T) {
 	assert.Empty(t, runs[0].WorkflowPath, "WorkflowPath should be empty when 'path' field is absent")
 }
 
+func TestFetchAndCacheWorkflowRunMetadata(t *testing.T) {
+	fakeBinDir := testutil.TempDir(t, "fake-gh-*")
+	outputDir := t.TempDir()
+	fakeGH := filepath.Join(fakeBinDir, "gh")
+	argsLogPath := filepath.Join(fakeBinDir, "gh-args.log")
+	fakeGHScript := "#!/bin/sh\n" +
+		"printf '%s\\n' \"$*\" >> \"" + argsLogPath + "\"\n" +
+		"cat <<'EOF'\n" +
+		`{"id":42,"run_number":7,"run_attempt":3,"html_url":"https://github.com/octo/repo/actions/runs/42","status":"completed","conclusion":"success","name":"Daily report","path":".github/workflows/daily-report.lock.yml","created_at":"2026-09-01T10:00:00Z","run_started_at":"2026-09-01T10:00:01Z","updated_at":"2026-09-01T10:02:00Z","event":"schedule","head_branch":"main","head_sha":"abc123","display_title":"Daily report","actor":{"login":"octocat"},"repository":{"full_name":"octo/repo"}}` + "\n" +
+		"EOF\n"
+	require.NoError(t, os.WriteFile(fakeGH, []byte(fakeGHScript), 0o755))
+	t.Setenv("PATH", fakeBinDir+string(os.PathListSeparator)+os.Getenv("PATH"))
+
+	currentRun := WorkflowRun{DatabaseID: 42, Attempt: 3, Status: "completed"}
+	run, err := fetchAndCacheWorkflowRunMetadata(context.Background(), currentRun, outputDir, "octo", "repo", "", false)
+	require.NoError(t, err)
+	assert.Equal(t, int64(42), run.DatabaseID)
+	assert.Equal(t, 3, run.Attempt)
+	assert.Equal(t, ".github/workflows/daily-report.lock.yml", run.WorkflowPath)
+	assert.Equal(t, "octo/repo", run.Repository)
+	assert.Equal(t, "octocat", run.Actor)
+
+	cachePath := filepath.Join(outputDir, runAPIResponseFileName)
+	cached, err := os.ReadFile(cachePath)
+	require.NoError(t, err)
+	assert.Contains(t, string(cached), `"run_attempt":3`)
+
+	require.NoError(t, os.Remove(fakeGH))
+	cachedRun, err := fetchAndCacheWorkflowRunMetadata(context.Background(), currentRun, outputDir, "octo", "repo", "", false)
+	require.NoError(t, err, "the second read should use the cache without invoking gh")
+	assert.Equal(t, run, cachedRun)
+
+	argsLog, err := os.ReadFile(argsLogPath)
+	require.NoError(t, err)
+	assert.Equal(t, "api repos/octo/repo/actions/runs/42\n", string(argsLog))
+
+	needsRefresh, err := workflowRunMetadataCacheNeedsRefresh(outputDir, currentRun, "octo", "other-repo")
+	require.NoError(t, err)
+	assert.True(t, needsRefresh, "a cache entry from another repository must be refreshed")
+
+	needsRefresh, err = workflowRunMetadataCacheNeedsRefresh(outputDir, WorkflowRun{
+		DatabaseID: 42,
+		Attempt:    4,
+		Status:     "completed",
+	}, "octo", "repo")
+	require.NoError(t, err)
+	assert.True(t, needsRefresh, "a cache entry from another attempt must be refreshed")
+
+	require.NoError(t, os.WriteFile(cachePath, []byte(strings.Replace(string(cached), `"status":"completed"`, `"status":"in_progress"`, 1)), 0o600))
+	needsRefresh, err = workflowRunMetadataCacheNeedsRefresh(outputDir, currentRun, "octo", "repo")
+	require.NoError(t, err)
+	assert.True(t, needsRefresh, "a mutable cache entry must be refreshed")
+
+	metadataApplied := applyWorkflowRunMetadata(&WorkflowRun{WorkflowName: "Daily report"}, WorkflowRun{
+		WorkflowName: ".github/workflows/daily-report.lock.yml",
+	})
+	assert.False(t, metadataApplied, "a path-like API name must not replace a resolved display name")
+}
+
 // TestBuildCreatedFilter verifies that buildCreatedFilter always produces a single
 // --created expression that enforces all supplied date bounds. The key invariant is that
 // StartDate is never silently dropped, which was the root cause of the bug where runs

pkg/cli/logs_json_test.go

@@ -12,6 +12,8 @@ import (
 	"time"
 
 	"github.com/github/gh-aw/pkg/testutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 // TestBuildLogsData tests the structured data creation for logs
@@ -994,6 +996,52 @@ func TestBuildLogsDataOrganizationEmptyWhenNoRepository(t *testing.T) {
 	}
 }
 
+func TestBuildLogsDataUsesGitHubRunMetadataWithoutAwInfo(t *testing.T) {
+	t.Parallel()
+	runDir := t.TempDir()
+	logsData := buildLogsData([]ProcessedRun{{
+		Run: WorkflowRun{
+			DatabaseID:   9003,
+			WorkflowName: "daily-report",
+			WorkflowPath: ".github/workflows/daily-report.lock.yml",
+			Status:       "completed",
+			Conclusion:   "failure",
+			Repository:   "myorg/myrepo",
+			Actor:        "octocat",
+			Attempt:      2,
+			HeadSha:      "abc123",
+			Event:        "schedule",
+			LogsPath:     runDir,
+		},
+	}}, runDir, nil)
+
+	require.Len(t, logsData.Runs, 1)
+	run := logsData.Runs[0]
+	assert.Equal(t, "myorg/myrepo", run.Repository)
+	assert.Equal(t, "myorg", run.Organization)
+	assert.Equal(t, "octocat", run.Actor)
+	assert.Equal(t, "2", run.RunAttempt)
+	assert.Equal(t, "abc123", run.SHA)
+	assert.Equal(t, "schedule", run.EventName)
+}
+
+func TestApplyGitHubMetadataToRunDataPreservesExistingValues(t *testing.T) {
+	t.Parallel()
+	runData := RunData{
+		Repository: "myorg/myrepo",
+		SHA:        "abc123",
+		Actor:      "octocat",
+		EventName:  "schedule",
+	}
+
+	applyGitHubMetadataToRunData(&runData, WorkflowRun{})
+
+	assert.Equal(t, "myorg/myrepo", runData.Repository)
+	assert.Equal(t, "abc123", runData.SHA)
+	assert.Equal(t, "octocat", runData.Actor)
+	assert.Equal(t, "schedule", runData.EventName)
+}
+
 // TestInferWorkflowPathFromDisplayName verifies that display names are correctly
 // slugified into conventional lock-file paths.
 func TestInferWorkflowPathFromDisplayName(t *testing.T) {