diff --git a/.github/workflows/breakage-suite.yml b/.github/workflows/breakage-suite.yml index b952c2f..a9c9261 100644 --- a/.github/workflows/breakage-suite.yml +++ b/.github/workflows/breakage-suite.yml @@ -1,6 +1,10 @@ name: Breakage suite -# Tests the downstream packages against the package under test, then reports once. Each +# Tests the downstream packages against the package under test, then reports once. The +# downstream packages are derived, not listed: every package of the organization that loads +# the package under test (its `deps`, `weakdeps`, `extras` or test environment, read from the +# Project.toml files by scripts/breakage/consumers.jl), plus the `targets` the caller declares +# for what cannot be derived (an application owned elsewhere, a documentation build). Each # cell of the matrix (a downstream package × its default branch `latest` or its newest tag # `stable`) runs its tests or builds its docs, and writes its result as a file; one `report` # job gathers them in one table (job summary, one PR comment per group, kept up to date @@ -19,9 +23,15 @@ on: workflow_call: inputs: targets: - description: 'JSON list of the downstream packages: {"name", "owner" (default control-toolbox), "break" ("test", default, or "doc")}' - required: true + description: 'JSON list of declared targets, added to the derived ones: {"name", "owner" (default control-toolbox), "break" ("test", default, or "doc")}' + required: false type: string + default: '[]' + consumers: + description: 'Derive the downstream packages from the Project.toml files of the organization' + required: false + type: boolean + default: true versions: description: 'JSON list of the versions of each downstream package' required: false @@ -46,7 +56,43 @@ on: required: false jobs: + plan: + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + outputs: + targets: ${{ steps.plan.outputs.targets }} + steps: + - uses: actions/checkout@v6 + + - uses: actions/checkout@v6 + with: + repository: control-toolbox/CTActions + ref: ${{ inputs.ctactions_ref }} + path: .ctactions + + - uses: julia-actions/setup-julia@v2 + with: + version: '1' + + # fails, rather than testing fewer packages, when the organization cannot be read or + # when there is nothing to test; the table of targets goes to the job summary + - name: Targets + id: plan + env: + GH_TOKEN: ${{ github.token }} + DECLARED: ${{ inputs.targets }} + DERIVE: ${{ inputs.consumers }} + run: | + set -euo pipefail + args=(plan --project Project.toml --declared "$DECLARED" --summary "$GITHUB_STEP_SUMMARY") + [ "$DERIVE" = true ] || args+=(--no-derive) + targets=$(julia --startup-file=no .ctactions/scripts/breakage/consumers.jl "${args[@]}") + echo "targets=$targets" >> "$GITHUB_OUTPUT" + break: + needs: plan runs-on: ubuntu-latest timeout-minutes: 60 permissions: @@ -54,7 +100,7 @@ jobs: strategy: fail-fast: false matrix: - target: ${{ fromJSON(inputs.targets) }} + target: ${{ fromJSON(needs.plan.outputs.targets) }} version: ${{ fromJSON(inputs.versions) }} env: NAME: ${{ matrix.target.name }} @@ -138,8 +184,8 @@ jobs: path: cells/ report: - needs: break - if: always() + needs: [plan, break] + if: ${{ always() && needs.plan.result == 'success' }} runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -189,7 +235,7 @@ jobs: - name: Report id: report env: - TARGETS: ${{ inputs.targets }} + TARGETS: ${{ needs.plan.outputs.targets }} VERSIONS: ${{ inputs.versions }} GROUP: ${{ inputs.group }} BASE_SHA: ${{ github.event.pull_request.base.sha }} diff --git a/docs/README.md b/docs/README.md index 19f0651..effc292 100644 --- a/docs/README.md +++ b/docs/README.md @@ -75,7 +75,7 @@ token. | [`coverage.yml`](workflows/coverage.md) | `Coverage.yml` | Test suite with coverage, uploaded to Codecov | no (push and tags) | | [`documentation.yml`](workflows/documentation.md) | `Documentation.yml` | Build and deploy the documentation; optional GPU upgrade pass; opt-in PR previews | label `run documentation` | | [`documentation-preview-cleanup.yml`](workflows/documentation-preview-cleanup.md) | `DocPreviewCleanup.yml` | Remove the previews of closed PRs from `gh-pages` | on close | -| [`breakage-suite.yml`](workflows/breakage-suite.md) | `Breakage.yml` | Test downstream packages against the PR, report once (one comment per group, an artifact); on a release PR, check REL1 against the run on the base | label `run breakage`; push to `main` | +| [`breakage-suite.yml`](workflows/breakage-suite.md) | `Breakage.yml` | Test the downstream packages (derived from the `Project.toml` files) against the PR, report once (one comment per group, an artifact); on a release PR, check REL1 against the run on the base | label `run breakage`; push to `main` | | [`breakage.yml`](workflows/breakage.md) | `Breakage.yml` | Former version of `breakage-suite.yml`, kept until every caller has moved | label `run breakage` | | [`formatter.yml`](workflows/formatter.md) | `Formatter.yml` | JuliaFormatter (BlueStyle), as an automatic PR | no (nightly) | | [`spell-check.yml`](workflows/spell-check.md) | `SpellCheck.yml` | Spell check with `typos` | every PR | diff --git a/docs/workflows/breakage-suite.md b/docs/workflows/breakage-suite.md index d98e625..ebba686 100644 --- a/docs/workflows/breakage-suite.md +++ b/docs/workflows/breakage-suite.md @@ -11,6 +11,13 @@ every caller has moved. ## What it does +- **`plan`**, once: the targets of the suite. They are **derived** by + [`scripts/breakage/consumers.jl`](../../scripts/breakage/consumers.jl), not listed in the + caller: every package of the organization that loads the package under test, read from + the `Project.toml` files (see [Which packages are tested](#which-packages-are-tested)), + plus the `targets` the caller declares. The table of targets, with where each comes from, + is in the job summary. The job fails, and nothing else runs, when the organization cannot + be read or when there is nothing to test. - **`break`**, one job per cell: a downstream package × `latest` (its default branch) or `stable` (its newest tag, a beta included). The cell runs the package's tests (`test`) or builds its docs (`doc`) with the package under test `dev`ed, and writes its result @@ -57,23 +64,52 @@ jobs: (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run breakage')) uses: control-toolbox/CTActions/.github/workflows/breakage-suite.yml@main with: - targets: >- - [{"name": "CTDirect"}, {"name": "CTFlows"}, {"name": "CTModels"}, - {"name": "CTParser"}, {"name": "CTSolvers"}, {"name": "OptimalControl"}] use_ct_registry: true secrets: SSH_KEY: ${{ secrets.SSH_KEY }} ``` -A package with two suites (OptimalControl: `run breakage` for its packages, `run breakage -applications` for the applications) calls it twice, with two `group`s; a heavy suite may -leave out the `push` trigger and be run by hand on `main` before a release. +No list of packages: the consumers are derived. A package with two suites (OptimalControl: +`run breakage` for its packages, `run breakage applications` for the applications) calls it +twice, with two `group`s; the second one declares its `targets` and sets `consumers: false` +(its applications are not packages that load OptimalControl). A heavy suite may leave out +the `push` trigger and be run by hand on `main` before a release. + +## Which packages are tested + +A package of the organization (a non-archived, non-fork repository `X.jl` whose +`Project.toml` has a `name` and a `uuid`) is a **consumer** of P, by uuid, when P is in: + +| Where | Why it counts | +| --- | --- | +| `[deps]` of `Project.toml` | the package loads P | +| `[weakdeps]` of `Project.toml` | an extension loads P | +| `[extras]` of `Project.toml` | its tests (or a test target) load P | +| `[deps]` of `test/Project.toml` | its tests load P | + +Left out: P itself, archived repositories, forks, and the repositories listed in `EXCLUDED` +in `consumers.jl`, each with its reason (the template of a new package; a package whose +compat is out of date until it is updated). What the `Project.toml` files do not show is +declared by the caller in `targets`: an application owned outside the organization, a +documentation build (`"break": "doc"`). A declared target replaces the derived one of the +same package, owner and check. + +```bash +# what a package would get, from its checkout (needs `gh`, authenticated) +julia scripts/breakage/consumers.jl list --project Project.toml +julia scripts/breakage/consumers.jl plan --project Project.toml --declared '[{"name": "Tutorials", "break": "doc"}]' +``` + +The `Project.toml` files read are those of the default branch of each package, at the time +of the run: a package that starts to load P is tested from then on, and the report says +`has no result on the base (a new target)` for it on the first release PR. ## Inputs and secrets | Input | Default | Meaning | | --- | --- | --- | -| `targets` | required | JSON list of `{"name", "owner", "break"}`: the package without `.jl`, its GitHub owner (default `control-toolbox`), `test` (default) or `doc` | +| `targets` | `'[]'` | JSON list of declared targets, added to the derived ones: `{"name", "owner", "break"}`, the package without `.jl`, its GitHub owner (default `control-toolbox`), `test` (default) or `doc` | +| `consumers` | `true` | Derive the consumers from the `Project.toml` files; `false` for the declared `targets` only | | `versions` | `'["latest", "stable"]'` | The versions of each package | | `group` | `'packages'` | Name of the suite: its comment and its artifact | | `use_ct_registry` | `false` | Add `ct-registry` (needs `SSH_KEY`) | @@ -83,6 +119,10 @@ leave out the `push` trigger and be run by hand on `main` before a release. `scripts/breakage/test/runtests.jl`: the verdict (not a release; a release breaking nothing, breaking something with or without a breaking number, only the stable column -counting, no baseline, missing cells, `test` and `doc` cells apart) and the files written. -`breakage-tests.yml` runs them when the script or the workflow change. The `break` job -itself is exercised by the callers. +counting, no baseline, missing cells, a new target, `test` and `doc` cells apart) and the +files written; the consumers (each place a package may be loaded from, the uuid and not the +name, archived repositories, forks, exclusions, a repository that cannot be read being an +error and not an empty list, the declared targets and their validation, the command line, +the ecosystem as it was read on 2026-10-19). The reading of GitHub is replaced by a fake, +as in the release tests. `breakage-tests.yml` runs them when the script or the workflow +change. The `plan` and `break` jobs themselves are exercised by the callers. diff --git a/scripts/breakage/consumers.jl b/scripts/breakage/consumers.jl new file mode 100644 index 0000000..ae11892 --- /dev/null +++ b/scripts/breakage/consumers.jl @@ -0,0 +1,242 @@ +# The downstream packages of a package: the targets of the breakage suite, read from the +# Project.toml files of the organization instead of written by hand in every package. +# +# usage: julia consumers.jl list --project Project.toml [--org control-toolbox] +# julia consumers.jl plan --project Project.toml [--declared JSON] [--no-derive] +# [--org control-toolbox] [--summary FILE] +# +# A package of the organization is a consumer of P when it lists P (by uuid) in the +# `[deps]`, `[weakdeps]` or `[extras]` of its `Project.toml`, or in the `[deps]` of its +# `test/Project.toml`: whatever loads P when its tests run. `list` prints those consumers; +# `plan` adds the `--declared` targets (a JSON list of {"name", "owner", "break"}) that +# cannot be derived — an application owned outside the organization, a documentation build — +# and prints the targets of the suite, one JSON line; `--summary` appends a table saying +# where each one comes from. A failure to read the organization is an error, never an +# empty list. Needs `gh`, authenticated (GH_TOKEN). Exit code 0, 1 (error), 2 (usage). +# Stdlib only. + +using TOML + +const ORG = "control-toolbox" + +"Repositories that are not part of the ecosystem to test, with the reason." +const EXCLUDED = Dict( + "CTAppTemplate" => "a template of a new package, not a package of the ecosystem", + "CTBenchmarks" => "its compat is out of date (CTBase 0.16, OptimalControl =1.1.6), so its tests fail whatever the change: remove this line once it is updated", +) + +"A target of the suite." +const Target = NamedTuple{(:name, :owner, :break_, :source), NTuple{4,String}} + +"The characters a package name and an owner may have: they end up in a git URL and in a shell." +valid_name(s::AbstractString) = occursin(r"^[A-Za-z0-9][A-Za-z0-9._-]*$", s) + +# What is read from GitHub, replaced in the tests. + +"Run `gh api …`: `(exit code, stdout, stderr)`." +function gh_api(args...) + out, err = IOBuffer(), IOBuffer() + p = run(pipeline(ignorestatus(`gh api $args`); stdout=out, stderr=err)) + return (p.exitcode, String(take!(out)), String(take!(err))) +end + +const API = Ref{Function}(gh_api) + +"The repositories of `org`, as `(name, archived, fork)`; throws when GitHub does not answer." +function list_repos(org::AbstractString) + code, out, err = API[]("--paginate", "orgs/$org/repos?per_page=100", "--jq", ".[] | \"\\(.name)\\t\\(.archived)\\t\\(.fork)\"") + code == 0 || error("cannot list the repositories of $org: $(strip(err))") + return [(n, a == "true", f == "true") for (n, a, f) in (split(l, '\t') for l in split(out, '\n'; keepempty=false))] +end + +"The text of `path` on the default branch of `org/repo`; `nothing` when the file does not exist, an error otherwise." +function read_file(org::AbstractString, repo::AbstractString, path::AbstractString) + code, out, err = API[]("repos/$org/$repo/contents/$path", "-H", "Accept: application/vnd.github.raw") + code == 0 && return out + occursin("HTTP 404", err) && return nothing + error("cannot read $path of $org/$repo: $(strip(err))") +end + +const LIST = Ref{Function}(list_repos) +const READ = Ref{Function}(read_file) + +"The parsed TOML `text` of `where`; `nothing`, with a warning, when it cannot be parsed." +function parse_toml(text::AbstractString, where::AbstractString) + try + return TOML.parse(text) + catch e + @warn "cannot parse $where, skipped: $(sprint(showerror, e))" + return nothing + end +end + +""" + package_of(project) -> (; name, uuid) | nothing + +The name and uuid of a parsed `Project.toml`; `nothing` when it is not a package (no +`name` or no `uuid`). +""" +function package_of(project::AbstractDict) + (haskey(project, "name") && haskey(project, "uuid")) || return nothing + return (; name=string(project["name"]), uuid=lowercase(string(project["uuid"]))) +end + +""" + loaded_by(project, test, uuid) -> String | nothing + +How the package of `uuid` is needed by a project (its parsed `Project.toml`, and its +`test/Project.toml` or `nothing`): `deps`, `weakdeps`, `extras` or `test`; `nothing` if not. +A table maps names to uuids, so that a package of the same name but another uuid is not it. +""" +function loaded_by(project::AbstractDict, test, uuid::AbstractString) + has(tbl) = tbl isa AbstractDict && any(u -> lowercase(string(u)) == uuid, values(tbl)) + for table in ("deps", "weakdeps", "extras") + has(get(project, table, nothing)) && return table + end + test !== nothing && has(get(test, "deps", nothing)) && return "test" + return nothing +end + +""" + consumers(name, uuid; org=ORG) -> Vector{Target} + +The packages of `org` that load the package `name` (of `uuid`), by their repository name +without `.jl`, sorted. Archived repositories, forks, `EXCLUDED` ones and any package of +the same uuid (the package itself) are left out. +""" +function consumers(name::AbstractString, uuid::AbstractString; org::AbstractString=ORG) + uuid = lowercase(uuid) + found = Target[] + for (repo, archived, fork) in LIST[](org) + endswith(repo, ".jl") || continue + pkg = repo[1:end-3] + (archived || fork || haskey(EXCLUDED, pkg)) && continue + valid_name(pkg) || error("the repository name $(repr(pkg)) of $org is not a valid package name") + text = READ[](org, repo, "Project.toml") + text === nothing && continue + project = parse_toml(text, "$repo/Project.toml") + project === nothing && continue + info = package_of(project) + (info === nothing || info.uuid == uuid) && continue + test_text = READ[](org, repo, "test/Project.toml") + test = test_text === nothing ? nothing : parse_toml(test_text, "$repo/test/Project.toml") + via = loaded_by(project, test, uuid) + via === nothing && continue + push!(found, (; name=pkg, owner=org, break_="test", source="derived (in the $via of $pkg)")) + end + return sort!(found; by=t -> (t.name, t.break_)) +end + +""" + parse_targets(text) -> Vector{Target} + +The targets of a JSON list of flat objects with string values, `{"name", "owner", "break"}` +(`owner` defaults to the organization, `break` to `test`); anything else is an error. +""" +function parse_targets(text::AbstractString; org::AbstractString=ORG) + s = strip(text) + isempty(s) && return Target[] + objects = collect(eachmatch(r"\{[^{}]*\}", s)) + rest = replace(s, r"\{[^{}]*\}" => "") + occursin(r"^\[[\s,]*\]$", rest) || throw(ArgumentError("targets must be a list of objects")) + out = Target[] + for m in objects + body = m.match + pairs = Dict{String,String}() + for kv in eachmatch(r"\"([^\"\\]*)\"\s*:\s*\"([^\"\\]*)\"", body) + pairs[kv[1]] = kv[2] + end + occursin(r"^\{[\s,]*\}$", replace(body, r"\"([^\"\\]*)\"\s*:\s*\"([^\"\\]*)\"" => "")) || + throw(ArgumentError("a target must have string values only: $body")) + unknown = setdiff(keys(pairs), ("name", "owner", "break")) + isempty(unknown) || throw(ArgumentError("unknown key $(first(unknown)) in a target: $body")) + haskey(pairs, "name") || throw(ArgumentError("a target has no name: $body")) + brk = get(pairs, "break", "test") + brk in ("test", "doc") || throw(ArgumentError("break must be test or doc, not $brk: $body")) + owner = get(pairs, "owner", org) + (valid_name(pairs["name"]) && valid_name(owner)) || throw(ArgumentError("invalid name or owner: $body")) + push!(out, (; name=pairs["name"], owner, break_=brk, source="declared")) + end + return out +end + +""" + plan(name, uuid; declared=Target[], derive=true, org=ORG) -> Vector{Target} + +The targets of the suite: the derived consumers and the declared targets. A declared target +replaces the derived one of the same package, owner and check. Throws when there is +nothing to test: a package without consumers has no use for the suite. +""" +function plan(name::AbstractString, uuid::AbstractString; declared::Vector{Target}=Target[], derive::Bool=true, org::AbstractString=ORG) + targets = declared + if derive + mine = Set((t.name, t.owner, t.break_) for t in declared) + targets = vcat(declared, [t for t in consumers(name, uuid; org) if (t.name, t.owner, t.break_) ∉ mine]) + end + isempty(targets) && error("no downstream package to test for $name: remove the Breakage workflow, or declare a target") + return sort!(targets; by=t -> (t.name, t.break_, t.owner)) +end + +"The targets as the JSON list of the `targets` input of breakage-suite.yml, on one line." +json_targets(targets) = "[" * join(("{\"name\":\"$(t.name)\",\"owner\":\"$(t.owner)\",\"break\":\"$(t.break_)\"}" for t in targets), ",") * "]" + +"A Markdown table of the targets and where each comes from." +function targets_table(name, targets) + io = IOBuffer() + println(io, "### Breakage targets — downstream of $name\n") + println(io, "| Package | Check | Owner | Source |\n| --- | --- | --- | --- |") + for t in targets + println(io, "| $(t.name) | $(t.break_) | $(t.owner) | $(t.source) |") + end + return String(take!(io)) +end + +"The value of `--name` in `args` (removed from it), or `default`." +function option!(args, name, default=nothing) + i = findfirst(==("--$name"), args) + i === nothing && return default + i == length(args) && throw(ArgumentError("--$name needs a value")) + value = args[i+1] + deleteat!(args, i:i+1) + return value +end + +const USAGE = """ +usage: julia consumers.jl list --project Project.toml [--org ORG] + julia consumers.jl plan --project Project.toml [--declared JSON] [--no-derive] [--org ORG] [--summary FILE]""" + +"Run the command in `args`; return the exit code." +function consumers_main(args) + args = collect(String, args) + isempty(args) && (println(stderr, USAGE); return 2) + command, args = args[1], args[2:end] + command in ("list", "plan") || (println(stderr, USAGE); return 2) + try + project = option!(args, "project") + project === nothing && throw(ArgumentError("--project is required")) + org = option!(args, "org", ORG) + declared = option!(args, "declared", "") + summary = option!(args, "summary") + derive = !("--no-derive" in args) + filter!(!=("--no-derive"), args) + isempty(args) || throw(ArgumentError("unexpected arguments: $(join(args, " "))")) + info = package_of(TOML.parsefile(project)) + info === nothing && throw(ArgumentError("$project has no name and uuid: not a package")) + if command == "list" + println(json_targets(consumers(info.name, info.uuid; org))) + else + targets = plan(info.name, info.uuid; declared=parse_targets(declared; org), derive, org) + println(json_targets(targets)) + summary === nothing || open(io -> print(io, targets_table(info.name, targets)), summary, "a") + end + return 0 + catch e + e isa ArgumentError && (println(stderr, "error: ", e.msg, "\n", USAGE); return 2) + println(stderr, "error: ", sprint(showerror, e)) + return 1 + end +end + +if abspath(PROGRAM_FILE) == @__FILE__ + exit(consumers_main(ARGS)) +end diff --git a/scripts/breakage/report.jl b/scripts/breakage/report.jl index d225403..e3ebbc1 100644 --- a/scripts/breakage/report.jl +++ b/scripts/breakage/report.jl @@ -59,6 +59,10 @@ function verdict(cells, baseline; expected::Int, base_version=nothing, head_vers for c in already push!(lines, "ℹ️ $(c["name"]) ($(c["break"])) already fails on the base: says nothing about the release") end + for c in cells + c["version"] == "stable" && !haskey(before, key(c)) && + push!(lines, "ℹ️ $(c["name"]) ($(c["break"])) has no result on the base (a new target): not compared") + end if isempty(broken) push!(lines, "✅ no downstream release broken by `$base_version` → `$head_version`") elseif breaking diff --git a/scripts/breakage/test/runtests.jl b/scripts/breakage/test/runtests.jl index 878b4b9..1421796 100644 --- a/scripts/breakage/test/runtests.jl +++ b/scripts/breakage/test/runtests.jl @@ -5,6 +5,7 @@ using Test, TOML include(joinpath(@__DIR__, "..", "report.jl")) +include(joinpath(@__DIR__, "..", "consumers.jl")) cell(name, version, status; brk="test") = Dict("name" => name, "owner" => "o", "version" => version, "break" => brk, "tag" => version == "stable" ? "v1.0.0" : "main", @@ -47,6 +48,8 @@ cell(name, version, status; brk="test") = notag = [cell("A", "latest", "pass"), cell("A", "stable", "no_tag"), cell("C", "latest", "pass"), cell("C", "stable", "fail")] v = verdict(notag, base; expected=4, base_version=V0, head_version=V1) @test v.ok && any(contains("no downstream release broken"), v.lines) + @test any(contains("C (test) has no result on the base (a new target): not compared"), v.lines) + @test !any(contains("A (test) has no result"), v.lines) # more cells than expected (a rerun) is not "missing" @test verdict(vcat(base, base[1:1]), base; expected=4, base_version=V0, head_version=V1).ok # a beta release is checked the same way @@ -85,3 +88,252 @@ cell(name, version, status; brk="test") = end end end + +# --- consumers.jl: the downstream packages, derived from the Project.toml files ------------ + +const UU = Dict(n => "0000000a-bcde-f000-0000-" * lpad(i, 12, '0') for (i, n) in enumerate( + ["Base", "Mid", "Leaf", "Other", "Twin", "Dead", "Forked"])) + +"The text of a Project.toml." +function toml_text(name; uuid=UU[name], deps=(), weakdeps=(), extras=(), version="1.0.0") + io = IOBuffer() + println(io, "name = \"$name\"") + uuid === nothing || println(io, "uuid = \"$uuid\"") + println(io, "version = \"$version\"") + for (table, names) in (("deps", deps), ("weakdeps", weakdeps), ("extras", extras)) + isempty(names) && continue + println(io, "\n[$table]") + for n in names + println(io, "$n = \"$(UU[n])\"") + end + end + return String(take!(io)) +end + +"Run `f` with an organization made of `repos` — `(name, archived, fork)` — and `files` — `(repo, path) => text | Exception`." +function with_org(f, repos, files) + list, read = LIST[], READ[] + LIST[] = org -> repos isa Exception ? throw(repos) : repos + READ[] = function (org, repo, path) + v = get(files, (repo, path), nothing) + v isa Exception && throw(v) + return v + end + try + return f() + finally + LIST[], READ[] = list, read + end +end + +"The standard output of `f()`, with its result." +function captured(f) + mktemp() do path, io + result = redirect_stdout(f, io) + flush(io) + return result, read(path, String) + end +end + +names(ts) = [t.name for t in ts] + +@testset "consumers" begin + @testset "what loads a package" begin + repos = [("Mid.jl", false, false), ("Leaf.jl", false, false), ("Twin.jl", false, false), ("Other.jl", false, false), + ("Dead.jl", true, false), ("Forked.jl", false, true), ("Base.jl", false, false), + ("website", false, false), ("Empty.jl", false, false), ("NoUuid.jl", false, false), ("Mirror.jl", false, false)] + files = Dict( + ("Base.jl", "Project.toml") => toml_text("Base"; extras=["Base"]), # lists itself: still not its own consumer + ("Mid.jl", "Project.toml") => toml_text("Mid"; deps=["Base"]), + ("Mirror.jl", "Project.toml") => toml_text("Base"; extras=["Base"]), # the same package, under another name + ("Leaf.jl", "Project.toml") => toml_text("Leaf"; extras=["Base"]), + ("Twin.jl", "Project.toml") => toml_text("Twin"; weakdeps=["Base"]), + ("Other.jl", "Project.toml") => toml_text("Other"), + ("Other.jl", "test/Project.toml") => "[deps]\nBase = \"$(UU["Base"])\"\n", + ("Dead.jl", "Project.toml") => toml_text("Dead"; deps=["Base"]), + ("Forked.jl", "Project.toml") => toml_text("Forked"; deps=["Base"]), + ("website", "Project.toml") => toml_text("Mid"; deps=["Base"]), + ("NoUuid.jl", "Project.toml") => toml_text("NoUuid"; uuid=nothing, deps=["Base"]), + ) + with_org(repos, files) do + found = consumers("Base", UU["Base"]) + # deps, extras, weakdeps and the test environment count; the package itself, an archived + # repository, a fork, an excluded one, a non-package and a non-.jl repository do not + @test names(found) == ["Leaf", "Mid", "Other", "Twin"] + @test all(t -> t.owner == "control-toolbox" && t.break_ == "test", found) + @test [t.source for t in found] == ["derived (in the extras of Leaf)", "derived (in the deps of Mid)", + "derived (in the test of Other)", "derived (in the weakdeps of Twin)"] + # the uuid decides, not the name: nobody loads a package of the same name + @test isempty(consumers("Base", "11111111-1111-1111-1111-111111111111")) + @test names(consumers("Base", uppercase(UU["Base"]))) == ["Leaf", "Mid", "Other", "Twin"] + @test package_of(TOML.parse(toml_text("Base"; uuid=uppercase(UU["Base"])))).uuid == UU["Base"] + # a package nobody loads has no consumer + @test isempty(consumers("Leaf", UU["Leaf"])) + end + # the excluded repositories are consumers by their Project.toml, and left out + for excluded in keys(EXCLUDED) + with_org([("$excluded.jl", false, false)], Dict(("$excluded.jl", "Project.toml") => toml_text("Mid"; deps=["Base"]))) do + @test isempty(consumers("Base", UU["Base"])) + end + end + @test all(!isempty, values(EXCLUDED)) # each with its reason + end + + @testset "reading GitHub" begin + calls = Vector{Vector{String}}() + api = Ref{Any}((0, "", "")) + old = API[] + API[] = (args...) -> (push!(calls, collect(String, args)); api[]) + try + api[] = (0, "A.jl\tfalse\tfalse\nB.jl\ttrue\tfalse\nC.jl\tfalse\ttrue\n", "") + @test list_repos("org") == [("A.jl", false, false), ("B.jl", true, false), ("C.jl", false, true)] + @test "--paginate" in calls[end] && any(contains("orgs/org/repos"), calls[end]) + api[] = (1, "", "HTTP 403: rate limit") + @test_throws ErrorException list_repos("org") + api[] = (0, "name = \"x\"\n", "") + @test read_file("org", "A.jl", "Project.toml") == "name = \"x\"\n" + @test any(contains("repos/org/A.jl/contents/Project.toml"), calls[end]) + api[] = (1, "", "gh: Not Found (HTTP 404)") + @test read_file("org", "A.jl", "test/Project.toml") === nothing # no such file: not an error + api[] = (1, "", "gh: Bad Gateway (HTTP 502)") + @test_throws ErrorException read_file("org", "A.jl", "Project.toml") # anything else is + finally + API[] = old + end + end + + @testset "errors are not empty lists" begin + @test_throws ErrorException with_org(() -> consumers("Base", UU["Base"]), ErrorException("cannot list"), Dict()) + files = Dict(("Mid.jl", "Project.toml") => ErrorException("cannot read Project.toml of x/Mid.jl: HTTP 502")) + @test_throws ErrorException with_org(() -> consumers("Base", UU["Base"]), [("Mid.jl", false, false)], files) + # a Project.toml that cannot be parsed is skipped, with a warning + files = Dict(("Mid.jl", "Project.toml") => "name = ", ("Leaf.jl", "Project.toml") => toml_text("Leaf"; deps=["Base"])) + with_org([("Mid.jl", false, false), ("Leaf.jl", false, false)], files) do + found = @test_logs (:warn, r"cannot parse Mid.jl/Project.toml") consumers("Base", UU["Base"]) + @test names(found) == ["Leaf"] + end + # a repository name that is not a name + with_org([("a b.jl", false, false)], Dict()) do + @test_throws ErrorException consumers("Base", UU["Base"]) + end + end + + @testset "declared targets" begin + t = parse_targets("""[{"name": "A"}, {"name": "B", "owner": "me", "break": "doc"}]""") + @test [(x.name, x.owner, x.break_, x.source) for x in t] == + [("A", "control-toolbox", "test", "declared"), ("B", "me", "doc", "declared")] + @test parse_targets("") == [] && parse_targets("[]") == [] && parse_targets(" [ ] ") == [] + @test parse_targets("""[{"name":"A"},{"name":"B"}]""") |> length == 2 + for bad in ("""{"name": "A"}""", """["A"]""", """[{"name": "A"}] x""", """[{"owner": "me"}]""", + """[{"name": "A", "color": "red"}]""", """[{"name": "A", "break": "docs"}]""", + """[{"name": "A B"}]""", """[{"name": "../x"}]""", """[{"name": "A", "owner": "a;b"}]""", + """[{"name": 3}]""", """[{"name": "A", "break": null}]""", """[{"name": "A"} {"name": "B"}""") + @test_throws ArgumentError parse_targets(bad) + end + end + + @testset "plan" begin + repos = [("Mid.jl", false, false), ("Leaf.jl", false, false)] + files = Dict(("Mid.jl", "Project.toml") => toml_text("Mid"; deps=["Base"]), ("Leaf.jl", "Project.toml") => toml_text("Leaf"; deps=["Base"])) + with_org(repos, files) do + @test names(plan("Base", UU["Base"])) == ["Leaf", "Mid"] + # a declared target is added; a doc check of a derived package is a target of its own + decl = parse_targets("""[{"name": "App", "owner": "me", "break": "doc"}, {"name": "Mid", "break": "doc"}]""") + p = plan("Base", UU["Base"]; declared=decl) + @test [(t.name, t.break_) for t in p] == [("App", "doc"), ("Leaf", "test"), ("Mid", "doc"), ("Mid", "test")] + # a declared target replaces the derived one of the same package, owner and check + p = plan("Base", UU["Base"]; declared=parse_targets("""[{"name": "Mid"}]""")) + @test [(t.name, t.source) for t in p] == [("Leaf", "derived (in the deps of Leaf)"), ("Mid", "declared")] + # without derivation, the declared targets only + p = plan("Base", UU["Base"]; declared=decl, derive=false) + @test [(t.name, t.break_) for t in p] == [("App", "doc"), ("Mid", "doc")] + # nothing to test is an error, not an empty matrix + @test_throws ErrorException plan("Leaf", UU["Leaf"]) + @test_throws ErrorException plan("Base", UU["Base"]; derive=false) + end + t = parse_targets("""[{"name": "A"}, {"name": "B", "owner": "me", "break": "doc"}]""") + @test json_targets(t) == """[{"name":"A","owner":"control-toolbox","break":"test"},{"name":"B","owner":"me","break":"doc"}]""" + table = targets_table("Base", t) + @test contains(table, "| A | test | control-toolbox | declared |") && contains(table, "| B | doc | me | declared |") + end + + @testset "command line" begin + mktempdir() do d + proj = joinpath(d, "Project.toml"); write(proj, toml_text("Base")) + summary_file = joinpath(d, "summary.md") + with_org(ErrorException("the organization must not be read"), Dict()) do + redirect_stderr(devnull) do + @test consumers_main(String[]) == 2 + @test consumers_main(["nope"]) == 2 + @test consumers_main(["list"]) == 2 # --project is required + @test consumers_main(["list", "--project", proj, "extra"]) == 2 + @test consumers_main(["plan", "--project", proj, "--declared", "{"]) == 2 + write(joinpath(d, "NoPkg.toml"), "x = 1\n") + @test consumers_main(["list", "--project", joinpath(d, "NoPkg.toml")]) == 2 + @test consumers_main(["plan", "--project", proj, "--no-derive"]) == 1 # nothing to test + end + code, out = captured() do + consumers_main(["plan", "--project", proj, "--no-derive", "--declared", """[{"name": "App", "owner": "me", "break": "doc"}]""", + "--summary", summary_file]) + end + @test code == 0 && out == """[{"name":"App","owner":"me","break":"doc"}]\n""" + @test contains(read(summary_file, String), "| App | doc | me | declared |") + end + # derived through the (faked) organization + files = Dict(("Mid.jl", "Project.toml") => toml_text("Mid"; deps=["Base"])) + with_org([("Mid.jl", false, false)], files) do + code, out = captured(() -> consumers_main(["list", "--project", proj])) + @test code == 0 && out == """[{"name":"Mid","owner":"control-toolbox","break":"test"}]\n""" + code, out = captured(() -> consumers_main(["plan", "--project", proj, "--declared", """[{"name": "App"}]"""])) + @test code == 0 && out == """[{"name":"App","owner":"control-toolbox","break":"test"},{"name":"Mid","owner":"control-toolbox","break":"test"}]\n""" + end + # a failure to read the organization is an error, not an empty list + redirect_stderr(devnull) do + with_org(ErrorException("HTTP 500"), Dict()) do + @test consumers_main(["list", "--project", proj]) == 1 + end + end + end + end + + @testset "the ecosystem, as read on 2026-10-19" begin + # name => (deps, weakdeps, extras, test deps), as in the Project.toml files then + eco = Dict( + "CTBase" => ((), (), (), ()), + "CTModels" => (("CTBase",), (), (), ()), + "CTSolvers" => (("CTBase", "CTModels"), (), (), ()), + "CTFlows" => (("CTBase", "CTModels", "CTSolvers"), (), ("CTLie",), ()), + "CTDirect" => (("CTBase", "CTModels", "CTParser", "CTSolvers"), (), ("CTParser",), ()), + "CTParser" => (("CTBase",), (), (), ("CTBase", "CTModels")), + "CTLie" => (("CTBase",), (), (), ()), + "OptimalControl" => (("CTBase", "CTDirect", "CTFlows", "CTLie", "CTModels", "CTParser", "CTSolvers"), (), (), ()), + "OptimalControlProblems" => (("CTBase", "CTDirect", "CTModels"), ("OptimalControl",), (), ("CTBase", "CTDirect", "OptimalControl")), + "CTBenchmarks" => (("CTBase", "OptimalControl", "OptimalControlProblems"), (), (), ()), + "CTDirectShooting" => (("CTBase", "CTFlows"), (), (), ("CTBase",)), # archived + ) + id(n) = "00000000-0000-0000-0001-" * lpad(findfirst(==(n), sort!(collect(keys(eco)))), 12, '0') + files = Dict{Tuple{String,String},Any}() + for (n, (deps, weak, extras, test)) in eco + tbl(names) = isempty(names) ? "" : join(("$x = \"$(id(x))\"" for x in names), "\n") * "\n" + text = "name = \"$n\"\nuuid = \"$(id(n))\"\n[deps]\n$(tbl(deps))[weakdeps]\n$(tbl(weak))[extras]\n$(tbl(extras))" + files[("$n.jl", "Project.toml")] = text + isempty(test) || (files[("$n.jl", "test/Project.toml")] = "[deps]\n$(tbl(test))") + end + expected = Dict( + "CTBase" => ["CTDirect", "CTFlows", "CTLie", "CTModels", "CTParser", "CTSolvers", "OptimalControl", "OptimalControlProblems"], + "CTModels" => ["CTDirect", "CTFlows", "CTParser", "CTSolvers", "OptimalControl", "OptimalControlProblems"], + "CTSolvers" => ["CTDirect", "CTFlows", "OptimalControl"], + "CTFlows" => ["OptimalControl"], # CTDirectShooting is archived + "CTDirect" => ["OptimalControl", "OptimalControlProblems"], + "CTParser" => ["CTDirect", "OptimalControl"], + "CTLie" => ["CTFlows", "OptimalControl"], + "OptimalControl" => ["OptimalControlProblems"], # CTBenchmarks is excluded + ) + with_org([("$n.jl", n == "CTDirectShooting", false) for n in keys(eco)], files) do + for (pkg, want) in expected + @test names(consumers(pkg, id(pkg))) == want + end + end + end +end +