diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 32c424fc1..1d8492f5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,8 +5,6 @@ on: branches: - master pull_request: - branches: - - master jobs: check: diff --git a/src/commands/cloud_agent/herdr/agents.rs b/src/commands/cloud_agent/herdr/agents.rs new file mode 100644 index 000000000..da07eb45e --- /dev/null +++ b/src/commands/cloud_agent/herdr/agents.rs @@ -0,0 +1,564 @@ +//! `railway ca herdr agents`: the picker herdr's popup runs. +//! +//! One list of every agent you own with its herdr machine state, then an +//! action on the one you pick. Sleep, wake and delete change the VM through +//! the same controller paths as `railway ca`, then keep the herdr machine in +//! step so the sidebar stops retrying a VM that is deliberately off. + +use std::fmt; + +use anyhow::{Result, bail}; +use clap::Parser; +use colored::Colorize; + +use super::herdr_cli::{Herdr, Machine}; +use super::state::Store; +use super::sync; +use super::target; +use crate::client::GQLClient; +use crate::commands::cloud_agent::lifecycle; +use crate::config::Configs; +use crate::controllers::cloud_agent as ca; +use crate::util::progress::create_spinner; +use crate::util::prompt::{prompt_confirm_with_default, prompt_select_with_cancel}; + +#[derive(Parser)] +pub struct Args { + /// Open the picker in a herdr popup pane instead of this terminal + #[clap(long)] + open: bool, + + /// Running on a cloud agent VM: no herdr machines here, so connect and new + /// are left to Local; the row for this VM is marked + #[clap(long)] + remote: bool, + + /// Only sleeping agents; with exactly one, wake it without asking + #[clap(long)] + wake: bool, +} + +struct Row { + agent: ca::Agent, + project: String, + machine: Option, + remote: bool, + this_vm: bool, +} + +impl Row { + fn machine_state(&self) -> &'static str { + match &self.machine { + Some(m) if m.enabled => "machine", + Some(_) => "machine (disabled)", + None => "no machine", + } + } +} + +impl fmt::Display for Row { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{:<9} {}/{}", + self.agent.status.label(), + self.project, + self.agent.name + )?; + if self.this_vm { + write!(f, " ← this VM") + } else if self.remote { + Ok(()) + } else { + write!(f, " {}", self.machine_state()) + } + } +} + +/// The list mixes the two things that are not about one agent with the agents. +enum Item { + New, + Sync, + Agent(Row), +} + +impl fmt::Display for Item { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Item::New => f.write_str("+ new agent"), + Item::Sync => f.write_str("↻ sync now"), + Item::Agent(row) => row.fmt(f), + } + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Action { + Connect, + Sleep, + Wake, + Delete, + New, + Quit, +} + +impl Action { + const ALL: [Action; 6] = [ + Action::Connect, + Action::Sleep, + Action::Wake, + Action::Delete, + Action::New, + Action::Quit, + ]; + + /// No machine catalog on a VM, so nothing to connect or add there. + const REMOTE: [Action; 4] = [Action::Sleep, Action::Wake, Action::Delete, Action::Quit]; +} + +impl fmt::Display for Action { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Action::Connect => "connect enable its herdr machine, waking it first if asleep", + Action::Sleep => "sleep stop the compute bill, keep the disk", + Action::Wake => "wake bring it back and re-enable its machine", + Action::Delete => "delete the agent, its disk and its machine", + Action::New => "new agent create one and add it to herdr", + Action::Quit => "quit", + }) + } +} + +struct Picker { + configs: Configs, + client: reqwest::Client, + backboard: String, + herdr: Herdr, + store: Store, + remote: bool, +} + +pub async fn command(args: Args) -> Result<()> { + let herdr = Herdr::from_env(); + if args.open { + let entrypoint = if args.wake { "wake" } else { "agents" }; + return herdr.plugin_pane_open(super::PLUGIN_ID, entrypoint); + } + + let configs = Configs::new()?; + let client = GQLClient::new_authorized(&configs)?; + let backboard = configs.get_backboard(); + let store = Store::new(&configs)?; + let mut picker = Picker { + configs, + client, + backboard, + herdr, + store, + remote: args.remote, + }; + let this_vm = std::env::var("RAILWAY_CLOUD_AGENT_ID").ok(); + let mut names = picker.store.load()?.project_names; + + loop { + let agents = ca::list_mine(&picker.client, &picker.backboard).await?; + if agents.iter().any(|a| !names.contains_key(&a.project_id)) { + names = lifecycle::place_names(&picker.client, &picker.configs) + .await + .into_iter() + .collect(); + if !names.is_empty() { + picker + .store + .update(|s| s.project_names = names.clone()) + .await?; + } + } + let machines = if args.remote { + Vec::new() + } else { + picker.herdr.machines()? + }; + let mut rows: Vec = agents + .into_iter() + .filter(|agent| !args.wake || matches!(agent.status, ca::Status::Sleeping)) + .map(|agent| Row { + machine: sync::machine_for(&agent, &machines).cloned(), + project: names + .get(&agent.project_id) + .cloned() + .unwrap_or_else(|| agent.project_id.clone()), + remote: args.remote, + this_vm: this_vm.as_deref() == Some(agent.id.as_str()), + agent, + }) + .collect(); + rows.sort_by(|a, b| { + a.project + .cmp(&b.project) + .then_with(|| a.agent.name.cmp(&b.agent.name)) + }); + + if args.wake { + return match rows.len() { + 0 => { + println!("No sleeping agents."); + Ok(()) + } + 1 => { + picker.wake(&rows[0]).await?; + picker.resync().await.map(drop) + } + _ => match inquire::Select::new("Wake", rows) + .with_render_config(Configs::get_render_config()) + .with_page_size(15) + .with_help_message("↑↓ move, type to filter, enter wakes, esc quits") + .prompt_skippable()? + { + Some(row) => { + picker.wake(&row).await?; + picker.resync().await.map(drop) + } + None => Ok(()), + }, + }; + } + + let items = picker_items(rows, args.remote); + if items.is_empty() { + println!( + "No cloud agents. {} creates one and adds it to herdr.", + "railway ca herdr new".cyan() + ); + return Ok(()); + } + let Some(item) = inquire::Select::new("Agent", items) + .with_render_config(Configs::get_render_config()) + .with_page_size(17) + .with_help_message("↑↓ move, type to filter, enter picks, esc quits") + .prompt_skippable()? + else { + return Ok(()); + }; + let row = match item { + Item::New => return super::new::command(super::new::Args::interactive()).await, + Item::Sync => { + match picker.resync().await { + Ok(applied) if applied.is_empty() => { + println!("✓ herdr machines match your agents.") + } + Ok(applied) => println!("✓ herdr sync: {}", applied.join(", ")), + Err(e) => eprintln!("{} {e:#}", "✗".red()), + } + continue; + } + Item::Agent(row) => row, + }; + let actions = if args.remote { + Action::REMOTE.to_vec() + } else { + Action::ALL.to_vec() + }; + let Some(action) = + prompt_select_with_cancel(&format!("{}/{}", row.project, row.agent.name), actions)? + else { + continue; + }; + + // connect and new leave a machine to look at, so the popup closes + // behind them; the rest stay on the list. + let result = match action { + Action::Connect => return picker.connect(&row).await, + Action::Sleep => picker + .sleep(&row) + .await + .and(picker.resync().await.map(drop)), + Action::Wake => match picker.wake(&row).await { + Ok(()) => { + picker.resync().await?; + return Ok(()); + } + Err(e) => Err(e), + }, + Action::Delete => picker + .delete(&row) + .await + .and(picker.resync().await.map(drop)), + Action::New => return super::new::command(super::new::Args::interactive()).await, + Action::Quit => return Ok(()), + }; + if let Err(e) = result { + eprintln!("{} {e:#}", "✗".red()); + } + } +} + +fn picker_items(rows: Vec, remote: bool) -> Vec { + if remote && rows.is_empty() { + return Vec::new(); + } + let mut items = Vec::with_capacity(rows.len() + 2); + if !remote { + items.push(Item::New); + } + items.push(Item::Sync); + items.extend(rows.into_iter().map(Item::Agent)); + items +} + +impl Picker { + async fn connect(&mut self, row: &Row) -> Result<()> { + let agent = self.ensure_awake(&row.agent).await?; + let spinner = create_spinner(format!("Waiting for {}'s ssh relay", agent.name)); + let ready = super::relay::wait_until_ready(&agent).await; + spinner.finish_and_clear(); + ready?; + match &row.machine { + Some(machine) => { + self.reconnect(&agent, machine).await?; + if let Some(harness) = self.store.load()?.bootstrap_pending.get(&agent.id).cloned() + { + super::bootstrap::run(&agent, &harness, &self.store).await?; + } + println!( + "✓ {} is enabled in herdr; pick it from the sidebar.", + machine.label.cyan() + ); + } + None => { + let harness = super::harness::choose(&Default::default(), true)?; + let target = target::target(&agent); + let label = target::label(&row.project, &agent.name); + super::known_hosts::ensure_relay_known_host()?; + self.herdr.machine_add(&target, &label)?; + let added = self + .herdr + .machines()? + .into_iter() + .find(|m| sync::is_machine_for(&agent, m)); + self.remember(&agent.id, added.as_ref().map(|m| m.id.as_str())) + .await?; + super::bootstrap::run(&agent, harness, &self.store).await?; + println!( + "✓ Added {} to herdr; pick it from the sidebar.", + label.cyan() + ); + } + } + Ok(()) + } + + async fn sleep(&mut self, row: &Row) -> Result<()> { + let agent = &row.agent; + self.herdr.notify( + &format!("Sleeping {}", agent.name), + "Requesting sleep; its disk is kept", + ); + let mut locked = self.store.lock().await?; + let spinner = create_spinner(format!("Sleeping agent {}", agent.name)); + let result = ca::sleep( + &self.client, + &self.backboard, + &agent.environment_id, + &agent.id, + ) + .await; + spinner.finish_and_clear(); + result?; + locked.state.sleep_until.insert( + agent.id.clone(), + chrono::Utc::now() + chrono::Duration::seconds(60), + ); + // Record the acknowledgement even if disabling Herdr subsequently fails. + locked.save()?; + if let Some(machine) = &row.machine { + self.herdr.machine_disable(&machine.id)?; + } + println!( + "✓ Sleep requested for agent {}; compute stops billing once it is asleep.", + agent.name.cyan() + ); + Ok(()) + } + + async fn wake(&mut self, row: &Row) -> Result<()> { + let agent = self.ensure_awake(&row.agent).await?; + match &row.machine { + Some(machine) => { + let spinner = create_spinner(format!("Waiting for {}'s ssh relay", agent.name)); + let ready = super::relay::wait_until_ready(&agent).await; + spinner.finish_and_clear(); + ready?; + // Off then on: a profile change makes herdr open a fresh + // connection, which is what clears a stuck Attention state. + self.reconnect(&agent, machine).await?; + println!( + "✓ Agent {} is running; {} is back in the sidebar.", + agent.name.cyan(), + machine.label.cyan() + ); + } + None if self.remote => println!("✓ Agent {} is running.", agent.name.cyan()), + None => println!( + "✓ Agent {} is running. It has no herdr machine; {} adds one.", + agent.name.cyan(), + "connect".cyan() + ), + } + Ok(()) + } + + async fn delete(&mut self, row: &Row) -> Result<()> { + let agent = &row.agent; + let confirmed = prompt_confirm_with_default( + &format!( + "Delete agent {} and everything on its disk?", + agent.name.cyan() + ), + false, + )?; + if !confirmed { + println!("Left agent {} alone.", agent.name); + return Ok(()); + } + + let mut locked = self.store.lock().await?; + let spinner = create_spinner(format!("Deleting agent {}", agent.name)); + let result = ca::delete(&self.client, &self.backboard, &agent.id).await; + spinner.finish_and_clear(); + // Same rule as `railway ca delete`: forget the pointer even when the + // mutation failed, so a gone agent is never reached for again. + ca::forget(&mut self.configs, &agent.environment_id)?; + result?; + + if let Some(machine) = &row.machine { + self.herdr.machine_remove(&machine.id)?; + } + locked.state.machines.remove(&agent.id); + locked.state.sleep_until.remove(&agent.id); + locked.state.bootstrap_pending.remove(&agent.id); + locked.save()?; + println!("✓ Deleted agent {}", agent.name.cyan()); + Ok(()) + } + + async fn ensure_awake(&mut self, agent: &ca::Agent) -> Result { + // Inventory can lag a sleep/wake elsewhere. The server reads live VM + // state and handles no-ops; always send the user's intent to it. + self.herdr.notify( + &format!("Waking {}", agent.name), + "The machine is re-enabled once its SSH relay answers", + ); + { + let mut locked = self.store.lock().await?; + ca::wake(&self.client, &self.backboard, &agent.id).await?; + locked.state.sleep_until.remove(&agent.id); + locked.save()?; + } + ca::remember(&mut self.configs, agent)?; + let spinner = create_spinner(format!("Waking agent {}", agent.name)); + let result = ca::wait_until_running( + &self.client, + &self.backboard, + &agent.environment_id, + &agent.id, + ) + .await; + spinner.finish_and_clear(); + result + } + + async fn resync(&mut self) -> Result> { + if self.remote { + return Ok(Vec::new()); + } + sync::resync(&self.client, &self.backboard, &self.herdr, &self.store).await + } + + async fn reconnect(&self, agent: &ca::Agent, machine: &Machine) -> Result<()> { + let mut locked = self.store.lock().await?; + if locked.state.sleep_pending(&agent.id, chrono::Utc::now()) { + bail!( + "A sleep was requested for {} while connecting; wake it again to reconnect.", + agent.name + ); + } + self.herdr.machine_disable(&machine.id)?; + self.herdr.machine_enable(&machine.id)?; + locked + .state + .machines + .insert(agent.id.clone(), machine.id.clone()); + locked.save() + } + + async fn remember(&self, agent_id: &str, profile_id: Option<&str>) -> Result<()> { + self.store + .update(|state| match profile_id { + Some(profile) => { + state + .machines + .insert(agent_id.to_string(), profile.to_string()); + } + None => { + state.machines.remove(agent_id); + } + }) + .await + } +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use crate::testkit::MockBackboard; + + #[tokio::test] + async fn an_observed_running_agent_still_sends_the_wake_request() { + let api = MockBackboard::spawn(); + let home = tempfile::tempdir().unwrap(); + let fake = super::super::herdr_cli::fake::FakeHerdr::with_machines("[]"); + let agent = ca::Agent { + id: "agent-1".into(), + name: "reviewer".into(), + status: ca::Status::Running, + project_id: "project-1".into(), + environment_id: "environment-1".into(), + created_at: chrono::Utc::now(), + }; + api.stub( + "CloudAgentWake", + serde_json::json!({"cloudAgentWake": {"id": agent.id, "status": "STARTING"}}), + ); + api.stub( + "CloudAgent", + serde_json::json!({"cloudAgent": { + "id": agent.id, "name": agent.name, "status": "RUNNING", + "projectId": agent.project_id, "environmentId": agent.environment_id, + "createdAt": agent.created_at, + }}), + ); + let mut picker = Picker { + configs: api.configs(&home), + client: reqwest::Client::new(), + backboard: api.url(), + herdr: fake.herdr(), + store: Store::at(home.path().join("state.json"), &api.url(), "test"), + remote: false, + }; + picker.ensure_awake(&agent).await.unwrap(); + assert!( + api.requests() + .iter() + .any(|r| r["operationName"] == "CloudAgentWake"), + "an inventory observation must not veto explicit wake intent" + ); + } + + #[test] + fn empty_local_picker_offers_creation() { + let items = picker_items(Vec::new(), false); + assert!(matches!(items.first(), Some(Item::New))); + assert!(picker_items(Vec::new(), true).is_empty()); + } +} diff --git a/src/commands/cloud_agent/herdr/bootstrap.rs b/src/commands/cloud_agent/herdr/bootstrap.rs new file mode 100644 index 000000000..33639b3f2 --- /dev/null +++ b/src/commands/cloud_agent/herdr/bootstrap.rs @@ -0,0 +1,816 @@ +//! One remote script over the relay, run once per agent: herdr integrations, +//! the two config.toml keys herdr's machine mode needs, and an `app` workspace. + +use anyhow::{Context, Result, bail}; +use clap::Parser; +use colored::Colorize; + +use super::state::Store; +use crate::client::GQLClient; +use crate::commands::code; +use crate::commands::code::{HARNESS_PATH, LaunchArgs, Progress}; +use crate::commands::ssh::native::run_native_ssh_captured; +use crate::config::Configs; +use crate::controllers::cloud_agent as ca; + +#[derive(Parser)] +pub struct Args { + /// Agent name or ID + pub agent: Option, + + #[clap(flatten)] + harness: super::harness::HarnessFlags, +} + +pub async fn command(args: Args) -> Result<()> { + let configs = Configs::new()?; + let client = GQLClient::new_authorized(&configs)?; + let (agent, _) = ca::resolve(&configs, &client, args.agent.as_deref(), None).await?; + let harness = super::harness::choose(&args.harness, false)?; + run(&agent, harness, &Store::new(&configs)?).await +} + +/// Prepare a running agent's VM for herdr. Idempotent; `new` calls this right +/// after `herdr machine add`. +pub(super) async fn run(agent: &ca::Agent, harness: &str, store: &Store) -> Result<()> { + track_bootstrap(store, &agent.id, harness, run_inner(agent, harness)).await.with_context(|| format!( + "Herdr setup is incomplete for {}. Connect again, or retry `railway ca herdr bootstrap {} --{harness}`", + agent.name, agent.id, + )) +} + +async fn track_bootstrap( + store: &Store, + agent_id: &str, + harness: &str, + run: impl std::future::Future>, +) -> Result<()> { + store + .update(|s| { + s.bootstrap_pending + .insert(agent_id.to_owned(), harness.to_owned()); + }) + .await?; + let result = run.await; + if result.is_ok() { + store + .update(|s| { + s.bootstrap_pending.remove(agent_id); + }) + .await?; + } + result +} + +async fn run_inner(agent: &ca::Agent, harness: &str) -> Result<()> { + if !matches!(agent.status, ca::Status::Running) { + bail!( + "Agent {} is {}. Wake it first: {}", + agent.name, + agent.status.label(), + format!("railway ca wake {}", agent.name).cyan() + ); + } + // App mode: the same provisioning `railway ca desktop` does (credential, + // skills, MCP) without the ~/.profile autostart, which would launch the + // harness on top of every herdr login shell. + let mut launch = LaunchArgs::for_app_mode( + harness, + Some(agent.project_id.clone()), + Some(agent.environment_id.clone()), + ); + launch.agent_id = Some(agent.id.clone()); + let progress = code::CliProgress::default(); + let prepared = code::prepare(&launch, &progress, code::SessionStyle::FullTerminal).await; + progress.finish(); + prepared.with_context(|| format!("Provisioning {} for {harness}", agent.name))?; + println!(" provisioned {harness}: skills, MCP, credential when one was carried"); + + let remote = Remote::new(&Configs::new()?.get_backboard(), harness)?; + let info = code::connect_info(&agent.environment_id, &agent.id).await?; + let (code, stdout, stderr) = tokio::task::spawn_blocking(move || { + run_native_ssh_captured( + &info.ssh_target, + &script(&remote), + info.identity.as_deref(), + None, + &info.relay_opts, + ) + }) + .await??; + let stdout = String::from_utf8_lossy(&stdout); + match outcome(&stdout) { + Outcome::Ok(lines) => { + for line in lines { + println!(" {line}"); + } + println!("✓ Bootstrapped agent {} for herdr", agent.name.cyan()); + Ok(()) + } + Outcome::Failed(lines) => { + for line in &lines { + println!(" {line}"); + } + let stderr = String::from_utf8_lossy(&stderr); + bail!( + "Bootstrap of agent {} did not complete; the FAILED lines above say where.{}", + agent.name, + if stderr.trim().is_empty() { + String::new() + } else { + format!("\n{}", stderr.trim()) + } + ) + } + Outcome::HerdrMissing => { + bail!( + "herdr is not installed on agent {}; connect it with `railway ca herdr agents` first", + agent.name + ) + } + Outcome::NoMarker => bail!( + "Bootstrap of agent {} produced no status marker (ssh exit {code}).\n{}\n{}", + agent.name, + stdout.trim(), + String::from_utf8_lossy(&stderr).trim() + ), + } +} + +enum Outcome<'a> { + /// Steps ran and at least one reported FAILED; the lines say which. + Failed(Vec<&'a str>), + Ok(Vec<&'a str>), + HerdrMissing, + NoMarker, +} + +fn outcome(stdout: &str) -> Outcome<'_> { + let lines: Vec<&str> = stdout.lines().map(str::trim).collect(); + if lines.contains(&"HERDR-MISSING") { + return Outcome::HerdrMissing; + } + if lines.contains(&"BOOTSTRAP-FAILED") { + return Outcome::Failed( + lines + .iter() + .copied() + .filter(|l| !l.is_empty() && *l != "BOOTSTRAP-FAILED") + .collect(), + ); + } + if !lines.contains(&"BOOTSTRAP-OK") { + return Outcome::NoMarker; + } + Outcome::Ok( + lines + .into_iter() + .filter(|l| !l.is_empty() && *l != "BOOTSTRAP-OK") + .collect(), + ) +} + +/// `set_table_key` edits one key inside one `[table]`: replaces the line +/// (commented or not) when present, appends to the table otherwise, and creates +/// the table at the end when it is missing. Other tables are never touched. +const BODY: &str = r##"if ! command -v herdr >/dev/null 2>&1; then echo HERDR-MISSING; exit 0; fi +fail=0 +for tool in claude codex; do + if herdr integration install "$tool" >/dev/null 2>&1; then + echo "integration $tool: ok" + else + echo "integration $tool: skipped (herdr integration install $tool failed)" + fi +done +cfg="$HOME/.config/herdr/config.toml" +mkdir -p "$(dirname "$cfg")" +[ -f "$cfg" ] || : > "$cfg" +before="$(cat "$cfg")" +set_table_key() { + awk -v table="$1" -v key="$2" -v line="$3" ' + BEGIN { in_t = 0; seen = 0; done = 0 } + /^[[:space:]]*\[/ { + if (in_t && !done) { print line; done = 1 } + in_t = ($0 ~ "^[[:space:]]*\\[" table "\\][[:space:]]*(#.*)?$") + if (in_t) seen = 1 + } + in_t && !done && $0 ~ "^[[:space:]]*#?[[:space:]]*" key "[[:space:]]*=" { print line; done = 1; next } + { print } + END { + if (!done) { + if (!seen) { if (NR > 0) print ""; print "[" table "]" } + print line + } + } + ' "$cfg" > "$cfg.tmp" && mv "$cfg.tmp" "$cfg" || { echo "config: FAILED to write $cfg"; fail=1; } +} +set_table_key terminal shell_mode 'shell_mode = "login"' +set_table_key terminal default_shell 'default_shell = "/bin/bash"' +set_table_key experimental pane_history 'pane_history = true' +if [ "$before" = "$(cat "$cfg")" ]; then + echo "config: pane_history, shell_mode = login, default_shell = /bin/bash (unchanged)" +else + echo "config: pane_history, shell_mode = login, default_shell = /bin/bash (updated)" + herdr server reload-config >/dev/null 2>&1 && echo "config: reloaded" +fi +prof="$HOME/.profile" +if grep -q "railway ca herdr env" "$prof" 2>/dev/null; then + echo "profile: env block present" +else + if cat >> "$prof" <<'PROFEOF' + +# railway ca herdr env +[ -f "$HOME/.claude-code-env" ] && set -a && . "$HOME/.claude-code-env" && set +a +[ -f "$HOME/.gh-token" ] && export GH_TOKEN="$(cat "$HOME/.gh-token")" +PROFEOF + then echo "profile: env block added"; else echo "profile: FAILED to write $prof"; fail=1; fi +fi +pending_launch="$HOME/.config/railway-ca-herdr-plugin/app-launch-pending.json" +if ws="$(herdr workspace list 2>/dev/null)"; then + if command -v python3 >/dev/null 2>&1; then + has="$(printf '%s' "$ws" | python3 -c 'import json,sys; d=json.load(sys.stdin); print(int(any(w.get("label")=="app" for w in d.get("result",{}).get("workspaces",[]))))' 2>/dev/null || echo 0)" + elif printf '%s' "$ws" | grep -Eq '"label"[[:space:]]*:[[:space:]]*"app"'; then + has=1 + else + has=0 + fi + if [ "$has" = 1 ]; then + echo "workspace app: exists" + elif ! command -v python3 >/dev/null 2>&1; then + echo "workspace app: FAILED (python3 is required to start @HARNESS_CMD@)"; fail=1 + elif ! mkdir -p "$(dirname "$pending_launch")"; then + echo "workspace app: FAILED to prepare launch tracking"; fail=1 + elif herdr workspace create --label app --cwd /app > "$pending_launch" 2>/dev/null; then + echo "workspace app: created (/app)" + if command -v python3 >/dev/null 2>&1; then + bare="$(herdr workspace list 2>/dev/null | python3 -c 'import json,sys; d=json.load(sys.stdin); print(" ".join(w["workspace_id"] for w in d.get("result",{}).get("workspaces",[]) if w.get("label")=="/" and w.get("pane_count")==1 and w.get("agent_status") in (None,"unknown")))' 2>/dev/null)" + for id in $bare; do + herdr workspace close "$id" >/dev/null 2>&1 && echo "workspace /: closed (bare startup shell)" + done + fi + else + echo "workspace app: FAILED to create"; fail=1 + fi + # Workspace creation and launching are separate operations. Preserve the + # target pane across failures so a retry can finish an existing workspace, + # and clear it only after launch succeeds to avoid restarting working agents. + if [ -f "$pending_launch" ]; then + root="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["result"]["root_pane"]["pane_id"])' "$pending_launch" 2>/dev/null)" + if [ -z "$root" ] || [ -z "@HARNESS_CMD@" ]; then + echo "workspace app: FAILED to resolve pending @HARNESS_CMD@ launch"; fail=1 + else + sleep 2 + if herdr pane run "$root" "@HARNESS_CMD@" >/dev/null 2>&1; then + if rm "$pending_launch"; then + echo "started @HARNESS_CMD@ in the app workspace" + else + echo "workspace app: FAILED to clear pending launch"; fail=1 + fi + else + echo "workspace app: FAILED to start @HARNESS_CMD@ in $root"; fail=1 + fi + fi + fi +else + echo "workspace app: FAILED (no herdr server running; connecting starts one)"; fail=1 +fi +"##; + +/// The plugin the VM's herdr server publishes, so the same keys work while +/// this machine is selected. `cfg` is the config path BODY established. +const REMOTE_SEGMENT: &str = r##"want="@CLI_VERSION@" +have="$(railway --version 2>/dev/null | awk '{print $2}')" +if [ -z "$have" ]; then + echo "railway cli: not installed on the VM" +elif [ "$have" != "$want" ] && [ "$(printf '%s\n%s\n' "$have" "$want" | sort -V | head -1)" = "$have" ]; then + if curl -fsSL https://railway.com/install.sh | bash -s -- -y --bin-dir "$(dirname "$(command -v railway)")" >/dev/null 2>&1; then + echo "railway cli: $have → $(railway --version 2>/dev/null | awk '{print $2}')" + else + echo "railway cli: upgrade from $have failed (kept it)" + fi +else + echo "railway cli: $have" +fi +pdir="$HOME/.config/railway-ca-herdr-plugin" +mkdir -p "$pdir" +cat > "$pdir/herdr-plugin.toml" <<'RAILWAY_CA_MANIFEST' +@MANIFEST@ +RAILWAY_CA_MANIFEST +cat > "$pdir/agents.sh" <<'RAILWAY_CA_AGENTS' +@AGENTS_SH@ +RAILWAY_CA_AGENTS +cat > "$pdir/sleep-self.sh" <<'RAILWAY_CA_SLEEP' +@SLEEP_SH@ +RAILWAY_CA_SLEEP +chmod 755 "$pdir"/*.sh +if herdr plugin list 2>/dev/null | grep -q "railway\.ca "; then + echo "remote plugin: linked" +elif herdr plugin link "$pdir" >/dev/null 2>&1; then + echo "remote plugin: linked (new)" +else + echo "remote plugin: FAILED to link $pdir"; fail=1 +fi +keys_block="$(printf '%s\n%s\n' "@KEYS_MARKER@" '@KEYS@')" +if grep -q "@KEYS_MARKER@" "$cfg" 2>/dev/null && command -v python3 >/dev/null 2>&1; then + KEYS_BLOCK="$keys_block" python3 - "$cfg" <<'RAILWAY_CA_PY' +import os, re, sys +path = sys.argv[1]; text = open(path).read(); block = os.environ["KEYS_BLOCK"] +marker = block.splitlines()[0] +kept = []; lines = text.split("\n"); i = 0 +while i < len(lines): + if lines[i].strip() == marker: + i += 1; continue + if lines[i].strip() == "[[keys.command]]": + j = i + 1 + while j < len(lines) and not lines[j].lstrip().startswith("["): j += 1 + if any('"railway.ca.' in l for l in lines[i:j]): + while j > i + 1 and not lines[j - 1].strip(): j -= 1 + i = j; continue + kept.append(lines[i]); i += 1 +base = "\n".join(kept).rstrip("\n") +new = (base + "\n\n" if base else "") + block + "\n" +if new != text: + open(path, "w").write(new); print("keys: updated") +else: + print("keys: present") +RAILWAY_CA_PY +elif grep -q "@KEYS_MARKER@" "$cfg" 2>/dev/null; then + echo "keys: present" +else + if printf '\n%s\n' "$keys_block" >> "$cfg"; then echo "keys: added"; else echo "keys: FAILED to write $cfg"; fail=1; fi +fi +herdr server reload-config >/dev/null 2>&1 && echo "config: reloaded" +"##; + +const AGENTS_SH: &str = r##"#!/bin/sh +export PATH="$HOME/.local/bin:/usr/local/bin:$PATH" +if [ "$1" = "--open" ]; then + exec "${HERDR_BIN_PATH:-herdr}" plugin pane open --plugin railway.ca --entrypoint agents +fi +if railway ca herdr --help >/dev/null 2>&1; then + exec railway ca herdr agents --remote +fi +echo "railway $(railway --version 2>/dev/null | awk '{print $2}') on this VM has no 'ca herdr' yet." +echo "Bootstrap upgrades it once a release ships it. Until then select Local and press the same key." +sleep 6"##; + +const SLEEP_SH: &str = r##"#!/bin/sh +set -eu +: "${RAILWAY_API_TOKEN:?RAILWAY_API_TOKEN is not in the herdr server env}" +: "${RAILWAY_CLOUD_AGENT_ID:?RAILWAY_CLOUD_AGENT_ID is not in the herdr server env}" +"${HERDR_BIN_PATH:-herdr}" notification show "Sleeping this agent" --body "railway: cloudAgentSleep issued from the VM; disable or re-sync its machine from Local" --sound none >/dev/null 2>&1 || true +sync +curl -fsS -m 20 "@BACKBOARD@" \ + -H "Authorization: Bearer $RAILWAY_API_TOKEN" -H "Content-Type: application/json" \ + -d "{\"query\":\"mutation(\$id: ID!) { cloudAgentSleep(id: \$id) { id status } }\",\"variables\":{\"id\":\"$RAILWAY_CLOUD_AGENT_ID\"}}" +echo"##; + +struct Remote { + manifest: String, + backboard: String, + harness_cmd: String, +} + +impl Remote { + fn new(backboard: &str, harness: &str) -> Result { + Ok(Self { + manifest: super::install::Manifest::remote().render()?, + backboard: backboard.to_string(), + harness_cmd: harness_command(harness).to_string(), + }) + } + + fn segment(&self) -> String { + REMOTE_SEGMENT + .replace("@MANIFEST@", self.manifest.trim_end()) + .replace("@AGENTS_SH@", AGENTS_SH) + .replace( + "@SLEEP_SH@", + &SLEEP_SH.replace("@BACKBOARD@", &self.backboard), + ) + .replace("@CLI_VERSION@", env!("CARGO_PKG_VERSION")) + .replace("@KEYS_MARKER@", super::install::KEYS_MARKER) + .replace("@KEYS@", super::install::REMOTE_KEYBINDING.trim_end()) + } +} + +/// What to type into the fresh /app pane. Railway's harness has its own binary +/// name; the others match their slug. Unknown slugs start nothing. +fn harness_command(harness: &str) -> &'static str { + match harness { + "claude" => "claude", + "codex" => "codex", + "grok" => "grok", + "railway" => "railway-agent-tui", + _ => "", + } +} + +fn script(remote: &Remote) -> String { + format!( + "{HARNESS_PATH}\n{}\n{}\nif [ \"$fail\" = 1 ]; then echo BOOTSTRAP-FAILED; else echo BOOTSTRAP-OK; fi\n", + BODY.replace("@HARNESS_CMD@", &remote.harness_cmd), + remote.segment() + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn failed_bootstrap_preserves_the_selected_harness_for_connect_to_retry() { + let dir = tempfile::tempdir().unwrap(); + let store = Store::at(dir.path().join("state.json"), "backboard", "account"); + store + .update(|s| { + s.machines.insert("agent".into(), "profile".into()); + }) + .await + .unwrap(); + let result = track_bootstrap(&store, "agent", "codex", async { + bail!("provisioning failed") + }) + .await; + assert!(result.is_err()); + let state = store.load().unwrap(); + assert_eq!(state.machines["agent"], "profile"); + // The same persisted value the existing-profile Connect path reads. + let harness = &state.bootstrap_pending["agent"]; + assert_eq!(harness, "codex"); + track_bootstrap(&store, "agent", harness, async { Ok(()) }) + .await + .unwrap(); + assert!(store.load().unwrap().bootstrap_pending.is_empty()); + } + + #[test] + fn outcome_reads_markers() { + assert!(matches!(outcome("HERDR-MISSING\n"), Outcome::HerdrMissing)); + assert!(matches!( + outcome("profile: FAILED to write /root/.profile\nBOOTSTRAP-FAILED\n"), + Outcome::Failed(lines) if lines == vec!["profile: FAILED to write /root/.profile"] + )); + assert!(matches!( + outcome("integration claude: ok\n"), + Outcome::NoMarker + )); + match outcome("integration claude: ok\n\nworkspace app: exists\nBOOTSTRAP-OK\n") { + Outcome::Ok(lines) => assert_eq!( + lines, + vec!["integration claude: ok", "workspace app: exists"] + ), + _ => panic!("expected Ok"), + } + } + + #[cfg(unix)] + mod script { + use std::process::Command; + + use super::super::{Remote, script}; + + fn remote() -> Remote { + Remote::new("https://backboard.example/graphql/v2", "claude").unwrap() + } + + fn keys_tail() -> String { + format!( + "\n{}\n{}", + crate::commands::cloud_agent::herdr::install::KEYS_MARKER, + crate::commands::cloud_agent::herdr::install::REMOTE_KEYBINDING + ) + } + + const WORKSPACES_WITHOUT_APP: &str = + r#"{"result":{"workspaces":[{"label":"default","cwd":"/root"}]}}"#; + const WORKSPACES_WITH_APP: &str = + r#"{"result":{"workspaces":[{"label":"default"},{"label":"app","cwd":"/app"}]}}"#; + + struct Vm { + home: tempfile::TempDir, + } + + impl Vm { + fn new(workspaces: &str) -> Self { + let vm = Self { + home: tempfile::tempdir().unwrap(), + }; + vm.install_herdr(&format!( + "#!/bin/bash\necho \"$*\" >> \"$HOME/herdr.log\"\nif [ \"$1 $2\" = \"workspace list\" ]; then cat <<'EOF'\n{workspaces}\nEOF\nfi\nif [ \"$1 $2\" = \"workspace create\" ]; then echo '{{\"result\":{{\"root_pane\":{{\"pane_id\":\"w9:p1\"}}}}}}'; fi\n" + )); + // Never discover or upgrade the host's real Railway CLI. The + // script prepends this directory to PATH just as it does on a VM. + for tool in ["railway", "curl"] { + use std::os::unix::fs::PermissionsExt; + let path = vm.home.path().join(".local/bin").join(tool); + std::fs::write(&path, "#!/bin/sh\nexit 127\n").unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)) + .unwrap(); + } + vm + } + + fn install_herdr(&self, shim: &str) { + use std::os::unix::fs::PermissionsExt; + let path = self.herdr_path(); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, shim).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + + fn herdr_path(&self) -> std::path::PathBuf { + self.home.path().join(".local/bin/herdr") + } + + fn run(&self) -> String { + let out = Command::new("bash") + .arg("-c") + .arg(script(&remote())) + .env_clear() + .env("HOME", self.home.path()) + .env("PATH", "/usr/bin:/bin") + .output() + .unwrap(); + assert!( + out.status.success(), + "stderr: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout).unwrap() + } + + fn config_path(&self) -> std::path::PathBuf { + self.home.path().join(".config/herdr/config.toml") + } + + fn config(&self) -> String { + std::fs::read_to_string(self.config_path()).unwrap_or_default() + } + + fn write_config(&self, text: &str) { + let path = self.config_path(); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, text).unwrap(); + } + + fn herdr_calls(&self) -> Vec { + std::fs::read_to_string(self.home.path().join("herdr.log")) + .unwrap_or_default() + .lines() + .map(str::to_owned) + .collect() + } + } + + #[test] + fn failed_launch_retries_the_existing_workspace_and_does_not_launch_twice() { + let vm = Vm::new(WORKSPACES_WITHOUT_APP); + vm.install_herdr( + r##"#!/bin/bash +echo "$*" >> "$HOME/herdr.log" +case "$1 $2" in + 'workspace list') + if [ -f "$HOME/app-created" ]; then + echo '{"result":{"workspaces":[{"label":"app","workspace_id":"w9"}]}}' + else + echo '{"result":{"workspaces":[]}}' + fi + ;; + 'workspace create') + touch "$HOME/app-created" + echo '{"result":{"root_pane":{"pane_id":"w9:p1"}}}' + ;; + 'pane run') + [ -f "$HOME/allow-launch" ] || exit 1 + ;; +esac +"##, + ); + let pending = vm + .home + .path() + .join(".config/railway-ca-herdr-plugin/app-launch-pending.json"); + let out = vm.run(); + assert!(out.contains("FAILED to start claude in w9:p1"), "{out}"); + assert!(out.trim_end().ends_with("BOOTSTRAP-FAILED"), "{out}"); + assert!(pending.exists()); + + std::fs::write(vm.home.path().join("allow-launch"), "").unwrap(); + let out = vm.run(); + assert!(out.contains("workspace app: exists"), "{out}"); + assert!(out.contains("started claude"), "{out}"); + assert!(out.trim_end().ends_with("BOOTSTRAP-OK"), "{out}"); + assert!(!pending.exists()); + + let out = vm.run(); + assert!(out.trim_end().ends_with("BOOTSTRAP-OK"), "{out}"); + let calls = vm.herdr_calls(); + assert_eq!( + calls + .iter() + .filter(|c| c.starts_with("workspace create")) + .count(), + 1, + "{calls:?}" + ); + assert_eq!( + calls + .iter() + .filter(|c| *c == "pane run w9:p1 claude") + .count(), + 2, + "{calls:?}" + ); + } + + #[test] + fn an_unreadable_pending_launch_does_not_report_success() { + let vm = Vm::new(WORKSPACES_WITH_APP); + let pending = vm + .home + .path() + .join(".config/railway-ca-herdr-plugin/app-launch-pending.json"); + std::fs::create_dir_all(pending.parent().unwrap()).unwrap(); + std::fs::write(&pending, "interrupted response").unwrap(); + let out = vm.run(); + assert!( + out.contains("FAILED to resolve pending claude launch"), + "{out}" + ); + assert!(out.trim_end().ends_with("BOOTSTRAP-FAILED"), "{out}"); + assert!(pending.exists()); + assert!(!vm.herdr_calls().iter().any(|c| c.starts_with("pane run"))); + } + + #[test] + fn fresh_vm_gets_config_integrations_and_workspace() { + let vm = Vm::new(WORKSPACES_WITHOUT_APP); + let out = vm.run(); + assert!(out.trim_end().ends_with("BOOTSTRAP-OK"), "{out}"); + assert!(out.contains("integration claude: ok"), "{out}"); + assert!(out.contains("integration codex: ok"), "{out}"); + assert!(out.contains("default_shell = /bin/bash (updated)"), "{out}"); + assert!(out.contains("profile: env block added"), "{out}"); + assert!(out.contains("workspace app: created"), "{out}"); + assert_eq!( + vm.config(), + format!( + "{}{}", + "[terminal]\nshell_mode = \"login\"\ndefault_shell = \"/bin/bash\"\n\n[experimental]\npane_history = true\n", + keys_tail() + ) + ); + let profile = std::fs::read_to_string(vm.home.path().join(".profile")).unwrap(); + assert!(profile.contains(".claude-code-env"), "{profile}"); + let calls = vm.herdr_calls(); + assert_eq!( + &calls[..5], + [ + "integration install claude", + "integration install codex", + "server reload-config", + "workspace list", + "workspace create --label app --cwd /app", + ] + ); + assert!( + calls.iter().any(|c| c.starts_with("plugin link ")), + "{calls:?}" + ); + assert!(out.contains("remote plugin: linked (new)"), "{out}"); + assert!(out.contains("railway cli: not installed"), "{out}"); + assert!(out.contains("keys: added"), "{out}"); + let pdir = vm.home.path().join(".config/railway-ca-herdr-plugin"); + let manifest = std::fs::read_to_string(pdir.join("herdr-plugin.toml")).unwrap(); + assert!(manifest.contains("id = \"railway.ca\""), "{manifest}"); + let sleep = std::fs::read_to_string(pdir.join("sleep-self.sh")).unwrap(); + assert!(sleep.contains("cloudAgentSleep"), "{sleep}"); + assert!( + sleep.contains("https://backboard.example/graphql/v2"), + "{sleep}" + ); + assert!( + std::fs::read_to_string(pdir.join("agents.sh")) + .unwrap() + .contains("--remote") + ); + } + + #[test] + fn second_run_changes_nothing() { + let vm = Vm::new(WORKSPACES_WITH_APP); + vm.run(); + let first = vm.config(); + let out = vm.run(); + assert_eq!(vm.config(), first); + assert!(out.contains("(unchanged)"), "{out}"); + assert!(out.contains("profile: env block present"), "{out}"); + assert!(out.contains("keys: present"), "{out}"); + assert!(out.contains("workspace app: exists"), "{out}"); + let profile = std::fs::read_to_string(vm.home.path().join(".profile")).unwrap(); + assert_eq!( + profile.matches("railway ca herdr env").count(), + 1, + "{profile}" + ); + assert!( + !vm.herdr_calls() + .iter() + .any(|c| c.starts_with("workspace create")), + "{:?}", + vm.herdr_calls() + ); + } + + #[test] + fn existing_config_keeps_other_keys_and_tables() { + let vm = Vm::new(WORKSPACES_WITH_APP); + vm.write_config( + "[theme]\nname = \"dark\"\n\n[terminal]\n# shell_mode = \"auto\"\nscrollback = 1\n\n[keys]\nshell_mode = \"x\"\n", + ); + vm.run(); + assert_eq!( + vm.config(), + format!( + "{}{}", + "[theme]\nname = \"dark\"\n\n[terminal]\nshell_mode = \"login\"\nscrollback = 1\n\ndefault_shell = \"/bin/bash\"\n[keys]\nshell_mode = \"x\"\n\n[experimental]\npane_history = true\n", + keys_tail() + ) + ); + } + + #[test] + fn existing_values_are_replaced_in_place() { + let vm = Vm::new(WORKSPACES_WITH_APP); + vm.write_config( + "[terminal]\nshell_mode = \"non_login\"\ndefault_shell = \"/bin/zsh\"\n[experimental]\npane_history = false\n", + ); + vm.run(); + assert_eq!( + vm.config(), + format!( + "{}{}", + "[terminal]\nshell_mode = \"login\"\ndefault_shell = \"/bin/bash\"\n[experimental]\npane_history = true\n", + keys_tail() + ) + ); + } + + #[test] + fn bare_startup_workspace_is_closed_after_app_is_created() { + let vm = Vm::new( + r#"{"result":{"workspaces":[{"workspace_id":"w1","label":"/","pane_count":1,"agent_status":"unknown"}]}}"#, + ); + let out = vm.run(); + assert!(out.contains("workspace /: closed"), "{out}"); + assert!( + vm.herdr_calls().contains(&"workspace close w1".to_string()), + "{:?}", + vm.herdr_calls() + ); + } + + #[test] + fn a_failed_step_ends_in_the_failed_marker() { + let vm = Vm::new(WORKSPACES_WITH_APP); + vm.install_herdr( + "#!/bin/bash\necho \"$*\" >> \"$HOME/herdr.log\"\nif [ \"$1 $2\" = \"plugin link\" ]; then exit 1; fi\nif [ \"$1 $2\" = \"workspace list\" ]; then cat <<'EOF'\n{\"result\":{\"workspaces\":[{\"label\":\"app\"}]}}\nEOF\nfi\n", + ); + let out = vm.run(); + assert!(out.contains("remote plugin: FAILED"), "{out}"); + assert!(out.trim_end().ends_with("BOOTSTRAP-FAILED"), "{out}"); + assert!(!out.contains("BOOTSTRAP-OK"), "{out}"); + } + + #[test] + fn a_commented_table_header_is_still_the_table() { + let vm = Vm::new(WORKSPACES_WITH_APP); + vm.write_config("[terminal] # mine\nscrollback = 1\n"); + vm.run(); + let cfg = vm.config(); + assert_eq!(cfg.matches("[terminal]").count(), 1, "{cfg}"); + assert!(cfg.contains("shell_mode = \"login\""), "{cfg}"); + } + + #[test] + fn no_server_reports_incomplete_bootstrap() { + let vm = Vm::new(""); + vm.install_herdr( + "#!/bin/bash\necho \"$*\" >> \"$HOME/herdr.log\"\n[ \"$1\" = workspace ] && exit 1\nexit 0\n", + ); + let out = vm.run(); + assert!(out.contains("workspace app: FAILED"), "{out}"); + assert!(out.trim_end().ends_with("BOOTSTRAP-FAILED"), "{out}"); + } + + #[test] + fn missing_herdr_reports_and_exits_zero() { + let vm = Vm::new(""); + std::fs::remove_file(vm.herdr_path()).unwrap(); + let out = vm.run(); + assert_eq!(out.trim(), "HERDR-MISSING"); + assert!(!vm.home.path().join(".config").exists()); + } + } +} diff --git a/src/commands/cloud_agent/herdr/harness.rs b/src/commands/cloud_agent/herdr/harness.rs new file mode 100644 index 000000000..b901f4162 --- /dev/null +++ b/src/commands/cloud_agent/herdr/harness.rs @@ -0,0 +1,104 @@ +//! Which coding agent a VM is provisioned for: flags first, then a pick that +//! defaults to `railway ca setup`'s choice. + +use std::fmt; + +use anyhow::{Context, Result}; +use clap::Args as ClapArgs; + +use crate::commands::code; +use crate::config::Configs; + +#[derive(ClapArgs, Default, Clone)] +pub struct HarnessFlags { + /// Provision for Claude Code + #[clap(long, conflicts_with_all = ["codex", "grok", "railway"])] + pub claude: bool, + + /// Provision for OpenAI Codex + #[clap(long, conflicts_with_all = ["grok", "railway"])] + pub codex: bool, + + /// Provision for xAI Grok + #[clap(long, conflicts_with = "railway")] + pub grok: bool, + + /// Provision for Railway's own agent (no sign-in needed) + #[clap(long)] + pub railway: bool, +} + +impl HarnessFlags { + pub fn slug(&self) -> Option<&'static str> { + [ + (self.claude, "claude"), + (self.codex, "codex"), + (self.grok, "grok"), + (self.railway, "railway"), + ] + .into_iter() + .find_map(|(on, slug)| on.then_some(slug)) + } +} + +const CHOICES: [(&str, &str); 4] = [ + ("claude", "Anthropic's Claude Code"), + ("codex", "OpenAI's Codex"), + ("grok", "xAI's Grok"), + ("railway", "Railway's own agent, no sign-in needed"), +]; + +struct Choice(&'static str, &'static str); + +impl fmt::Display for Choice { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:<8} {}", self.0, self.1) + } +} + +/// A flag wins; otherwise ask (starting on the saved default) or, when asking +/// is not wanted, take the saved default. +pub fn choose(flags: &HarnessFlags, ask: bool) -> Result<&'static str> { + if let Some(slug) = flags.slug() { + return Ok(slug); + } + let default = code::default_harness()?; + if !ask { + return Ok(default); + } + let options: Vec = CHOICES.iter().map(|(s, b)| Choice(s, b)).collect(); + let picked = inquire::Select::new("Coding agent", options) + .with_starting_cursor(default_cursor(default)) + .with_render_config(Configs::get_render_config()) + .prompt() + .context("Failed to prompt for the coding agent")?; + Ok(picked.0) +} + +fn default_cursor(default: &str) -> usize { + CHOICES + .iter() + .position(|(slug, _)| *slug == default) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_flag_maps_to_its_slug() { + let flags = HarnessFlags { + codex: true, + ..Default::default() + }; + assert_eq!(flags.slug(), Some("codex")); + assert_eq!(HarnessFlags::default().slug(), None); + } + + #[test] + fn the_saved_default_is_the_starting_row() { + assert_eq!(default_cursor("grok"), 2); + assert_eq!(default_cursor("shell"), 0); + } +} diff --git a/src/commands/cloud_agent/herdr/herdr_cli.rs b/src/commands/cloud_agent/herdr/herdr_cli.rs new file mode 100644 index 000000000..4993d83e1 --- /dev/null +++ b/src/commands/cloud_agent/herdr/herdr_cli.rs @@ -0,0 +1,220 @@ +//! The one place this module runs the `herdr` binary. +//! +//! herdr injects `HERDR_BIN_PATH` into plugin commands so they reach the binary +//! that started them regardless of PATH; tests point it at `tests/fakes/herdr`. + +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +use anyhow::{Context, Result, bail}; +use serde::Deserialize; + +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +pub struct Machine { + pub id: String, + pub label: String, + pub target: String, + #[serde(default)] + pub session: String, + pub enabled: bool, + #[serde(default)] + pub selected: bool, +} + +pub struct Herdr { + bin: PathBuf, + env: Vec<(String, String)>, +} + +impl Herdr { + pub fn from_env() -> Self { + let bin = std::env::var_os("HERDR_BIN_PATH") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from("herdr")); + Self { + bin, + env: Vec::new(), + } + } + + #[cfg(test)] + pub fn at(bin: impl Into) -> Self { + Self { + bin: bin.into(), + env: Vec::new(), + } + } + + #[cfg(test)] + pub fn with_env(mut self, key: &str, value: impl Into) -> Self { + self.env.push((key.to_string(), value.into())); + self + } + + fn command(&self, args: &[&str]) -> Command { + let mut cmd = Command::new(&self.bin); + cmd.args(args).envs(self.env.iter().map(|(k, v)| (k, v))); + cmd + } + + pub fn machines(&self) -> Result> { + let out = self.output(&["machine", "list", "--json"])?; + serde_json::from_str(&out).context("Unexpected `herdr machine list --json` output") + } + + /// Interactive: herdr asks about host keys and installing its server on the + /// VM, so stdio is inherited and the caller must be on a terminal. + pub fn machine_add(&self, target: &str, label: &str) -> Result<()> { + self.run_inherited(&["machine", "add", target, "--label", label]) + } + + pub fn machine_enable(&self, id: &str) -> Result<()> { + self.output(&["machine", "enable", id]).map(drop) + } + + pub fn machine_disable(&self, id: &str) -> Result<()> { + self.output(&["machine", "disable", id]).map(drop) + } + + pub fn machine_remove(&self, id: &str) -> Result<()> { + self.output(&["machine", "remove", id]).map(drop) + } + + pub fn plugin_link(&self, dir: &Path) -> Result<()> { + let dir = dir.to_string_lossy(); + self.output(&["plugin", "link", &dir]).map(drop) + } + + pub fn plugin_unlink(&self, plugin_id: &str) -> Result<()> { + self.output(&["plugin", "unlink", plugin_id]).map(drop) + } + + /// Best effort: a toast is never worth failing the action for. + pub fn notify(&self, title: &str, body: &str) { + let _ = self.output(&[ + "notification", + "show", + title, + "--body", + body, + "--sound", + "none", + ]); + } + + pub fn server_reload_config(&self) -> Result<()> { + self.output(&["server", "reload-config"]).map(drop) + } + + pub fn plugin_pane_open(&self, plugin_id: &str, entrypoint: &str) -> Result<()> { + self.output(&[ + "plugin", + "pane", + "open", + "--plugin", + plugin_id, + "--entrypoint", + entrypoint, + ]) + .map(drop) + } + + fn output(&self, args: &[&str]) -> Result { + let out = self + .command(args) + .stdin(Stdio::null()) + .output() + .with_context(|| format!("Failed to run {} {}", self.bin.display(), args.join(" ")))?; + if !out.status.success() { + bail!( + "`herdr {}` failed ({}): {}", + args.join(" "), + out.status, + String::from_utf8_lossy(&out.stderr).trim() + ); + } + Ok(String::from_utf8_lossy(&out.stdout).into_owned()) + } + + fn run_inherited(&self, args: &[&str]) -> Result<()> { + let status = self + .command(args) + .status() + .with_context(|| format!("Failed to run {} {}", self.bin.display(), args.join(" ")))?; + if !status.success() { + bail!("`herdr {}` failed ({status})", args.join(" ")); + } + Ok(()) + } +} + +#[cfg(all(test, unix))] +pub(crate) mod fake { + //! `tests/fakes/herdr` answers `machine list --json` from + //! `$FAKE_HERDR_MACHINES` and appends every argv line to `$FAKE_HERDR_LOG`. + //! Both ride the child's env, so parallel tests never share state. + + use std::path::PathBuf; + + use super::Herdr; + + pub struct FakeHerdr { + dir: tempfile::TempDir, + } + + impl FakeHerdr { + pub fn with_machines(json: &str) -> Self { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("machines.json"), json).unwrap(); + Self { dir } + } + + pub fn herdr(&self) -> Herdr { + let bin = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fakes/herdr"); + Herdr::at(bin) + .with_env( + "FAKE_HERDR_MACHINES", + self.dir.path().join("machines.json").to_string_lossy(), + ) + .with_env( + "FAKE_HERDR_LOG", + self.dir.path().join("log").to_string_lossy(), + ) + } + + pub fn calls(&self) -> Vec { + std::fs::read_to_string(self.dir.path().join("log")) + .unwrap_or_default() + .lines() + .map(str::to_owned) + .collect() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // The fake herdr is a shebang script: unix only. + + #[cfg(unix)] + #[test] + fn machine_list_parses_and_calls_are_logged() { + let fake = fake::FakeHerdr::with_machines( + r#"[{"id":"0123456789abcdef0123456789abcdef","label":"p/a","target":"agent:e:i@ssh.railway.com","session":"default","enabled":true,"selected":false}]"#, + ); + let herdr = fake.herdr(); + let machines = herdr.machines().unwrap(); + assert_eq!(machines.len(), 1); + assert_eq!(machines[0].target, "agent:e:i@ssh.railway.com"); + herdr.machine_disable(&machines[0].id).unwrap(); + assert_eq!( + fake.calls(), + vec![ + "machine list --json".to_string(), + "machine disable 0123456789abcdef0123456789abcdef".to_string(), + ] + ); + } +} diff --git a/src/commands/cloud_agent/herdr/install.rs b/src/commands/cloud_agent/herdr/install.rs new file mode 100644 index 000000000..6290927e2 --- /dev/null +++ b/src/commands/cloud_agent/herdr/install.rs @@ -0,0 +1,673 @@ +//! The plugin manifest herdr links. Every command in it is argv with an +//! absolute path to this binary: herdr starts plugin commands with the +//! server's env, not the user's shell PATH. + +use std::ffi::OsString; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use clap::Parser; +use colored::Colorize; +use serde::{Deserialize, Serialize}; + +use super::PLUGIN_ID; +use super::herdr_cli::Herdr; + +pub const MANIFEST_FILE: &str = "herdr-plugin.toml"; + +// Two keys, unbound in herdr's defaults: the picker (which also offers new +// agent and sync now) and wake. The descriptions are what `prefix+?` lists. +const KEYBINDING: &str = r#"[[keys.command]] +key = "prefix+shift+a" +type = "plugin_action" +command = "railway.ca.agents" +description = "railway agents" + +[[keys.command]] +key = "prefix+shift+s" +type = "plugin_action" +command = "railway.ca.wake" +description = "railway wake agent" +"#; + +/// On a VM the same keys mean: the picker in remote mode, and sleep THIS agent. +/// Same plugin id on both servers, so a binding reads the same wherever the +/// client is pointed. +pub(super) const REMOTE_KEYBINDING: &str = r#"[[keys.command]] +key = "prefix+shift+a" +type = "plugin_action" +command = "railway.ca.agents" +description = "railway agents" + +[[keys.command]] +key = "prefix+shift+s" +type = "plugin_action" +command = "railway.ca.sleep-self" +description = "railway sleep this agent" +"#; + +#[derive(Parser)] +pub struct Args { + /// Unlink the plugin and delete its manifest + #[clap(long)] + remove: bool, + + /// Print the manifest and exit without writing or linking + #[clap(long, conflicts_with = "remove")] + print: bool, + + /// Leave herdr's config.toml alone (no keybindings added or removed) + #[clap(long)] + no_keys: bool, +} + +pub async fn command(args: Args) -> Result<()> { + let dir = super::plugin_dir()?; + let herdr = Herdr::from_env(); + if args.remove { + if let Some(pid) = super::watch::stop() { + println!("✓ Stopped the watcher (pid {pid})"); + } + remove_from(&herdr, &dir)?; + println!("✓ Unlinked herdr plugin {}", PLUGIN_ID.cyan()); + if !args.no_keys && remove_keybindings(&herdr_config_path()?)? { + println!("✓ Removed the Railway keybindings from herdr's config.toml"); + let _ = herdr.server_reload_config(); + } + return Ok(()); + } + let manifest = Manifest::new(railway_binary()?); + if args.print { + print!("{}", manifest.render()?); + return Ok(()); + } + install_into(&herdr, &dir, &manifest)?; + println!( + "✓ Linked herdr plugin {} from {}", + PLUGIN_ID.cyan(), + dir.join(MANIFEST_FILE).display() + ); + if manifest.binary().contains("/target/") { + println!( + "{}", + format!( + "The manifest points at a build directory ({}); rerun install after moving or cleaning it.", + manifest.binary() + ) + .yellow() + ); + } + if let Some(warning) = super::known_hosts::ssh_config_warning() { + println!("{} {warning}", "!".yellow()); + } + match super::known_hosts::ensure_relay_known_host()? { + super::known_hosts::Seeded::Added => { + println!("✓ Added the Railway ssh relay to ~/.ssh/known_hosts") + } + super::known_hosts::Seeded::Present => {} + super::known_hosts::Seeded::NoSource => println!( + "{}", + "The relay's host key is not cached yet; connect once with `railway ca ssh` and rerun install." + .yellow() + ), + } + if args.no_keys { + println!( + "\nTo bind the pickers, add to your herdr config.toml:\n\n{}", + KEYBINDING.dimmed() + ); + return Ok(()); + } + super::watch::spawn_detached(); + std::thread::sleep(std::time::Duration::from_millis(300)); + match super::watch::running() { + Some(pid) => println!( + "✓ Watching cloud agent state for this herdr session (pid {pid}, log in {})", + dir.join("watch*.log").display() + ), + None if std::env::var_os("HERDR_SOCKET_PATH").is_none() => println!( + "{}", + "Not inside herdr: the watcher starts with herdr's next launch.".dimmed() + ), + None => println!( + "{}", + "The watcher did not start; see the watch log.".yellow() + ), + } + let config = herdr_config_path()?; + if ensure_keybindings(&config)? { + println!( + "✓ Bound {} agents (new agent and sync live in the picker), {} wake in {}", + "prefix+shift+a".cyan(), + "prefix+shift+s".cyan(), + config.display() + ); + if herdr.server_reload_config().is_err() { + println!( + "{}", + "herdr is not running; the bindings apply when it starts.".dimmed() + ); + } + } + Ok(()) +} + +pub(super) const KEYS_MARKER: &str = "# railway ca herdr keys"; + +/// herdr's own rule: `$XDG_CONFIG_HOME/herdr`, else `~/.config/herdr`. +fn herdr_config_path() -> Result { + let dir = match std::env::var_os("XDG_CONFIG_HOME") { + Some(xdg) if !xdg.is_empty() => PathBuf::from(xdg), + _ => dirs::home_dir() + .ok_or_else(|| anyhow::anyhow!("Unable to get home directory"))? + .join(".config"), + }; + Ok(dir.join("herdr").join("config.toml")) +} + +/// Our marker block is rewritten when the bindings changed; a config that +/// binds the actions on its own, without the marker, is left alone. +fn ensure_keybindings(path: &Path) -> Result { + let existing = std::fs::read_to_string(path).unwrap_or_default(); + let current = existing.contains(KEYS_MARKER); + if !current && existing.contains("railway.ca.agents") { + return Ok(false); + } + let base = if current { + strip_keybindings(&existing) + } else { + existing.clone() + }; + let wanted = with_keybindings(&base); + if wanted == existing { + return Ok(false); + } + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(path, wanted).with_context(|| format!("Writing {}", path.display()))?; + Ok(true) +} + +fn with_keybindings(existing: &str) -> String { + let mut out = existing.to_string(); + if !out.is_empty() && !out.ends_with('\n') { + out.push('\n'); + } + if !out.is_empty() { + out.push('\n'); + } + out.push_str(KEYS_MARKER); + out.push('\n'); + out.push_str(KEYBINDING); + out +} + +/// Drops the marker line and every `[[keys.command]]` table bound to a +/// `railway.ca.*` action. Anything else in the file is kept byte for byte. +fn remove_keybindings(path: &Path) -> Result { + let Ok(text) = std::fs::read_to_string(path) else { + return Ok(false); + }; + if !text.contains(KEYS_MARKER) && !text.contains("\"railway.ca.") { + return Ok(false); + } + let stripped = strip_keybindings(&text); + if stripped == text { + return Ok(false); + } + std::fs::write(path, stripped).with_context(|| format!("Writing {}", path.display()))?; + Ok(true) +} + +fn strip_keybindings(text: &str) -> String { + let lines: Vec<&str> = text.lines().collect(); + let mut out: Vec<&str> = Vec::with_capacity(lines.len()); + let mut i = 0; + while i < lines.len() { + let line = lines[i]; + if line.trim() == KEYS_MARKER { + i += 1; + continue; + } + if line.trim() == "[[keys.command]]" { + let mut end = i + 1; + while end < lines.len() && !lines[end].trim_start().starts_with('[') { + end += 1; + } + let ours = lines[i..end] + .iter() + .any(|l| l.trim_start().starts_with("command") && l.contains("\"railway.ca.")); + if ours { + while end > i + 1 && lines[end - 1].trim().is_empty() { + end -= 1; + } + i = end; + continue; + } + } + out.push(line); + i += 1; + } + let mut joined = out.join("\n"); + joined.truncate(joined.trim_end_matches('\n').len()); + if !joined.is_empty() && text.ends_with('\n') { + joined.push('\n'); + } + joined +} + +fn install_into(herdr: &Herdr, dir: &Path, manifest: &Manifest) -> Result<()> { + std::fs::create_dir_all(dir).with_context(|| format!("Creating {}", dir.display()))?; + let path = dir.join(MANIFEST_FILE); + let text = manifest.render()?; + let unchanged = std::fs::read_to_string(&path).ok().as_deref() == Some(text.as_str()); + std::fs::write(&path, &text).with_context(|| format!("Writing {}", path.display()))?; + match herdr.plugin_link(dir) { + Err(e) if is_already_linked(&e) && !unchanged => { + herdr.plugin_unlink(PLUGIN_ID)?; + herdr.plugin_link(dir) + } + Err(e) if is_already_linked(&e) => Ok(()), + other => other, + } +} + +fn is_already_linked(e: &anyhow::Error) -> bool { + e.to_string().to_lowercase().contains("already") +} + +/// The manifest goes regardless: `plugin unlink` needs a running herdr, and a +/// stopped one must not leave the files behind. +fn remove_from(herdr: &Herdr, dir: &Path) -> Result<()> { + let unlinked = herdr.plugin_unlink(PLUGIN_ID); + match std::fs::remove_file(dir.join(MANIFEST_FILE)) { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => return Err(e.into()), + _ => {} + } + if let Err(e) = unlinked { + println!( + "{} herdr did not unlink the plugin ({e:#}); run `herdr plugin unlink {PLUGIN_ID}` once it is up.", + "!".yellow() + ); + } + Ok(()) +} + +/// The PATH entry when it is this same binary (a stable symlink such as +/// `/opt/homebrew/bin/railway`), otherwise the executable itself (a dev build). +fn railway_binary() -> Result { + let exe = std::env::current_exe()? + .canonicalize() + .context("Resolving the railway binary path")?; + Ok(binary_path(exe, std::env::var_os("PATH"))) +} + +fn binary_path(exe: PathBuf, path: Option) -> PathBuf { + find_in_path("railway", path) + .filter(|found| found.canonicalize().ok().as_deref() == Some(exe.as_path())) + .unwrap_or(exe) +} + +fn find_in_path(name: &str, path: Option) -> Option { + std::env::split_paths(&path?) + .map(|dir| dir.join(name)) + .find(|candidate| candidate.is_file()) + .and_then(|found| std::path::absolute(found).ok()) +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +pub(super) struct Manifest { + id: String, + name: String, + version: String, + min_herdr_version: String, + description: String, + platforms: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + startup: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + events: Vec, + actions: Vec, + panes: Vec, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +struct Event { + on: String, + command: Vec, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +struct Startup { + command: Vec, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +struct Action { + id: String, + title: String, + command: Vec, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +struct Pane { + id: String, + title: String, + placement: String, + width: String, + height: u32, + command: Vec, +} + +impl Manifest { + fn new(railway: PathBuf) -> Self { + let railway = railway.to_string_lossy().into_owned(); + let cmd = |args: &[&str]| -> Vec { + std::iter::once(railway.clone()) + .chain(["ca", "herdr"].into_iter().map(str::to_owned)) + .chain(args.iter().map(|s| s.to_string())) + .collect() + }; + let pane = |id: &str, title: &str| Pane { + id: id.into(), + title: title.into(), + placement: "popup".into(), + width: "80%".into(), + height: 24, + command: cmd(&[id]), + }; + Self { + id: PLUGIN_ID.into(), + name: "Railway cloud agents".into(), + version: env!("CARGO_PKG_VERSION").into(), + min_herdr_version: "0.9.0".into(), + description: "Railway cloud agents as herdr machines: create, connect, sleep, wake" + .into(), + platforms: vec!["linux".into(), "macos".into()], + startup: vec![Startup { + command: cmd(&["sync", "--spawn-watch"]), + }], + // Local agent activity is the one server-side event that fires + // often (workspace focus is client-local in herdr 0.9 and never + // reaches hooks); 30 s keeps a busy agent cheap. + events: vec![Event { + on: "pane.agent_status_changed".into(), + command: cmd(&["sync", "--debounce", "30"]), + }], + actions: vec![ + Action { + id: "sync".into(), + title: "Railway: sync agents".into(), + command: cmd(&["sync"]), + }, + Action { + id: "agents".into(), + title: "Railway: agents".into(), + command: cmd(&["agents", "--open"]), + }, + Action { + id: "new".into(), + title: "Railway: new agent".into(), + command: cmd(&["new", "--open"]), + }, + Action { + id: "wake".into(), + title: "Railway: wake agent".into(), + command: cmd(&["agents", "--wake", "--open"]), + }, + ], + panes: vec![ + pane("agents", "Railway agents"), + pane("new", "New Railway agent"), + Pane { + id: "wake".into(), + title: "Wake Railway agent".into(), + placement: "popup".into(), + width: "80%".into(), + height: 24, + command: cmd(&["agents", "--wake"]), + }, + ], + } + } + + /// The manifest bootstrap writes on the VM. Commands are the two scripts + /// bootstrap drops next to it, so it works before the VM's railway binary + /// knows `ca herdr`. + pub(super) fn remote() -> Self { + let sh = |script: &str, extra: &[&str]| -> Vec { + ["sh", script] + .into_iter() + .chain(extra.iter().copied()) + .map(str::to_owned) + .collect() + }; + Self { + id: PLUGIN_ID.into(), + name: "Railway cloud agents (this VM)".into(), + version: env!("CARGO_PKG_VERSION").into(), + min_herdr_version: "0.9.0".into(), + description: "Railway agents picker and sleep for this VM".into(), + platforms: vec!["linux".into()], + startup: Vec::new(), + events: Vec::new(), + actions: vec![ + Action { + id: "agents".into(), + title: "Railway: agents".into(), + command: sh("agents.sh", &["--open"]), + }, + Action { + id: "sleep-self".into(), + title: "Railway: sleep this agent".into(), + command: sh("sleep-self.sh", &[]), + }, + ], + panes: vec![Pane { + id: "agents".into(), + title: "Railway agents".into(), + placement: "popup".into(), + width: "80%".into(), + height: 24, + command: sh("agents.sh", &[]), + }], + } + } + + fn binary(&self) -> &str { + self.actions + .first() + .and_then(|a| a.command.first()) + .map(String::as_str) + .unwrap_or_default() + } + + pub(super) fn render(&self) -> Result { + toml::to_string(self).context("Rendering the herdr plugin manifest") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn manifest_round_trips_with_one_binary_path() { + let manifest = Manifest::new(PathBuf::from("/opt/homebrew/bin/railway")); + let text = manifest.render().unwrap(); + let parsed: Manifest = toml::from_str(&text).unwrap(); + assert_eq!(parsed, manifest); + assert_eq!(parsed.id, "railway.ca"); + assert_eq!( + parsed + .actions + .iter() + .map(|a| a.id.as_str()) + .collect::>(), + ["sync", "agents", "new", "wake"] + ); + assert_eq!( + parsed + .panes + .iter() + .map(|p| p.id.as_str()) + .collect::>(), + ["agents", "new", "wake"] + ); + assert_eq!( + parsed.startup[0].command, + [ + "/opt/homebrew/bin/railway", + "ca", + "herdr", + "sync", + "--spawn-watch" + ] + ); + assert_eq!(parsed.events[0].on, "pane.agent_status_changed"); + assert_eq!( + parsed.actions[1].command, + [ + "/opt/homebrew/bin/railway", + "ca", + "herdr", + "agents", + "--open" + ] + ); + assert_eq!( + parsed.panes[1].command, + ["/opt/homebrew/bin/railway", "ca", "herdr", "new"] + ); + let commands = parsed + .startup + .iter() + .map(|s| &s.command) + .chain(parsed.actions.iter().map(|a| &a.command)) + .chain(parsed.panes.iter().map(|p| &p.command)); + for command in commands { + assert_eq!(command[0], "/opt/homebrew/bin/railway", "{command:?}"); + } + assert!(text.contains("[[startup]]"), "{text}"); + assert!(text.contains("[[actions]]"), "{text}"); + assert!(text.contains("[[panes]]"), "{text}"); + assert!(text.contains("placement = \"popup\""), "{text}"); + } + + #[cfg(unix)] + #[test] + fn path_symlink_to_the_running_exe_wins_over_the_exe() { + let tmp = tempfile::tempdir().unwrap(); + let exe = tmp.path().join("target").join("railway"); + std::fs::create_dir_all(exe.parent().unwrap()).unwrap(); + std::fs::write(&exe, "").unwrap(); + let exe = exe.canonicalize().unwrap(); + let bin = tmp.path().join("bin"); + std::fs::create_dir_all(&bin).unwrap(); + std::os::unix::fs::symlink(&exe, bin.join("railway")).unwrap(); + + let path = Some(OsString::from(bin.to_string_lossy().into_owned())); + assert_eq!(binary_path(exe.clone(), path), bin.join("railway")); + + let other = tmp.path().join("other"); + std::fs::create_dir_all(&other).unwrap(); + std::fs::write(other.join("railway"), "").unwrap(); + let path = Some(OsString::from(other.to_string_lossy().into_owned())); + assert_eq!(binary_path(exe.clone(), path), exe); + assert_eq!(binary_path(exe.clone(), None), exe); + } + + // The fake herdr is a shebang script: unix only. + + #[cfg(unix)] + #[test] + fn install_writes_the_manifest_and_links_the_dir() { + let fake = super::super::herdr_cli::fake::FakeHerdr::with_machines("[]"); + let herdr = fake.herdr(); + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("herdr-plugin"); + let manifest = Manifest::new(PathBuf::from("/usr/local/bin/railway")); + + install_into(&herdr, &dir, &manifest).unwrap(); + install_into(&herdr, &dir, &manifest).unwrap(); + + let link = format!("plugin link {}", dir.display()); + assert_eq!(fake.calls(), vec![link.clone(), link]); + let written = std::fs::read_to_string(dir.join(MANIFEST_FILE)).unwrap(); + assert_eq!(toml::from_str::(&written).unwrap(), manifest); + + remove_from(&herdr, &dir).unwrap(); + assert!(!dir.join(MANIFEST_FILE).exists()); + assert_eq!(fake.calls().last().unwrap(), "plugin unlink railway.ca"); + } + + #[test] + fn keybindings_are_added_once_and_removed_cleanly() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("herdr").join("config.toml"); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let before = "[ui]\naccent = \"cyan\"\n\n[[keys.command]]\nkey = \"prefix+alt+g\"\ntype = \"popup\"\ncommand = \"lazygit\"\n"; + std::fs::write(&path, before).unwrap(); + assert!(ensure_keybindings(&path).unwrap()); + assert!(!ensure_keybindings(&path).unwrap()); + let text = std::fs::read_to_string(&path).unwrap(); + assert_eq!(text.matches("railway.ca.agents").count(), 1, "{text}"); + assert!(text.contains("railway.ca.wake"), "{text}"); + assert!(text.contains("lazygit"), "{text}"); + assert!(remove_keybindings(&path).unwrap()); + assert_eq!(std::fs::read_to_string(&path).unwrap(), before); + assert!(!remove_keybindings(&path).unwrap()); + } + + #[test] + fn remote_manifest_uses_the_dropped_scripts_and_the_same_plugin_id() { + let text = Manifest::remote().render().unwrap(); + let parsed: Manifest = toml::from_str(&text).unwrap(); + assert_eq!(parsed.id, PLUGIN_ID); + assert!(parsed.startup.is_empty() && parsed.events.is_empty()); + let ids: Vec<&str> = parsed.actions.iter().map(|a| a.id.as_str()).collect(); + assert_eq!(ids, vec!["agents", "sleep-self"]); + for command in parsed + .actions + .iter() + .map(|a| &a.command) + .chain(parsed.panes.iter().map(|p| &p.command)) + { + assert_eq!(command[0], "sh", "{command:?}"); + assert!(command[1].ends_with(".sh"), "{command:?}"); + } + } + + #[test] + fn an_outdated_block_is_replaced_and_a_hand_binding_is_respected() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.toml"); + let old = format!( + "x = 1\n\n{KEYS_MARKER}\n[[keys.command]]\nkey = \"prefix+shift+a\"\ntype = \"plugin_action\"\ncommand = \"railway.ca.agents\"\n" + ); + std::fs::write(&path, &old).unwrap(); + assert!(ensure_keybindings(&path).unwrap()); + let text = std::fs::read_to_string(&path).unwrap(); + assert!(text.starts_with("x = 1\n\n"), "{text}"); + assert_eq!(text.matches("[[keys.command]]").count(), 2, "{text}"); + assert!(text.contains("railway.ca.wake"), "{text}"); + assert!(!ensure_keybindings(&path).unwrap()); + + let hand = "[[keys.command]]\nkey = \"prefix+m\"\ntype = \"plugin_action\"\ncommand = \"railway.ca.agents\"\n"; + std::fs::write(&path, hand).unwrap(); + assert!(!ensure_keybindings(&path).unwrap()); + assert_eq!(std::fs::read_to_string(&path).unwrap(), hand); + } + + #[test] + fn missing_config_is_created_with_only_our_block() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("herdr").join("config.toml"); + assert!(ensure_keybindings(&path).unwrap()); + let text = std::fs::read_to_string(&path).unwrap(); + assert!(text.starts_with(KEYS_MARKER), "{text}"); + assert!(remove_keybindings(&path).unwrap()); + assert_eq!(std::fs::read_to_string(&path).unwrap(), ""); + } +} diff --git a/src/commands/cloud_agent/herdr/known_hosts.rs b/src/commands/cloud_agent/herdr/known_hosts.rs new file mode 100644 index 000000000..2ea860e8a --- /dev/null +++ b/src/commands/cloud_agent/herdr/known_hosts.rs @@ -0,0 +1,151 @@ +//! herdr's saved-machine probe and every background reconnect run plain `ssh` +//! with `StrictHostKeyChecking=yes`, which never prompts, so the relay's key +//! has to be in `~/.ssh/known_hosts` before `machine add`. The railway CLI keeps +//! its verified copy in `~/.railway/known_hosts_relay`; this copies it across. + +use std::path::Path; + +use anyhow::{Context, Result}; + +use crate::config::Configs; + +/// herdr's background reconnects run `ssh` with `StrictHostKeyChecking=yes` +/// and no config of their own, so a user `Host` block that points the relay +/// at `UserKnownHostsFile /dev/null` parks every machine in Attention after +/// the first network blip. `ssh -G` shows what would actually be used. +pub fn ssh_config_warning() -> Option { + let (host, _) = Configs::get_ssh_relay(); + let out = std::process::Command::new("ssh") + .args(["-G", host]) + .stderr(std::process::Stdio::null()) + .output() + .ok()?; + let text = String::from_utf8_lossy(&out.stdout); + let files: Vec<&str> = text + .lines() + .find_map(|l| l.strip_prefix("userknownhostsfile ")) + .map(|v| v.split_whitespace().collect()) + .unwrap_or_default(); + if files.iter().all(|f| *f == "/dev/null") { + return Some(format!( + "Your ssh config sends {host}'s host keys to /dev/null (UserKnownHostsFile). herdr reconnects with strict checking and will park every Railway machine in Attention; remove {host} from that Host block." + )); + } + None +} + +pub fn ensure_relay_known_host() -> Result { + let home = dirs::home_dir().ok_or_else(|| anyhow::anyhow!("Unable to get home directory"))?; + let (host, _) = Configs::get_ssh_relay(); + ensure_in( + &home.join(".railway").join("known_hosts_relay"), + &home.join(".ssh").join("known_hosts"), + host, + ) +} + +#[derive(Debug, PartialEq, Eq)] +pub enum Seeded { + Present, + Added, + /// The CLI has not connected to the relay yet, so there is nothing to copy. + NoSource, +} + +fn ensure_in(source: &Path, known_hosts: &Path, host: &str) -> Result { + let Ok(relay) = std::fs::read_to_string(source) else { + return Ok(Seeded::NoSource); + }; + let Some(line) = relay + .lines() + .find(|l| is_ed25519_for(l, host)) + .map(str::trim) + else { + return Ok(Seeded::NoSource); + }; + let existing = std::fs::read_to_string(known_hosts).unwrap_or_default(); + if existing.lines().any(|l| is_ed25519_for(l, host)) { + return Ok(Seeded::Present); + } + if let Some(parent) = known_hosts.parent() { + std::fs::create_dir_all(parent)?; + } + let mut out = existing; + if !out.is_empty() && !out.ends_with('\n') { + out.push('\n'); + } + out.push_str(line); + out.push_str(" railway-ca-herdr"); + out.push('\n'); + std::fs::write(known_hosts, out) + .with_context(|| format!("Writing {}", known_hosts.display()))?; + Ok(Seeded::Added) +} + +fn is_ed25519_for(line: &str, host: &str) -> bool { + let mut parts = line.split_whitespace(); + let hosts = parts.next().unwrap_or_default(); + parts.next() == Some("ssh-ed25519") && hosts.split(',').any(|h| h == host) +} + +#[cfg(test)] +mod tests { + use super::*; + + const RELAY: &str = "ssh.railway.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJ8X3z81relaykey\n"; + + #[test] + fn adds_once_and_keeps_existing_lines() { + let dir = tempfile::tempdir().unwrap(); + let src = dir.path().join("known_hosts_relay"); + let kh = dir.path().join("ssh").join("known_hosts"); + std::fs::write(&src, RELAY).unwrap(); + assert_eq!( + ensure_in(&src, &kh, "ssh.railway.com").unwrap(), + Seeded::Added + ); + std::fs::write( + &kh, + format!( + "{}other ssh-rsa AAAA\n", + std::fs::read_to_string(&kh).unwrap() + ), + ) + .unwrap(); + assert_eq!( + ensure_in(&src, &kh, "ssh.railway.com").unwrap(), + Seeded::Present + ); + let text = std::fs::read_to_string(&kh).unwrap(); + assert_eq!(text.matches("ssh.railway.com").count(), 1, "{text}"); + assert!(text.contains("relaykey railway-ca-herdr\n"), "{text}"); + assert!(text.contains("other ssh-rsa"), "{text}"); + } + + #[test] + fn rsa_only_entry_does_not_count() { + let dir = tempfile::tempdir().unwrap(); + let src = dir.path().join("relay"); + let kh = dir.path().join("known_hosts"); + std::fs::write(&src, RELAY).unwrap(); + std::fs::write(&kh, "ssh.railway.com ssh-rsa AAAAB3\n").unwrap(); + assert_eq!( + ensure_in(&src, &kh, "ssh.railway.com").unwrap(), + Seeded::Added + ); + } + + #[test] + fn missing_source_is_reported_not_fatal() { + let dir = tempfile::tempdir().unwrap(); + assert_eq!( + ensure_in( + &dir.path().join("nope"), + &dir.path().join("kh"), + "ssh.railway.com" + ) + .unwrap(), + Seeded::NoSource + ); + } +} diff --git a/src/commands/cloud_agent/herdr/mod.rs b/src/commands/cloud_agent/herdr/mod.rs new file mode 100644 index 000000000..549339bff --- /dev/null +++ b/src/commands/cloud_agent/herdr/mod.rs @@ -0,0 +1,71 @@ +//! `railway ca herdr`: Railway cloud agents as herdr saved machines. +//! +//! herdr 0.9 can hold several SSH machines in one window, each running its own +//! herdr server. A cloud agent VM is exactly such a machine: the server on the +//! VM owns the panes and detects the coding agent natively, and after a sleep +//! herdr's own restore brings the layout and `claude --resume` back. What +//! herdr cannot do is create, wake, sleep or delete the VM, so these verbs are +//! the control plane, and the herdr plugin is a generated manifest whose every +//! command calls one of them. + +mod agents; +mod bootstrap; +mod harness; +mod herdr_cli; +mod install; +mod known_hosts; +mod new; +mod relay; +mod state; +mod sync; +mod target; +mod watch; + +use anyhow::Result; +use clap::Parser; + +pub const PLUGIN_ID: &str = "railway.ca"; + +#[derive(Parser)] +pub struct Args { + #[clap(subcommand)] + command: Command, +} + +#[derive(Parser)] +enum Command { + /// Register the herdr plugin: write its manifest and link it + Install(install::Args), + + /// Create a cloud agent and add it to herdr as a machine + New(new::Args), + + /// Pick an agent: connect, sleep, wake, delete + Agents(agents::Args), + + /// Reconcile herdr's saved machines with your cloud agents + Sync(sync::Args), + + /// Prepare an agent's VM for herdr: integrations, config, workspace + Bootstrap(bootstrap::Args), + + /// Follow cloud agent state and keep herdr's machines in step + Watch(watch::Args), +} + +pub async fn command(args: Args) -> Result<()> { + match args.command { + Command::Install(a) => install::command(a).await, + Command::New(a) => new::command(a).await, + Command::Agents(a) => agents::command(a).await, + Command::Sync(a) => sync::command(a).await, + Command::Bootstrap(a) => bootstrap::command(a).await, + Command::Watch(a) => watch::command(a).await, + } +} + +/// `~/.railway/herdr-plugin`: the manifest herdr links, and the plugin's state. +pub fn plugin_dir() -> Result { + let home = dirs::home_dir().ok_or_else(|| anyhow::anyhow!("Unable to get home directory"))?; + Ok(home.join(".railway").join("herdr-plugin")) +} diff --git a/src/commands/cloud_agent/herdr/new.rs b/src/commands/cloud_agent/herdr/new.rs new file mode 100644 index 000000000..87a08e669 --- /dev/null +++ b/src/commands/cloud_agent/herdr/new.rs @@ -0,0 +1,436 @@ +//! `railway ca herdr new`: pick a place, name the agent, create it, and hand +//! the VM to herdr as a saved machine. + +use std::fmt::Display; + +use anyhow::{Context, Result, bail}; +use clap::Parser; +use colored::Colorize; +use inquire::validator::Validation; + +use crate::client::GQLClient; +use crate::config::Configs; +use crate::controllers::cloud_agent as ca; +use crate::util::progress::create_spinner; +use crate::workspace::{self, Workspace}; + +use super::herdr_cli::{Herdr, Machine}; +use super::state::Store; +use super::target; + +#[derive(Parser)] +pub struct Args { + /// Name for the agent (defaults to a generated one) + #[clap(value_name = "NAME")] + name: Option, + + /// Project ID (skips the project picker) + #[clap(long, short)] + project: Option, + + /// Environment ID (skips the environment picker) + #[clap(long, short)] + environment: Option, + + #[clap(flatten)] + harness: super::harness::HarnessFlags, + + /// Open this flow in a herdr popup pane instead of running it here + #[clap(long)] + open: bool, + + /// Pick and name only; print what would be created without creating it + #[clap(long)] + dry_run: bool, +} + +impl Args { + /// The picker's "new agent": every step asked, nothing skipped. + pub(super) fn interactive() -> Self { + Self { + name: None, + project: None, + environment: None, + harness: Default::default(), + open: false, + dry_run: false, + } + } +} + +pub async fn command(args: Args) -> Result<()> { + if args.open { + return Herdr::from_env().plugin_pane_open(super::PLUGIN_ID, "new"); + } + + let mut configs = Configs::new()?; + let client = GQLClient::new_authorized(&configs)?; + let store = Store::new(&configs)?; + let workspaces = workspace::workspaces_with_client(&client, &configs).await?; + let rows = rows(&workspaces); + let row = pick_project(rows, args.project.as_deref(), args.environment.as_deref())?; + let environment = + match choose_environment(row.environments.clone(), args.environment.as_deref())? { + EnvChoice::Chosen(env) => env, + EnvChoice::Ask(envs) => inquire::Select::new("Environment", envs) + .with_render_config(Configs::get_render_config()) + .prompt() + .context("Failed to prompt for environment")?, + }; + let name = match args.name { + Some(name) => { + if !valid_name(&name) { + bail!("Invalid agent name {name:?}: {NAME_RULE}"); + } + Some(name) + } + None => prompt_name()?, + }; + + let harness = super::harness::choose(&args.harness, true)?; + + if args.dry_run { + let shown = name.as_deref().unwrap_or(""); + let target = target::target_for(&environment.id, ""); + let label = target::label(&row.project_name, shown); + println!("{}", "Dry run, nothing created.".dimmed()); + println!(" agent {shown}"); + println!(" project {} ({})", row.project_name, row.project_id); + println!(" environment {} ({})", environment.name, environment.id); + println!(" agent type {harness}"); + println!(" herdr machine add {target} --label {label:?}"); + return Ok(()); + } + + let backboard = configs.get_backboard(); + let spinner = create_spinner("Creating a cloud agent".to_string()); + let agent = ca::create( + &client, + &backboard, + &environment.id, + name, + None, + ca::CreateOptions::default(), + ) + .await + .inspect_err(|_| spinner.finish_and_clear())?; + ca::remember(&mut configs, &agent)?; + spinner.set_message(format!("Waiting for agent {} to start", agent.name)); + let agent = ca::wait_until_running(&client, &backboard, &environment.id, &agent.id) + .await + .inspect_err(|_| spinner.finish_and_clear())?; + spinner.finish_and_clear(); + println!("✓ Created agent {}", agent.name.cyan()); + super::watch::nudge(); + + let herdr = Herdr::from_env(); + let target = target::target(&agent); + let label = target::label(&row.project_name, &agent.name); + super::known_hosts::ensure_relay_known_host()?; + let spinner = create_spinner(format!("Waiting for {}'s ssh relay", agent.name)); + let ready = super::relay::wait_until_ready(&agent).await; + spinner.finish_and_clear(); + ready?; + herdr.machine_add(&target, &label)?; + + match herdr + .machines() + .map(|machines| profile_id_for(&machines, &target)) + { + Ok(Some(profile)) => { + store + .update(|s| { + s.machines.insert(agent.id.clone(), profile); + }) + .await?; + } + Ok(None) => { + warn("herdr did not list the new machine; `railway ca herdr sync` will record it") + } + Err(e) => warn(&format!("could not read herdr machines: {e:#}")), + } + + super::bootstrap::run(&agent, harness, &store).await?; + + println!( + "\n{} is in the herdr sidebar as {}, provisioned for {}.", + agent.name.cyan(), + label.cyan(), + harness.cyan() + ); + Ok(()) +} + +fn warn(message: &str) { + eprintln!("{} {message}", "warning:".yellow()); +} + +const NAME_RULE: &str = + "1-63 characters, letters, digits, '.', '_' or '-', starting with a letter or digit"; + +/// Backboard's grammar: `^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$`. +fn valid_name(name: &str) -> bool { + let mut chars = name.chars(); + let Some(first) = chars.next() else { + return false; + }; + first.is_ascii_alphanumeric() + && name.len() <= 63 + && chars.all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-')) +} + +fn prompt_name() -> Result> { + let validator = |input: &str| { + if input.trim().is_empty() || valid_name(input.trim()) { + Ok(Validation::Valid) + } else { + Ok(Validation::Invalid(NAME_RULE.into())) + } + }; + let name = inquire::Text::new("Agent name") + .with_render_config(Configs::get_render_config()) + .with_placeholder("leave empty for a generated name") + .with_validator(validator) + .prompt() + .context("Failed to prompt for agent name")?; + let name = name.trim(); + Ok((!name.is_empty()).then(|| name.to_string())) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct Row { + workspace: String, + project_id: String, + project_name: String, + /// Only the environments this user can act in. + environments: Vec, +} + +impl Display for Row { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} / {}", self.workspace, self.project_name) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct Env { + id: String, + name: String, +} + +impl Display for Env { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.name) + } +} + +fn rows(workspaces: &[Workspace]) -> Vec { + let mut rows: Vec = workspaces + .iter() + .flat_map(|w| { + w.projects() + .into_iter() + .filter(|p| p.deleted_at().is_none()) + .map(move |p| Row { + workspace: w.name().to_string(), + project_id: p.id().to_string(), + project_name: p.name().to_string(), + environments: p + .environments() + .into_iter() + .filter(|e| e.can_access) + .map(|e| Env { + id: e.id, + name: e.name, + }) + .collect(), + }) + }) + .collect(); + rows.sort_by(|a, b| { + a.workspace + .to_lowercase() + .cmp(&b.workspace.to_lowercase()) + .then_with(|| { + a.project_name + .to_lowercase() + .cmp(&b.project_name.to_lowercase()) + }) + }); + rows +} + +fn pick_project( + rows: Vec, + project_id: Option<&str>, + environment_id: Option<&str>, +) -> Result { + if rows.is_empty() { + bail!("No projects found in any of your workspaces."); + } + if let Some(id) = project_id { + return rows + .into_iter() + .find(|r| r.project_id == id) + .ok_or_else(|| anyhow::anyhow!("No project with id {id} in your workspaces.")); + } + if let Some(id) = environment_id { + return rows + .into_iter() + .find(|r| r.environments.iter().any(|e| e.id == id)) + .ok_or_else(|| anyhow::anyhow!("No environment with id {id} in your workspaces.")); + } + inquire::Select::new("Project", rows) + .with_render_config(Configs::get_render_config()) + .with_page_size(15) + .prompt() + .context("Failed to prompt for project") +} + +#[derive(Debug, PartialEq, Eq)] +enum EnvChoice { + Chosen(Env), + Ask(Vec), +} + +fn choose_environment(mut environments: Vec, requested: Option<&str>) -> Result { + if let Some(id) = requested { + return match environments.iter().position(|e| e.id == id || e.name == id) { + Some(i) => Ok(EnvChoice::Chosen(environments.swap_remove(i))), + None => bail!("Environment {id} is not one you can access in this project."), + }; + } + match environments.len() { + 0 => bail!("You have no accessible environments in this project."), + 1 => Ok(EnvChoice::Chosen(environments.remove(0))), + _ => Ok(EnvChoice::Ask(environments)), + } +} + +fn profile_id_for(machines: &[Machine], target: &str) -> Option { + machines + .iter() + .find(|m| m.target == target) + .map(|m| m.id.clone()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn env(id: &str, name: &str) -> Env { + Env { + id: id.into(), + name: name.into(), + } + } + + fn row(project_id: &str, project_name: &str, environments: Vec) -> Row { + Row { + workspace: "Railway".into(), + project_id: project_id.into(), + project_name: project_name.into(), + environments, + } + } + + #[test] + fn name_grammar_matches_backboard() { + assert!(valid_name("a")); + assert!(valid_name("Reviewer-1.2_x")); + assert!(valid_name(&"a".repeat(63))); + assert!(!valid_name("")); + assert!(!valid_name("-lead")); + assert!(!valid_name(".dot")); + assert!(!valid_name("has space")); + assert!(!valid_name("naïve")); + assert!(!valid_name(&"a".repeat(64))); + } + + #[test] + fn rows_render_workspace_slash_project() { + assert_eq!( + row("p1", "orchestrator", vec![]).to_string(), + "Railway / orchestrator" + ); + } + + #[test] + fn sole_environment_is_chosen_without_asking() { + assert_eq!( + choose_environment(vec![env("e1", "production")], None).unwrap(), + EnvChoice::Chosen(env("e1", "production")) + ); + } + + #[test] + fn several_environments_are_asked_about() { + let envs = vec![env("e1", "production"), env("e2", "staging")]; + assert!(matches!( + choose_environment(envs, None).unwrap(), + EnvChoice::Ask(v) if v.len() == 2 + )); + } + + #[test] + fn requested_environment_resolves_by_id_or_name() { + let envs = vec![env("e1", "production"), env("e2", "staging")]; + assert_eq!( + choose_environment(envs.clone(), Some("e2")).unwrap(), + EnvChoice::Chosen(env("e2", "staging")) + ); + assert_eq!( + choose_environment(envs.clone(), Some("production")).unwrap(), + EnvChoice::Chosen(env("e1", "production")) + ); + assert!(choose_environment(envs, Some("nope")).is_err()); + } + + #[test] + fn no_environment_is_an_error() { + assert!(choose_environment(vec![], None).is_err()); + } + + #[test] + fn project_or_environment_flag_skips_the_picker() { + let rows = vec![ + row("p1", "one", vec![env("e1", "prod")]), + row("p2", "two", vec![env("e2", "prod")]), + ]; + assert_eq!( + pick_project(rows.clone(), Some("p2"), None) + .unwrap() + .project_id, + "p2" + ); + assert_eq!( + pick_project(rows.clone(), None, Some("e1")) + .unwrap() + .project_id, + "p1" + ); + assert!(pick_project(rows.clone(), Some("p9"), None).is_err()); + assert!(pick_project(rows, None, Some("e9")).is_err()); + assert!(pick_project(vec![], None, None).is_err()); + } + + // The fake herdr is a shebang script: unix only. + + #[cfg(unix)] + #[test] + fn profile_id_is_looked_up_by_target_after_add() { + let target = target::target_for("env-1", "agent-1"); + let fake = super::super::herdr_cli::fake::FakeHerdr::with_machines(&format!( + r#"[{{"id":"0123456789abcdef0123456789abcdef","label":"p/a","target":"{target}","enabled":true}}]"# + )); + let herdr = fake.herdr(); + herdr.machine_add(&target, "p/a").unwrap(); + let machines = herdr.machines().unwrap(); + assert_eq!( + profile_id_for(&machines, &target).as_deref(), + Some("0123456789abcdef0123456789abcdef") + ); + assert_eq!(profile_id_for(&machines, "someone@elsewhere"), None); + assert_eq!(fake.calls()[0], format!("machine add {target} --label p/a")); + } +} diff --git a/src/commands/cloud_agent/herdr/relay.rs b/src/commands/cloud_agent/herdr/relay.rs new file mode 100644 index 000000000..e85ff6a27 --- /dev/null +++ b/src/commands/cloud_agent/herdr/relay.rs @@ -0,0 +1,150 @@ +//! The relay says an agent is running before it executes commands for it; in +//! between it answers ssh with a JSON status document. herdr reads that as an +//! unsupported platform and parks the machine in Attention, which it never +//! retries on its own. So a machine is only handed to herdr once a real +//! command has round-tripped. + +use std::path::Path; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result, bail}; + +use crate::commands::code; +use crate::commands::ssh::config as ssh_config; +use crate::commands::ssh::native::run_native_ssh_captured; +use crate::config::Configs; +use crate::controllers::cloud_agent as ca; + +const TIMEOUT: Duration = Duration::from_secs(150); +const PAUSE: Duration = Duration::from_secs(3); + +pub async fn wait_until_ready(agent: &ca::Agent) -> Result<()> { + let info = code::connect_info(&agent.environment_id, &agent.id).await?; + // Herdr persists only the URI and invokes OpenSSH itself. Carry over the + // identity used by this probe so its later connections use the same key. + let config = ssh_config::expand_tilde(Path::new("~/.ssh/config"))?; + let (host, _) = Configs::get_ssh_relay(); + ensure_identity(&config, host, &info.ssh_target, info.identity.as_deref()).await?; + let nonce = format!("herdr-ready-{}", rand::random::()); + let started = Instant::now(); + loop { + let target = info.ssh_target.clone(); + let identity = info.identity.clone(); + let mut opts = info.relay_opts.clone(); + opts.push("-o".into()); + opts.push("ConnectTimeout=10".into()); + let command = format!("echo {nonce}"); + let seen = tokio::task::spawn_blocking(move || { + run_native_ssh_captured(&target, &command, identity.as_deref(), None, &opts) + }) + .await? + .map(|(_, stdout, _)| String::from_utf8_lossy(&stdout).contains(&nonce)) + .unwrap_or(false); + if seen { + return Ok(()); + } + if started.elapsed() > TIMEOUT { + bail!( + "Agent {} is running but its ssh relay is not executing commands yet; retry in a minute.", + agent.name + ); + } + tokio::time::sleep(PAUSE).await; + } +} + +async fn ensure_identity( + config: &Path, + host: &str, + user: &str, + identity: Option<&Path>, +) -> Result<()> { + let marker = format!("herdr-{host}-{user}"); + let _lock = super::state::lock_file(&config.with_extension("herdr.lock")).await?; + let Some(identity) = identity else { + ssh_config::remove_marked_block(config, &marker)?; + return Ok(()); + }; + // Match arguments are patterns. Relay hostnames and agent usernames must + // remain literal rather than broadening the rule to other SSH connections. + for value in [host, user] { + if value.is_empty() + || value + .chars() + .any(|c| c.is_whitespace() || matches!(c, '*' | '?' | '!' | ',' | '"' | '\\')) + { + bail!("Cannot write an SSH identity rule for {value:?}"); + } + } + let identity = + ssh_config::quote_ssh_config_value(&identity.to_string_lossy().replace('%', "%%")); + let block = format!( + "# BEGIN railway:{marker}\n\ + Match host {host} user {user}\n\ + IdentityFile {identity}\n\ + IdentitiesOnly yes\n\ + Host *\n\ + # END railway:{marker}\n" + ); + ssh_config::upsert_marked_block(config, &marker, &block) + .with_context(|| format!("Configuring Herdr's SSH identity in {}", config.display())) +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + + #[tokio::test] + async fn raw_herdr_uri_uses_the_selected_key_only_for_that_agent() { + let dir = tempfile::tempdir().unwrap(); + let config = dir.path().join("config"); + let key = dir.path().join("custom key"); + std::fs::write(&config, "Host workbox\n IdentityFile /work/key\n").unwrap(); + ensure_identity(&config, "relay.example", "agent:env:a1", Some(&key)) + .await + .unwrap(); + // An update replaces the rule rather than accumulating identity keys. + let selected = dir.path().join("selected key"); + ensure_identity(&config, "relay.example", "agent:env:a1", Some(&selected)) + .await + .unwrap(); + let resolved = |target| { + let out = std::process::Command::new("ssh") + .arg("-G") + .arg("-F") + .arg(&config) + .arg(target) + .output() + .unwrap(); + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout).unwrap() + }; + let output = resolved("ssh://agent%3Aenv%3Aa1@relay.example:2222"); + assert!(output.contains("user agent:env:a1\n"), "{output}"); + assert!( + output.contains(&format!("identityfile {}\n", selected.display())), + "{output}" + ); + assert!(output.contains("identitiesonly yes\n"), "{output}"); + assert!(!output.contains(&key.to_string_lossy().to_string())); + for target in [ + "ssh://agent%3Aenv%3Aa2@relay.example", + "ssh://agent%3Aenv%3Aa1@other.example", + "workbox", + ] { + assert!(!resolved(target).contains(&selected.to_string_lossy().to_string())); + } + assert!(resolved("workbox").contains("identityfile /work/key\n")); + ensure_identity(&config, "relay.example", "agent:env:a1", None) + .await + .unwrap(); + assert!( + !resolved("ssh://agent%3Aenv%3Aa1@relay.example") + .contains(&selected.to_string_lossy().to_string()) + ); + } +} diff --git a/src/commands/cloud_agent/herdr/state.rs b/src/commands/cloud_agent/herdr/state.rs new file mode 100644 index 000000000..2a063e9c6 --- /dev/null +++ b/src/commands/cloud_agent/herdr/state.rs @@ -0,0 +1,274 @@ +//! Plugin bookkeeping: cached profile matches, acknowledged transitions and +//! incomplete bootstrap steps. Inventory comes from Railway and Herdr on sync. + +use std::collections::BTreeMap; +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{Context, Result}; +use chrono::{DateTime, Utc}; +use fs2::FileExt; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::config::Configs; + +#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct State { + #[serde(default)] + scope: Option, + #[serde(default)] + revision: u64, + /// agent id → herdr profile id + #[serde(default)] + pub machines: BTreeMap, + #[serde(default)] + pub last_sync: Option, + /// project id → name, so the picker skips the workspace tree query + /// (~2 s for a couple of hundred projects) when it already knows them all. + #[serde(default)] + pub project_names: BTreeMap, + /// agent id → status label at the last sync, so a wake done elsewhere is + /// noticed and its machine reconnected. + #[serde(default)] + pub agent_status: BTreeMap, + /// An acknowledged sleep must not be undone by a lagging RUNNING read. + /// Cleared on a sleeping observation, an explicit wake, or the bounded + /// transition deadline (a wake elsewhere may have overtaken the sleep). + #[serde(default)] + pub sleep_until: BTreeMap>, + /// A saved machine is not proof that its Railway bootstrap completed. + #[serde(default)] + pub bootstrap_pending: BTreeMap, +} + +impl State { + /// One file per watcher session, so a named session's sync observations + /// do not overwrite the default session's memory. + pub fn path() -> Result { + Ok(super::plugin_dir()?.join(file_name(std::env::var_os("HERDR_SOCKET_PATH").as_deref()))) + } + + pub fn load_from(path: &Path) -> Result { + match std::fs::read_to_string(path) { + Ok(text) => serde_json::from_str(&text) + .with_context(|| format!("Unreadable {}", path.display())), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(Self::default()), + Err(e) => Err(e).with_context(|| format!("Reading {}", path.display())), + } + } + + pub fn save_to(&self, path: &Path) -> Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + crate::util::write_atomic(path, &serde_json::to_string_pretty(self)?) + .with_context(|| format!("Writing {}", path.display())) + } + + fn bind_scope(&mut self, scope: &str) { + if self.scope.as_deref() != Some(scope) { + *self = Self { + scope: Some(scope.to_owned()), + revision: self.revision, + ..Default::default() + }; + } + } + + pub fn same_revision(&self, other: &Self) -> bool { + self.scope == other.scope && self.revision == other.revision + } + + pub fn sleep_pending(&self, id: &str, now: DateTime) -> bool { + self.sleep_until.get(id).is_some_and(|until| *until > now) + } +} + +/// Capture the same account/backend as the request client, including token +/// overrides. Raw credentials never go into the state file. A token rotation +/// without a stable user ID conservatively discards the old pruning evidence. +#[derive(Clone)] +pub(super) struct Store { + pub path: PathBuf, + scope: String, +} + +impl Store { + pub fn new(configs: &Configs) -> Result { + let principal = if let Some(token) = Configs::get_railway_token() { + format!("project:{token}") + } else if let Some(token) = Configs::get_railway_api_token() { + format!("api:{token}") + } else if let Some(id) = &configs.root_config.user.id { + format!("user:{id}") + } else { + format!( + "session:{}", + configs.get_railway_auth_token().unwrap_or_default() + ) + }; + Ok(Self::at( + State::path()?, + &configs.get_backboard(), + &principal, + )) + } + + pub(super) fn at(path: PathBuf, backboard: &str, principal: &str) -> Self { + let encoded = serde_json::to_vec(&(backboard, principal)).expect("strings serialize"); + Self { + path, + scope: format!("{:x}", Sha256::digest(encoded)), + } + } + + pub fn load(&self) -> Result { + let mut state = State::load_from(&self.path)?; + state.bind_scope(&self.scope); + Ok(state) + } + + pub async fn lock(&self) -> Result { + let file = lock_file(&self.path.with_extension("lock")).await?; + Ok(LockedState { + state: self.load()?, + path: self.path.clone(), + _file: file, + }) + } + + pub async fn update(&self, change: impl FnOnce(&mut State)) -> Result<()> { + let mut locked = self.lock().await?; + change(&mut locked.state); + locked.save() + } +} + +pub(super) struct LockedState { + pub state: State, + path: PathBuf, + _file: File, +} + +impl LockedState { + pub fn save(&mut self) -> Result<()> { + self.state.revision = self.state.revision.wrapping_add(1); + self.state.save_to(&self.path) + } +} + +/// A sibling lock survives atomic replacement of the state file. Waiting is +/// asynchronous, and closing the file releases the lock on every error path. +pub(super) async fn lock_file(path: &Path) -> Result { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let file = File::options() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(path)?; + loop { + match file.try_lock_exclusive() { + Ok(()) => return Ok(file), + Err(e) if e.kind() == fs2::lock_contended_error().kind() => { + tokio::time::sleep(Duration::from_millis(25)).await; + } + Err(e) => return Err(e).context("Locking herdr state"), + } + } +} + +fn file_name(socket: Option<&std::ffi::OsStr>) -> String { + let session = socket + .map(Path::new) + .and_then(|p| p.parent()) + .filter(|dir| dir.parent().and_then(|d| d.file_name()) == Some("sessions".as_ref())) + .and_then(|dir| dir.file_name()) + .map(|n| n.to_string_lossy().into_owned()); + match session { + Some(name) => format!("state-{name}.json"), + None => "state.json".to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn named_sessions_get_their_own_state_file() { + assert_eq!(file_name(None), "state.json"); + assert_eq!( + file_name(Some("/Users/me/.config/herdr/herdr.sock".as_ref())), + "state.json" + ); + assert_eq!( + file_name(Some("/x/herdr/sessions/rca/herdr.sock".as_ref())), + "state-rca.json" + ); + } + + #[test] + fn missing_file_is_empty_state_and_roundtrips() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("nested").join("state.json"); + assert_eq!(State::load_from(&path).unwrap(), State::default()); + let mut s = State::default(); + s.machines.insert("agent-1".into(), "profile-1".into()); + s.save_to(&path).unwrap(); + assert_eq!(State::load_from(&path).unwrap(), s); + } + + #[tokio::test] + async fn account_and_backend_changes_discard_pruning_evidence() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("state.json"); + let a = Store::at(path.clone(), "https://production", "account-a"); + a.update(|s| { + s.machines.insert("agent-a".into(), "profile-a".into()); + }) + .await + .unwrap(); + assert_eq!(a.load().unwrap().machines.len(), 1); + for (host, account) in [ + ("https://production", "account-b"), + ("https://staging", "account-a"), + ] { + let other = Store::at(path.clone(), host, account); + assert!(other.load().unwrap().machines.is_empty()); + } + assert!(!std::fs::read_to_string(path).unwrap().contains("account-a")); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn concurrent_updates_preserve_each_registration_and_invalidate_old_plans() { + let dir = tempfile::tempdir().unwrap(); + let store = Store::at(dir.path().join("state.json"), "backboard", "account"); + let before = store.load().unwrap(); + let start = std::sync::Arc::new(tokio::sync::Barrier::new(20)); + let writes = (0..20).map(|n| { + let store = store.clone(); + let start = start.clone(); + tokio::spawn(async move { + start.wait().await; + store + .update(|s| { + s.machines + .insert(format!("agent-{n}"), format!("profile-{n}")); + }) + .await + .unwrap(); + }) + }); + for result in futures::future::join_all(writes).await { + result.unwrap(); + } + let after = store.load().unwrap(); + assert_eq!(after.machines.len(), 20); + assert!(!after.same_revision(&before)); + } +} diff --git a/src/commands/cloud_agent/herdr/sync.rs b/src/commands/cloud_agent/herdr/sync.rs new file mode 100644 index 000000000..90e130c62 --- /dev/null +++ b/src/commands/cloud_agent/herdr/sync.rs @@ -0,0 +1,1085 @@ +//! Reconcile herdr's saved machines with the cloud agents you own. +//! +//! Runs as herdr's startup hook as well as an action, so it never prompts and +//! says one line when nothing is wrong. Machines are matched to agents on the +//! ssh target only; labels are display text and anyone can edit them. + +use std::collections::BTreeMap; + +use anyhow::{Result, bail}; +use chrono::Utc; +use clap::Parser; +use colored::Colorize; + +use super::herdr_cli::{Herdr, Machine}; +use super::state::{State, Store}; +use super::target; +use crate::client::GQLClient; +use crate::config::Configs; +use crate::controllers::cloud_agent as ca; + +#[derive(Parser)] +pub struct Args { + /// Print what would change and change nothing + #[clap(long)] + dry_run: bool, + + /// Output as JSON + #[clap(long)] + json: bool, + + /// Do nothing when the last sync was this many seconds ago or less + #[clap(long, value_name = "SECONDS")] + debounce: Option, + + /// Also make sure this session's watcher is running + #[clap(long)] + spawn_watch: bool, +} + +#[derive(Debug, Clone)] +pub(super) enum Op { + /// A machine pointing at an agent that no longer exists. + Remove(String), + /// An enabled machine whose agent is not awake. + Disable(String), + /// A disabled machine whose agent is running. + Enable(String), + /// An enabled machine whose agent went sleeping → running since the last + /// sync. herdr parked it in Attention while the VM was down and never + /// retries that on its own; off and on again makes it reconnect. + Kick(String), + /// An agent with no machine. Adding one is interactive, so only reported. + Missing(ca::Agent), +} + +impl Op { + fn profile_id(&self) -> Option<&str> { + match self { + Op::Remove(id) | Op::Disable(id) | Op::Enable(id) | Op::Kick(id) => Some(id), + Op::Missing(_) => None, + } + } + + fn verb(&self) -> &'static str { + match self { + Op::Remove(_) => "remove", + Op::Disable(_) => "disable", + Op::Enable(_) => "enable", + Op::Kick(_) => "reconnect", + Op::Missing(_) => "missing", + } + } + + fn describe(&self, machines: &[Machine]) -> String { + match self { + Op::Missing(agent) => format!( + "missing agent {} ({}) has no herdr machine", + agent.name, + agent.status.label() + ), + op => { + let id = op.profile_id().unwrap_or_default(); + let label = machines + .iter() + .find(|m| m.id == id) + .map(|m| m.label.as_str()) + .unwrap_or(id); + format!("{:<8} machine {label} ({id})", op.verb()) + } + } + } + + fn to_json(&self) -> serde_json::Value { + match self { + Op::Missing(agent) => serde_json::json!({ + "op": "missing", + "agent": { "id": agent.id, "name": agent.name, "status": agent.status.label() }, + }), + op => serde_json::json!({ "op": op.verb(), "profile": op.profile_id() }), + } + } +} + +#[derive(Debug, Default, Clone)] +pub(super) struct Plan { + pub ops: Vec, + /// agent id → herdr profile id, for every agent that has a machine + pub matches: BTreeMap, + /// profile id → agent, for the ops that must wait for its ssh relay first + pub agents: BTreeMap, + /// agent id → status label, remembered for the next run + pub statuses: BTreeMap, +} + +pub(super) fn is_machine_for(agent: &ca::Agent, machine: &Machine) -> bool { + machine.target == target::target(agent) + || target::agent_id_of(&machine.target).as_deref() == Some(agent.id.as_str()) +} + +pub(super) fn machine_for<'a>(agent: &ca::Agent, machines: &'a [Machine]) -> Option<&'a Machine> { + machines.iter().find(|m| is_machine_for(agent, m)) +} + +/// `known` is agent id → profile id from a previous sync in the SAME account +/// and backend. Store discards this evidence on a scope change. +pub(super) fn reconcile( + agents: &[ca::Agent], + machines: &[Machine], + previous: &BTreeMap, + known: &BTreeMap, +) -> Plan { + let mut plan = Plan::default(); + for agent in agents { + plan.statuses.insert(agent.id.clone(), agent.status.label()); + } + for machine in machines { + if target::agent_id_of(&machine.target).is_none() { + continue; + } + let Some(agent) = agents.iter().find(|a| is_machine_for(a, machine)) else { + if !agents.is_empty() && known.values().any(|p| p == &machine.id) { + plan.ops.push(Op::Remove(machine.id.clone())); + } + continue; + }; + plan.matches.insert(agent.id.clone(), machine.id.clone()); + let awake = matches!(agent.status, ca::Status::Running | ca::Status::Starting); + let was_awake = previous + .get(&agent.id) + .is_none_or(|s| s == "running" || s == "starting"); + if machine.enabled && !awake { + plan.ops.push(Op::Disable(machine.id.clone())); + } else if !machine.enabled && agent.status == ca::Status::Running { + plan.agents.insert(machine.id.clone(), agent.clone()); + plan.ops.push(Op::Enable(machine.id.clone())); + } else if machine.enabled && agent.status == ca::Status::Running && !was_awake { + plan.agents.insert(machine.id.clone(), agent.clone()); + plan.ops.push(Op::Kick(machine.id.clone())); + } + } + for agent in agents { + if agent.status.is_live() && !plan.matches.contains_key(&agent.id) { + plan.ops.push(Op::Missing(agent.clone())); + } + } + plan +} + +fn plan_for(agents: &[ca::Agent], machines: &[Machine], state: &State) -> Plan { + let mut plan = reconcile(agents, machines, &state.agent_status, &state.machines); + plan.ops.retain(|op| { + !matches!(op, Op::Enable(_) | Op::Kick(_)) + || !op + .profile_id() + .and_then(|id| plan.agents.get(id)) + .is_some_and(|agent| state.sleep_pending(&agent.id, Utc::now())) + }); + for agent in agents + .iter() + .filter(|a| state.sleep_pending(&a.id, Utc::now())) + { + if let Some(machine) = machine_for(agent, machines) + && machine.enabled + && !plan + .ops + .iter() + .any(|op| matches!(op, Op::Disable(id) if id == &machine.id)) + { + plan.ops.push(Op::Disable(machine.id.clone())); + } + } + plan +} + +#[derive(Debug, Default)] +pub(super) struct Outcome { + pub applied: Vec, + pub failed: Vec<(Op, String)>, +} + +/// Apply only after relay readiness and the state revision have been checked. +fn apply(herdr: &Herdr, ops: &[Op]) -> Outcome { + let mut outcome = Outcome::default(); + for op in ops { + let result = match op { + Op::Remove(id) => herdr.machine_remove(id), + Op::Disable(id) => herdr.machine_disable(id), + Op::Enable(id) => herdr.machine_enable(id), + Op::Kick(id) => herdr + .machine_disable(id) + .and_then(|()| herdr.machine_enable(id)), + Op::Missing(_) => continue, + }; + match result { + Ok(()) => outcome.applied.push(op.clone()), + Err(e) => outcome.failed.push((op.clone(), format!("{e:#}"))), + } + } + outcome +} + +pub async fn command(args: Args) -> Result<()> { + if args.spawn_watch { + super::watch::spawn_detached(); + } + let configs = Configs::new()?; + let client = GQLClient::new_authorized(&configs)?; + let backboard = configs.get_backboard(); + let herdr = Herdr::from_env(); + let store = Store::new(&configs)?; + let Some(report) = run_sync(&client, &backboard, &herdr, &store, &args, relay_ready).await? + else { + return Ok(()); + }; + if !args.dry_run { + super::watch::nudge(); + } + let Report { + agents, + machines, + plan, + outcome, + } = report; + + if args.json { + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ + "dryRun": args.dry_run, + "plan": plan.ops.iter().map(Op::to_json).collect::>(), + "applied": outcome.applied.iter().map(Op::to_json).collect::>(), + "failed": outcome + .failed + .iter() + .map(|(op, err)| { + let mut v = op.to_json(); + v["error"] = serde_json::Value::String(err.clone()); + v + }) + .collect::>(), + "machines": plan.matches, + }))? + ); + } else if let Some(reason) = toast_reason(&outcome) { + // Started by herdr (a key or the focus hook), not a terminal: the + // summary goes to a toast. The hook stays quiet unless it changed something. + println!("{}", summary(&plan, &outcome, agents.len())); + if reason { + herdr.notify( + "Railway sync", + &plain(&summary(&plan, &outcome, agents.len())), + ); + } + } else if args.dry_run { + if plan.ops.is_empty() { + println!( + "herdr machines match your {} agent{}; nothing to do.", + agents.len(), + plural(agents.len()) + ); + } + for op in &plan.ops { + println!("would {}", op.describe(&machines)); + } + } else { + println!("{}", summary(&plan, &outcome, agents.len())); + } + + if !outcome.failed.is_empty() { + bail!( + "{} herdr change{} failed:\n{}", + outcome.failed.len(), + plural(outcome.failed.len()), + outcome + .failed + .iter() + .map(|(op, err)| format!(" {}: {err}", op.describe(&machines))) + .collect::>() + .join("\n") + ); + } + Ok(()) +} + +fn summary(plan: &Plan, outcome: &Outcome, agent_count: usize) -> String { + let count = |verb: &str| { + outcome + .applied + .iter() + .filter(|op| op.verb() == verb) + .count() + }; + let mut parts = Vec::new(); + for verb in ["enable", "disable", "remove", "reconnect"] { + let n = count(verb); + if n > 0 { + parts.push(format!( + "{verb}{} {n}", + if verb.ends_with('e') { "d" } else { "ed" } + )); + } + } + let missing: Vec<&str> = plan + .ops + .iter() + .filter_map(|op| match op { + Op::Missing(agent) => Some(agent.name.as_str()), + _ => None, + }) + .collect(); + let mut line = if !outcome.failed.is_empty() { + format!( + "✗ herdr sync: {} failed{}{}.", + outcome.failed.len(), + if parts.is_empty() { "" } else { "; " }, + parts.join(", ") + ) + } else if parts.is_empty() { + format!( + "✓ herdr machines match your {agent_count} agent{}.", + plural(agent_count) + ) + } else { + format!("✓ herdr sync: {}.", parts.join(", ")) + }; + if !missing.is_empty() { + line.push_str( + &format!( + " {} agent{} without a machine: {} ({} adds one)", + missing.len(), + plural(missing.len()), + missing.join(", "), + "railway ca herdr agents".cyan() + ) + .dimmed() + .to_string(), + ); + } + line +} + +fn plural(n: usize) -> &'static str { + if n == 1 { "" } else { "s" } +} + +/// The whole reconcile, quietly: what the picker runs after a sleep or wake so +/// every machine row reflects the agent it points at. +pub(super) async fn resync( + client: &reqwest::Client, + backboard: &str, + herdr: &Herdr, + store: &Store, +) -> Result> { + let args = Args { + dry_run: false, + json: false, + debounce: None, + spawn_watch: false, + }; + let report = run_sync(client, backboard, herdr, store, &args, relay_ready) + .await? + .expect("an undebounced sync always runs"); + let Report { + machines, outcome, .. + } = report; + if let Some((op, err)) = outcome.failed.first() { + bail!("{} failed: {err}", op.describe(&machines)); + } + Ok(outcome + .applied + .iter() + .map(|op| op.describe(&machines)) + .collect()) +} + +struct Report { + agents: Vec, + machines: Vec, + plan: Plan, + outcome: Outcome, +} + +async fn relay_ready(agent: ca::Agent) -> Result<()> { + super::relay::wait_until_ready(&agent).await +} + +/// One sync at a time, but never hold the state lock during a relay wait: +/// sleep/wake and registration can proceed while a machine is unresponsive. +/// Their writes advance the revision, so a waiting sync must refetch before +/// applying its old plan. Both CLI and watcher take this same path. +async fn run_sync( + client: &reqwest::Client, + backboard: &str, + herdr: &Herdr, + store: &Store, + args: &Args, + mut ready: F, +) -> Result> +where + F: FnMut(ca::Agent) -> Fut, + Fut: std::future::Future>, +{ + let _sync = super::state::lock_file(&store.path.with_extension("sync.lock")).await?; + let mut applied = Vec::new(); + 'retry: for _ in 0..3 { + let baseline = store.load()?; + if args + .debounce + .is_some_and(|secs| synced_within(&baseline, secs, Utc::now())) + { + return Ok(None); + } + let agents = ca::list_mine(client, backboard).await?; + let machines = herdr.machines()?; + let plan = plan_for(&agents, &machines, &baseline); + if args.dry_run { + return Ok(Some(Report { + agents, + machines, + plan, + outcome: Outcome::default(), + })); + } + + let mut outcome = Outcome::default(); + for op in &plan.ops { + if matches!(op, Op::Enable(_) | Op::Kick(_)) + && let Some(agent) = op.profile_id().and_then(|id| plan.agents.get(id)) + && let Err(e) = ready(agent.clone()).await + { + outcome.failed.push((op.clone(), format!("{e:#}"))); + continue; + } + let locked = store.lock().await?; + if !locked.state.same_revision(&baseline) { + continue 'retry; + } + // Apply in plan order without delaying earlier disables behind + // later relay probes. Keep the lock only for the catalog edit. + let change = apply(herdr, std::slice::from_ref(op)); + applied.extend(change.applied); + outcome.failed.extend(change.failed); + } + + let mut locked = store.lock().await?; + if !locked.state.same_revision(&baseline) { + continue; + } + outcome.applied = applied; + remember(&mut locked.state, &plan, &outcome); + locked.save()?; + return Ok(Some(Report { + agents, + machines, + plan, + outcome, + })); + } + bail!("Agent actions changed while syncing; retry `railway ca herdr sync`.") +} + +/// `Some(true)` when a toast is due, `Some(false)` for a quiet hook run, +/// `None` when we are on a terminal and print as usual. +fn toast_reason(outcome: &Outcome) -> Option { + let manual = std::env::var("HERDR_PLUGIN_ACTION_ID").is_ok(); + let hook = std::env::var("HERDR_PLUGIN_EVENT").is_ok(); + if !manual && !hook { + return None; + } + Some(manual || !outcome.applied.is_empty() || !outcome.failed.is_empty()) +} + +fn plain(s: &str) -> String { + s.trim_start_matches('✓').trim().to_string() +} + +/// A failed Enable or Kick keeps the agent's previous status, so the next run +/// plans it again instead of believing the machine already followed. +fn remember(state: &mut State, plan: &Plan, outcome: &Outcome) { + let mut statuses = plan.statuses.clone(); + for (op, _) in &outcome.failed { + if let Some(agent) = op.profile_id().and_then(|id| plan.agents.get(id)) { + match state.agent_status.get(&agent.id) { + Some(old) => statuses.insert(agent.id.clone(), old.clone()), + None => statuses.remove(&agent.id), + }; + } + } + // A deleted agent cannot appear in plan.matches. Keep its ownership + // evidence when removal failed, or the next sync will treat the leftover + // profile as someone else's and never retry it. + state.machines.retain(|_, profile| { + outcome + .failed + .iter() + .any(|(op, _)| matches!(op, Op::Remove(id) if id == profile)) + }); + state.machines.extend(plan.matches.clone()); + state.agent_status = statuses; + state.sleep_until.retain(|id, until| { + *until > Utc::now() && plan.statuses.get(id).is_none_or(|s| s != "sleeping") + }); + state.last_sync = Some(Utc::now().to_rfc3339()); +} + +pub(super) fn synced_within(state: &State, secs: u64, now: chrono::DateTime) -> bool { + state + .last_sync + .as_deref() + .and_then(|t| chrono::DateTime::parse_from_rfc3339(t).ok()) + .map(|t| now.signed_duration_since(t).num_seconds().unsigned_abs() <= secs) + .unwrap_or(false) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::controllers::cloud_agent::Status; + + fn agent(id: &str, status: Status) -> ca::Agent { + ca::Agent { + id: id.into(), + name: format!("name-{id}"), + status, + project_id: "project".into(), + environment_id: "env".into(), + created_at: Utc::now(), + } + } + + /// What a previous sync would have recorded: every machine matched to an + /// agent, plus any `agent:*` machine as if it had been seen before. + fn known(agents: &[ca::Agent], machines: &[Machine]) -> BTreeMap { + machines + .iter() + .filter_map(|m| { + let id = target::agent_id_of(&m.target)?; + let agent = agents.iter().find(|a| a.id == id).map(|a| a.id.clone()); + Some((agent.unwrap_or(id), m.id.clone())) + }) + .collect() + } + + fn machine(id: &str, target: &str, enabled: bool) -> Machine { + Machine { + id: id.into(), + label: format!("label-{id}"), + target: target.into(), + session: "default".into(), + enabled, + selected: false, + } + } + + fn ours(id: &str, agent_id: &str, enabled: bool) -> Machine { + machine(id, &target::target_for("env", agent_id), enabled) + } + + fn ops_of(plan: &Plan) -> Vec { + plan.ops + .iter() + .map(|op| match op { + Op::Missing(agent) => format!("missing {}", agent.id), + op => format!("{} {}", op.verb(), op.profile_id().unwrap()), + }) + .collect() + } + + fn after(machines: &[Machine], plan: &Plan) -> Vec { + machines + .iter() + .filter(|m| { + !plan + .ops + .iter() + .any(|op| matches!(op, Op::Remove(id) if id == &m.id)) + }) + .map(|m| { + let mut m = m.clone(); + for op in &plan.ops { + match op { + Op::Disable(id) if id == &m.id => m.enabled = false, + Op::Enable(id) | Op::Kick(id) if id == &m.id => m.enabled = true, + _ => {} + } + } + m + }) + .collect() + } + + #[test] + fn reconcile_cases() { + let cases: Vec<(&str, Vec, Vec, Vec<&str>)> = vec![ + ( + "running agent with enabled machine", + vec![agent("a1", Status::Running)], + vec![ours("p1", "a1", true)], + vec![], + ), + ( + "sleeping agent with enabled machine", + vec![agent("a1", Status::Sleeping)], + vec![ours("p1", "a1", true)], + vec!["disable p1"], + ), + ( + "crashed agent with enabled machine", + vec![agent("a1", Status::Crashed)], + vec![ours("p1", "a1", true)], + vec!["disable p1"], + ), + ( + "running agent with disabled machine", + vec![agent("a1", Status::Running)], + vec![ours("p1", "a1", false)], + vec!["enable p1"], + ), + ( + "starting agent with disabled machine waits", + vec![agent("a1", Status::Starting)], + vec![ours("p1", "a1", false)], + vec![], + ), + ( + "our machine with no agent (account still has others)", + vec![agent("a2", Status::Running)], + vec![ours("p1", "gone", true)], + vec!["remove p1", "missing a2"], + ), + ( + "agent with no machine", + vec![agent("a1", Status::Running)], + vec![], + vec!["missing a1"], + ), + ( + "deleting agent with no machine is not missing", + vec![agent("a1", Status::Deleting)], + vec![], + vec![], + ), + ( + "foreign machines are never touched", + vec![agent("a1", Status::Sleeping)], + vec![ + machine("w", "workbox", true), + machine("m", "me@workbox", true), + ours("p1", "a1", true), + ], + vec!["disable p1"], + ), + ( + "match on agent id inside a target from another relay host", + vec![agent("a1", Status::Sleeping)], + vec![machine("p1", "agent:env:a1@ssh.elsewhere.example", true)], + vec!["disable p1"], + ), + ( + "labels do not match", + vec![agent("a1", Status::Running)], + vec![machine("p1", "workbox", true)], + vec!["missing a1"], + ), + ]; + for (name, agents, machines, expected) in cases { + let plan = reconcile( + &agents, + &machines, + &BTreeMap::new(), + &known(&agents, &machines), + ); + assert_eq!(ops_of(&plan), expected, "{name}"); + } + } + + #[test] + fn matches_map_every_matched_agent_to_its_profile() { + let agents = vec![agent("a1", Status::Running), agent("a2", Status::Sleeping)]; + let machines = vec![ + ours("p1", "a1", true), + ours("p2", "a2", true), + ours("p3", "gone", true), + machine("w", "workbox", true), + ]; + let plan = reconcile( + &agents, + &machines, + &BTreeMap::new(), + &known(&agents, &machines), + ); + assert_eq!( + plan.matches, + BTreeMap::from([ + ("a1".to_string(), "p1".to_string()), + ("a2".to_string(), "p2".to_string()) + ]) + ); + } + + #[test] + fn second_run_over_the_result_is_a_no_op() { + let agents = vec![ + agent("a1", Status::Running), + agent("a2", Status::Sleeping), + agent("a3", Status::Running), + ]; + let machines = vec![ + ours("p1", "a1", false), + ours("p2", "a2", true), + ours("p3", "gone", true), + machine("w", "workbox", true), + ]; + let plan = reconcile( + &agents, + &machines, + &BTreeMap::new(), + &known(&agents, &machines), + ); + assert_eq!( + ops_of(&plan), + vec!["enable p1", "disable p2", "remove p3", "missing a3"] + ); + + let machines = after(&machines, &plan); + assert_eq!(machines.len(), 3); + let again = reconcile( + &agents, + &machines, + &BTreeMap::new(), + &known(&agents, &machines), + ); + assert_eq!(ops_of(&again), vec!["missing a3"]); + assert_eq!(again.matches, plan.matches); + } + + #[test] + fn a_wake_done_elsewhere_reconnects_the_enabled_machine() { + let agents = [agent("a1", Status::Running)]; + let machines = [machine("p1", &target::target(&agents[0]), true)]; + let mut previous = BTreeMap::new(); + previous.insert("a1".to_string(), "sleeping".to_string()); + let plan = reconcile(&agents, &machines, &previous, &BTreeMap::new()); + assert!( + matches!(plan.ops.as_slice(), [Op::Kick(id)] if id == "p1"), + "{:?}", + plan.ops + ); + assert_eq!(plan.statuses.get("a1").map(String::as_str), Some("running")); + assert!(plan.agents.contains_key("p1")); + + previous.insert("a1".to_string(), "running".to_string()); + assert!( + reconcile(&agents, &machines, &previous, &BTreeMap::new()) + .ops + .is_empty() + ); + assert!( + reconcile( + &agents, + &machines, + &BTreeMap::new(), + &known(&agents, &machines) + ) + .ops + .is_empty() + ); + } + + #[test] + fn unknown_machines_and_empty_agent_lists_never_trigger_removal() { + let orphan = machine("p9", "ssh://agent%3Aenv%3Anobody@ssh.railway.com", true); + let removals = |plan: &Plan| { + plan.ops + .iter() + .filter(|op| matches!(op, Op::Remove(_))) + .count() + }; + // Never recorded by a previous sync: someone else's agent, or a hand-added machine. + let plan = reconcile( + &[agent("a1", Status::Running)], + std::slice::from_ref(&orphan), + &BTreeMap::new(), + &BTreeMap::new(), + ); + assert_eq!(removals(&plan), 0, "{:?}", plan.ops); + // Recorded before, but the agent list came back empty (wrong account, API blip). + let mut known = BTreeMap::new(); + known.insert("nobody".to_string(), "p9".to_string()); + let plan = reconcile(&[], std::slice::from_ref(&orphan), &BTreeMap::new(), &known); + assert_eq!(removals(&plan), 0, "{:?}", plan.ops); + // Recorded before and the account still has agents: the agent is really gone. + let plan = reconcile( + &[agent("a1", Status::Running)], + &[orphan], + &BTreeMap::new(), + &known, + ); + assert_eq!(removals(&plan), 1, "{:?}", plan.ops); + } + + #[test] + fn a_failed_kick_keeps_the_old_status_so_it_is_retried() { + let agents = [agent("a1", Status::Running)]; + let machines = [machine("p1", &target::target(&agents[0]), true)]; + let mut previous = BTreeMap::new(); + previous.insert("a1".to_string(), "sleeping".to_string()); + let plan = reconcile(&agents, &machines, &previous, &BTreeMap::new()); + assert!(matches!(plan.ops.as_slice(), [Op::Kick(_)])); + let outcome = Outcome { + applied: vec![], + failed: vec![(plan.ops[0].clone(), "relay never answered".into())], + }; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("state.json"); + let mut state = State::default(); + state.agent_status = previous.clone(); + remember(&mut state, &plan, &outcome); + state.save_to(&path).unwrap(); + let state = State::load_from(&path).unwrap(); + assert_eq!( + state.agent_status.get("a1").map(String::as_str), + Some("sleeping") + ); + assert!(matches!( + reconcile(&agents, &machines, &state.agent_status, &BTreeMap::new()) + .ops + .as_slice(), + [Op::Kick(_)] + )); + } + + // The fake herdr is a shebang script: unix only. + + #[cfg(unix)] + #[tokio::test] + async fn apply_runs_exactly_the_planned_herdr_commands() { + let fake = super::super::herdr_cli::fake::FakeHerdr::with_machines(&format!( + r#"[ + {{"id":"p1","label":"proj/one","target":"{}","enabled":true}}, + {{"id":"p2","label":"proj/two","target":"{}","enabled":false}}, + {{"id":"p3","label":"proj/gone","target":"{}","enabled":true}}, + {{"id":"w","label":"workbox","target":"workbox","enabled":true}} + ]"#, + target::target_for("env", "a1"), + target::target_for("env", "a2"), + target::target_for("env", "gone"), + )); + let herdr = fake.herdr(); + let agents = vec![ + agent("a1", Status::Sleeping), + agent("a2", Status::Running), + agent("a3", Status::Running), + ]; + let machines = herdr.machines().unwrap(); + let plan = reconcile( + &agents, + &machines, + &BTreeMap::new(), + &known(&agents, &machines), + ); + let outcome = apply(&herdr, &plan.ops); + assert!(outcome.failed.is_empty(), "{:?}", outcome.failed); + assert_eq!(outcome.applied.len(), 3); + assert_eq!( + fake.calls(), + vec![ + "machine list --json".to_string(), + "machine disable p1".to_string(), + "machine enable p2".to_string(), + "machine remove p3".to_string(), + ] + ); + } + + // The fake herdr is a shebang script: unix only. + + #[cfg(unix)] + #[tokio::test] + async fn apply_collects_failures_instead_of_stopping() { + let herdr = Herdr::at("/nonexistent/herdr"); + let plan = Plan { + ops: vec![Op::Disable("p1".into()), Op::Remove("p2".into())], + ..Default::default() + }; + let outcome = apply(&herdr, &plan.ops); + assert!(outcome.applied.is_empty()); + assert_eq!(outcome.failed.len(), 2); + assert!(matches!(outcome.failed[0].0, Op::Disable(_))); + assert!(matches!(outcome.failed[1].0, Op::Remove(_))); + } + + #[test] + fn dry_run_lines_name_the_machine() { + let machines = vec![ours("p1", "a1", true)]; + let plan = reconcile( + &[agent("a1", Status::Sleeping)], + &machines, + &BTreeMap::new(), + &BTreeMap::new(), + ); + assert_eq!( + plan.ops[0].describe(&machines), + "disable machine label-p1 (p1)" + ); + } + + #[test] + fn debounce_window_reads_last_sync() { + let now = Utc::now(); + let mut state = State::default(); + assert!(!synced_within(&state, 30, now)); + state.last_sync = Some((now - chrono::Duration::seconds(10)).to_rfc3339()); + assert!(synced_within(&state, 30, now)); + state.last_sync = Some((now - chrono::Duration::seconds(45)).to_rfc3339()); + assert!(!synced_within(&state, 30, now)); + state.last_sync = Some("not a date".into()); + assert!(!synced_within(&state, 30, now)); + } + + #[test] + fn failed_removal_is_retried_until_it_succeeds() { + let agents = [agent("a1", Status::Running)]; + let machines = [ours("p1", "a1", true), ours("p9", "gone", true)]; + let mut state = State::default(); + state.machines = known(&agents, &machines); + let first = plan_for(&agents, &machines, &state); + assert_eq!(ops_of(&first), ["remove p9"]); + let failed = Outcome { + applied: Vec::new(), + failed: vec![(Op::Remove("p9".into()), "temporary write failure".into())], + }; + remember(&mut state, &first, &failed); + let retry = plan_for(&agents, &machines, &state); + assert_eq!(ops_of(&retry), ["remove p9"]); + remember( + &mut state, + &retry, + &Outcome { + applied: vec![Op::Remove("p9".into())], + failed: Vec::new(), + }, + ); + assert!(!state.machines.contains_key("gone")); + assert_eq!(state.machines.get("a1").map(String::as_str), Some("p1")); + } + + #[test] + fn pending_sleep_overrides_lagging_running_inventory() { + let agents = [agent("a1", Status::Running)]; + let mut state = State::default(); + state + .sleep_until + .insert("a1".into(), Utc::now() + chrono::Duration::seconds(60)); + assert!( + plan_for(&agents, &[ours("p1", "a1", false)], &state) + .ops + .is_empty() + ); + assert_eq!( + ops_of(&plan_for(&agents, &[ours("p1", "a1", true)], &state)), + ["disable p1"] + ); + let asleep = plan_for( + &[agent("a1", Status::Sleeping)], + &[ours("p1", "a1", false)], + &state, + ); + remember(&mut state, &asleep, &Outcome::default()); + assert!(state.sleep_until.is_empty()); + assert_eq!( + ops_of(&plan_for(&agents, &[ours("p1", "a1", false)], &state)), + ["enable p1"] + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn account_switch_does_not_remove_the_previous_accounts_profiles() { + let dir = tempfile::tempdir().unwrap(); + let api = crate::testkit::MockBackboard::spawn(); + let path = dir.path().join("state.json"); + let old = Store::at(path.clone(), &api.url(), "account-a"); + old.update(|s| { + s.machines.insert("a1".into(), "p1".into()); + }) + .await + .unwrap(); + let new = Store::at(path, &api.url(), "account-b"); + api.stub( + "MyCloudAgents", + serde_json::json!({"myCloudAgents": [{ + "id": "a2", "name": "other-account", "status": "RUNNING", + "projectId": "p", "environmentId": "e", "createdAt": Utc::now(), + }]}), + ); + let fake = super::super::herdr_cli::fake::FakeHerdr::with_machines(&format!( + r#"[{{"id":"p1","label":"first","target":"{}","enabled":true}}]"#, + target::target_for("env", "a1") + )); + resync(&reqwest::Client::new(), &api.url(), &fake.herdr(), &new) + .await + .unwrap(); + assert_eq!(fake.calls(), ["machine list --json"]); + assert!(!new.load().unwrap().machines.contains_key("a1")); + } + + #[cfg(unix)] + #[tokio::test] + async fn sleep_during_relay_wait_invalidates_the_enable_plan_without_blocking_sleep() { + let dir = tempfile::tempdir().unwrap(); + let api = crate::testkit::MockBackboard::spawn(); + let store = Store::at(dir.path().join("state.json"), &api.url(), "account"); + api.stub( + "MyCloudAgents", + serde_json::json!({"myCloudAgents": [{ + "id": "a1", "name": "first", "status": "RUNNING", + "projectId": "p", "environmentId": "env", "createdAt": Utc::now(), + }]}), + ); + let fake = super::super::herdr_cli::fake::FakeHerdr::with_machines(&format!( + r#"[{{"id":"p1","label":"first","target":"{}","enabled":false}}]"#, + target::target_for("env", "a1") + )); + let args = Args { + dry_run: false, + json: false, + debounce: None, + spawn_watch: false, + }; + let report = tokio::time::timeout( + std::time::Duration::from_secs(3), + run_sync( + &reqwest::Client::new(), + &api.url(), + &fake.herdr(), + &store, + &args, + |_| async { + // Represents a successful sleep acknowledgement while SSH is + // being probed. This would deadlock if sync held the state lock. + store + .update(|s| { + s.sleep_until + .insert("a1".into(), Utc::now() + chrono::Duration::seconds(60)); + }) + .await?; + Ok(()) + }, + ), + ) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!(report.outcome.applied.is_empty()); + assert_eq!(fake.calls(), ["machine list --json", "machine list --json"]); + assert_eq!( + api.requests().len(), + 2, + "the old observation must be refetched" + ); + assert!(store.load().unwrap().sleep_pending("a1", Utc::now())); + } +} diff --git a/src/commands/cloud_agent/herdr/target.rs b/src/commands/cloud_agent/herdr/target.rs new file mode 100644 index 000000000..4bd34793a --- /dev/null +++ b/src/commands/cloud_agent/herdr/target.rs @@ -0,0 +1,112 @@ +//! The two strings herdr sees for an agent: its ssh target and its label. + +use crate::config::Configs; +use crate::controllers::cloud_agent as ca; + +/// herdr hands the target straight to `ssh`, and the relay reads +/// `agent::` as the username. herdr rejects a literal `:` in the +/// user part as a password, so the colons ride percent-encoded in the URI +/// form, which OpenSSH decodes (verified against the relay). +pub fn target(agent: &ca::Agent) -> String { + target_for(&agent.environment_id, &agent.id) +} + +pub fn target_for(environment_id: &str, agent_id: &str) -> String { + let (host, port) = Configs::get_ssh_relay(); + let user = format!("agent%3A{environment_id}%3A{agent_id}"); + match port { + Some(port) => format!("ssh://{user}@{host}:{port}"), + None => format!("ssh://{user}@{host}"), + } +} + +/// The agent id inside a target this module wrote, `None` for anyone else's. +pub fn agent_id_of(target: &str) -> Option { + let user = target.strip_prefix("ssh://").unwrap_or(target); + let user = user.rsplit_once('@')?.0; + let user = user.replace("%3A", ":").replace("%3a", ":"); + let mut parts = user.splitn(3, ':'); + if parts.next()? != "agent" { + return None; + } + parts.next()?; + parts.next().map(str::to_owned) +} + +/// Sidebar budget. herdr clips labels rather than wrapping, and the header row +/// also carries the connection signal, so a 26-column sidebar shows about 24 +/// characters of label. Project first for grouping; the agent name is the part +/// you search for, so it keeps its characters and the project absorbs the cut. +pub const LABEL_WIDTH: usize = 24; + +pub fn label(project: &str, name: &str) -> String { + label_within(project, name, LABEL_WIDTH) +} + +fn label_within(project: &str, name: &str, width: usize) -> String { + let project = project.trim(); + let name = name.trim(); + let plen = project.chars().count(); + let nlen = name.chars().count(); + if plen + 1 + nlen <= width { + return format!("{project}/{name}"); + } + const PROJECT_FLOOR: usize = 6; + let name = cut(name, (width - 1).saturating_sub(PROJECT_FLOOR).max(1)); + let project = cut( + project, + (width - 1).saturating_sub(name.chars().count()).max(1), + ); + format!("{project}/{name}") +} + +fn cut(s: &str, width: usize) -> String { + if s.chars().count() <= width { + return s.to_string(); + } + let keep = width.saturating_sub(1); + let mut out: String = s.chars().take(keep).collect(); + out.push('…'); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn short_labels_are_untouched() { + assert_eq!(label("orch", "reviewer"), "orch/reviewer"); + } + + #[test] + fn long_pairs_fit_the_budget_with_project_cut_first() { + let l = label("railway-sandboxes", "content-wonder"); + assert!(l.chars().count() <= LABEL_WIDTH, "{l}"); + assert!(l.ends_with("/content-wonder"), "{l}"); + assert!(l.starts_with("railway-"), "{l}"); + } + + #[test] + fn both_halves_are_cut_when_both_are_long() { + let l = label("Hello World Page", "Build a simple hello world"); + assert!(l.chars().count() <= LABEL_WIDTH, "{l}"); + assert!(l.contains('…'), "{l}"); + assert!(l.contains('/'), "{l}"); + } + + #[test] + fn target_round_trips_the_agent_id() { + let t = target_for("env-1", "agent-1"); + assert!(t.starts_with("ssh://agent%3Aenv-1%3Aagent-1@"), "{t}"); + let userinfo = t.trim_start_matches("ssh://").rsplit_once('@').unwrap().0; + assert!(!userinfo.contains(':'), "{t}"); + assert_eq!(agent_id_of(&t).as_deref(), Some("agent-1")); + assert_eq!( + agent_id_of("agent:env-1:agent-1@ssh.railway.com").as_deref(), + Some("agent-1") + ); + assert_eq!(agent_id_of("workbox"), None); + assert_eq!(agent_id_of("me@workbox"), None); + } +} diff --git a/src/commands/cloud_agent/herdr/watch.rs b/src/commands/cloud_agent/herdr/watch.rs new file mode 100644 index 000000000..a77507be8 --- /dev/null +++ b/src/commands/cloud_agent/herdr/watch.rs @@ -0,0 +1,475 @@ +//! Keeps herdr's machines in step with the cloud agents behind them as they +//! change, so a sleep from anywhere disables the machine before herdr's next +//! reconnect can park it in Attention, and a wake re-enables it once the +//! relay answers. The signal is backboard's `cloudAgentInvalidation` +//! subscription, one per environment holding one of the user's agents, on the +//! unpublished internal graph the web and mobile apps use. Reconnects refetch, +//! and a periodic sweep covers missed events or an unavailable subscription. One watcher per +//! herdr session, started by `install` and the plugin's startup hook, told to +//! re-list environments by SIGUSR1 (`nudge`), gone when the session's socket is. + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use clap::Parser; +use colored::Colorize; +use futures::StreamExt; +use graphql_client::{GraphQLQuery, QueryBody}; +use serde::{Deserialize, Serialize}; +use tokio::sync::mpsc; +use tokio::task::JoinSet; + +use super::herdr_cli::Herdr; +use super::state::{State, Store}; +use super::sync; +use crate::client::GQLClient; +use crate::config::Configs; +use crate::controllers::cloud_agent as ca; +use crate::subscription::subscribe_graphql_internal; + +const DEBOUNCE: Duration = Duration::from_secs(2); +const LIVENESS: Duration = Duration::from_secs(30); +const RESUBSCRIBE_MIN: Duration = Duration::from_secs(5); +const RESUBSCRIBE_MAX: Duration = Duration::from_secs(120); + +#[derive(Parser)] +pub struct Args { + /// Stay attached to this terminal and print every event; detached, only + /// subscriptions, failures and applied changes are logged + #[clap(long)] + foreground: bool, +} + +struct CloudAgentInvalidation; + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct Variables { + environment_id: String, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +struct ResponseData { + cloud_agent_invalidation: Invalidation, +} + +#[derive(Deserialize, Debug)] +struct Invalidation { + #[allow(dead_code)] + id: String, + agent: Option, +} + +#[derive(Deserialize, Debug)] +struct Snapshot { + id: String, + status: String, +} + +impl GraphQLQuery for CloudAgentInvalidation { + type Variables = Variables; + type ResponseData = ResponseData; + + fn build_query(variables: Variables) -> QueryBody { + QueryBody { + variables, + query: "subscription CloudAgentInvalidation($environmentId: String!) { cloudAgentInvalidation(environmentId: $environmentId) { id agent { id status } } }", + operation_name: "CloudAgentInvalidation", + } + } +} + +enum Signal { + Changed(String), +} + +pub async fn command(args: Args) -> Result<()> { + let socket = PathBuf::from( + std::env::var_os("HERDR_SOCKET_PATH") + .context("HERDR_SOCKET_PATH is not set; run this from inside herdr")?, + ); + let pidfile = pidfile_path()?; + if let Some(pid) = running_pid(&pidfile) { + bail!("a watcher for this herdr session is already running (pid {pid})"); + } + std::fs::write(&pidfile, std::process::id().to_string())?; + let result = run(&socket, args.foreground).await; + let _ = std::fs::remove_file(&pidfile); + result +} + +async fn run(socket: &Path, verbose: bool) -> Result<()> { + let (tx, mut rx) = mpsc::channel::(64); + let mut tasks: JoinSet<()> = JoinSet::new(); + let mut watched: BTreeSet = BTreeSet::new(); + let mut liveness = tokio::time::interval(LIVENESS); + liveness.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut nudged = nudge_signal()?; + let mut relist = true; + + loop { + if relist { + relist = false; + match watched_environments().await { + Ok(envs) if envs != watched => { + tasks.shutdown().await; + for env in &envs { + tasks.spawn(subscribe(env.clone(), tx.clone(), verbose)); + } + say(verbose, &format!("watching {} environment(s)", envs.len())); + watched = envs; + } + Ok(_) => {} + Err(e) => say(true, &format!("could not list agents: {e:#}")), + } + } + tokio::select! { + _ = liveness.tick() => { + if !server_alive(socket) { + say(true, "herdr session gone; exiting"); + return Ok(()); + } + // Also discovers agents created in a previously empty + // environment, even when no local pane produces hook events. + relist = true; + resync(verbose, "periodic reconciliation").await; + } + _ = nudge_recv(&mut nudged) => { + say(verbose, "nudged: re-listing environments"); + relist = true; + } + Some(Signal::Changed(what)) = rx.recv() => { + say(verbose, &what); + tokio::time::sleep(DEBOUNCE).await; + while rx.try_recv().is_ok() {} + resync(verbose, &what).await; + } + } + } +} + +/// Every environment holding one of the user's agents: a machine added later +/// in any of them is covered without re-listing. +async fn watched_environments() -> Result> { + let configs = Configs::new()?; + let client = GQLClient::new_authorized(&configs)?; + let agents = ca::list_mine(&client, &configs.get_backboard()).await?; + Ok(agents.into_iter().map(|a| a.environment_id).collect()) +} + +async fn subscribe(environment_id: String, tx: mpsc::Sender, verbose: bool) { + subscribe_with(environment_id, tx, verbose, |environment_id| { + subscribe_graphql_internal::(Variables { environment_id }) + }) + .await; +} + +async fn subscribe_with( + environment_id: String, + tx: mpsc::Sender, + verbose: bool, + mut connect: F, +) where + F: FnMut(String) -> Fut, + Fut: std::future::Future>, + S: futures::Stream< + Item = Result, graphql_ws_client::Error>, + > + Unpin, +{ + let mut backoff = RESUBSCRIBE_MIN; + loop { + let stream = connect(environment_id.clone()).await; + let mut stream = match stream { + Ok(s) => s, + Err(e) => { + say( + true, + &format!( + "subscribe {environment_id}: {e:#}; retrying in {}s", + backoff.as_secs() + ), + ); + tokio::time::sleep(backoff).await; + backoff = (backoff * 2).min(RESUBSCRIBE_MAX); + continue; + } + }; + backoff = RESUBSCRIBE_MIN; + say(true, &format!("subscribed {environment_id}")); + if tx + .send(Signal::Changed(format!( + "subscribed {environment_id}: refetching" + ))) + .await + .is_err() + { + return; + } + while let Some(item) = stream.next().await { + let what = match item { + Ok(response) => match response.data { + Some(data) => describe(&environment_id, data.cloud_agent_invalidation.agent), + None => format!("{environment_id}: invalidation without data"), + }, + Err(e) => { + say(verbose, &format!("stream {environment_id}: {e}")); + break; + } + }; + if tx.send(Signal::Changed(what)).await.is_err() { + return; + } + } + tokio::time::sleep(RESUBSCRIBE_MIN).await; + } +} + +fn describe(environment_id: &str, agent: Option) -> String { + match agent { + Some(a) => format!("agent {} is now {}", a.id, a.status.to_lowercase()), + None => format!("environment {environment_id}: agents changed"), + } +} + +async fn resync(verbose: bool, cause: &str) { + let run = async { + let configs = Configs::new()?; + let client = GQLClient::new_authorized(&configs)?; + let store = Store::new(&configs)?; + sync::resync( + &client, + &configs.get_backboard(), + &Herdr::from_env(), + &store, + ) + .await + }; + match run.await { + Ok(applied) if applied.is_empty() => say(verbose, "synced, nothing to change"), + Ok(applied) => say(true, &format!("{cause}: {}", applied.join(", "))), + Err(e) => say(true, &format!("sync failed after \"{cause}\": {e:#}")), + } +} + +/// A unix socket file can outlive its server; only a connection proves one. +#[cfg(unix)] +fn server_alive(socket: &Path) -> bool { + std::os::unix::net::UnixStream::connect(socket).is_ok() +} + +#[cfg(not(unix))] +fn server_alive(socket: &Path) -> bool { + socket.exists() +} + +#[cfg(unix)] +type NudgeSignal = tokio::signal::unix::Signal; +#[cfg(not(unix))] +type NudgeSignal = (); + +#[cfg(unix)] +fn nudge_signal() -> Result { + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::user_defined1()) + .context("Installing the SIGUSR1 handler") +} + +#[cfg(not(unix))] +fn nudge_signal() -> Result { + Ok(()) +} + +#[cfg(unix)] +async fn nudge_recv(signal: &mut NudgeSignal) { + signal.recv().await; +} + +#[cfg(not(unix))] +async fn nudge_recv(_: &mut NudgeSignal) { + std::future::pending::<()>().await +} + +fn say(verbose: bool, line: &str) { + if verbose { + println!( + "{} {line}", + chrono::Local::now().format("%H:%M:%S").to_string().dimmed() + ); + } +} + +fn pidfile_path() -> Result { + let state = State::path()?; + let stem = state + .file_stem() + .map(|s| s.to_string_lossy().replace("state", "watch")) + .unwrap_or_else(|| "watch".into()); + Ok(state.with_file_name(format!("{stem}.pid"))) +} + +/// The recorded pid, only while that pid is still one of our watchers: pids +/// are reused after a crash or reboot, and a signal to a stranger is fatal. +#[cfg(unix)] +fn running_pid(pidfile: &Path) -> Option { + let pid: u32 = std::fs::read_to_string(pidfile).ok()?.trim().parse().ok()?; + let out = Command::new("ps") + .args(["-o", "command=", "-p", &pid.to_string()]) + .stderr(Stdio::null()) + .output() + .ok()?; + let command = String::from_utf8_lossy(&out.stdout); + is_watcher_command(&command).then_some(pid) +} + +#[cfg(not(unix))] +fn running_pid(_pidfile: &Path) -> Option { + None +} + +#[allow(dead_code)] +fn is_watcher_command(command: &str) -> bool { + let mut words = command.split_whitespace(); + words + .next() + .is_some_and(|exe| exe.ends_with("railway") || exe.contains("railway")) + && command.contains(" ca herdr watch") +} + +/// Stop this session's watcher, if one of ours is running. +pub fn stop() -> Option { + if !cfg!(unix) { + return None; + } + let pidfile = pidfile_path().ok()?; + let pid = running_pid(&pidfile)?; + let _ = Command::new("kill") + .args(["-TERM", &pid.to_string()]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + let _ = std::fs::remove_file(&pidfile); + Some(pid) +} + +/// Tell this session's watcher that the set of environments may have changed. +pub fn nudge() { + if !cfg!(unix) { + return; + } + let Ok(pidfile) = pidfile_path() else { return }; + if let Some(pid) = running_pid(&pidfile) { + let _ = Command::new("kill") + .args(["-USR1", &pid.to_string()]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + } +} + +/// The watcher's pid for this session, if one is running. +pub fn running() -> Option { + pidfile_path().ok().and_then(|p| running_pid(&p)) +} + +/// Start this session's watcher in the background unless one is up. Quiet on +/// every failure: without a watcher the plugin degrades to the manual sync key. +pub fn spawn_detached() { + if !cfg!(unix) { + return; + } + let Ok(pidfile) = pidfile_path() else { return }; + if running_pid(&pidfile).is_some() || std::env::var_os("HERDR_SOCKET_PATH").is_none() { + return; + } + let Ok(exe) = std::env::current_exe() else { + return; + }; + let Ok(out) = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(pidfile.with_extension("log")) + else { + return; + }; + let Ok(err) = out.try_clone() else { return }; + let mut cmd = Command::new(exe); + cmd.args(["ca", "herdr", "watch", "--foreground"]) + .stdin(Stdio::null()) + .stdout(out) + .stderr(err); + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + cmd.process_group(0); + } + let _ = cmd.spawn(); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn reconnect_refetches_even_when_no_invalidation_is_replayed() { + let (tx, mut rx) = mpsc::channel(4); + let task = tokio::spawn(subscribe_with("env".into(), tx, false, |_| async { + // Each successful subscription closes without replaying any event. + Ok(futures::stream::empty()) + })); + let result = tokio::time::timeout(RESUBSCRIBE_MIN + Duration::from_secs(2), async { + for _ in 0..2 { + let Some(Signal::Changed(cause)) = rx.recv().await else { + panic!("watcher exited") + }; + assert!(cause.contains("refetching"), "{cause}"); + } + }) + .await; + task.abort(); + result.expect("both the initial subscription and reconnect must refetch"); + } + + #[test] + fn the_query_is_the_one_the_apps_send() { + let body = CloudAgentInvalidation::build_query(Variables { + environment_id: "env-1".into(), + }); + assert_eq!(body.operation_name, "CloudAgentInvalidation"); + assert!( + body.query + .contains("cloudAgentInvalidation(environmentId: $environmentId)") + ); + assert_eq!( + serde_json::to_value(&body.variables).unwrap(), + serde_json::json!({ "environmentId": "env-1" }) + ); + let data: ResponseData = serde_json::from_value(serde_json::json!({ + "cloudAgentInvalidation": { "id": "x", "agent": { "id": "a1", "status": "SLEEPING" } } + })) + .unwrap(); + assert_eq!( + describe("e", data.cloud_agent_invalidation.agent), + "agent a1 is now sleeping" + ); + } + + #[test] + fn only_a_live_watcher_process_counts_as_running() { + let dir = tempfile::tempdir().unwrap(); + let pidfile = dir.path().join("watch.pid"); + std::fs::write(&pidfile, "999999").unwrap(); + assert_eq!(running_pid(&pidfile), None); + // This test binary is alive but is not `railway ca herdr watch`: + // a reused pid must never be mistaken for our watcher. + std::fs::write(&pidfile, std::process::id().to_string()).unwrap(); + assert_eq!(running_pid(&pidfile), None); + assert!(is_watcher_command( + "/opt/homebrew/bin/railway ca herdr watch --foreground" + )); + assert!(!is_watcher_command("/usr/bin/sleep 30")); + assert!(!is_watcher_command("railway ca herdr sync")); + } +} diff --git a/src/commands/cloud_agent/lifecycle.rs b/src/commands/cloud_agent/lifecycle.rs index bed0d6e3f..4a42557ab 100644 --- a/src/commands/cloud_agent/lifecycle.rs +++ b/src/commands/cloud_agent/lifecycle.rs @@ -1085,7 +1085,10 @@ fn describe_sessions(sessions: &[ca::ConsoleSession]) -> String { /// /// Best-effort: the workspace listing is a second request, and a list that /// prints ids because it failed is better than a list that errors. -async fn place_names(client: &reqwest::Client, configs: &Configs) -> HashMap { +pub(crate) async fn place_names( + client: &reqwest::Client, + configs: &Configs, +) -> HashMap { let mut names = HashMap::new(); let Ok(workspaces) = crate::workspace::workspaces_with_client(client, configs).await else { return names; diff --git a/src/commands/cloud_agent/mod.rs b/src/commands/cloud_agent/mod.rs index 38477e023..2d4e6c3fc 100644 --- a/src/commands/cloud_agent/mod.rs +++ b/src/commands/cloud_agent/mod.rs @@ -13,6 +13,7 @@ pub mod bootstrap; pub(crate) mod client_sessions; pub(crate) mod codex; pub mod desktop; +mod herdr; pub mod lifecycle; pub mod mcp_sync; pub(crate) mod opencode; @@ -123,6 +124,9 @@ Guide: https://github.com/railwayapp/cli/blob/master/docs/cloud-agents.md"#)] /// Delete an agent and everything on its disk #[clap(visible_alias = "rm")] Delete(lifecycle::DeleteArgs), + + /// Show cloud agents in herdr as machines (herdr 0.9+) + Herdr(herdr::Args), } /// Shared launch arguments have different help in CA's in-VM execution path. @@ -175,6 +179,8 @@ pub async fn command(args: Args) -> Result<()> { Some(Command::Wake(a)) => tracked("wake", lifecycle::wake(a)).await, Some(Command::Sleep(a)) => tracked("sleep", lifecycle::sleep(a)).await, Some(Command::Delete(a)) => tracked("delete", lifecycle::delete(a)).await, + // Untracked: herdr runs these as hooks many times an hour. + Some(Command::Herdr(a)) => herdr::command(a).await, None if args.launch.is_bare() && is_stdout_terminal() => browse().await, // Flags given, or no terminal to draw on: behave like `railway code`, // which means the pane on a terminal and a plain ssh session off one. diff --git a/src/commands/code.rs b/src/commands/code.rs index 6bdeed247..adb137e64 100644 --- a/src/commands/code.rs +++ b/src/commands/code.rs @@ -916,7 +916,7 @@ cat > ~/.grok/auth.json"#; /// Deliberately not `. ~/.profile`: that sources `.bashrc`, whose starship/mise/ /// zoxide init writes to stdout and would corrupt the AGENT-READY marker this /// command parses. Mirrors the image's own export line instead. -const HARNESS_PATH: &str = r#"export PATH="$HOME/.opencode/bin:$HOME/.local/bin:$HOME/.grok/bin:$HOME/.local/share/mise/shims:$PATH""#; +pub(crate) const HARNESS_PATH: &str = r#"export PATH="$HOME/.opencode/bin:$HOME/.local/bin:$HOME/.grok/bin:$HOME/.local/share/mise/shims:$PATH""#; /// Saved OpenCode conversations are resumed by the VM's `opencode`, which must /// be V2. A resume goes straight to the binary rather than through the runtime diff --git a/src/commands/ssh/config.rs b/src/commands/ssh/config.rs index 7b91548fe..a9c390ad6 100644 --- a/src/commands/ssh/config.rs +++ b/src/commands/ssh/config.rs @@ -734,7 +734,7 @@ fn sanitize_alias(input: &str) -> String { } } -fn quote_ssh_config_value(value: &str) -> String { +pub(crate) fn quote_ssh_config_value(value: &str) -> String { if !value .chars() .any(|c| c.is_whitespace() || matches!(c, '"' | '\\')) diff --git a/src/main.rs b/src/main.rs index 9a565dd37..0f7f5f915 100644 --- a/src/main.rs +++ b/src/main.rs @@ -938,6 +938,21 @@ mod cli_tests { assert_parses(&["ca", "setup", "--show"]); assert_parses(&["ca", "start", "--claude"]); assert_parses(&["ca", "start", "--codex", "--new"]); + for verb in ["install", "new", "agents", "sync"] { + assert_parses(&["ca", "herdr", verb]); + } + assert_parses(&["ca", "herdr", "bootstrap"]); + assert_parses(&["ca", "herdr", "install", "--print"]); + assert_parses(&["ca", "herdr", "install", "--remove"]); + assert_parses(&["ca", "herdr", "new", "--dry-run"]); + assert_parses(&["ca", "herdr", "new", "--codex", "-p", "proj"]); + assert_parses(&["ca", "herdr", "agents", "--remote"]); + assert_parses(&["ca", "herdr", "agents", "--wake", "--open"]); + assert_parses(&["ca", "herdr", "sync", "--debounce", "30"]); + assert_parses(&["ca", "herdr", "sync", "--spawn-watch"]); + assert_parses(&["ca", "herdr", "watch", "--foreground"]); + assert_parses(&["ca", "herdr", "bootstrap", "my-box", "--claude"]); + assert_parses(&["ca", "herdr", "sync", "--dry-run", "--json"]); } /// `railway ca` browses and `railway code` launches, but every launch diff --git a/src/subscription.rs b/src/subscription.rs index 6942e4ba2..c54f0764e 100644 --- a/src/subscription.rs +++ b/src/subscription.rs @@ -14,7 +14,19 @@ where ::Variables: Send + Sync + Unpin, ::ResponseData: std::fmt::Debug, { - Ok(Client::build(connect_websocket().await?) + Ok(Client::build(connect_websocket("/graphql/v2").await?) + .subscribe(StreamingOperation::::new(variables)) + .await?) +} + +pub async fn subscribe_graphql_internal( + variables: T::Variables, +) -> Result>> +where + ::Variables: Send + Sync + Unpin, + ::ResponseData: std::fmt::Debug, +{ + Ok(Client::build(connect_websocket("/graphql/internal").await?) .subscribe(StreamingOperation::::new(variables)) .await?) } @@ -22,10 +34,10 @@ where /// Open one connection for multiple operations. The caller must drive the /// actor for as long as it needs the subscriptions. pub async fn connect_graphql() -> Result<(Client, ConnectionActor)> { - Ok(Client::build(connect_websocket().await?).await?) + Ok(Client::build(connect_websocket("/graphql/v2").await?).await?) } -async fn connect_websocket() -> Result { +async fn connect_websocket(path: &str) -> Result { let configs = Configs::new()?; let hostname = configs.get_host(); let client = reqwest::Client::default(); @@ -34,7 +46,7 @@ async fn connect_websocket() -> Result { // Railway VM mid-build — 1s is routinely missed even when the network // is fine, and every retry misses it the same way. let mut request = client - .get(format!("wss://backboard.{hostname}/graphql/v2")) + .get(format!("wss://backboard.{hostname}{path}")) .timeout(Duration::from_secs(10)); if let Some(token) = &Configs::get_railway_token() { diff --git a/tests/fakes/herdr b/tests/fakes/herdr new file mode 100755 index 000000000..d055710b2 --- /dev/null +++ b/tests/fakes/herdr @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +"""Fake herdr for tests: canned `machine list --json`, every argv logged.""" +import json, os, sys + +args = sys.argv[1:] +log = os.environ.get("FAKE_HERDR_LOG") +if log: + with open(log, "a") as f: + f.write(" ".join(args) + "\n") + +if args[:3] == ["machine", "list", "--json"]: + path = os.environ.get("FAKE_HERDR_MACHINES") + print(open(path).read() if path and os.path.exists(path) else "[]") +elif args[:2] == ["machine", "add"]: + print(json.dumps({"id": "f" * 32, "label": args[args.index("--label") + 1], "target": args[2]})) +elif args[:1] in (["machine"], ["plugin"], ["workspace"], ["integration"]): + pass +else: + sys.stderr.write(f"fake herdr: unsupported {args}\n") + sys.exit(2)