feat(cache): cache tasks that rewrite files they read

CHANGELOG.md

@@ -1,5 +1,6 @@
 # Changelog
 
+- **Changed** Tasks that write a file they also read are now cached, instead of being reported as `not cached because it modified its inputs`. For example, `tsc --incremental` reads and rewrites `tsconfig.tsbuildinfo`: the next run is a cache hit if the file is as `tsc` left it, or if it is missing, as on a fresh clone, in which case the hit restores it ([#913](https://github.com/voidzero-dev/vite-task/pull/913)).
 - **Fixed** A cache hit restores the symbolic links a task's `output` globs match as links, instead of dropping them, so a replayed output that links a directory, such as a bundler's `node_modules/<package>` link, is complete ([#885](https://github.com/voidzero-dev/vite-task/pull/885)).
 - **Fixed** On Linux and macOS, programs using automatic file-access tracking no longer abort when they call `execve`, `execv`, `execl`, `execle`, or `posix_spawn` with an empty path; the call fails with `ENOENT`, as it does without tracking. On Linux, `execveat` with an empty path and without `AT_EMPTY_PATH` now also fails with `ENOENT`, instead of trying to run what its directory descriptor refers to ([#896](https://github.com/voidzero-dev/vite-task/pull/896)).
 - **Perf** `vp run` starts faster in workspaces whose tasks share cache `input` and `output` patterns: each distinct pattern is now parsed once instead of once for every task that uses it. In a 138-package workspace, this saves about 2 seconds of CPU on every run ([#887](https://github.com/voidzero-dev/vite-task/pull/887), [#890](https://github.com/voidzero-dev/vite-task/pull/890)).

crates/vt/src/session/cache/mod.rs

@@ -276,7 +276,7 @@ pub fn split_path(path: &str) -> (Option<&str>, &str) {
 /// its own cache warm across branch switches, and a cache from a different
 /// version is simply ignored (it lives in a directory this build never looks
 /// at) rather than aborting the run. Bumping the version starts a fresh cache.
-const CACHE_SCHEMA_VERSION: u32 = 19;
+const CACHE_SCHEMA_VERSION: u32 = 20;
 
 /// Name of the per-version subdirectory (e.g. `v14`) under the task-cache
 /// directory that holds the database and output archives for the current

crates/vt/src/session/event.rs

@@ -4,7 +4,6 @@ use std::{
     time::Duration,
 };
 
-use vt_path::RelativePathBuf;
 use vt_server::Error as IpcServerError;
 use vt_str::Str;
 
@@ -100,12 +99,6 @@ pub enum CacheNotUpdatedReason {
     /// - Fast-fail: a sibling task failed, triggering cancellation
     ///   while this task was still running.
     Cancelled,
-    /// Task modified files it read during execution (read-write overlap detected by fspy).
-    /// Caching such tasks is unsound because the prerun input hashes become stale.
-    InputModified {
-        /// First path that was both read and written during execution.
-        path: RelativePathBuf,
-    },
     /// fspy isn't compiled in on this build and the task requires fspy
     /// (its `input` config includes auto-inference). Task ran but cannot
     /// be cached without tracked path accesses.

crates/vt/src/session/execute/cache_update.rs

@@ -14,6 +14,7 @@ use super::{
     CacheState,
     fingerprint::{PathRead, PostRunFingerprint, TrackedEnvQuery},
     glob,
+    rebuildable::Rebuildable,
     spawn::ChildOutcome,
 };
 use crate::{
@@ -32,9 +33,9 @@ struct TrackingOutcome {
     /// Auto-output writes after output exclusions are applied. Empty when
     /// `output_config.includes_auto` is false.
     path_writes: FxHashSet<RelativePathBuf>,
-    /// First path that was both read and written during execution, if any.
-    /// A non-empty value means caching this task is unsound.
-    read_write_overlap: Option<RelativePathBuf>,
+    /// Every write fspy recorded, before output exclusions and `ignoreOutput`
+    /// paths are applied.
+    rebuildable: Rebuildable,
 }
 
 type TrackedEnvValues = BTreeMap<Str, Option<EnvValueHash>>;
@@ -110,20 +111,6 @@ pub(super) async fn update_cache(
         workspace_root,
     );
 
-    if let Some(TrackingOutcome { read_write_overlap: Some(path), .. }) = &fspy_outcome {
-        // fspy-inferred read-write overlap: the task wrote to a file it also
-        // read, so the prerun input hashes are stale and caching is unsound.
-        // (We only check fspy-inferred reads, not globbed_inputs. A task that
-        // writes to a glob-matched file without reading it produces perpetual
-        // cache misses but not a correctness bug.)
-        return (
-            CacheUpdateStatus::NotUpdated(CacheNotUpdatedReason::InputModified {
-                path: path.clone(),
-            }),
-            None,
-        );
-    }
-
     if fspy_outcome.is_none() && fspy.is_some() {
         // Task requested fspy auto-inference but this binary was built without
         // `cfg(fspy)`. Task ran, but we can't compute a valid cache entry
@@ -144,13 +131,16 @@ pub(super) async fn update_cache(
         }
     };
 
-    // Paths already in globbed_inputs are skipped: the overlap check above
-    // guarantees no input modification, so the prerun hash is the correct
-    // post-exec hash.
+    // Paths already in globbed_inputs are skipped: they are hashed before
+    // the run, so one that the task rewrote makes the next run miss instead
+    // of hitting a stale entry.
     let empty_path_reads = HashMap::default();
     let path_reads = fspy_outcome.as_ref().map_or(&empty_path_reads, |o| &o.path_reads);
+    let no_writes = Rebuildable::default();
+    let rebuildable = fspy_outcome.as_ref().map_or(&no_writes, |o| &o.rebuildable);
     let post_run_fingerprint = match PostRunFingerprint::create(
         path_reads,
+        rebuildable,
         workspace_root,
         &globbed_inputs,
         tracked_envs,
@@ -204,7 +194,9 @@ pub(super) async fn update_cache(
 /// `path_reads` is gated on `input_config.includes_auto`, filtered by
 /// user-configured input negatives, and by tool-reported `ignoreInput` paths.
 /// `path_writes` is filtered by user-configured output negatives and
-/// tool-reported `ignoreOutput` paths before read-write overlap detection.
+/// tool-reported `ignoreOutput` paths, which decide what is archived.
+/// `rebuildable` takes every write unfiltered, since what the task can
+/// rebuild doesn't depend on what is archived.
 #[cfg_attr(
     not(fspy),
     expect(clippy::missing_const_for_fn, reason = "the body is a no-op without fspy")
@@ -243,7 +235,7 @@ fn observe_fspy(
                 };
             let filtered_path_writes: FxHashSet<RelativePathBuf> =
                 // fspy can also be attached for auto-input-only tasks. In that
-                // mode writes must not become auto outputs or overlap candidates.
+                // mode writes must not become auto outputs.
                 if metadata.output_config.includes_auto
                     && let Some(fspy) = fspy
                 {
@@ -259,12 +251,10 @@ fn observe_fspy(
                 } else {
                     FxHashSet::default()
                 };
-            let read_write_overlap =
-                filtered_path_reads.keys().find(|p| filtered_path_writes.contains(*p)).cloned();
             TrackingOutcome {
                 path_reads: filtered_path_reads,
                 path_writes: filtered_path_writes,
-                read_write_overlap,
+                rebuildable: Rebuildable::new(tracked.path_writes),
             }
         })
     }

crates/vt/src/session/execute/fingerprint.rs

@@ -20,6 +20,7 @@ use vt_plan::cache_metadata::EnvValueHash;
 use vt_str::Str;
 use wincode::{SchemaRead, SchemaWrite};
 
+use super::rebuildable::Rebuildable;
 use crate::{
     collections::HashMap,
     session::cache::{EnvMismatch, InputChangeKind},
@@ -43,9 +44,9 @@ pub struct PathRead {
 /// Used to validate whether cached outputs are still valid.
 #[derive(SchemaWrite, SchemaRead, Debug, Default, Serialize)]
 pub struct PostRunFingerprint {
-    /// Paths inferred from fspy during execution with their content fingerprints.
-    /// Only populated when `input_config.includes_auto` is true.
-    pub inferred_inputs: HashMap<RelativePathBuf, PathFingerprint>,
+    /// Paths inferred from fspy during execution, fingerprinted as the task
+    /// left them. Only populated when `input_config.includes_auto` is true.
+    pub inferred_inputs: HashMap<RelativePathBuf, InferredInput>,
 
     /// Env vars observed via runner-aware IPC `getEnv` with `tracked: true`.
     /// Key is the env name; value is the env value hash at execution time, or
@@ -79,6 +80,15 @@ pub enum PostRunMismatch {
     TrackedEnvQuery { query: TrackedEnvQuery, mismatch: EnvMismatch },
 }
 
+/// A path the task read, as the task left it.
+#[derive(SchemaWrite, SchemaRead, PartialEq, Eq, Debug, Clone, Serialize, Deserialize)]
+pub struct InferredInput {
+    pub fingerprint: PathFingerprint,
+    /// The task wrote the path, so a later run may find it missing, as on a
+    /// clean checkout. A path that is there must still match `fingerprint`.
+    pub may_be_missing: bool,
+}
+
 /// Fingerprint for a single path (file or directory)
 #[derive(SchemaWrite, SchemaRead, PartialEq, Eq, Debug, Clone, Serialize, Deserialize)]
 pub enum PathFingerprint {
@@ -107,19 +117,20 @@ impl PostRunFingerprint {
     /// Negative glob filtering is done upstream (see
     /// [`super::tracked_accesses::TrackedPathAccesses::from_raw`]).
     /// Paths already present in `globbed_inputs` are skipped — they are
-    /// already tracked by the prerun glob fingerprint, and the read-write
-    /// overlap check in `execute_spawn` guarantees the task did not modify
-    /// them, so the prerun hash is still correct.
+    /// already tracked by the prerun glob fingerprint. Every other path is
+    /// fingerprinted as the task left it, including a file the task rewrote.
     ///
     /// # Arguments
     /// * `inferred_path_reads` - Map of paths that were read during execution (from fspy)
+    /// * `rebuildable` - What the task can rebuild, so a later run may find it missing
     /// * `base_dir` - Workspace root for resolving relative paths
     /// * `globbed_inputs` - Prerun glob fingerprint; paths here are skipped
     /// * `tracked_envs` - Tool-requested env vars (name -> value hash), validated on lookup
     /// * `tracked_env_queries` - Tool-requested bulk env queries (query -> match-set hashes)
     #[tracing::instrument(level = "debug", skip_all, name = "create_post_run_fingerprint")]
     pub fn create(
         inferred_path_reads: &HashMap<RelativePathBuf, PathRead>,
+        rebuildable: &Rebuildable,
         base_dir: &AbsolutePath,
         globbed_inputs: &BTreeMap<RelativePathBuf, u64>,
         tracked_envs: BTreeMap<Str, Option<EnvValueHash>>,
@@ -131,7 +142,8 @@ impl PostRunFingerprint {
             .map(|(relative_path, path_read)| {
                 let full_path = Arc::<AbsolutePath>::from(base_dir.join(relative_path));
                 let fingerprint = fingerprint_path(&full_path, *path_read)?;
-                Ok((relative_path.clone(), fingerprint))
+                let may_be_missing = rebuildable.may_be_missing(relative_path);
+                Ok((relative_path.clone(), InferredInput { fingerprint, may_be_missing }))
             })
             .collect::<anyhow::Result<HashMap<_, _>>>()?;
 
@@ -153,8 +165,9 @@ impl PostRunFingerprint {
         base_dir: &AbsolutePath,
         unfiltered_envs: &FxHashMap<EnvName<Arc<OsStr>>, Arc<OsStr>>,
     ) -> anyhow::Result<Option<PostRunMismatch>> {
-        let input_mismatch = self.inferred_inputs.par_iter().find_map_any(
-            |(input_relative_path, path_fingerprint)| {
+        let input_mismatch =
+            self.inferred_inputs.par_iter().find_map_any(|(input_relative_path, input)| {
+                let InferredInput { fingerprint: path_fingerprint, may_be_missing } = input;
                 let input_full_path = Arc::<AbsolutePath>::from(base_dir.join(input_relative_path));
                 let path_read = PathRead {
                     read_dir_entries: matches!(path_fingerprint, PathFingerprint::Folder(Some(_))),
@@ -163,7 +176,9 @@ impl PostRunFingerprint {
                     Ok(ok) => ok,
                     Err(err) => return Some(Err(err)),
                 };
-                if path_fingerprint == &current_path_fingerprint {
+                if path_fingerprint == &current_path_fingerprint
+                    || (*may_be_missing && current_path_fingerprint == PathFingerprint::NotFound)
+                {
                     None
                 } else {
                     let (kind, entry_name) =
@@ -180,8 +195,7 @@ impl PostRunFingerprint {
                     };
                     Some(Ok(PostRunMismatch::Input { kind, path }))
                 }
-            },
-        );
+            });
         if let Some(result) = input_mismatch {
             return result.map(Some);
         }
@@ -500,6 +514,9 @@ fn process_directory_unix(file: &File, path_read: PathRead) -> anyhow::Result<Pa
 mod tests {
     use std::ffi::{OsStr, OsString};
 
+    use rustc_hash::FxHashSet;
+    use vt_path::AbsolutePathBuf;
+
     use super::*;
 
     #[cfg(unix)]
@@ -651,4 +668,44 @@ mod tests {
 
         assert_eq!(mismatch.is_none(), cfg!(windows));
     }
+
+    #[test]
+    fn validate_allows_a_written_input_to_be_missing_but_not_changed() {
+        let tmp = tempfile::TempDir::new().unwrap();
+        let workspace_root = AbsolutePathBuf::new(tmp.path().to_path_buf()).unwrap();
+        let path = RelativePathBuf::new("state.txt").unwrap();
+        let full_path = workspace_root.join(&path);
+        std::fs::write(full_path.as_path(), "v1").unwrap();
+
+        let path_reads = HashMap::from([(path.clone(), PathRead { read_dir_entries: false })]);
+        let create = |written| {
+            PostRunFingerprint::create(
+                &path_reads,
+                &Rebuildable::new(written),
+                &workspace_root,
+                &BTreeMap::new(),
+                BTreeMap::new(),
+                BTreeMap::new(),
+            )
+            .unwrap()
+        };
+        let written = create(FxHashSet::from_iter([path]));
+        let read_only = create(FxHashSet::default());
+        let validate = |fingerprint: &PostRunFingerprint| {
+            fingerprint.validate(&workspace_root, &FxHashMap::default()).unwrap()
+        };
+
+        std::fs::remove_file(full_path.as_path()).unwrap();
+        assert!(validate(&written).is_none());
+        assert!(matches!(
+            validate(&read_only),
+            Some(PostRunMismatch::Input { kind: InputChangeKind::Removed, .. })
+        ));
+
+        std::fs::write(full_path.as_path(), "garbage").unwrap();
+        assert!(matches!(
+            validate(&written),
+            Some(PostRunMismatch::Input { kind: InputChangeKind::ContentModified, .. })
+        ));
+    }
 }

crates/vt/src/session/execute/mod.rs

@@ -3,6 +3,7 @@ pub mod fingerprint;
 pub mod glob;
 mod hash;
 pub mod pipe;
+mod rebuildable;
 mod redact;
 mod scheduler;
 pub mod spawn;

crates/vt/src/session/execute/rebuildable.rs

@@ -0,0 +1,31 @@
+//! What a task can rebuild, so a later run may find it missing.
+//!
+//! Inputs are fingerprinted as the task left them, so they include the files
+//! a task read and then wrote, such as a tool's cache file. A clean checkout
+//! (CI, a fresh clone) doesn't have those files yet, and running the task
+//! would write them again, so a later run may find them missing. A file that
+//! is there must still be as the task left it.
+//!
+//! The written paths are fspy's records before output exclusions and
+//! tool-reported `ignoreOutput` paths are applied: those decide what is
+//! archived, not what the task can rebuild.
+
+use rustc_hash::FxHashSet;
+use vt_path::{RelativePath, RelativePathBuf};
+
+/// The paths a task wrote, relative to the workspace root.
+#[derive(Debug, Default)]
+pub struct Rebuildable {
+    written: FxHashSet<RelativePathBuf>,
+}
+
+impl Rebuildable {
+    pub const fn new(written: FxHashSet<RelativePathBuf>) -> Self {
+        Self { written }
+    }
+
+    /// Whether a later run may find `path` missing: the task wrote it.
+    pub fn may_be_missing(&self, path: &RelativePath) -> bool {
+        self.written.contains(path)
+    }
+}