diff --git a/.github/workflows/paper.yml b/.github/workflows/paper.yml new file mode 100644 index 0000000..4204af6 --- /dev/null +++ b/.github/workflows/paper.yml @@ -0,0 +1,102 @@ +name: paper + +# Checks for the manuscript branch: the generated figures still reproduce +# their committed sidecars, the survey bibliography regenerates identically, +# and the PDF builds without unresolved references or the artefacts the +# iteration-2 review caught. The pre-submission reviewer itself stays on +# demand (review/REVISION_PLAN_ITER3.md, P3). + +on: + push: + branches: [docs/sign-convention-manuscript] + pull_request: + branches: [docs/sign-convention-manuscript] + workflow_dispatch: + +concurrency: + group: paper-${{ github.ref }} + cancel-in-progress: true + +jobs: + sidecars: + name: figure sidecars reproduce + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + OMP_NUM_THREADS: "1" + OPENBLAS_NUM_THREADS: "1" + MPLBACKEND: Agg + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + - name: install + run: | + python -m pip install --upgrade pip + pip install -e ".[test]" + - name: regenerate the fast figures in memory and diff against the sidecars + run: python -m codameter.figures --check --skip-slow --rtol 1e-5 + + survey: + name: survey bibliography regenerates identically + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: rebuild survey.bib and appendix_table.tex + run: python paper/build_survey.py + - name: no drift from the committed files + run: git diff --exit-code -- paper/survey.bib paper/appendix_table.tex + + pdf: + name: PDF builds and passes the text checks + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - uses: quarto-dev/quarto-actions/setup@v2 + with: + tinytex: true + - name: TeX packages the rendered manuscript loads + # quarto's TinyTeX installs missing packages on demand during the render; + # this pre-installs the ones the gji class, natbib and the tables need so + # the render does not stop on the first missing .sty. + run: | + tlmgr install latexmk natbib tabularx booktabs longtable caption \ + lineno xcolor amsmath amsfonts siunitx float fontspec unicode-math \ + lm-math upquote microtype etoolbox parskip footnote setspace \ + iftex textcomp lmodern calc || true + - name: install codameter (for the version string) + run: | + python -m pip install --upgrade pip + pip install -e . + - name: build + run: python paper/build.py --no-survey + - name: text checks + run: | + sudo apt-get update -qq && sudo apt-get install -y -qq poppler-utils + pdftotext -layout paper/manuscript_marine.pdf /tmp/paper.txt + n_unresolved=$(grep -c '??' /tmp/paper.txt || true) + n_commapct=$(grep -c ',%' /tmp/paper.txt || true) + n_lt=$(grep -c '<<' /tmp/paper.txt || true) + n_tilde=$(grep -cE '(Table|Fig\.|Section|eq\.)~' /tmp/paper.txt || true) + echo "pages: $(pdfinfo paper/manuscript_marine.pdf | awk '/Pages/{print $2}')" + echo "unresolved: $n_unresolved commapct: $n_commapct lt: $n_lt tilde: $n_tilde" + test "$n_unresolved" -eq 0 && test "$n_commapct" -eq 0 && test "$n_lt" -eq 0 && test "$n_tilde" -eq 0 + - name: upload the PDF + if: always() + uses: actions/upload-artifact@v4 + with: + name: manuscript-pdf + path: | + paper/manuscript_marine.pdf + paper/manuscript_marine.tex + if-no-files-found: ignore diff --git a/literature/figs/SOURCES.md b/literature/figs/SOURCES.md index 501f360..8445b51 100644 --- a/literature/figs/SOURCES.md +++ b/literature/figs/SOURCES.md @@ -18,14 +18,29 @@ prints the registry. `paper/build.py --figures` runs the driver. | `demo_11_multiverse` | `codameter.deviations.multiverse` + `fig_multiverse_full` (slow) | | `demo_12_bayes` | `codameter.uq_bayes._build_bayes` + `_fig_bayes` (slow) | +| `realdata_1_validation` | `codameter.gate1.fig_gate1_comparison`: needs the untracked daily products under `paper/data/gate1/dvv2y/` (skipped with a message where they are absent); every plotted array is in the sidecar | + +Every `.json` sidecar records `git_commit`; sidecars written since +2026-09-13 also record `git_dirty` (true when tracked files under `src/` +differed from that commit when the figure was made) and `generator_digest` +(a digest of the package version and the figure-generating modules, so two +sidecars with the same digest came from the same figure code). Older sidecars +gain the two fields when their figure is next regenerated; the release step +regenerates every figure from a clean tree at the tag. +`python -m codameter.figures --check [--skip-slow]` regenerates the figures in +memory and reports any array that differs from the committed sidecar; the +`paper` workflow runs it on every pull request to the manuscript branch. + ## Produced outside this repository -`realdata_1_validation.png`, `realdata_2_interferograms.png` and -`realdata_3_warmup.png` come from the noisepy-dvv-cloud Gate 1 run -(CI.LJR / CI.RXH / CI.ARV, 2018-2019; see `paper/data/gate1/README.md`). -Their inputs are the daily ensemble products under `paper/data/gate1/dvv2y/` -(not tracked by git) and the published Clements and Denolle (2022) product -under `paper/data/gate1/legacy_cd2022/`. The comparison script -(`scripts/compare_cd2022.py`) and the figure scripts live in that repository; -the commit they were run at is to be pinned here (audit finding REP-02). -Until then these three figures cannot be regenerated from this checkout. +`realdata_2_interferograms.png` and `realdata_3_warmup.png` come from the +noisepy-dvv-cloud Gate 1 run (CI.LJR / CI.RXH / CI.ARV, 2018-2019; see +`paper/data/gate1/README.md`). Their inputs are the daily correlations of that +run, which are not archived here, so they are committed as produced. The +comparison figure (`realdata_1_validation`) used to be produced there as well; +it is now generated in this repository from the archived daily products and +`paper/data/gate1/comparison.json`. The Gate 1 run commit and `--use-case` +are to be pinned in `paper/data/gate1/README.md` (issue #46). +Until then the two interferogram-based figures cannot be regenerated from +this checkout; the comparison figure can, wherever the daily products are +present. diff --git a/literature/figs/realdata_1_validation.json b/literature/figs/realdata_1_validation.json new file mode 100644 index 0000000..0b1f670 --- /dev/null +++ b/literature/figs/realdata_1_validation.json @@ -0,0 +1,256 @@ +{ + "figure": "realdata_1_validation", + "generator": "codameter.gate1.fig_gate1_comparison (needs paper/data/gate1/dvv2y)", + "codameter_version": "0.4.0", + "git_commit": "319e57ac8e6c34e1833b5342d7a7e7d2b214c957", + "git_dirty": false, + "generator_digest": "7d5cd15e6b62", + "generated_utc": "2026-09-13T23:22:31+00:00", + "python": "3.12.13", + "numpy": "2.4.3", + "matplotlib": "3.10.9", + "axes": [ + { + "axes": "ax0", + "title": "Gate 1, 2.0-4.0 Hz, 2018-2019: single-station ensemble against the published product (demeaned over the overlap)", + "xlabel": "", + "ylabel": "CI.LJR\ndv/v (%)", + "lines": [ + "daily dv/v", + "codameter, trailing 90-day mean", + "Clements and Denolle (2022)", + "_child4" + ], + "images": 0, + "collections": [ + "daily $\\pm1.96\\,\\sigma$ (corrected floor)" + ], + "patches": 0 + }, + { + "axes": "ax1", + "title": "", + "xlabel": "", + "ylabel": "CI.ARV\ndv/v (%)", + "lines": [ + "daily dv/v", + "codameter, trailing 90-day mean", + "Clements and Denolle (2022)", + "_child4" + ], + "images": 0, + "collections": [ + "daily $\\pm1.96\\,\\sigma$ (corrected floor)" + ], + "patches": 0 + }, + { + "axes": "ax2", + "title": "", + "xlabel": "date", + "ylabel": "CI.RXH\ndv/v (%)", + "lines": [ + "daily dv/v", + "codameter, trailing 90-day mean", + "Clements and Denolle (2022)", + "_child4" + ], + "images": 0, + "collections": [ + "daily $\\pm1.96\\,\\sigma$ (corrected floor)" + ], + "patches": 0 + } + ], + "arrays": [ + "ax0/collection0/offsets", + "ax0/collection0/path_lengths", + "ax0/collection0/vertices", + "ax0/line0/x", + "ax0/line0/y", + "ax0/line1/x", + "ax0/line1/y", + "ax0/line2/x", + "ax0/line2/y", + "ax0/line3/x", + "ax0/line3/y", + "ax1/collection0/offsets", + "ax1/collection0/path_lengths", + "ax1/collection0/vertices", + "ax1/line0/x", + "ax1/line0/y", + "ax1/line1/x", + "ax1/line1/y", + "ax1/line2/x", + "ax1/line2/y", + "ax1/line3/x", + "ax1/line3/y", + "ax2/collection0/offsets", + "ax2/collection0/path_lengths", + "ax2/collection0/vertices", + "ax2/line0/x", + "ax2/line0/y", + "ax2/line1/x", + "ax2/line1/y", + "ax2/line2/x", + "ax2/line2/y", + "ax2/line3/x", + "ax2/line3/y", + "data/arv/daily_dvv_pct", + "data/arv/daily_err_pct", + "data/arv/days", + "data/arv/published_days", + "data/arv/published_dvv_pct", + "data/arv/trailing_days", + "data/arv/trailing_dvv_pct", + "data/ljr/daily_dvv_pct", + "data/ljr/daily_err_pct", + "data/ljr/days", + "data/ljr/published_days", + "data/ljr/published_dvv_pct", + "data/ljr/trailing_days", + "data/ljr/trailing_dvv_pct", + "data/rxh/daily_dvv_pct", + "data/rxh/daily_err_pct", + "data/rxh/days", + "data/rxh/published_days", + "data/rxh/published_dvv_pct", + "data/rxh/trailing_days", + "data/rxh/trailing_dvv_pct" + ], + "gate1": { + "rules": { + "band_hz": "2.0-4.0", + "burn_in_days": 150, + "trailing_days": 90, + "trailing_min_finite": 45, + "centered_days": 45, + "centered_min_finite": 23, + "join": "inner join on calendar date; published product restricted to 2018-2019", + "r": "Pearson, on the overlap, after removing each series' overlap mean", + "slope": "OLS slope of the published product on the codameter series" + }, + "stations": { + "CI.LJR": { + "station": "CI.LJR", + "daily_rows": 681, + "daily_first": "2018-01-01", + "daily_last": "2019-12-30", + "legacy_rows_2018_2019": 729, + "burn_in_until": "2018-05-31", + "matched": { + "n": 579, + "first": "2018-05-31", + "last": "2019-12-30", + "r": 0.9850624256185397, + "rms_diff": 0.04616500786170553, + "slope": 1.0691502357580198 + }, + "matched_no_burn_in": { + "n": 639, + "first": "2018-04-01", + "last": "2019-12-30", + "r": 0.9842861169792171, + "rms_diff": 0.04475023506362687, + "slope": 1.06604616277162 + }, + "centered": { + "n": 682, + "first": "2018-02-16", + "last": "2019-12-30", + "r": 0.8653456948495024, + "rms_diff": 0.11961154037286262, + "slope": 0.8767149750561822 + }, + "raw": { + "n": 681, + "first": "2018-02-15", + "last": "2019-12-30", + "r": 0.8482459139276761, + "rms_diff": 0.12926786732877105, + "slope": 0.8270457775608157 + } + }, + "CI.ARV": { + "station": "CI.ARV", + "daily_rows": 394, + "daily_first": "2018-01-01", + "daily_last": "2019-12-30", + "legacy_rows_2018_2019": 729, + "burn_in_until": "2018-05-31", + "matched": { + "n": 357, + "first": "2018-05-31", + "last": "2019-08-17", + "r": 0.9045574746252278, + "rms_diff": 0.1499582242219106, + "slope": 2.1497390972385637 + }, + "matched_no_burn_in": { + "n": 390, + "first": "2018-04-28", + "last": "2019-08-17", + "r": 0.8991749046935936, + "rms_diff": 0.14417421369707928, + "slope": 2.01388560869008 + }, + "centered": { + "n": 424, + "first": "2018-03-20", + "last": "2019-10-31", + "r": 0.36862022477672945, + "rms_diff": 0.21417922543897486, + "slope": 0.7964142472626492 + }, + "raw": { + "n": 394, + "first": "2018-02-15", + "last": "2019-12-14", + "r": 0.3050023982841881, + "rms_diff": 0.23344247992602937, + "slope": 0.48000644317005864 + } + }, + "CI.RXH": { + "station": "CI.RXH", + "daily_rows": 563, + "daily_first": "2018-01-01", + "daily_last": "2019-12-30", + "legacy_rows_2018_2019": 640, + "burn_in_until": "2018-05-31", + "matched": { + "n": 446, + "first": "2018-05-31", + "last": "2019-12-30", + "r": 0.8288596923048149, + "rms_diff": 0.020031985799333026, + "slope": 0.8061950785354097 + }, + "matched_no_burn_in": { + "n": 507, + "first": "2018-03-31", + "last": "2019-12-30", + "r": 0.7639616357430739, + "rms_diff": 0.024834967608089075, + "slope": 0.6416129386867837 + }, + "centered": { + "n": 554, + "first": "2018-02-15", + "last": "2019-11-28", + "r": 0.620432199974238, + "rms_diff": 0.03468605759921402, + "slope": 0.5227569651774386 + }, + "raw": { + "n": 560, + "first": "2018-02-15", + "last": "2019-12-30", + "r": 0.4911444632042665, + "rms_diff": 0.045858139775157306, + "slope": 0.35447454696414116 + } + } + } + } +} diff --git a/literature/figs/realdata_1_validation.npz b/literature/figs/realdata_1_validation.npz new file mode 100644 index 0000000..1e6be99 Binary files /dev/null and b/literature/figs/realdata_1_validation.npz differ diff --git a/literature/figs/realdata_1_validation.png b/literature/figs/realdata_1_validation.png index cd2c753..b19620f 100644 Binary files a/literature/figs/realdata_1_validation.png and b/literature/figs/realdata_1_validation.png differ diff --git a/paper/data/gate1/README.md b/paper/data/gate1/README.md index 0c2ac19..49eb1b7 100644 --- a/paper/data/gate1/README.md +++ b/paper/data/gate1/README.md @@ -43,8 +43,12 @@ is the archived one times a constant per band (0.463, 0.328, 0.232 and `scripts/correct_gate1_within_error.py`, `dvv_err` recomputed, the originals kept as `CI..v040.parquet`, and the factors and before/after medians logged in `dvv2y/correction.json`. The comparison -statistics do not use these columns. The figures in `../figures/gate1/` -still show the uncorrected bars. +statistics do not use these columns. The manuscript's comparison figure +(`literature/figs/realdata_1_validation.png`) is now generated in this +repository by `codameter.gate1` from the rescaled products and +`comparison.json`, so it shows the corrected bars and the matched-rule r; +the cloud-run figures in `../figures/gate1/` still show the uncorrected +bars and the centred-rule r. Ensemble members (from `noisepy_dvv_cloud/dvv.py`): the codameter recommendation for the `--use-case` passed to the run, at the product's diff --git a/paper/manuscript_marine.qmd b/paper/manuscript_marine.qmd index 43fea18..5c800b4 100644 --- a/paper/manuscript_marine.qmd +++ b/paper/manuscript_marine.qmd @@ -1316,9 +1316,11 @@ After the correction, single-station \dvv\ (NoisePy correlations, a codameter 5-member ensemble, 2--4$\,$Hz, 2018--2019) agrees in shape with the published @Clements2023 product. The comparison is not trivial to get right: the CD2023 90-day-comp product is a *trailing* 90-day stack, so it lags a -centered-smoothed daily series by about 45 days, and comparing without -matching that smoothing caps the correlation near 0.7 even on a real annual -cycle (Fig.\ \ref{fig:realdata-validation}). We match by applying the same +centered-smoothed daily series by about 45 days, and comparing a centred +45-day mean against it without matching the smoothing lowers the +correlation to 0.87, 0.37 and 0.62 at the three stations (the centred rule +of Table\ \ref{tab:gate1}) against 0.99, 0.90 and 0.83 under the matched +rule, even on a real annual cycle. We match by applying the same trailing 90-day mean to the daily series (at least 45 finite days in the window), compare demeaned --- the two products reference different epochs, and a constant offset is bookkeeping, not error --- and exclude the first 150 @@ -1367,20 +1369,18 @@ CI.RXH & centred & 554 & 0.620 & 0.035 & 0.52 \\ \begin{figure} \centering \includegraphics[width=\textwidth]{realdata_1_validation.png} - \caption{Single-station \dvv\ at three CI stations, 2018--2019, against the - published \citet{Clements2023} product (dashed, reference-shifted; the - panel labels name it by its 2022 data release). Daily \dvv\ (points) with the - between-configuration spread of the five-member ensemble (shaded) and the - within-measurement error bars (Table~\ref{tab:estimands}), and a centred - 45-day-smoothed curve. This figure was produced by the Gate 1 cloud run - itself and is not regenerated from this repository; its annotated $r$ values - correspond to the centred rule of Table~\ref{tab:gate1} (0.87, 0.37 and - 0.62 for LJR, ARV and RXH from the archived products), not to the matched - rule quoted in the text. The error bars drawn here predate the Weaver-floor - correction of the present revision and are too large by a factor of 3.1 at - 2--4\,Hz (the archived columns have since been rescaled by - \texttt{scripts/correct\_gate1\_within\_error.py}; the externally produced - figure has not been regenerated).} + \caption{Single-station \dvv\ at three CI stations (LJR, ARV, RXH from + top to bottom), 2--4\,Hz, 2018--2019, against the published + \citet{Clements2023} product (dashed; the legend names it by its 2022 data + release). Points: the daily ensemble \dvv\ of the Gate 1 run; shading: its + $\pm1.96\,\sigma$ band from the archived per-day error after the + Weaver-floor correction of the present revision + (Table~\ref{tab:estimands}); solid: the trailing 90-day mean of the daily + series after the 150-day burn-in (dotted line), the matched rule of + Table~\ref{tab:gate1}, whose Pearson $r$ and overlap are printed on each + panel. Both smoothed series are demeaned over their overlap. Generated + from the archived products and the archived comparison statistics; the + plotted arrays are in the figure's sidecar.} \label{fig:realdata-validation} \end{figure} @@ -1518,9 +1518,12 @@ valid downstream intervals. **Make it executable.** All synthetics, estimators and generated figures in this paper are released in the open codameter package; one driver regenerates every generated figure together with a numerical sidecar holding every plotted -array and the run's provenance, and the package is unit-tested. The three -real-data figures were produced by the cloud run and are archived, not -regenerated (Section\ \ref{sec:deployment}). An executable record turns an undocumented +array and the run's provenance, and the package is unit-tested. Of the +three real-data figures, the comparison (Fig.\ \ref{fig:realdata-validation}) +is regenerated by the same driver from the archived daily products; the +interferograms and the warm-up figure were produced by the cloud run from +correlations that are not archived here and are committed as produced +(Section\ \ref{sec:deployment}). An executable record turns an undocumented choice into a versioned, inspectable one, and lets a reader re-run a study's pipeline on the truth-known synthetic to see its bias before trusting it on data. @@ -1753,23 +1756,27 @@ under-reporting; the verified rate is the one among the 82 full-text rows. All synthetics, estimators, and generated figures in this paper are implemented in the open-source Python package codameter (MIT license), openly available at -; each generated figure's sidecar -records its generating commit and numerical arrays. The figures and manuscript -can originate from different revisions. \texttt{python -m codameter.figures} -regenerates every generated figure with a \texttt{.npz} sidecar of its plotted -arrays and a \texttt{.json} sidecar of its provenance; -\texttt{python -m codameter.calibration} reproduces -Table\ \ref{tab:calibration} (\texttt{paper/data/calibration/}); -\texttt{scripts/compare\_gate1.py} reproduces Table\ \ref{tab:gate1} from the -daily products under \texttt{paper/data/gate1/} and the published -\citet{Clements2023} series archived there. The daily products and the three +; each generated figure carries a +numerical sidecar with its plotted arrays, its generating commit, whether +the source tree was clean at that commit and a digest of the +figure-generating sources, so the figures and the manuscript can originate +from different revisions and be told apart. The repository documents how +one command regenerates every generated figure and how a check regenerates +them in memory and reports any array that differs from its committed +sidecar; how the repeated-realisation calibration of +Table\ \ref{tab:calibration} is rerun for each of its three scenarios +(about two hours per scenario on six worker processes; the settings and +seeds of each run are stored with the archived run); and how +Table\ \ref{tab:gate1} and Fig.\ \ref{fig:realdata-validation} are +recomputed from the daily products and the published \citet{Clements2023} +series archived with the paper. The daily products and the two remaining real-data figures were produced by the noisepy-dvv-cloud Gate 1 run from -public Southern California Earthquake Data Center waveforms; the products are -available locally during this analysis but are not tracked in the repository. -The exact Gate 1 run commit, run configuration and redistribution archive -remain to be supplied; their current provenance limits are recorded in -\texttt{paper/data/gate1/README.md}. A versioned archive is planned but -has not yet been deposited. +public Southern California Earthquake Data Center waveforms; the products +are available locally during this analysis but are not tracked in the +repository. The exact Gate 1 run commit, run configuration and +redistribution archive remain to be supplied; their current provenance +limits are recorded with the archived products. A versioned archive is +planned but has not yet been deposited. # Acknowledgements {.unnumbered} diff --git a/paper/manuscript_marine.tex b/paper/manuscript_marine.tex index 21093a5..c004fa9 100644 --- a/paper/manuscript_marine.tex +++ b/paper/manuscript_marine.tex @@ -251,7 +251,7 @@ \title{The reproducibility cost of ad-hoc processing choices in ambient-noise seismic velocity-change monitoring} \author{M. A. Denolle} -\date{2026-09-13} +\date{2026-09-14} \begin{document} \maketitle \begin{abstract} @@ -1658,16 +1658,18 @@ \section{Real-data comparison and deployment}\label{sec:deployment} with the published \citet{Clements2023} product. The comparison is not trivial to get right: the CD2023 90-day-comp product is a \emph{trailing} 90-day stack, so it lags a centered-smoothed daily -series by about 45 days, and comparing without matching that smoothing -caps the correlation near 0.7 even on a real annual cycle -(Fig.~\ref{fig:realdata-validation}). We match by applying the same -trailing 90-day mean to the daily series (at least 45 finite days in the -window), compare demeaned --- the two products reference different -epochs, and a constant offset is bookkeeping, not error --- and exclude -the first 150 days of each station's series as reference burn-in. The -comparison is computed by a script archived with the paper -(\texttt{scripts/compare\_gate1.py}) from the daily products and the -published series, with the join, smoothing and mask stated in its output +series by about 45 days, and comparing a centred 45-day mean against it +without matching the smoothing lowers the correlation to 0.87, 0.37 and +0.62 at the three stations (the centred rule of Table~\ref{tab:gate1}) +against 0.99, 0.90 and 0.83 under the matched rule, even on a real +annual cycle. We match by applying the same trailing 90-day mean to the +daily series (at least 45 finite days in the window), compare demeaned +--- the two products reference different epochs, and a constant offset +is bookkeeping, not error --- and exclude the first 150 days of each +station's series as reference burn-in. The comparison is computed by a +script archived with the paper (\texttt{scripts/compare\_gate1.py}) from +the daily products and the published series, with the join, smoothing +and mask stated in its output (\texttt{paper/data/gate1/comparison.json}); Table~\ref{tab:gate1} lists the result. Under the matched rule CI.LJR reaches \(r=0.985\) on 579 days with an amplitude slope of 1.07 (published on codameter); CI.ARV @@ -1707,20 +1709,18 @@ \section{Real-data comparison and deployment}\label{sec:deployment} \begin{figure} \centering \includegraphics[width=\textwidth]{realdata_1_validation.png} - \caption{Single-station \dvv\ at three CI stations, 2018--2019, against the - published \citet{Clements2023} product (dashed, reference-shifted; the - panel labels name it by its 2022 data release). Daily \dvv\ (points) with the - between-configuration spread of the five-member ensemble (shaded) and the - within-measurement error bars (Table~\ref{tab:estimands}), and a centred - 45-day-smoothed curve. This figure was produced by the Gate 1 cloud run - itself and is not regenerated from this repository; its annotated $r$ values - correspond to the centred rule of Table~\ref{tab:gate1} (0.87, 0.37 and - 0.62 for LJR, ARV and RXH from the archived products), not to the matched - rule quoted in the text. The error bars drawn here predate the Weaver-floor - correction of the present revision and are too large by a factor of 3.1 at - 2--4\,Hz (the archived columns have since been rescaled by - \texttt{scripts/correct\_gate1\_within\_error.py}; the externally produced - figure has not been regenerated).} + \caption{Single-station \dvv\ at three CI stations (LJR, ARV, RXH from + top to bottom), 2--4\,Hz, 2018--2019, against the published + \citet{Clements2023} product (dashed; the legend names it by its 2022 data + release). Points: the daily ensemble \dvv\ of the Gate 1 run; shading: its + $\pm1.96\,\sigma$ band from the archived per-day error after the + Weaver-floor correction of the present revision + (Table~\ref{tab:estimands}); solid: the trailing 90-day mean of the daily + series after the 150-day burn-in (dotted line), the matched rule of + Table~\ref{tab:gate1}, whose Pearson $r$ and overlap are printed on each + panel. Both smoothed series are demeaned over their overlap. Generated + from the archived products and the archived comparison statistics; the + plotted arrays are in the figure's sidecar.} \label{fig:realdata-validation} \end{figure} @@ -1861,8 +1861,11 @@ \section{Discussion}\label{sec:discussion} figures in this paper are released in the open codameter package; one driver regenerates every generated figure together with a numerical sidecar holding every plotted array and the run's provenance, and the -package is unit-tested. The three real-data figures were produced by the -cloud run and are archived, not regenerated +package is unit-tested. Of the three real-data figures, the comparison +(Fig.~\ref{fig:realdata-validation}) is regenerated by the same driver +from the archived daily products; the interferograms and the warm-up +figure were produced by the cloud run from correlations that are not +archived here and are committed as produced (Section~\ref{sec:deployment}). An executable record turns an undocumented choice into a versioned, inspectable one, and lets a reader re-run a study's pipeline on the truth-known synthetic to see its bias @@ -2139,24 +2142,26 @@ \section*{Data availability}\label{data-availability} All synthetics, estimators, and generated figures in this paper are implemented in the open-source Python package codameter (MIT license), openly available at \url{https://github.com/Denolle-Lab/codameter}; each -generated figure's sidecar records its generating commit and numerical -arrays. The figures and manuscript can originate from different -revisions. \texttt{python -m codameter.figures} regenerates every -generated figure with a \texttt{.npz} sidecar of its plotted arrays and -a \texttt{.json} sidecar of its provenance; -\texttt{python -m codameter.calibration} reproduces -Table~\ref{tab:calibration} (\texttt{paper/data/calibration/}); -\texttt{scripts/compare\_gate1.py} reproduces Table~\ref{tab:gate1} from -the daily products under \texttt{paper/data/gate1/} and the published -\citet{Clements2023} series archived there. The daily products and the -three real-data figures were produced by the noisepy-dvv-cloud Gate 1 -run from public Southern California Earthquake Data Center waveforms; -the products are available locally during this analysis but are not -tracked in the repository. The exact Gate 1 run commit, run -configuration and redistribution archive remain to be supplied; their -current provenance limits are recorded in -\texttt{paper/data/gate1/README.md}. A versioned archive is planned but -has not yet been deposited. +generated figure carries a numerical sidecar with its plotted arrays, +its generating commit, whether the source tree was clean at that commit +and a digest of the figure-generating sources, so the figures and the +manuscript can originate from different revisions and be told apart. The +repository documents how one command regenerates every generated figure +and how a check regenerates them in memory and reports any array that +differs from its committed sidecar; how the repeated-realisation +calibration of Table~\ref{tab:calibration} is rerun for each of its +three scenarios (about two hours per scenario on six worker processes; +the settings and seeds of each run are stored with the archived run); +and how Table~\ref{tab:gate1} and Fig.~\ref{fig:realdata-validation} are +recomputed from the daily products and the published +\citet{Clements2023} series archived with the paper. The daily products +and the two remaining real-data figures were produced by the +noisepy-dvv-cloud Gate 1 run from public Southern California Earthquake +Data Center waveforms; the products are available locally during this +analysis but are not tracked in the repository. The exact Gate 1 run +commit, run configuration and redistribution archive remain to be +supplied; their current provenance limits are recorded with the archived +products. A versioned archive is planned but has not yet been deposited. \section*{Acknowledgements}\label{acknowledgements} \addcontentsline{toc}{section}{Acknowledgements} diff --git a/review/REVISION_PLAN_ITER3.md b/review/REVISION_PLAN_ITER3.md index 6296028..e180da0 100644 --- a/review/REVISION_PLAN_ITER3.md +++ b/review/REVISION_PLAN_ITER3.md @@ -157,3 +157,4 @@ The submission decision should rest on a defensible measurement claim, transpare - 2026-09-11: Rewritten for Marine Denolle's observational priorities, with explicit measurement targets, diagnostic failure responses, provenance requirements, and evidence-based completion criteria. No experiments or implementation tasks are marked complete by this rewrite. - 2026-09-11: GitHub issues opened under milestone "Iteration 3 revision": P0 #38, P1 #39, P2 #40, P3 #41, P4 #42, P5 #43, P6 #44, P7 #45; author inputs #46 (Gate 1 provenance and rights, blocks P3), #47 (Yuan 2021 Table B3, blocks P1 and P4), #48 (survey rows and search rules, blocks P4), #49 (Fig 8 keep, trim or drop, blocks P0), #50 (step-error metric, blocks P1). Status is tracked in the issues; this file changes only by ticking boxes and appending to this record. - 2026-09-13: P0 executed on branch `iter3/P0` (commits 10a9db3, c66b409) for issue #38; every P0 item addressed except the Fig 8 keep/trim/drop decision (#49), where the composite was kept and made legible pending the decision. PDF rebuilt: 83 pages, no `??`, no `,%`, no `<<`; the Acknowledgements placeholder remains for P6. +- 2026-09-13: P3 part 1 executed on branch `iter3/P3` (PR #54) for issue #41: sidecar `git_dirty` and `generator_digest`, `codameter.figures --check`, shard generator digests, `codameter.gate1` regenerating Fig 15 from the archived products with the corrected error band and the matched-rule r, the calibration invocations in Data availability, and `.github/workflows/paper.yml` (sidecar check, survey drift, PDF build with text checks). Archive, release and DOI wait on #46 and the other packages. diff --git a/src/codameter/bench.py b/src/codameter/bench.py index d4950c1..9853651 100644 --- a/src/codameter/bench.py +++ b/src/codameter/bench.py @@ -26,6 +26,7 @@ from __future__ import annotations import argparse +import functools import json import os import re @@ -43,6 +44,7 @@ from . import use_cases as uc from ._version import __version__ from .deviations import metrics +from .provenance import git_commit # --------------------------------------------------------------------------- # Config grids, built relative to each case's recommended config. @@ -195,6 +197,12 @@ def _case(case_id: str) -> dict: return golden.generate(case_id) +@functools.lru_cache(maxsize=1) +def _generator_hash_cached() -> str: + """golden._generator_hash() once per process (it rereads source files).""" + return golden._generator_hash() + + def score_cell(case_id: str, config_index: int, cfg: dict) -> dict: """Score one ``(case, config)`` cell into a JSON-serializable row.""" case = golden.CASES_BY_ID[case_id] @@ -213,6 +221,11 @@ def score_cell(case_id: str, config_index: int, cfg: dict) -> dict: "target": case.get("target"), "eps_max": uc.eps_max(use_case), "codameter_version": __version__, + # The generator digest and commit pin the synthesis code the golden + # arrays were built with (audit S-RP.4); check_shards refuses to merge + # rows whose digests differ. + "generator_hash": _generator_hash_cached(), + "git_commit": git_commit(), } try: d = _case(case_id) @@ -334,8 +347,11 @@ def check_shards(pairs: list[tuple[str, dict]]) -> dict: A merge is complete only if every shard ``k`` of the declared ``N`` is present, every ``(case_id, config_index)`` cell appears exactly once, and - all rows come from one codameter version (audit SCALE-02). Retries that - rewrite a shard file are fine; a shard appended twice is not. + all rows come from one codameter version and one generator digest + (audits SCALE-02 and S-RP.4). Retries that rewrite a shard file are fine; + a shard appended twice is not. The commits the rows were produced at are + listed but not required to agree: a digest pins the synthesis code, a + commit only the tree it was run from. """ names = sorted({n for n, _ in pairs}) ks: set[int] = set() @@ -366,12 +382,23 @@ def check_shards(pairs: list[tuple[str, dict]]) -> dict: versions = sorted({str(r.get("codameter_version")) for _, r in pairs}) if len(versions) > 1: problems.append(f"rows from different codameter versions: {versions}") + n_no_hash = sum(1 for _, r in pairs if not r.get("generator_hash")) + if n_no_hash: + problems.append(f"{n_no_hash} row(s) carry no generator digest") + hashes = sorted( + {str(r["generator_hash"]) for _, r in pairs if r.get("generator_hash")} + ) + if len(hashes) > 1: + problems.append(f"rows from different generator digests: {hashes}") + commits = sorted({r["git_commit"] for _, r in pairs if r.get("git_commit")}) return { "n_shards": n_shards, "shards_present": sorted(ks), "missing": missing, "duplicate_cells": len(dups), "codameter_versions": versions, + "generator_hashes": hashes, + "git_commits": commits, "n_rows": len(pairs), "unique_cells": len(cells), "problems": problems, diff --git a/src/codameter/errors.py b/src/codameter/errors.py new file mode 100644 index 0000000..2456e70 --- /dev/null +++ b/src/codameter/errors.py @@ -0,0 +1,8 @@ +"""Exceptions shared across modules (kept apart so ``python -m`` runs of a +module never see two copies of a class).""" + +from __future__ import annotations + + +class MissingInputs(RuntimeError): + """A generator's inputs are not available on this machine (data not tracked).""" diff --git a/src/codameter/figures.py b/src/codameter/figures.py index 3ef3b09..ef0ef4c 100644 --- a/src/codameter/figures.py +++ b/src/codameter/figures.py @@ -24,9 +24,10 @@ import argparse import dataclasses +import functools import json import platform -import subprocess +import warnings from collections.abc import Callable, Iterable from datetime import datetime, timezone from pathlib import Path @@ -36,6 +37,8 @@ import numpy as np from ._version import __version__ +from .errors import MissingInputs +from .provenance import git_commit, git_dirty __all__ = [ "EXTERNAL", @@ -50,10 +53,12 @@ #: Figures the paper includes that are produced outside this repository. EXTERNAL = ( - "realdata_1_validation", "realdata_2_interferograms", "realdata_3_warmup", ) +#: Generated figures whose inputs are not tracked by git (skipped with a +#: message where the inputs are absent). +NEEDS_DATA = {"realdata_1_validation"} #: Generators that take minutes rather than seconds. SLOW = {"demo_10_deviations", "demo_11_multiverse", "demo_12_bayes"} #: float64 arrays with more elements than this are stored as float32 in the @@ -63,20 +68,39 @@ Generator = Callable[[], tuple[Any, dict[str, Any], dict[str, Any]]] -def _git_commit() -> str | None: - try: - out = subprocess.run( - ["git", "rev-parse", "HEAD"], - capture_output=True, - text=True, - cwd=Path(__file__).resolve().parent, - timeout=10, - check=False, - ) - except (OSError, subprocess.SubprocessError): - return None - sha = out.stdout.strip() - return sha if out.returncode == 0 and sha else None +_git_commit = git_commit # kept for callers of the old private names +_git_dirty = git_dirty + + +#: Modules whose source enters the generator digest: every figure builder +#: lives in one of them, so a change to any of them changes the digest. +_DIGEST_MODULES = ( + "figures", + "synthetic_demo", + "deviations", + "uq_bayes", + "uq_measurement", + "gate1", +) + + +@functools.lru_cache(maxsize=1) +def generator_digest() -> str: + """Short digest of the package version and the figure-generating sources. + Computed once per process (the sources do not change during a build). + + Recorded in every sidecar so that a figure can be matched to the exact + generator code, as :func:`codameter.golden._generator_hash` does for the + golden datasets; two sidecars with the same digest were produced by the + same figure code whatever the commit says. + """ + import hashlib + + h = hashlib.sha1(__version__.encode()) + here = Path(__file__).resolve().parent + for mod in _DIGEST_MODULES: + h.update((here / f"{mod}.py").read_bytes()) + return h.hexdigest()[:12] def _jsonable(obj: Any) -> Any: @@ -182,6 +206,8 @@ def save_figure( "generator": generator, "codameter_version": __version__, "git_commit": _git_commit(), + "git_dirty": _git_dirty(), + "generator_digest": generator_digest(), "generated_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"), "python": platform.python_version(), "numpy": np.__version__, @@ -248,6 +274,13 @@ def _gen_demo_12(): return fig, arrays, meta +def _gen_realdata_1(): + from .gate1 import fig_gate1_comparison + + fig = fig_gate1_comparison() + return fig, dict(fig.codameter_arrays), dict(fig.codameter_meta) + + def generators() -> dict[str, tuple[str, Generator]]: """Every generated figure: ``name -> (generator description, callable)``.""" from . import synthetic_demo as sd @@ -271,9 +304,28 @@ def _gen(b=builder): "codameter.uq_bayes._build_bayes + _fig_bayes", _gen_demo_12, ) + gens["realdata_1_validation"] = ( + "codameter.gate1.fig_gate1_comparison (needs paper/data/gate1/dvv2y)", + _gen_realdata_1, + ) return gens +def _select( + only: Iterable[str] | None, + skip_slow: bool, + gens: dict[str, tuple[str, Generator]] | None = None, +) -> list[str]: + gens = generators() if gens is None else gens + wanted = list(gens) if only is None else list(only) + unknown = sorted(set(wanted) - set(gens)) + if unknown: + raise KeyError(f"unknown figure(s): {unknown}; known: {sorted(gens)}") + if skip_slow: + wanted = [n for n in wanted if n not in SLOW] + return wanted + + def build_all_figures( outdir: str | Path, *, @@ -286,18 +338,17 @@ def build_all_figures( from .synthetic_demo import apply_style gens = generators() - wanted = list(gens) if only is None else list(only) - unknown = sorted(set(wanted) - set(gens)) - if unknown: - raise KeyError(f"unknown figure(s): {unknown}; known: {sorted(gens)}") - if skip_slow: - wanted = [n for n in wanted if n not in SLOW] + wanted = _select(only, skip_slow, gens) apply_style() written = [] for name in wanted: desc, gen = gens[name] print(f"[{name}] {desc}", flush=True) - fig, arrays, meta = gen() + try: + fig, arrays, meta = gen() + except MissingInputs as exc: + print(f" skipped: {exc}", flush=True) + continue written.append( save_figure( fig, outdir, name, generator=desc, extra_arrays=arrays, extra_meta=meta @@ -308,12 +359,124 @@ def build_all_figures( return written +def _max_abs(x: np.ndarray) -> float: + """Largest finite magnitude in ``x`` (0.0 when there is none). + + Allocates one temporary the size of ``x`` (the masked magnitudes), not a + concatenation of both arrays being compared. + """ + if x.size == 0: + return 0.0 + with np.errstate(invalid="ignore"), warnings.catch_warnings(): + warnings.simplefilter("ignore", RuntimeWarning) # all-NaN input + m = np.nanmax(np.where(np.isfinite(x), np.abs(x), np.nan)) + return float(m) if np.isfinite(m) else 0.0 + + +def compare_sidecar(arrays: dict[str, Any], npz_path: Path, *, rtol: float = 1e-6): + """Compare freshly generated arrays with a committed ``.npz`` sidecar. + + Returns a list of human-readable differences (empty when they agree). + Floating arrays agree when every entry is within ``rtol`` of the stored + value or within ``rtol`` times the largest magnitude in either array (so the + rounding noise of an entry that is zero up to platform arithmetic, such + as the zero-change point of a sweep, does not count as a difference). + Large float64 arrays are compared at float32 precision, the precision the + sidecar stores them at (:data:`LARGE_ARRAY`); NaNs must match in position. + """ + diffs: list[str] = [] + if not npz_path.exists(): + return [f"no committed sidecar at {npz_path}"] + with np.load(npz_path, allow_pickle=False) as z: + stored = {k: z[k] for k in z.files} + fresh = {k: compact_array(np.asarray(v)) for k, v in arrays.items()} + for k in sorted(set(stored) | set(fresh)): + if k not in stored: + diffs.append(f"{k}: new array not in the sidecar") + continue + if k not in fresh: + diffs.append(f"{k}: in the sidecar but no longer generated") + continue + a, b = stored[k], fresh[k] + if a.shape != b.shape: + diffs.append(f"{k}: shape {a.shape} in sidecar, {b.shape} generated") + continue + if a.dtype.kind in "fc" and b.dtype.kind in "fc": + tol = max(rtol, 1e-6 if a.dtype == np.float32 else rtol) + scale = max(_max_abs(a), _max_abs(b)) + if not np.allclose(a, b, rtol=tol, atol=tol * scale, equal_nan=True): + with np.errstate( + invalid="ignore", divide="ignore" + ), warnings.catch_warnings(): + warnings.simplefilter("ignore", RuntimeWarning) # all-NaN ratio + denom = np.maximum(np.maximum(np.abs(a), np.abs(b)), tol * scale) + rel = np.nanmax(np.abs(a - b) / denom) + if np.isfinite(rel): + diffs.append( + f"{k}: values differ (max relative difference {rel:.3g})" + ) + else: + diffs.append(f"{k}: values differ only in NaN placement") + elif not np.array_equal(a, b): + diffs.append(f"{k}: values differ") + return diffs + + +def check_all_figures( + outdir: str | Path, + *, + only: Iterable[str] | None = None, + skip_slow: bool = False, + rtol: float = 1e-6, +) -> dict[str, list[str]]: + """Regenerate the selected figures in memory and diff their arrays against + the committed sidecars in ``outdir``; returns ``{name: differences}``.""" + import matplotlib.pyplot as plt + + from .synthetic_demo import apply_style + + gens = generators() + outdir = Path(outdir) + apply_style() + report: dict[str, list[str]] = {} + for name in _select(only, skip_slow, gens): + desc, gen = gens[name] + print(f"[{name}] {desc}", flush=True) + try: + fig, arrays, meta = gen() + except MissingInputs as exc: + print(f" skipped: {exc}", flush=True) + continue + all_arrays: dict[str, Any] = dict(figure_arrays(fig)) + for k, v in (arrays or {}).items(): + all_arrays[f"data/{k}"] = np.asarray(v) + plt.close(fig) + report[name] = compare_sidecar(all_arrays, outdir / f"{name}.npz", rtol=rtol) + status = "ok" if not report[name] else f"{len(report[name])} difference(s)" + print(f" {status}", flush=True) + for d in report[name]: + print(f" {d}", flush=True) + return report + + def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) ap.add_argument("--out", default="literature/figs", help="output directory") ap.add_argument("--only", default=None, help="comma-separated figure names") ap.add_argument("--skip-slow", action="store_true", help=f"skip {sorted(SLOW)}") ap.add_argument("--list", action="store_true", help="list figures and exit") + ap.add_argument( + "--check", + action="store_true", + help="regenerate in memory and diff against the committed sidecars; " + "exit 1 on any difference", + ) + ap.add_argument( + "--rtol", + type=float, + default=1e-6, + help="relative tolerance for --check (float32-stored arrays use at least 1e-6)", + ) args = ap.parse_args(argv) if args.list: for name, (desc, _) in generators().items(): @@ -322,6 +485,16 @@ def main(argv: list[str] | None = None) -> int: print(f"{name:<28} external; see literature/figs/SOURCES.md") return 0 only = [s.strip() for s in args.only.split(",")] if args.only else None + if args.check: + report = check_all_figures( + args.out, only=only, skip_slow=args.skip_slow, rtol=args.rtol + ) + bad = {k: v for k, v in report.items() if v} + print( + f"checked {len(report)} figure(s): {len(report) - len(bad)} match, " + f"{len(bad)} differ" + ) + return 1 if bad else 0 build_all_figures(args.out, only=only, skip_slow=args.skip_slow) return 0 diff --git a/src/codameter/gate1.py b/src/codameter/gate1.py new file mode 100644 index 0000000..49697e4 --- /dev/null +++ b/src/codameter/gate1.py @@ -0,0 +1,192 @@ +"""The Gate 1 field comparison figure, generated from the archived products. + +Inputs (see ``paper/data/gate1/README.md``): + +- ``paper/data/gate1/dvv2y/band=2.0-4.0/CI..parquet``: the daily + single-station ensemble dv/v of the noisepy-dvv-cloud Gate 1 run, with the + error columns rescaled to the corrected Weaver floor + (``scripts/correct_gate1_within_error.py``); not tracked by git until the + redistribution terms are settled (issue #46), so the generator raises + :class:`codameter.errors.MissingInputs` where they are absent. +- ``paper/data/gate1/legacy_cd2022/CI..arrow``: the published + Clements and Denolle (2022) product (tracked). +- ``paper/data/gate1/comparison.json``: the statistics of + ``scripts/compare_gate1.py`` under its stated rules; the figure quotes the + matched rule (trailing 90-day mean, 150-day burn-in, inner join on + calendar days) and applies the same smoothing to the plotted series. + +Every plotted array goes to the figure sidecar, so the manuscript's numbers +about this figure can be checked without the parquet inputs. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np + +ROOT = Path(__file__).resolve().parents[2] +GATE1 = ROOT / "paper" / "data" / "gate1" +STATIONS = ("LJR", "ARV", "RXH") +BAND = "2.0-4.0" +Z95 = 1.959964 + + +def _rules(gate1: Path = GATE1) -> dict: + """The archived comparison statistics and rules (``comparison.json``).""" + from .errors import MissingInputs + + path = gate1 / "comparison.json" + if not path.exists(): + raise MissingInputs(f"{path} is not available (archived comparison statistics)") + out: dict = json.loads(path.read_text()) + return out + + +def load_station(sta: str, *, gate1: Path = GATE1, band: str = BAND): + """Daily codameter series (with its error) and the published product. + + Returns ``(daily, err, legacy)`` as pandas Series on calendar-day indices; + ``legacy`` is restricted to 2018-2019 as in ``scripts/compare_gate1.py``. + """ + import pandas as pd + import pyarrow.ipc as ipc + + from .errors import MissingInputs + + pq = gate1 / "dvv2y" / f"band={band}" / f"CI.{sta}.parquet" + if not pq.exists(): + raise MissingInputs(f"{pq} is not available (untracked Gate 1 product)") + d = pd.read_parquet(pq) + d["date"] = pd.to_datetime(d["date"]) + d = d.set_index("date").sort_index() + grid = pd.date_range(d.index.min(), d.index.max(), freq="D") + daily = d["dvv"].reindex(grid) + err = d["dvv_err"].reindex(grid) + legacy = ( + ipc.open_file(gate1 / "legacy_cd2022" / f"CI.{sta}.arrow") + .read_all() + .to_pandas() + ) + legacy["DATE"] = pd.to_datetime(legacy["DATE"]) + legacy = legacy.set_index("DATE")["DVV"].sort_index() + legacy = legacy[(legacy.index >= "2018-01-01") & (legacy.index <= "2019-12-31")] + return daily, err, legacy + + +def fig_gate1_comparison(*, gate1: Path = GATE1): + """Three-station comparison with the published product under the matched rule. + + Per station: the daily codameter dv/v with its $\\pm1.96\\sigma$ band + (the archived, corrected ``dvv_err``), the trailing 90-day mean after the + 150-day burn-in, and the published product; both smoothed series are + demeaned over their overlap, as the comparison statistics are. The panel + annotation quotes the matched-rule Pearson r and the number of overlapping + days from ``comparison.json``; the sidecar carries every plotted array + plus the per-station statistics. + """ + import matplotlib.pyplot as plt + import pandas as pd + + from .synthetic_demo import C + + cmp = _rules(gate1) + rules = cmp["rules"] + by_station = {s["station"]: s for s in cmp["stations"]} + burn_days = int(rules["burn_in_days"]) + trailing = int(rules["trailing_days"]) + trailing_min = int(rules["trailing_min_finite"]) + + fig, axes = plt.subplots(len(STATIONS), 1, figsize=(10.5, 8.4), sharex=True) + extra: dict[str, np.ndarray] = {} + meta: dict[str, dict] = {"rules": rules, "stations": {}} + for ax, sta in zip(axes, STATIONS, strict=True): + daily, err, legacy = load_station(sta, gate1=gate1) + burn = daily.index.min() + pd.Timedelta(days=burn_days) + smooth = daily.rolling(f"{trailing}D", min_periods=trailing_min).mean() + smooth = smooth[smooth.index >= burn] + joined = pd.DataFrame({"ours": smooth, "published": legacy}).dropna() + ours_dm = smooth - joined["ours"].mean() + pub_dm = legacy - joined["published"].mean() + daily_dm = daily - joined["ours"].mean() + + t = daily.index.to_numpy() + ax.fill_between( + t, + (daily_dm - Z95 * err).to_numpy(), + (daily_dm + Z95 * err).to_numpy(), + color=C["alt"], + alpha=0.25, + lw=0, + label=r"daily $\pm1.96\,\sigma$ (corrected floor)", + ) + ax.plot(t, daily_dm.to_numpy(), ".", ms=2.5, color=C["alt"], label="daily dv/v") + ax.plot( + smooth.index.to_numpy(), + ours_dm.to_numpy(), + color=C["truth"], + lw=2.0, + label=f"codameter, trailing {trailing}-day mean", + ) + ax.plot( + legacy.index.to_numpy(), + pub_dm.to_numpy(), + "--", + color="0.2", + lw=1.4, + label="Clements and Denolle (2022)", + ) + ax.axvline(burn, color="0.6", lw=1.0, ls=":") + st = by_station[f"CI.{sta}"]["matched"] + ax.text( + 0.99, + 0.93, + f"matched r = {st['r']:.2f} on {st['n']} days", + transform=ax.transAxes, + ha="right", + va="top", + fontsize=12, + fontweight="bold", + ) + ax.set_ylabel(f"CI.{sta}\ndv/v (%)") + ax.grid(alpha=0.3) + key = sta.lower() + extra[f"{key}/days"] = ( + (t - np.datetime64("2018-01-01")).astype("timedelta64[D]").astype(float) + ) + extra[f"{key}/daily_dvv_pct"] = daily.to_numpy(float) + extra[f"{key}/daily_err_pct"] = err.to_numpy(float) + extra[f"{key}/trailing_days"] = ( + (smooth.index.to_numpy() - np.datetime64("2018-01-01")) + .astype("timedelta64[D]") + .astype(float) + ) + extra[f"{key}/trailing_dvv_pct"] = smooth.to_numpy(float) + extra[f"{key}/published_days"] = ( + (legacy.index.to_numpy() - np.datetime64("2018-01-01")) + .astype("timedelta64[D]") + .astype(float) + ) + extra[f"{key}/published_dvv_pct"] = legacy.to_numpy(float) + meta["stations"][f"CI.{sta}"] = by_station[f"CI.{sta}"] + handles, labels = axes[0].get_legend_handles_labels() + fig.legend( + handles, + labels, + loc="lower center", + ncol=4, + fontsize=11, + frameon=False, + bbox_to_anchor=(0.5, 0.0), + ) + axes[0].set_title( + f"Gate 1, {BAND} Hz, 2018-2019: single-station ensemble against the " + "published product (demeaned over the overlap)", + fontsize=13, + ) + axes[-1].set_xlabel("date") + fig.tight_layout(rect=(0, 0.04, 1, 1)) + fig.codameter_arrays = extra # type: ignore[attr-defined] + fig.codameter_meta = {"gate1": meta} # type: ignore[attr-defined] + return fig diff --git a/src/codameter/provenance.py b/src/codameter/provenance.py new file mode 100644 index 0000000..b7389c5 --- /dev/null +++ b/src/codameter/provenance.py @@ -0,0 +1,59 @@ +"""Git provenance helpers shared by the figure driver and the benchmark. + +Kept free of plotting imports so that a benchmark row can record the commit +without importing matplotlib. +""" + +from __future__ import annotations + +import functools +import subprocess +from pathlib import Path + +_HERE = Path(__file__).resolve().parent + + +def _git(*args: str) -> str | None: + try: + out = subprocess.run( + ["git", *args], + capture_output=True, + text=True, + cwd=_HERE, + timeout=10, + check=False, + ) + except (OSError, subprocess.SubprocessError): + return None + return out.stdout if out.returncode == 0 else None + + +@functools.lru_cache(maxsize=1) +def git_commit() -> str | None: + """Full SHA of HEAD in the repository this package is imported from (None outside git). + + Cached for the life of the process: HEAD does not change during a run, + and a benchmark sweep calls this once per scored cell. + """ + out = _git("rev-parse", "HEAD") + sha = (out or "").strip() + return sha or None + + +@functools.lru_cache(maxsize=1) +def git_dirty() -> bool | None: + """True when tracked files under ``src/`` differ from HEAD (None outside git). + + Cached for the life of the process, like :func:`git_commit`: a figure build + renders many figures from one tree state. + + The pathspec is the parent of the package directory, so every tracked + source under ``src/`` counts, not only ``src/codameter``. A record whose + ``git_commit`` names a commit but whose ``git_dirty`` is true was produced + by code that commit does not contain (audit S-RP.1). + """ + # Relative pathspec from the package directory (the subprocess cwd): src/. + out = _git("status", "--porcelain", "--untracked-files=no", "--", "..") + if out is None: + return None + return bool(out.strip()) diff --git a/tests/test_bench.py b/tests/test_bench.py index 781e36d..4ea6cdb 100644 --- a/tests/test_bench.py +++ b/tests/test_bench.py @@ -109,3 +109,44 @@ def test_best_config_is_the_recommended_one(): best = min(ok, key=lambda r: r["rms"]) assert best["estimator"] == "stretching (TS)" assert best["reference"] == "fixed" + + +def test_score_cell_rows_carry_generator_digest_and_commit(): + from codameter import golden + from codameter import use_cases as uc + + row = bench.score_cell(EASY, 0, uc.recommend("volcano")) + assert row["generator_hash"] == golden._generator_hash() + assert "git_commit" in row + + +def test_check_shards_refuses_rows_from_different_generators(): + base = { + "case_id": "c", + "codameter_version": "1", + "generator_hash": "aaaa", + "git_commit": "x", + } + pairs = [ + ("shard-00000-of-00002.jsonl", dict(base, config_index=0)), + ("shard-00001-of-00002.jsonl", dict(base, config_index=1)), + ] + assert bench.check_shards(pairs)["complete"] + pairs[1] = (pairs[1][0], dict(pairs[1][1], generator_hash="bbbb")) + inv = bench.check_shards(pairs) + assert not inv["complete"] + assert any("generator digest" in p for p in inv["problems"]) + # A different commit alone is listed, not refused. + pairs[1] = (pairs[1][0], dict(pairs[1][1], generator_hash="aaaa", git_commit="y")) + inv = bench.check_shards(pairs) + assert inv["complete"] and inv["git_commits"] == ["x", "y"] + # a missing commit is omitted from the list, not stringified + row = dict(pairs[1][1], git_commit=None) + assert bench.check_shards([pairs[0], (pairs[1][0], row)])["git_commits"] == ["x"] + # A row without a digest is refused, not treated as the digest "None". + row = dict(pairs[1][1]) + del row["generator_hash"] + inv = bench.check_shards([pairs[0], (pairs[1][0], row)]) + assert not inv["complete"] and any( + "no generator digest" in p for p in inv["problems"] + ) diff --git a/tests/test_figures.py b/tests/test_figures.py index 782a8d7..30ce9a4 100644 --- a/tests/test_figures.py +++ b/tests/test_figures.py @@ -76,3 +76,102 @@ def test_cli_lists_and_rejects_unknown(tmp_path, capsys): assert "demo_12_bayes" in out and "realdata_1_validation" in out with pytest.raises(KeyError): F.build_all_figures(tmp_path, only=["no_such_figure"]) + + +def test_sidecar_records_dirty_flag_and_generator_digest(tmp_path): + import matplotlib.pyplot as plt + + fig, ax = plt.subplots() + ax.plot([0, 1], [0, 1]) + F.save_figure(fig, tmp_path, "prov", generator="test") + plt.close(fig) + meta = json.loads((tmp_path / "prov.json").read_text()) + assert meta["generator_digest"] == F.generator_digest() + assert re.fullmatch(r"[0-9a-f]{12}", meta["generator_digest"]) + assert meta["git_dirty"] in (True, False, None) + assert meta["git_commit"] is None or re.fullmatch( + r"[0-9a-f]{40}", meta["git_commit"] + ) + + +def test_compare_sidecar_reports_differences(tmp_path): + big = np.linspace(0, 1, F.LARGE_ARRAY + 1) + arrays = {"ax0/line0/y": np.array([1.0, 2.0, np.nan]), "data/big": big} + np.savez_compressed( + tmp_path / "s.npz", **{k: F.compact_array(v) for k, v in arrays.items()} + ) + assert F.compare_sidecar(arrays, tmp_path / "s.npz") == [] + # float32 storage of the large array is not a difference + assert ( + F.compare_sidecar( + dict(arrays, **{"data/big": big * (1 + 1e-8)}), tmp_path / "s.npz" + ) + == [] + ) + # rounding noise on an entry that is zero up to arithmetic is not a difference + tiny = {"ax0/line0/y": np.array([1e-18, 0.5]), "data/big": big} + np.savez_compressed(tmp_path / "t.npz", **tiny) + shifted = dict(tiny, **{"ax0/line0/y": np.array([-3e-18, 0.5])}) + assert F.compare_sidecar(shifted, tmp_path / "t.npz") == [] + moved = dict(tiny, **{"ax0/line0/y": np.array([1e-18, 0.5001])}) + assert F.compare_sidecar(moved, tmp_path / "t.npz") != [] + # the scale comes from either array: a stored zero against a large value differs + zeros = {"ax0/line0/y": np.zeros(2), "data/big": big} + np.savez_compressed(tmp_path / "z.npz", **zeros) + assert ( + F.compare_sidecar( + dict(zeros, **{"ax0/line0/y": np.array([0.0, 0.5])}), tmp_path / "z.npz" + ) + != [] + ) + assert ( + F.compare_sidecar( + dict(zeros, **{"ax0/line0/y": np.array([0.0, 1e-18])}), tmp_path / "z.npz" + ) + != [] + ) + # arrays differing only in NaN placement: one message, no warning + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("error") + d = F.compare_sidecar( + dict(zeros, **{"ax0/line0/y": np.array([0.0, np.nan])}), tmp_path / "z.npz" + ) + assert d == ["ax0/line0/y: values differ only in NaN placement"] + diffs = F.compare_sidecar( + {"ax0/line0/y": np.array([1.0, 2.5, np.nan]), "data/other": big}, + tmp_path / "s.npz", + ) + assert any("values differ" in d for d in diffs) + assert any("no longer generated" in d for d in diffs) + assert any("new array" in d for d in diffs) + assert F.compare_sidecar(arrays, tmp_path / "missing.npz") == [ + f"no committed sidecar at {tmp_path / 'missing.npz'}" + ] + + +def test_missing_inputs_are_skipped_not_fatal(tmp_path, monkeypatch, capsys): + from codameter.errors import MissingInputs + + def _gen(): + raise MissingInputs("no data here") + + monkeypatch.setattr(F, "generators", lambda: {"needs_data": ("test", _gen)}) + assert F.build_all_figures(tmp_path, only=["needs_data"]) == [] + assert "skipped: no data here" in capsys.readouterr().out + assert F.check_all_figures(tmp_path, only=["needs_data"]) == {} + + +def test_gate1_generator_is_registered_and_needs_data(): + assert "realdata_1_validation" in F.generators() + assert "realdata_1_validation" not in F.EXTERNAL + assert "realdata_1_validation" in F.NEEDS_DATA + + +def test_gate1_generator_skips_when_inputs_are_absent(tmp_path): + from codameter.errors import MissingInputs + from codameter.gate1 import fig_gate1_comparison + + with pytest.raises(MissingInputs): + fig_gate1_comparison(gate1=tmp_path) # no comparison.json, no products