Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions browser/data-browser/src/App.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
} from './helpers/originNode';

import { useEffect, type JSX } from 'react';
import { reportRepeatedCommitFailures } from './helpers/sentry';
import { RouterProvider } from '@tanstack/react-router';
import { ProxyConnectReturn } from './chunks/AppPage/ProxyConnectReturn';
import { router } from './routes/Router';
Expand Down Expand Up @@ -241,6 +242,7 @@ window.store = store;
// traffic shows up alongside React render counts. Cmd/Ctrl+Shift+P to
// dump a snapshot.
attachStoreToProfiler(store);
reportRepeatedCommitFailures(store);

if (isDev()) {
const { attachDevtools } = await import('./helpers/devtools');
Expand Down
36 changes: 34 additions & 2 deletions browser/data-browser/src/helpers/sentry.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import * as Sentry from '@sentry/react';
import { initSentry } from './sentry';
vi.mock('@sentry/react', () => ({ init: vi.fn() }));
import { StoreEvents } from '@tomic/react';
import { initSentry, reportRepeatedCommitFailures } from './sentry';
vi.mock('@sentry/react', () => ({
init: vi.fn(),
isEnabled: vi.fn(() => true),
captureMessage: vi.fn(),
}));
describe('Sentry configuration', () => {
beforeEach(() => {
vi.clearAllMocks();
Expand Down Expand Up @@ -77,4 +82,31 @@ describe('Sentry configuration', () => {
expect.objectContaining({ environment: 'staging' }),
);
});
it('groups repeated commit failures by server and message, not by resource', () => {
let handler: ((failure: unknown) => void) | undefined;
const store = {
on: vi.fn((event: string, cb: (failure: unknown) => void) => {
if (event === StoreEvents.CommitRepeatedlyFailing) handler = cb;

return () => {};
}),
};
reportRepeatedCommitFailures(store as never);
handler?.({
subject: 'atomic:abc',
error: new Error('Parent of atomic:abc (atomic:def) not found'),
failures: 4,
server: 'https://node1.example',
});
expect(Sentry.captureMessage).toHaveBeenCalledWith(
'Commit keeps failing',
expect.objectContaining({
fingerprint: [
'commit-keeps-failing',
'https://node1.example',
'Parent of <id> (<id>) not found',
],
}),
);
});
});
30 changes: 30 additions & 0 deletions browser/data-browser/src/helpers/sentry.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import * as Sentry from '@sentry/react';
import { StoreEvents, type Store } from '@tomic/react';

/**
* Browser error reporting. Strictly opt-in: nothing is initialised, and no
Expand Down Expand Up @@ -61,3 +62,32 @@ export function initSentry(): void {
tracesSampleRate: 0,
});
}

/** Identifiers differ per resource; the failure itself is what groups. */
function withoutIdentifiers(message: string): string {
return message.replace(/(?:atomic|did:ad):[\w-]+/g, '<id>');
}

/**
* Report a write that keeps failing while the outbox retries it. Those
* failures only ever reached the console, so a drive that could not sync (for
* instance a server that never received its root) was invisible to us.
*/
export function reportRepeatedCommitFailures(store: Store): () => void {
return store.on(StoreEvents.CommitRepeatedlyFailing, failure => {
if (!Sentry.isEnabled()) return;

const message = withoutIdentifiers(failure.error.message);

Sentry.captureMessage('Commit keeps failing', {
level: 'warning',
fingerprint: ['commit-keeps-failing', failure.server, message],
tags: { server: failure.server },
extra: {
subject: failure.subject,
failures: failure.failures,
error: failure.error.message,
},
});
});
}
39 changes: 39 additions & 0 deletions browser/lib/src/local-outbox.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import {
groupTiers,
runBounded,
BLOCK_AFTER_FAILURES,
REPORT_AFTER_FAILURES,
type OutboxEntry,
} from './local-outbox.js';
import { commitToJsonADObject, type Commit } from './commit.js';
Expand Down Expand Up @@ -696,6 +697,44 @@ describe('LocalOutbox blocking', () => {
}
});

it('reports a persistently failing write once, and keeps retrying it', async ({
expect,
}) => {
vi.useFakeTimers();

try {
const outbox = new LocalOutbox();
outbox.markDirty(SUBJECT);
// The entry is a live object, so read its count at the moment of the call.
const failuresAtReport: (number | undefined)[] = [];
const onRepeatedFailure = vi.fn((entry: OutboxEntry) => {
failuresAtReport.push(entry.failures);
});
const drainSubject = vi.fn(async () => {
throw new Error('Parent of x not found: Resource not found.');
});
const ctx = { ...blockingCtx(drainSubject), onRepeatedFailure };

let now = 0;
vi.setSystemTime(now);

for (let i = 1; i <= REPORT_AFTER_FAILURES + 2; i++) {
await outbox.drain(ctx);
now += drainBackoffMs(i) + 1;
vi.setSystemTime(now);
}

// Not on the first failures, exactly once at the threshold, and the
// entry is neither blocked nor dropped: reporting changes nothing else.
expect(onRepeatedFailure).toHaveBeenCalledTimes(1);
expect(failuresAtReport).toEqual([REPORT_AFTER_FAILURES]);
expect(outbox.getEntry(SUBJECT)?.blocked).toBeFalsy();
expect(drainSubject).toHaveBeenCalledTimes(REPORT_AFTER_FAILURES + 2);
} finally {
vi.useRealTimers();
}
});

it('a successful retry clears the failure streak before blocking', async ({
expect,
}) => {
Expand Down
13 changes: 13 additions & 0 deletions browser/lib/src/local-outbox.ts
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,11 @@ export function drainBackoffMs(failures: number): number {
// few-second window an ordering race needs.
export const BLOCK_AFTER_FAILURES = 8;

/** Consecutive failed drains of one entry before it counts as persistently
* failing and is reported once through `onRepeatedFailure`. Earlier than
* `BLOCK_AFTER_FAILURES`, which only applies to errors classed as blocking. */
export const REPORT_AFTER_FAILURES = 4;

/** How many subjects of one tier are drained at once when the caller
* supplies `tierOf`. Bounded so a reconnect with hundreds of dirty
* subjects does not open hundreds of concurrent COMMITs. */
Expand Down Expand Up @@ -171,6 +176,10 @@ export interface OutboxDrainContext {
/** Notification hook fired once when an entry transitions to blocked —
* caller typically surfaces a persistent "could not sync" message. */
onBlocked?: (entry: OutboxEntry, error: unknown) => void;
/** Fired once per failure streak, when an entry has failed
* `REPORT_AFTER_FAILURES` drains in a row, whatever the error. For
* reporting only: the entry keeps retrying as before. */
onRepeatedFailure?: (entry: OutboxEntry, error: unknown) => void;
}

/**
Expand Down Expand Up @@ -1076,6 +1085,10 @@ export class LocalOutbox {
// unauthorized ("stop + surface"); a later `markDirty` re-arms it.
live.failures = (live.failures ?? 0) + 1;

if (live.failures === REPORT_AFTER_FAILURES) {
ctx.onRepeatedFailure?.(live, e);
}

if (
live.failures >= BLOCK_AFTER_FAILURES &&
ctx.isBlockingError?.(live, e)
Expand Down
24 changes: 24 additions & 0 deletions browser/lib/src/store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,16 @@ type ConnectionStateCallback = (connected: boolean) => void;
type SyncStatusCallback = (status: StoreSyncStatus) => void;
type CommitLogCallback = (entries: CommitLogEntry[]) => void;

export interface RepeatedCommitFailure {
subject: string;
error: Error;
failures: number;
/** Origin the writes are sent to. */
server: string;
}

type RepeatedCommitFailureCallback = (failure: RepeatedCommitFailure) => void;

type ServerURLCallback = (serverURL: string) => void;
type DriveCallback = (drive: string) => void;

Expand Down Expand Up @@ -299,6 +309,11 @@ export enum StoreEvents {
* Use `store.on(StoreEvents.Indexing, driveSubject, callback)`.
*/
Indexing = 'indexing',
/**
* A queued write has failed several drains in a row and is still being
* retried. Meant for error reporting, not for the user.
*/
CommitRepeatedlyFailing = 'commit-repeatedly-failing',
}

export interface ImportJsonADOptions {
Expand Down Expand Up @@ -374,6 +389,7 @@ type StoreEventHandlers = {
[StoreEvents.CommitLogChanged]: CommitLogCallback;
[StoreEvents.ResourceUpdated]: ResourceCallback;
[StoreEvents.Error]: ErrorCallback;
[StoreEvents.CommitRepeatedlyFailing]: RepeatedCommitFailureCallback;
/** Only used via `on(Indexing, drive, cb)`; not emitted through EventManager. */
[StoreEvents.Indexing]: IndexingStatusCallback;
};
Expand Down Expand Up @@ -1376,6 +1392,14 @@ export class Store {

return isUnrecoverableCommitError(msg, code);
},
onRepeatedFailure: (entry, e) => {
this.eventManager.emit(StoreEvents.CommitRepeatedlyFailing, {
subject: entry.subject,
error: e instanceof Error ? e : new Error(String(e)),
failures: entry.failures ?? 0,
server: this.getServerUrl(),
});
},
onBlocked: (entry, e) => {
const msg = e instanceof Error ? e.message : String(e);
// Stopped retrying, but the entry stays queued + visible. Tell the
Expand Down