From 064068829b4d986b71cca9fd7786f9469dfd14c1 Mon Sep 17 00:00:00 2001 From: Matt Rubashkin Date: Tue, 28 Jul 2026 16:50:06 -0700 Subject: [PATCH 1/7] feat: add opt-in incremental element data subscription to the client Adds subscribeToIncrementalElementData, which advertises incremental (append-semantics) delivery by posting the existing wb:plugin:element:subscribe:data message with a { mode: 'incremental' } capability option, and delivers WorkbookElementDataChunk envelopes ({ data, offset, isComplete, totalRows? }) to its callback. Hosts without incremental support ignore the extra subscribe argument and keep sending cumulative WorkbookElementData payloads; those are detected (every value in a legacy payload is a column array, so typed non-array offset/isComplete/data fields unambiguously identify the envelope) and normalized into replace-everything chunks at offset 0, so consumers need no branching code. This defines the plugin half of the protocol and is a safe no-op against current hosts. All existing methods, events, and types are unchanged. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UtbehD5eW7ci5awuCz1QcM --- .../src/client/__tests__/initialize.test.ts | 60 +++++++++++++++++++ packages/plugin-sdk/src/client/initialize.ts | 46 ++++++++++++++ packages/plugin-sdk/src/types.ts | 33 ++++++++++ 3 files changed, 139 insertions(+) diff --git a/packages/plugin-sdk/src/client/__tests__/initialize.test.ts b/packages/plugin-sdk/src/client/__tests__/initialize.test.ts index aa7ff0f..d6797a5 100644 --- a/packages/plugin-sdk/src/client/__tests__/initialize.test.ts +++ b/packages/plugin-sdk/src/client/__tests__/initialize.test.ts @@ -638,6 +638,66 @@ describe('initialize', () => { expect(callback).not.toHaveBeenCalled(); }); + it('subscribeToIncrementalElementData subscribes with the incremental capability, dispatches chunks, and unsubscribes', () => { + const callback = vi.fn(); + const unsub = client.elements.subscribeToIncrementalElementData( + 'el1', + callback, + ); + + const sub = findPostMessage( + postMessageSpy, + 'wb:plugin:element:subscribe:data', + ); + expect(sub?.data.args).toEqual(['el1', { mode: 'incremental' }]); + + const chunk = { + data: { c1: [1, 2, 3] }, + offset: 0, + isComplete: false, + totalRows: 6, + }; + sendWindowMessage({ + type: 'wb:plugin:element:el1:data', + result: chunk, + error: null, + }); + expect(callback).toHaveBeenCalledWith(chunk); + + postMessageSpy.mockClear(); + callback.mockClear(); + unsub(); + const unsubMsg = findPostMessage( + postMessageSpy, + 'wb:plugin:element:unsubscribe:data', + ); + expect(unsubMsg?.data.args).toEqual(['el1']); + + sendWindowMessage({ + type: 'wb:plugin:element:el1:data', + result: chunk, + error: null, + }); + expect(callback).not.toHaveBeenCalled(); + }); + + it('subscribeToIncrementalElementData normalizes legacy cumulative payloads into replace chunks at offset 0', () => { + const callback = vi.fn(); + client.elements.subscribeToIncrementalElementData('el1', callback); + + const legacyData = { c1: [1, 2, 3], c2: ['a', 'b', 'c'] }; + sendWindowMessage({ + type: 'wb:plugin:element:el1:data', + result: legacyData, + error: null, + }); + expect(callback).toHaveBeenCalledWith({ + data: legacyData, + offset: 0, + isComplete: false, + }); + }); + it('fetchMoreElementData posts wb:plugin:element:fetch-more', () => { client.elements.fetchMoreElementData('el1'); const msg = findPostMessage( diff --git a/packages/plugin-sdk/src/client/initialize.ts b/packages/plugin-sdk/src/client/initialize.ts index 8f9c9fe..ee62b38 100644 --- a/packages/plugin-sdk/src/client/initialize.ts +++ b/packages/plugin-sdk/src/client/initialize.ts @@ -4,11 +4,30 @@ import { PluginMessageResponse, PluginStyle, UrlParameter, + WorkbookElementData, + WorkbookElementDataChunk, WorkbookSelection, WorkbookVariable, } from '../types'; import { validateConfigId } from '../utils/error'; +// Every value in a legacy cumulative WorkbookElementData payload is a column +// array, so typed non-array `offset`/`isComplete`/`data` fields can only come +// from the incremental chunk envelope. +function isElementDataChunk( + result: WorkbookElementData | WorkbookElementDataChunk, +): result is WorkbookElementDataChunk { + const chunk = result as Partial; + return ( + result != null && + typeof chunk.offset === 'number' && + typeof chunk.isComplete === 'boolean' && + typeof chunk.data === 'object' && + chunk.data !== null && + !Array.isArray(chunk.data) + ); +} + export function initialize(): PluginInstance { const pluginConfig: Partial> = { config: {} as T, @@ -255,6 +274,33 @@ export function initialize(): PluginInstance { void execPromise('wb:plugin:element:unsubscribe:data', configId); }; }, + subscribeToIncrementalElementData(configId, callback) { + validateConfigId(configId, 'element'); + const eventName = `wb:plugin:element:${configId}:data`; + const onData = ( + result: WorkbookElementData | WorkbookElementDataChunk, + ) => { + if (isElementDataChunk(result)) { + callback(result); + } else { + // A host without incremental support ignores the subscribe + // options and keeps sending cumulative payloads. Deliver those as + // replace-everything chunks so consumers behave identically + // against either host. Legacy hosts never signal completion, so + // isComplete stays false. + callback({ data: result, offset: 0, isComplete: false }); + } + }; + on(eventName, onData); + void execPromise('wb:plugin:element:subscribe:data', configId, { + mode: 'incremental', + }); + + return () => { + off(eventName, onData); + void execPromise('wb:plugin:element:unsubscribe:data', configId); + }; + }, fetchMoreElementData(configId) { validateConfigId(configId, 'element'); void execPromise('wb:plugin:element:fetch-more', configId); diff --git a/packages/plugin-sdk/src/types.ts b/packages/plugin-sdk/src/types.ts index f6fecc6..37d5369 100644 --- a/packages/plugin-sdk/src/types.ts +++ b/packages/plugin-sdk/src/types.ts @@ -70,6 +70,21 @@ export interface WorkbookElementData { [colId: string]: any[]; } +/** + * A chunk of rows delivered through an incremental element data subscription + * @typedef {object} WorkbookElementDataChunk + * @property {WorkbookElementData} data Rows contained in this chunk only + * @property {number} offset Absolute row offset of the first row in this chunk + * @property {boolean} isComplete True when no more rows are available to fetch + * @property {(number | undefined)} totalRows Total rows in the source element, if known + */ +export interface WorkbookElementDataChunk { + data: WorkbookElementData; + offset: number; + isComplete: boolean; + totalRows?: number; +} + /** * Column data * @typedef {object} WorkbookElementColumn @@ -383,6 +398,24 @@ export interface PluginInstance { callback: (data: WorkbookElementData) => void, ): Unsubscriber; + /** + * Subscriber for the data within a given sheet, delivered incrementally. + * Advertises incremental (append-semantics) delivery to the host: hosts + * that support it deliver each page as a chunk of new rows at an absolute + * row offset, while hosts that do not silently keep sending cumulative + * payloads, which are delivered as replace-everything chunks at offset 0 + * with isComplete false. Callers can treat both hosts identically. This + * method defines the plugin half of the protocol and behaves like + * subscribeToElementData against hosts without incremental support. + * @param {string} configId ID from config of type: 'element' + * @callback callback Function to call with each chunk of data + * @returns {Unsubscriber} A callable unsubscriber to changes in the data + */ + subscribeToIncrementalElementData( + configId: string, + callback: (chunk: WorkbookElementDataChunk) => void, + ): Unsubscriber; + /** * Ask sigma to load more data * @param {string} configId ID from config of type: 'element' From fc5c421c917b3414bea904b2ae69816bacb0b5bc Mon Sep 17 00:00:00 2001 From: Matt Rubashkin Date: Tue, 28 Jul 2026 16:50:24 -0700 Subject: [PATCH 2/7] feat: add useIncrementalElementData React hook MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accumulates WorkbookElementDataChunk payloads internally and returns [data, loadMore, { rowCount, isComplete, totalRows }], making migration from usePaginatedElementData a one-line hook swap. Assembly trusts each chunk's absolute offset — rows before it are kept, rows at or after it are overwritten — so overlapping or re-sent chunks apply idempotently and cumulative payloads from hosts without incremental support (which arrive normalized to offset 0) replace state wholesale, matching today's behavior exactly with no branching in plugin code. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UtbehD5eW7ci5awuCz1QcM --- .../src/react/__tests__/hooks.test.tsx | 131 ++++++++++++++++++ packages/plugin-sdk/src/react/hooks.ts | 94 +++++++++++++ 2 files changed, 225 insertions(+) diff --git a/packages/plugin-sdk/src/react/__tests__/hooks.test.tsx b/packages/plugin-sdk/src/react/__tests__/hooks.test.tsx index 99f4cf7..3e6e6b8 100644 --- a/packages/plugin-sdk/src/react/__tests__/hooks.test.tsx +++ b/packages/plugin-sdk/src/react/__tests__/hooks.test.tsx @@ -11,6 +11,7 @@ import { useEditorPanelConfig, useElementColumns, useElementData, + useIncrementalElementData, useInteraction, useLoadingState, usePaginatedElementData, @@ -272,6 +273,136 @@ describe('react/hooks', () => { }); }); + describe('useIncrementalElementData', () => { + it('subscribes to incremental data and concatenates chunks by offset', () => { + const sub = stubSubscription( + client.elements, + 'subscribeToIncrementalElementData', + ); + const { result } = renderHook(() => useIncrementalElementData('el1'), { + wrapper: withProvider(client), + }); + expect(sub.spy).toHaveBeenCalledWith('el1', expect.any(Function)); + expect(result.current[0]).toEqual({}); + expect(result.current[2]).toEqual({ rowCount: 0, isComplete: false }); + + act(() => + sub.emit({ + data: { c1: [1, 2], c2: ['a', 'b'] }, + offset: 0, + isComplete: false, + totalRows: 4, + }), + ); + act(() => + sub.emit({ + data: { c1: [3, 4], c2: ['c', 'd'] }, + offset: 2, + isComplete: true, + }), + ); + + expect(result.current[0]).toEqual({ + c1: [1, 2, 3, 4], + c2: ['a', 'b', 'c', 'd'], + }); + expect(result.current[2]).toEqual({ + rowCount: 4, + isComplete: true, + totalRows: 4, + }); + }); + + it('applies overlapping chunks idempotently by trusting the offset', () => { + const sub = stubSubscription( + client.elements, + 'subscribeToIncrementalElementData', + ); + const { result } = renderHook(() => useIncrementalElementData('el1'), { + wrapper: withProvider(client), + }); + + act(() => + sub.emit({ data: { c1: [1, 2, 3] }, offset: 0, isComplete: false }), + ); + const overlapping = { + data: { c1: [3, 4] }, + offset: 2, + isComplete: false, + }; + act(() => sub.emit(overlapping)); + act(() => sub.emit(overlapping)); + + expect(result.current[0]).toEqual({ c1: [1, 2, 3, 4] }); + expect(result.current[2].rowCount).toBe(4); + }); + + it('matches legacy cumulative payloads exactly when the host lacks incremental support', () => { + // Exercise the real client end-to-end: the host ignores the capability + // option and re-sends the entire accumulated data set on every page. + const { result } = renderHook(() => useIncrementalElementData('el1'), { + wrapper: withProvider(client), + }); + + const sendLegacyData = (data: Record) => { + window.dispatchEvent( + new MessageEvent('message', { + data: { + type: 'wb:plugin:element:el1:data', + result: data, + error: null, + }, + }), + ); + }; + + act(() => sendLegacyData({ c1: [1, 2, 3] })); + act(() => sendLegacyData({ c1: [1, 2, 3, 4, 5, 6] })); + + expect(result.current[0]).toEqual({ c1: [1, 2, 3, 4, 5, 6] }); + expect(result.current[2]).toEqual({ rowCount: 6, isComplete: false }); + }); + + it('returns a loadMore callback that fetches more data', () => { + stubSubscription( + client.elements, + 'subscribeToIncrementalElementData', + ); + const fetchSpy = vi.spyOn(client.elements, 'fetchMoreElementData'); + const { result } = renderHook(() => useIncrementalElementData('el1'), { + wrapper: withProvider(client), + }); + act(() => result.current[1]()); + expect(fetchSpy).toHaveBeenCalledWith('el1'); + }); + + it('does not subscribe and loadMore is a no-op when configId is falsy', () => { + const subSpy = vi.spyOn( + client.elements, + 'subscribeToIncrementalElementData', + ); + const fetchSpy = vi.spyOn(client.elements, 'fetchMoreElementData'); + const { result } = renderHook(() => useIncrementalElementData(''), { + wrapper: withProvider(client), + }); + expect(subSpy).not.toHaveBeenCalled(); + act(() => result.current[1]()); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('unsubscribes on unmount', () => { + const sub = stubSubscription( + client.elements, + 'subscribeToIncrementalElementData', + ); + const { unmount } = renderHook(() => useIncrementalElementData('el1'), { + wrapper: withProvider(client), + }); + unmount(); + expect(sub.unsubscribe).toHaveBeenCalled(); + }); + }); + describe('useConfig', () => { it('returns the full config when no key is provided', () => { vi.spyOn(client.config, 'get').mockReturnValue({ a: 1 }); diff --git a/packages/plugin-sdk/src/react/hooks.ts b/packages/plugin-sdk/src/react/hooks.ts index f3b9700..2a9ee93 100644 --- a/packages/plugin-sdk/src/react/hooks.ts +++ b/packages/plugin-sdk/src/react/hooks.ts @@ -6,6 +6,7 @@ import { CustomPluginConfigOptions, WorkbookElementColumns, WorkbookElementData, + WorkbookElementDataChunk, WorkbookSelection, WorkbookVariable, PluginStyle, @@ -130,6 +131,99 @@ export function usePaginatedElementData( return [data, loadMore]; } +/** + * Progress metadata for incrementally accumulated element data + * @typedef {object} IncrementalElementDataInfo + * @property {number} rowCount Number of rows accumulated so far + * @property {boolean} isComplete True once the host reports no more rows are available + * @property {(number | undefined)} totalRows Total rows in the source element, if the host reports it + */ +export interface IncrementalElementDataInfo { + rowCount: number; + isComplete: boolean; + totalRows?: number; +} + +interface IncrementalElementDataState { + data: WorkbookElementData; + info: IncrementalElementDataInfo; +} + +const INITIAL_INCREMENTAL_STATE: IncrementalElementDataState = { + data: {}, + info: { rowCount: 0, isComplete: false }, +}; + +// Applies a chunk by trusting its absolute offset: rows before the offset are +// kept and rows at or after it are overwritten, so re-sent or overlapping +// chunks apply idempotently. Cumulative payloads from hosts without +// incremental support arrive normalized to offset 0 and therefore replace the +// accumulated state wholesale, preserving today's replace-semantics. +function applyElementDataChunk( + prev: IncrementalElementDataState, + chunk: WorkbookElementDataChunk, +): IncrementalElementDataState { + const data: WorkbookElementData = {}; + for (const colId of Object.keys(chunk.data)) { + data[colId] = (prev.data[colId] ?? []) + .slice(0, chunk.offset) + .concat(chunk.data[colId]); + } + const rowCount = Object.values(data).reduce( + (max, rows) => Math.max(max, rows.length), + 0, + ); + return { + data, + info: { + rowCount, + isComplete: chunk.isComplete, + totalRows: chunk.totalRows ?? prev.info.totalRows, + }, + }; +} + +/** + * Provides the data values from the corresponding config element, accumulated + * from incremental chunks, with a callback to fetch more in chunks of 25_000 + * data points. Drop-in replacement for usePaginatedElementData that avoids + * re-delivering already received rows when the host supports incremental + * delivery, and behaves identically to usePaginatedElementData when it does + * not (isComplete then remains false since legacy hosts never signal + * completion). + * @param {string} configId ID from the config for fetching incremental + * element data, with type: 'element' + * @returns {[WorkbookElementData, Function, IncrementalElementDataInfo]} + * Accumulated Element Data for the config element, a callback to fetch more + * data, and progress metadata + */ +export function useIncrementalElementData( + configId: string, +): [WorkbookElementData, () => void, IncrementalElementDataInfo] { + const client = usePlugin(); + const [state, setState] = React.useState( + INITIAL_INCREMENTAL_STATE, + ); + + const loadMore = React.useCallback(() => { + if (configId) { + client.elements.fetchMoreElementData(configId); + } + }, [configId, client.elements]); + + React.useEffect(() => { + if (configId) { + setState(INITIAL_INCREMENTAL_STATE); + return client.elements.subscribeToIncrementalElementData( + configId, + chunk => setState(prev => applyElementDataChunk(prev, chunk)), + ); + } + }, [client, configId]); + + return [state.data, loadMore, state.info]; +} + /** * Provides the latest value for entire config or certain key within the config * @param {string} key Key within Plugin Config, optional From 4d465a52feeb84f46fd5be50679366deaf3b596b Mon Sep 17 00:00:00 2001 From: Matt Rubashkin Date: Tue, 28 Jul 2026 16:50:30 -0700 Subject: [PATCH 3/7] docs: document useIncrementalElementData and incremental subscription Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UtbehD5eW7ci5awuCz1QcM --- packages/plugin-sdk/README.md | 53 +++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/packages/plugin-sdk/README.md b/packages/plugin-sdk/README.md index 7b0a0d2..9c71c35 100644 --- a/packages/plugin-sdk/README.md +++ b/packages/plugin-sdk/README.md @@ -795,6 +795,59 @@ interface WorkbookElementData { } ``` +#### useIncrementalElementData() + +Drop-in replacement for `usePaginatedElementData()` that opts in to +incremental (append-semantics) data delivery. When the host supports it, each +page is delivered as a chunk containing only the new rows, so loading a large +element costs each row once instead of re-delivering the entire accumulated +data set on every page. When the host does not support incremental delivery, +the hook transparently falls back to today's cumulative behavior — no +branching code is required in the plugin. + +```ts +function useIncrementalElementData( + configId: string, +): [WorkbookElementData, () => void, IncrementalElementDataInfo]; +``` + +Arguments + +- `configId : string` - A workbook element’s unique identifier from the plugin config. + +Returns the accumulated row data from the specified element, a callback for +fetching more data, and progress metadata: + +```ts +interface IncrementalElementDataInfo { + rowCount: number; // rows accumulated so far + isComplete: boolean; // true once the host reports no more rows (always false on hosts without incremental support) + totalRows?: number; // total rows in the source element, if the host reports it +} +``` + +Example + +```ts +const [data, loadMore, { rowCount, isComplete }] = + useIncrementalElementData('source'); +``` + +Framework Agnostic Usage + +```ts +const unsubscribe = client.elements.subscribeToIncrementalElementData( + 'source', + chunk => { + // chunk.data contains only this chunk's rows; chunk.offset is the + // absolute row offset to apply them at. Hosts without incremental + // support deliver their cumulative payloads as replace-everything + // chunks at offset 0. + applyRowsAtOffset(chunk.data, chunk.offset); + }, +); +``` + #### useVariable() Returns a given variable's value and a setter to update that variable From 575f7aa46eef49ad6593c7ea7fdfceff40649cdb Mon Sep 17 00:00:00 2001 From: Matt Rubashkin Date: Tue, 28 Jul 2026 16:50:30 -0700 Subject: [PATCH 4/7] chore: bump version to 1.3.0 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UtbehD5eW7ci5awuCz1QcM --- CHANGELOG.md | 14 ++++++++++++++ package.json | 2 +- packages/plugin-sdk/package.json | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e5b4ae..86ee95e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +## v1.3.0 (July 28th, 2026) + +- Added opt-in incremental (append-semantics) element data delivery: + - `subscribeToIncrementalElementData` on the client, which advertises + incremental delivery to the host and invokes its callback with + `WorkbookElementDataChunk` envelopes (`data`, `offset`, `isComplete`, + `totalRows`). + - `useIncrementalElementData` React hook, which accumulates chunks + internally and is a drop-in replacement for `usePaginatedElementData`. + - Hosts that do not support incremental delivery are unaffected: their + cumulative payloads are transparently delivered as replace-everything + chunks at offset 0, so plugins using the new API work against both host + behaviors. All existing APIs are unchanged. + ## v1.0.0 (September 23rd, 2022) `@sigmacomputing/plugin` has moved to https://github.com/sigmacomputing/plugin and diff --git a/package.json b/package.json index 4dcdf85..c601e76 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@sigmacomputing/plugin-sdk-root", - "version": "1.2.0", + "version": "1.3.0", "private": true, "description": "Sigma Computing Plugin Client SDK", "license": "MIT", diff --git a/packages/plugin-sdk/package.json b/packages/plugin-sdk/package.json index 36830b2..a5f3771 100644 --- a/packages/plugin-sdk/package.json +++ b/packages/plugin-sdk/package.json @@ -1,6 +1,6 @@ { "name": "@sigmacomputing/plugin", - "version": "1.2.0", + "version": "1.3.0", "description": "Sigma Computing Plugin Client SDK", "license": "MIT", "type": "module", From 3b1b11b74644826a8f0165173627b0bbdd607d65 Mon Sep 17 00:00:00 2001 From: Matt Rubashkin Date: Tue, 28 Jul 2026 17:31:15 -0700 Subject: [PATCH 5/7] fix: harden incremental chunk assembly against partial and malformed chunks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Findings from a multi-perspective review of the initial implementation: - A chunk omitting a column — or an empty terminal chunk that only flips isComplete — silently dropped accumulated data. Chunks at offset > 0 now carry the accumulated columns forward; offset 0 remains a wholesale replace, preserving degraded-mode and refresh semantics. - Rows now always land at their absolute offset: the head is padded when a column first appears mid-stream or a host skips ahead, instead of silently compacting the gap and misaligning rows. - Column ids are applied own-property-safely: '__proto__' is skipped (a crafted payload could otherwise reparent the accumulator) and ids like 'constructor' can no longer collide with inherited members and throw. - The chunk type guard requires offset to be a non-negative integer, so NaN/negative/fractional offsets degrade to legacy normalization instead of corrupting assembly. - totalRows re-baselines on an offset-0 restart rather than carrying a stale value across a refresh; hook state resets when configId goes falsy. - Docs now warn prominently that isComplete stays false forever on hosts without incremental support (never gate load-more loops on it alone), state the per-element single-subscription-mode constraint, and spell out the chunk invariants hosts must satisfy. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UtbehD5eW7ci5awuCz1QcM --- packages/plugin-sdk/README.md | 13 ++- .../src/client/__tests__/initialize.test.ts | 22 ++++ packages/plugin-sdk/src/client/initialize.ts | 7 +- .../src/react/__tests__/hooks.test.tsx | 104 ++++++++++++++++++ packages/plugin-sdk/src/react/hooks.ts | 40 +++++-- packages/plugin-sdk/src/types.ts | 20 +++- 6 files changed, 187 insertions(+), 19 deletions(-) diff --git a/packages/plugin-sdk/README.md b/packages/plugin-sdk/README.md index 9c71c35..76ab7ec 100644 --- a/packages/plugin-sdk/README.md +++ b/packages/plugin-sdk/README.md @@ -821,11 +821,17 @@ fetching more data, and progress metadata: ```ts interface IncrementalElementDataInfo { rowCount: number; // rows accumulated so far - isComplete: boolean; // true once the host reports no more rows (always false on hosts without incremental support) + isComplete: boolean; // true once the host reports no more rows totalRows?: number; // total rows in the source element, if the host reports it } ``` +> **Warning:** on hosts without incremental support, `isComplete` stays +> `false` forever — completion is a signal only incremental-capable hosts can +> send. Never drive an auto-load loop or a "load more" affordance from +> `isComplete` alone; use `rowCount` to detect whether a fetch actually made +> progress (if it stops growing, there is no more data). + Example ```ts @@ -848,6 +854,11 @@ const unsubscribe = client.elements.subscribeToIncrementalElementData( ); ``` +Use one subscription style per element: the delivery mode belongs to the +(plugin, element) subscription, so mixing `subscribeToElementData` and +`subscribeToIncrementalElementData` (or their hooks) on the same config +element is unsupported. + #### useVariable() Returns a given variable's value and a setter to update that variable diff --git a/packages/plugin-sdk/src/client/__tests__/initialize.test.ts b/packages/plugin-sdk/src/client/__tests__/initialize.test.ts index d6797a5..ad4035e 100644 --- a/packages/plugin-sdk/src/client/__tests__/initialize.test.ts +++ b/packages/plugin-sdk/src/client/__tests__/initialize.test.ts @@ -698,6 +698,28 @@ describe('initialize', () => { }); }); + it('subscribeToIncrementalElementData treats envelopes with malformed offsets as legacy payloads', () => { + const callback = vi.fn(); + client.elements.subscribeToIncrementalElementData('el1', callback); + + for (const offset of [-1, 1.5, Number.NaN]) { + callback.mockClear(); + const malformed = { data: { c1: [1] }, offset, isComplete: true }; + sendWindowMessage({ + type: 'wb:plugin:element:el1:data', + result: malformed, + error: null, + }); + // Not recognized as a chunk: falls back to replace-at-0 normalization + // instead of corrupting chunk assembly downstream. + expect(callback).toHaveBeenCalledWith({ + data: malformed, + offset: 0, + isComplete: false, + }); + } + }); + it('fetchMoreElementData posts wb:plugin:element:fetch-more', () => { client.elements.fetchMoreElementData('el1'); const msg = findPostMessage( diff --git a/packages/plugin-sdk/src/client/initialize.ts b/packages/plugin-sdk/src/client/initialize.ts index ee62b38..73fb902 100644 --- a/packages/plugin-sdk/src/client/initialize.ts +++ b/packages/plugin-sdk/src/client/initialize.ts @@ -13,14 +13,17 @@ import { validateConfigId } from '../utils/error'; // Every value in a legacy cumulative WorkbookElementData payload is a column // array, so typed non-array `offset`/`isComplete`/`data` fields can only come -// from the incremental chunk envelope. +// from the incremental chunk envelope. Offsets must be non-negative integers; +// a payload with a malformed offset is treated as legacy data rather than +// letting a NaN/negative/fractional value corrupt chunk assembly downstream. function isElementDataChunk( result: WorkbookElementData | WorkbookElementDataChunk, ): result is WorkbookElementDataChunk { const chunk = result as Partial; return ( result != null && - typeof chunk.offset === 'number' && + Number.isInteger(chunk.offset) && + (chunk.offset as number) >= 0 && typeof chunk.isComplete === 'boolean' && typeof chunk.data === 'object' && chunk.data !== null && diff --git a/packages/plugin-sdk/src/react/__tests__/hooks.test.tsx b/packages/plugin-sdk/src/react/__tests__/hooks.test.tsx index 3e6e6b8..ccdc0e5 100644 --- a/packages/plugin-sdk/src/react/__tests__/hooks.test.tsx +++ b/packages/plugin-sdk/src/react/__tests__/hooks.test.tsx @@ -337,6 +337,110 @@ describe('react/hooks', () => { expect(result.current[2].rowCount).toBe(4); }); + it('preserves accumulated data when a terminal chunk is empty or omits a column', () => { + const sub = stubSubscription( + client.elements, + 'subscribeToIncrementalElementData', + ); + const { result } = renderHook(() => useIncrementalElementData('el1'), { + wrapper: withProvider(client), + }); + + act(() => + sub.emit({ + data: { c1: [1, 2], c2: ['a', 'b'] }, + offset: 0, + isComplete: false, + }), + ); + // A chunk omitting c2 must not delete c2's accumulated rows. + act(() => + sub.emit({ data: { c1: [3, 4] }, offset: 2, isComplete: false }), + ); + // An empty terminal chunk only flips isComplete. + act(() => sub.emit({ data: {}, offset: 4, isComplete: true })); + + expect(result.current[0]).toEqual({ c1: [1, 2, 3, 4], c2: ['a', 'b'] }); + expect(result.current[2]).toEqual({ rowCount: 4, isComplete: true }); + }); + + it('replaces state wholesale and re-baselines totalRows on an offset-0 restart', () => { + const sub = stubSubscription( + client.elements, + 'subscribeToIncrementalElementData', + ); + const { result } = renderHook(() => useIncrementalElementData('el1'), { + wrapper: withProvider(client), + }); + + act(() => + sub.emit({ + data: { c1: [1, 2, 3] }, + offset: 0, + isComplete: true, + totalRows: 3, + }), + ); + // Host refresh: new column set, no totalRows reported. + act(() => + sub.emit({ data: { c9: ['x'] }, offset: 0, isComplete: false }), + ); + + expect(result.current[0]).toEqual({ c9: ['x'] }); + expect(result.current[2]).toEqual({ rowCount: 1, isComplete: false }); + }); + + it('keeps rows at their absolute offsets for gaps and columns appearing mid-stream', () => { + const sub = stubSubscription( + client.elements, + 'subscribeToIncrementalElementData', + ); + const { result } = renderHook(() => useIncrementalElementData('el1'), { + wrapper: withProvider(client), + }); + + act(() => + sub.emit({ data: { c1: [1, 2] }, offset: 0, isComplete: false }), + ); + // c2 first appears at offset 2; its rows must not land at index 0. + act(() => + sub.emit({ + data: { c1: [3, 4], c2: ['c', 'd'] }, + offset: 2, + isComplete: false, + }), + ); + + expect(result.current[0].c1).toEqual([1, 2, 3, 4]); + expect(result.current[0].c2.length).toBe(4); + expect(result.current[0].c2[2]).toBe('c'); + expect(result.current[0].c2[3]).toBe('d'); + expect(result.current[0].c2[0]).toBeUndefined(); + }); + + it('tolerates column ids that collide with Object.prototype members', () => { + const sub = stubSubscription( + client.elements, + 'subscribeToIncrementalElementData', + ); + const { result } = renderHook(() => useIncrementalElementData('el1'), { + wrapper: withProvider(client), + }); + + // JSON.parse creates '__proto__' as an own enumerable property, which + // is exactly what a (hostile or buggy) wire payload can carry. + const data = JSON.parse( + '{"constructor": [1, 2], "toString": [3, 4], "__proto__": [5, 6]}', + ); + act(() => sub.emit({ data, offset: 0, isComplete: true })); + + expect(result.current[0]['constructor']).toEqual([1, 2]); + expect(result.current[0]['toString']).toEqual([3, 4]); + // '__proto__' is skipped rather than reparenting the accumulator. + expect(Object.getPrototypeOf(result.current[0])).toBe(Object.prototype); + expect(result.current[2].rowCount).toBe(2); + }); + it('matches legacy cumulative payloads exactly when the host lacks incremental support', () => { // Exercise the real client end-to-end: the host ignores the capability // option and re-sends the entire accumulated data set on every page. diff --git a/packages/plugin-sdk/src/react/hooks.ts b/packages/plugin-sdk/src/react/hooks.ts index 2a9ee93..ccbe030 100644 --- a/packages/plugin-sdk/src/react/hooks.ts +++ b/packages/plugin-sdk/src/react/hooks.ts @@ -156,18 +156,30 @@ const INITIAL_INCREMENTAL_STATE: IncrementalElementDataState = { // Applies a chunk by trusting its absolute offset: rows before the offset are // kept and rows at or after it are overwritten, so re-sent or overlapping -// chunks apply idempotently. Cumulative payloads from hosts without -// incremental support arrive normalized to offset 0 and therefore replace the -// accumulated state wholesale, preserving today's replace-semantics. +// chunks apply idempotently. An offset-0 chunk replaces the accumulated state +// wholesale (host refresh, or a cumulative payload from a host without +// incremental support, normalized upstream); a chunk at offset > 0 starts +// from the accumulated columns, so a chunk that omits a column — or an empty +// terminal chunk that only flips isComplete — cannot drop received rows. function applyElementDataChunk( prev: IncrementalElementDataState, chunk: WorkbookElementDataChunk, ): IncrementalElementDataState { - const data: WorkbookElementData = {}; + const data: WorkbookElementData = chunk.offset === 0 ? {} : { ...prev.data }; for (const colId of Object.keys(chunk.data)) { - data[colId] = (prev.data[colId] ?? []) - .slice(0, chunk.offset) - .concat(chunk.data[colId]); + // '__proto__' is never a real column id; assigning it would swap the + // object's prototype instead of adding a column. + if (colId === '__proto__') continue; + // Own-property check so inherited members (e.g. a column named + // 'constructor') can never be mistaken for accumulated rows. + const prevRows = Object.prototype.hasOwnProperty.call(data, colId) + ? data[colId] + : []; + const head = prevRows.slice(0, chunk.offset); + // Pad so rows always land at their absolute offset, even for a column + // first appearing mid-stream or a host that skips ahead. + head.length = chunk.offset; + data[colId] = head.concat(chunk.data[colId]); } const rowCount = Object.values(data).reduce( (max, rows) => Math.max(max, rows.length), @@ -178,7 +190,11 @@ function applyElementDataChunk( info: { rowCount, isComplete: chunk.isComplete, - totalRows: chunk.totalRows ?? prev.info.totalRows, + // An offset-0 restart re-baselines the total instead of carrying a + // stale value from the previous load. + totalRows: + chunk.totalRows ?? + (chunk.offset === 0 ? undefined : prev.info.totalRows), }, }; } @@ -189,8 +205,10 @@ function applyElementDataChunk( * data points. Drop-in replacement for usePaginatedElementData that avoids * re-delivering already received rows when the host supports incremental * delivery, and behaves identically to usePaginatedElementData when it does - * not (isComplete then remains false since legacy hosts never signal - * completion). + * not. IMPORTANT: hosts without incremental support never signal completion, + * so isComplete stays false forever there — never drive an auto-load loop or + * a "load more" affordance from isComplete alone; use rowCount to detect + * whether a fetch actually made progress. * @param {string} configId ID from the config for fetching incremental * element data, with type: 'element' * @returns {[WorkbookElementData, Function, IncrementalElementDataInfo]} @@ -212,8 +230,8 @@ export function useIncrementalElementData( }, [configId, client.elements]); React.useEffect(() => { + setState(INITIAL_INCREMENTAL_STATE); if (configId) { - setState(INITIAL_INCREMENTAL_STATE); return client.elements.subscribeToIncrementalElementData( configId, chunk => setState(prev => applyElementDataChunk(prev, chunk)), diff --git a/packages/plugin-sdk/src/types.ts b/packages/plugin-sdk/src/types.ts index 37d5369..f23f263 100644 --- a/packages/plugin-sdk/src/types.ts +++ b/packages/plugin-sdk/src/types.ts @@ -71,10 +71,15 @@ export interface WorkbookElementData { } /** - * A chunk of rows delivered through an incremental element data subscription + * A chunk of rows delivered through an incremental element data subscription. + * Hosts must deliver chunks in non-decreasing offset order (offset 0 restarts + * and replaces all accumulated state), include the subscription's full column + * set in every chunk with all column arrays the same length, and may send an + * empty data object at offset > 0 to update isComplete/totalRows without + * appending rows. * @typedef {object} WorkbookElementDataChunk * @property {WorkbookElementData} data Rows contained in this chunk only - * @property {number} offset Absolute row offset of the first row in this chunk + * @property {number} offset Absolute row offset of the first row in this chunk; a non-negative integer * @property {boolean} isComplete True when no more rows are available to fetch * @property {(number | undefined)} totalRows Total rows in the source element, if known */ @@ -404,9 +409,14 @@ export interface PluginInstance { * that support it deliver each page as a chunk of new rows at an absolute * row offset, while hosts that do not silently keep sending cumulative * payloads, which are delivered as replace-everything chunks at offset 0 - * with isComplete false. Callers can treat both hosts identically. This - * method defines the plugin half of the protocol and behaves like - * subscribeToElementData against hosts without incremental support. + * with isComplete false. Callers can treat both hosts identically, but + * must not assume isComplete ever becomes true: hosts without incremental + * support never signal completion. This method defines the plugin half of + * the protocol and behaves like subscribeToElementData against hosts + * without incremental support. Use one subscription style per element: + * the delivery mode belongs to the (plugin, element) subscription, so + * mixing this with subscribeToElementData on the same configId is + * unsupported. * @param {string} configId ID from config of type: 'element' * @callback callback Function to call with each chunk of data * @returns {Unsubscriber} A callable unsubscriber to changes in the data From d944a2af4208cf3f2ea940f215139c61dc492918 Mon Sep 17 00:00:00 2001 From: Matt Rubashkin Date: Tue, 11 Aug 2026 09:51:00 -0700 Subject: [PATCH 6/7] fix: normalize failed-eval null payloads and widen the chunk type guard Addresses review feedback on #70. - The host sends null when an element's data eval fails. The incremental subscription now normalizes that to an empty replace-at-0 chunk, so it degrades the way the non-incremental subscription does instead of throwing during chunk assembly downstream. - isElementDataChunk now takes unknown and narrows to a non-null object in the body, which is accurate about what actually arrives on the wire. - Adds a hook-level test dispatching result: null end-to-end plus a client-level normalization test. Both fail against the pre-fix code with "TypeError: Cannot convert undefined or null to object". - Drops the CHANGELOG entry. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JS3J7PZCg9pTYfTSNY5UjY --- CHANGELOG.md | 14 -------- .../src/client/__tests__/initialize.test.ts | 17 +++++++++ packages/plugin-sdk/src/client/initialize.ts | 21 ++++++----- .../src/react/__tests__/hooks.test.tsx | 36 +++++++++++++++++++ 4 files changed, 66 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86ee95e..5e5b4ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,17 +1,3 @@ -## v1.3.0 (July 28th, 2026) - -- Added opt-in incremental (append-semantics) element data delivery: - - `subscribeToIncrementalElementData` on the client, which advertises - incremental delivery to the host and invokes its callback with - `WorkbookElementDataChunk` envelopes (`data`, `offset`, `isComplete`, - `totalRows`). - - `useIncrementalElementData` React hook, which accumulates chunks - internally and is a drop-in replacement for `usePaginatedElementData`. - - Hosts that do not support incremental delivery are unaffected: their - cumulative payloads are transparently delivered as replace-everything - chunks at offset 0, so plugins using the new API work against both host - behaviors. All existing APIs are unchanged. - ## v1.0.0 (September 23rd, 2022) `@sigmacomputing/plugin` has moved to https://github.com/sigmacomputing/plugin and diff --git a/packages/plugin-sdk/src/client/__tests__/initialize.test.ts b/packages/plugin-sdk/src/client/__tests__/initialize.test.ts index ad4035e..63595b6 100644 --- a/packages/plugin-sdk/src/client/__tests__/initialize.test.ts +++ b/packages/plugin-sdk/src/client/__tests__/initialize.test.ts @@ -698,6 +698,23 @@ describe('initialize', () => { }); }); + it('subscribeToIncrementalElementData normalizes a null payload into an empty replace chunk', () => { + const callback = vi.fn(); + client.elements.subscribeToIncrementalElementData('el1', callback); + + // The host sends null when the element's data eval fails. + sendWindowMessage({ + type: 'wb:plugin:element:el1:data', + result: null, + error: null, + }); + expect(callback).toHaveBeenCalledWith({ + data: {}, + offset: 0, + isComplete: false, + }); + }); + it('subscribeToIncrementalElementData treats envelopes with malformed offsets as legacy payloads', () => { const callback = vi.fn(); client.elements.subscribeToIncrementalElementData('el1', callback); diff --git a/packages/plugin-sdk/src/client/initialize.ts b/packages/plugin-sdk/src/client/initialize.ts index 73fb902..3c71ec9 100644 --- a/packages/plugin-sdk/src/client/initialize.ts +++ b/packages/plugin-sdk/src/client/initialize.ts @@ -17,11 +17,11 @@ import { validateConfigId } from '../utils/error'; // a payload with a malformed offset is treated as legacy data rather than // letting a NaN/negative/fractional value corrupt chunk assembly downstream. function isElementDataChunk( - result: WorkbookElementData | WorkbookElementDataChunk, + result: unknown, ): result is WorkbookElementDataChunk { + if (typeof result !== 'object' || result === null) return false; const chunk = result as Partial; return ( - result != null && Number.isInteger(chunk.offset) && (chunk.offset as number) >= 0 && typeof chunk.isComplete === 'boolean' && @@ -280,18 +280,23 @@ export function initialize(): PluginInstance { subscribeToIncrementalElementData(configId, callback) { validateConfigId(configId, 'element'); const eventName = `wb:plugin:element:${configId}:data`; - const onData = ( - result: WorkbookElementData | WorkbookElementDataChunk, - ) => { + const onData = (result: unknown) => { if (isElementDataChunk(result)) { callback(result); } else { // A host without incremental support ignores the subscribe // options and keeps sending cumulative payloads. Deliver those as // replace-everything chunks so consumers behave identically - // against either host. Legacy hosts never signal completion, so - // isComplete stays false. - callback({ data: result, offset: 0, isComplete: false }); + // against either host. A host also sends null when the element's + // data eval fails, which normalizes to an empty chunk so it + // degrades the same way the non-incremental subscription does + // rather than throwing during chunk assembly. Legacy hosts never + // signal completion, so isComplete stays false. + callback({ + data: (result ?? {}) as WorkbookElementData, + offset: 0, + isComplete: false, + }); } }; on(eventName, onData); diff --git a/packages/plugin-sdk/src/react/__tests__/hooks.test.tsx b/packages/plugin-sdk/src/react/__tests__/hooks.test.tsx index ccdc0e5..ff2b46d 100644 --- a/packages/plugin-sdk/src/react/__tests__/hooks.test.tsx +++ b/packages/plugin-sdk/src/react/__tests__/hooks.test.tsx @@ -467,6 +467,42 @@ describe('react/hooks', () => { expect(result.current[2]).toEqual({ rowCount: 6, isComplete: false }); }); + it('does not throw when the host reports a failed data eval as null', () => { + // Exercise the real client end-to-end: the host sends null when the + // element's data eval fails, which must degrade to empty data the way + // the non-incremental hooks do rather than throwing. + const { result } = renderHook(() => useIncrementalElementData('el1'), { + wrapper: withProvider(client), + }); + + const sendData = (data: unknown) => { + window.dispatchEvent( + new MessageEvent('message', { + data: { + type: 'wb:plugin:element:el1:data', + result: data, + error: null, + }, + }), + ); + }; + + expect(() => act(() => sendData(null))).not.toThrow(); + expect(result.current[0]).toEqual({}); + expect(result.current[2]).toEqual({ rowCount: 0, isComplete: false }); + + // A failed eval after rows arrived also degrades to empty rather than + // leaving stale rows or throwing, and later data still lands. + act(() => sendData({ c1: [1, 2, 3] })); + expect(() => act(() => sendData(null))).not.toThrow(); + expect(result.current[0]).toEqual({}); + expect(result.current[2].rowCount).toBe(0); + + act(() => sendData({ c1: [4, 5] })); + expect(result.current[0]).toEqual({ c1: [4, 5] }); + expect(result.current[2].rowCount).toBe(2); + }); + it('returns a loadMore callback that fetches more data', () => { stubSubscription( client.elements, From c950614f11b42c3de7e308f442c31bb3276ca25d Mon Sep 17 00:00:00 2001 From: Matt Rubashkin Date: Tue, 11 Aug 2026 13:08:14 -0700 Subject: [PATCH 7/7] fix: drop rows from chunks that start past the accumulated stream Normalizing a failed eval to an empty offset-0 chunk clears the accumulated rows, which makes a host that resumes mid-stream instead of restarting land its rows past everything held. applyElementDataChunk padded that gap to the absolute offset, so a consumer saw holes reported as real rows: four rows in, a failed eval, then a resume at offset 4 produced [<4 holes>, 4, 5] with rowCount 6 and isComplete true. Chunks starting past the accumulated rows now have their rows dropped rather than backfilled. isComplete and totalRows still apply, so the gap surfaces as rowCount < totalRows instead of leaving the host re-sending the same rejected offset forever. Per-column padding is unchanged: a contiguous chunk can still backfill a column that first appears mid-stream. Documents the contract this rests on in WorkbookElementDataChunk: offset may be at most the number of rows delivered so far, and a failed eval arrives as a null payload that terminates the stream, so the host must restart at offset 0 to resume. Both new tests fail against the previous commit with the phantom rows visible in the diff. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JS3J7PZCg9pTYfTSNY5UjY --- .../src/react/__tests__/hooks.test.tsx | 90 +++++++++++++++++++ packages/plugin-sdk/src/react/hooks.ts | 22 ++++- packages/plugin-sdk/src/types.ts | 10 ++- 3 files changed, 119 insertions(+), 3 deletions(-) diff --git a/packages/plugin-sdk/src/react/__tests__/hooks.test.tsx b/packages/plugin-sdk/src/react/__tests__/hooks.test.tsx index ff2b46d..330e451 100644 --- a/packages/plugin-sdk/src/react/__tests__/hooks.test.tsx +++ b/packages/plugin-sdk/src/react/__tests__/hooks.test.tsx @@ -467,6 +467,44 @@ describe('react/hooks', () => { expect(result.current[2]).toEqual({ rowCount: 6, isComplete: false }); }); + it('drops rows from a chunk that starts past the accumulated rows', () => { + const sub = stubSubscription( + client.elements, + 'subscribeToIncrementalElementData', + ); + const { result } = renderHook(() => useIncrementalElementData('el1'), { + wrapper: withProvider(client), + }); + + act(() => + sub.emit({ + data: { c1: [1, 2] }, + offset: 0, + isComplete: false, + totalRows: 6, + }), + ); + // The host skips ahead of everything accumulated. Padding to offset 4 + // would present two holes as real rows, so the chunk's rows are dropped. + act(() => + sub.emit({ + data: { c1: [5, 6] }, + offset: 4, + isComplete: true, + totalRows: 6, + }), + ); + + expect(result.current[0]).toEqual({ c1: [1, 2] }); + // Progress flags still apply, so the gap is visible as rowCount < + // totalRows rather than the host re-sending the rejected offset forever. + expect(result.current[2]).toEqual({ + rowCount: 2, + isComplete: true, + totalRows: 6, + }); + }); + it('does not throw when the host reports a failed data eval as null', () => { // Exercise the real client end-to-end: the host sends null when the // element's data eval fails, which must degrade to empty data the way @@ -503,6 +541,58 @@ describe('react/hooks', () => { expect(result.current[2].rowCount).toBe(2); }); + it('does not fabricate rows when the host resumes mid-stream after a failed eval', () => { + // End-to-end through the real client: a failed eval clears the stream, + // so a host that resumes where it left off instead of restarting at + // offset 0 must not have its gap backfilled with phantom rows. + const { result } = renderHook(() => useIncrementalElementData('el1'), { + wrapper: withProvider(client), + }); + + const sendData = (data: unknown) => { + window.dispatchEvent( + new MessageEvent('message', { + data: { + type: 'wb:plugin:element:el1:data', + result: data, + error: null, + }, + }), + ); + }; + + act(() => + sendData({ + data: { c1: [0, 1] }, + offset: 0, + isComplete: false, + totalRows: 8, + }), + ); + act(() => + sendData({ data: { c1: [2, 3] }, offset: 2, isComplete: false }), + ); + expect(result.current[2].rowCount).toBe(4); + + act(() => sendData(null)); + expect(result.current[0]).toEqual({}); + + act(() => + sendData({ + data: { c1: [4, 5] }, + offset: 4, + isComplete: true, + totalRows: 8, + }), + ); + + // Without the contiguity guard this is [<4 holes>, 4, 5] with + // rowCount 6 — four fabricated rows reported as real data. + expect(result.current[0]).toEqual({}); + expect(result.current[2].rowCount).toBe(0); + expect(result.current[2].totalRows).toBe(8); + }); + it('returns a loadMore callback that fetches more data', () => { stubSubscription( client.elements, diff --git a/packages/plugin-sdk/src/react/hooks.ts b/packages/plugin-sdk/src/react/hooks.ts index ccbe030..e20e19b 100644 --- a/packages/plugin-sdk/src/react/hooks.ts +++ b/packages/plugin-sdk/src/react/hooks.ts @@ -165,6 +165,23 @@ function applyElementDataChunk( prev: IncrementalElementDataState, chunk: WorkbookElementDataChunk, ): IncrementalElementDataState { + // A chunk starting past every row accumulated so far means the host skipped + // ahead of the stream, which the contract forbids. Landing its rows at their + // absolute offset would pad the gap with holes that read as real rows, so + // the rows are dropped and only the progress flags are taken. isComplete and + // totalRows still apply: suppressing them would leave the host re-sending + // the same rejected offset forever, while keeping them lets a consumer see + // the gap as rowCount < totalRows. + if (chunk.offset > prev.info.rowCount) { + return { + data: prev.data, + info: { + rowCount: prev.info.rowCount, + isComplete: chunk.isComplete, + totalRows: chunk.totalRows ?? prev.info.totalRows, + }, + }; + } const data: WorkbookElementData = chunk.offset === 0 ? {} : { ...prev.data }; for (const colId of Object.keys(chunk.data)) { // '__proto__' is never a real column id; assigning it would swap the @@ -176,8 +193,9 @@ function applyElementDataChunk( ? data[colId] : []; const head = prevRows.slice(0, chunk.offset); - // Pad so rows always land at their absolute offset, even for a column - // first appearing mid-stream or a host that skips ahead. + // Pad so rows always land at their absolute offset even when the column + // first appears mid-stream. The chunk itself is known to be contiguous + // with the accumulated rows, so this can only backfill a new column. head.length = chunk.offset; data[colId] = head.concat(chunk.data[colId]); } diff --git a/packages/plugin-sdk/src/types.ts b/packages/plugin-sdk/src/types.ts index f23f263..d3c84e7 100644 --- a/packages/plugin-sdk/src/types.ts +++ b/packages/plugin-sdk/src/types.ts @@ -76,7 +76,15 @@ export interface WorkbookElementData { * and replaces all accumulated state), include the subscription's full column * set in every chunk with all column arrays the same length, and may send an * empty data object at offset > 0 to update isComplete/totalRows without - * appending rows. + * appending rows. A chunk must never start past the rows already delivered: + * offset may be at most the number of rows sent so far, so the stream stays + * contiguous and no gap is ever left to guess at. A consumer that receives + * one anyway keeps its accumulated rows and discards the chunk's rows rather + * than fabricating the missing ones. + * + * A failed data eval is reported as a null payload rather than a chunk, which + * terminates the stream; the host must restart at offset 0 to resume, since + * the consumer has no rows left to append to. * @typedef {object} WorkbookElementDataChunk * @property {WorkbookElementData} data Rows contained in this chunk only * @property {number} offset Absolute row offset of the first row in this chunk; a non-negative integer