From 91b26bf9ed7ed47bea7fbd0b0773a33536a0cc16 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 5 Feb 2026 13:22:38 -0500 Subject: [PATCH 01/12] Add protocol stream with stateless reorg detection Adds streamProtocol method to ArrowFlight service that wraps streamQuery with protocol-level message handling: - Data: New records with block ranges - Reorg: Chain reorganization detected with invalidation ranges - Watermark: Confirmation that block ranges are complete Includes validation logic ported from Rust implementation: - validatePrevHash: Genesis vs non-genesis block rules - validateNetworks: Network consistency across batches - validateConsecutiveness: Hash chain and gap detection Co-Authored-By: Claude Opus 4.5 --- packages/amp/src/arrow-flight.ts | 227 +++++++++++++- packages/amp/src/index.ts | 9 + packages/amp/src/protocol-stream/errors.ts | 237 ++++++++++++++ packages/amp/src/protocol-stream/index.ts | 115 +++++++ packages/amp/src/protocol-stream/messages.ts | 235 ++++++++++++++ .../amp/src/protocol-stream/validation.ts | 224 +++++++++++++ .../test/protocol-stream/validation.test.ts | 293 ++++++++++++++++++ 7 files changed, 1339 insertions(+), 1 deletion(-) create mode 100644 packages/amp/src/protocol-stream/errors.ts create mode 100644 packages/amp/src/protocol-stream/index.ts create mode 100644 packages/amp/src/protocol-stream/messages.ts create mode 100644 packages/amp/src/protocol-stream/validation.ts create mode 100644 packages/amp/test/protocol-stream/validation.test.ts diff --git a/packages/amp/src/arrow-flight.ts b/packages/amp/src/arrow-flight.ts index 54d85fe..4c78ed3 100644 --- a/packages/amp/src/arrow-flight.ts +++ b/packages/amp/src/arrow-flight.ts @@ -26,6 +26,20 @@ import { parseRecordBatch } from "./internal/arrow-flight-ipc/RecordBatch.ts" import { type ArrowSchema, getMessageType, MessageHeaderType, parseSchema } from "./internal/arrow-flight-ipc/Schema.ts" import type { AuthInfo, BlockRange, RecordBatchMetadata } from "./models.ts" import { RecordBatchMetadataFromUint8Array } from "./models.ts" +import { + type ProtocolStreamError, + ProtocolArrowFlightError, + ProtocolValidationError +} from "./protocol-stream/errors.ts" +import { + type InvalidationRange, + type ProtocolMessage, + data as protocolData, + makeInvalidationRange, + reorg as protocolReorg, + watermark as protocolWatermark +} from "./protocol-stream/messages.ts" +import { validateAll } from "./protocol-stream/validation.ts" import { FlightDescriptor_DescriptorType, FlightDescriptorSchema, FlightService } from "./protobuf/Flight_pb.ts" import { CommandStatementQuerySchema } from "./protobuf/FlightSql_pb.ts" @@ -270,8 +284,57 @@ export class ArrowFlight extends Context.Tag("Amp/ArrowFlight") Stream.Stream, ArrowFlightError> + + /** + * Executes an Arrow Flight SQL query and returns a stream of protocol messages + * with stateless reorg detection. + * + * Protocol messages include: + * - `Data`: New records to process with block ranges + * - `Reorg`: Chain reorganization detected with invalidation ranges + * - `Watermark`: Confirmation that block ranges are complete + * + * @example + * ```typescript + * const arrowFlight = yield* ArrowFlight + * + * yield* arrowFlight.streamProtocol("SELECT * FROM eth.logs").pipe( + * Stream.runForEach((message) => { + * switch (message._tag) { + * case "Data": + * return Effect.log(`Data: ${message.data.length} records`) + * case "Reorg": + * return Effect.log(`Reorg: ${message.invalidation.length} ranges`) + * case "Watermark": + * return Effect.log(`Watermark confirmed`) + * } + * }) + * ) + * ``` + */ + readonly streamProtocol: ( + sql: string, + options?: ProtocolStreamOptions + ) => Stream.Stream }>() {} +/** + * Options for creating a protocol stream. + */ +export interface ProtocolStreamOptions { + /** + * Schema to validate and decode the record batch data. + * If provided, data will be validated against this schema. + */ + readonly schema?: QueryOptions["schema"] + + /** + * Resume watermark from a previous session. + * Allows resumption of streaming queries from a known position. + */ + readonly resumeWatermark?: ReadonlyArray +} + const make = Effect.gen(function*() { const auth = yield* Effect.serviceOption(Auth) const transport = yield* Transport @@ -408,10 +471,148 @@ const make = Effect.gen(function*() { } ) as any + /** + * Internal state maintained by the protocol stream for reorg detection. + */ + interface ProtocolStreamState { + readonly previous: ReadonlyArray + readonly initialized: boolean + } + + /** + * Detects reorgs by comparing incoming ranges to previous ranges. + */ + const detectReorgs = ( + previous: ReadonlyArray, + incoming: ReadonlyArray + ): ReadonlyArray => { + const invalidations: Array = [] + + for (const incomingRange of incoming) { + const prevRange = previous.find((p) => p.network === incomingRange.network) + if (!prevRange) continue + + // Skip identical ranges (watermarks can repeat) + if ( + incomingRange.network === prevRange.network && + incomingRange.numbers.start === prevRange.numbers.start && + incomingRange.numbers.end === prevRange.numbers.end && + incomingRange.hash === prevRange.hash && + incomingRange.prevHash === prevRange.prevHash + ) { + continue + } + + const incomingStart = incomingRange.numbers.start + const prevEnd = prevRange.numbers.end + + // Detect backwards jump (reorg indicator) + if (incomingStart < prevEnd + 1) { + invalidations.push( + makeInvalidationRange( + incomingRange.network, + incomingStart, + Math.max(incomingRange.numbers.end, prevEnd) + ) + ) + } + } + + return invalidations + } + + const streamProtocol = ( + sql: string, + options?: ProtocolStreamOptions + ): Stream.Stream => { + // Get the underlying Arrow Flight stream + const rawStream = streamQuery(sql, { + schema: options?.schema, + stream: true, + resumeWatermark: options?.resumeWatermark + }) as unknown as Stream.Stream< + QueryResult>>, + ArrowFlightError + > + + const initialState: ProtocolStreamState = { + previous: [], + initialized: false + } + + const ZERO_HASH = "0x0000000000000000000000000000000000000000000000000000000000000000" + + return rawStream.pipe( + // Map Arrow Flight errors to protocol errors + Stream.mapError((error: ArrowFlightError) => new ProtocolArrowFlightError({ cause: error })), + + // Process each batch with state tracking + Stream.mapAccumEffect(initialState, (state, queryResult) => + Effect.gen(function*() { + const batchData = queryResult.data + const metadata = queryResult.metadata + const incoming = metadata.ranges + + // Validate the incoming batch + if (state.initialized) { + yield* validateAll(state.previous, incoming).pipe( + Effect.mapError((error) => new ProtocolValidationError({ cause: error })) + ) + } else { + // Validate prevHash for first batch + for (const range of incoming) { + const isGenesis = range.numbers.start === 0 + if (isGenesis) { + if (range.prevHash !== undefined && range.prevHash !== ZERO_HASH) { + return yield* Effect.fail( + new ProtocolValidationError({ + cause: { _tag: "InvalidPrevHashError", network: range.network } + }) + ) + } + } else { + if (range.prevHash === undefined || range.prevHash === ZERO_HASH) { + return yield* Effect.fail( + new ProtocolValidationError({ + cause: { _tag: "MissingPrevHashError", network: range.network, block: range.numbers.start } + }) + ) + } + } + } + } + + // Detect reorgs + const invalidations = state.initialized ? detectReorgs(state.previous, incoming) : [] + + // Determine message type + let message: ProtocolMessage + + if (invalidations.length > 0) { + message = protocolReorg(state.previous, incoming, invalidations) + } else if (metadata.rangesComplete && batchData.length === 0) { + message = protocolWatermark(incoming) + } else { + message = protocolData(batchData as unknown as ReadonlyArray>, incoming) + } + + const newState: ProtocolStreamState = { + previous: incoming, + initialized: true + } + + return [newState, message] as const + })), + + Stream.withSpan("ArrowFlight.streamProtocol") + ) + } + return { client, query, - streamQuery + streamQuery, + streamProtocol } as const }) @@ -441,3 +642,27 @@ const blockRangesToResumeWatermark = (ranges: ReadonlyArray): string } return JSON.stringify(watermarks) } + +// ============================================================================= +// Protocol Stream Re-exports +// ============================================================================= + +export type { ProtocolStreamError } + +export { + ProtocolArrowFlightError, + ProtocolValidationError +} from "./protocol-stream/errors.ts" + +export { + InvalidationRange, + ProtocolMessage, + ProtocolMessageData, + ProtocolMessageReorg, + ProtocolMessageWatermark, + data as protocolMessageData, + invalidates, + makeInvalidationRange, + reorg as protocolMessageReorg, + watermark as protocolMessageWatermark +} from "./protocol-stream/messages.ts" diff --git a/packages/amp/src/index.ts b/packages/amp/src/index.ts index 02f7213..d019dd8 100644 --- a/packages/amp/src/index.ts +++ b/packages/amp/src/index.ts @@ -1,5 +1,7 @@ /** * An implementation of the Arrow Flight protocol. + * + * Includes `streamProtocol` method for protocol-level streaming with reorg detection. */ export * as ArrowFlight from "./arrow-flight.ts" @@ -22,3 +24,10 @@ export * as AdminApi from "./admin/api.ts" * Operations for interacting with the Amp registry API. */ export * as RegistryApi from "./registry/api.ts" + +/** + * Protocol stream types and validation for reorg detection. + * + * These types are used by `ArrowFlight.streamProtocol()`. + */ +export * as ProtocolStream from "./protocol-stream/index.ts" diff --git a/packages/amp/src/protocol-stream/errors.ts b/packages/amp/src/protocol-stream/errors.ts new file mode 100644 index 0000000..6d0d6db --- /dev/null +++ b/packages/amp/src/protocol-stream/errors.ts @@ -0,0 +1,237 @@ +/** + * This module contains error definitions for the ProtocolStream service. + * + * Errors are organized into two categories: + * - ValidationError: Protocol invariant violations detected during stream processing + * - ProtocolStreamError: Top-level errors that can occur in the ProtocolStream service + * + * @module + */ +import * as Schema from "effect/Schema" + +// ============================================================================= +// Validation Errors +// ============================================================================= + +/** + * Represents an error that occurs when a batch contains duplicate networks. + * + * Each batch should contain at most one range per network. + */ +export class DuplicateNetworkError extends Schema.TaggedError( + "Amp/ProtocolStream/DuplicateNetworkError" +)("DuplicateNetworkError", { + /** + * The network that appeared more than once in the batch. + */ + network: Schema.String +}) { + override get message(): string { + return `Duplicate network in batch: ${this.network}` + } +} + +/** + * Represents an error that occurs when the number of networks changes between batches. + * + * The network set must remain stable across all batches in a stream. + */ +export class NetworkCountChangedError extends Schema.TaggedError( + "Amp/ProtocolStream/NetworkCountChangedError" +)("NetworkCountChangedError", { + /** + * The expected number of networks (from previous batch). + */ + expected: Schema.Number, + /** + * The actual number of networks received. + */ + actual: Schema.Number +}) { + override get message(): string { + return `Network count changed: expected ${this.expected}, got ${this.actual}` + } +} + +/** + * Represents an error that occurs when an unexpected network appears in a batch. + * + * All networks must be established in the first batch and remain consistent. + */ +export class UnexpectedNetworkError extends Schema.TaggedError( + "Amp/ProtocolStream/UnexpectedNetworkError" +)("UnexpectedNetworkError", { + /** + * The unexpected network that was encountered. + */ + network: Schema.String +}) { + override get message(): string { + return `Unexpected network in batch: ${this.network}` + } +} + +/** + * Represents an error that occurs when a non-genesis block is missing its prevHash. + * + * Non-genesis blocks (start > 0) must have a prevHash to enable hash chain validation. + */ +export class MissingPrevHashError extends Schema.TaggedError( + "Amp/ProtocolStream/MissingPrevHashError" +)("MissingPrevHashError", { + /** + * The network where the missing prevHash was detected. + */ + network: Schema.String, + /** + * The block number that is missing a prevHash. + */ + block: Schema.Number +}) { + override get message(): string { + return `Missing prevHash for non-genesis block ${this.block} on network ${this.network}` + } +} + +/** + * Represents an error that occurs when a genesis block has an invalid prevHash. + * + * Genesis blocks (start = 0) must have either no prevHash or a zero hash. + */ +export class InvalidPrevHashError extends Schema.TaggedError( + "Amp/ProtocolStream/InvalidPrevHashError" +)("InvalidPrevHashError", { + /** + * The network where the invalid prevHash was detected. + */ + network: Schema.String +}) { + override get message(): string { + return `Genesis block has invalid prevHash on network ${this.network}` + } +} + +/** + * Represents an error that occurs when consecutive blocks have mismatched hashes. + * + * For consecutive blocks (incoming.start === prev.end + 1), the incoming prevHash + * must match the previous block's hash. + */ +export class HashMismatchOnConsecutiveBlocksError extends Schema.TaggedError( + "Amp/ProtocolStream/HashMismatchOnConsecutiveBlocksError" +)("HashMismatchOnConsecutiveBlocksError", { + /** + * The network where the hash mismatch was detected. + */ + network: Schema.String, + /** + * The expected hash (from the previous block). + */ + expectedHash: Schema.String, + /** + * The actual prevHash from the incoming block. + */ + actualPrevHash: Schema.String +}) { + override get message(): string { + return `Hash mismatch on consecutive blocks for network ${this.network}: expected ${this.expectedHash}, got ${this.actualPrevHash}` + } +} + +/** + * Represents an error that occurs when a backwards jump has matching hashes. + * + * A backwards jump (incoming.start < prev.end + 1) indicates a reorg, which + * requires different hashes. If hashes match, it's an invalid protocol state. + */ +export class InvalidReorgError extends Schema.TaggedError( + "Amp/ProtocolStream/InvalidReorgError" +)("InvalidReorgError", { + /** + * The network where the invalid reorg was detected. + */ + network: Schema.String +}) { + override get message(): string { + return `Invalid reorg detected on network ${this.network}: backwards jump with matching hashes` + } +} + +/** + * Represents an error that occurs when there is a gap in block numbers. + * + * Forward gaps (incoming.start > prev.end + 1) are always protocol violations. + */ +export class GapError extends Schema.TaggedError( + "Amp/ProtocolStream/GapError" +)("GapError", { + /** + * The network where the gap was detected. + */ + network: Schema.String, + /** + * The first missing block number (prev.end + 1). + */ + missingStart: Schema.Number, + /** + * The last missing block number (incoming.start - 1). + */ + missingEnd: Schema.Number +}) { + override get message(): string { + return `Gap in block numbers for network ${this.network}: missing blocks ${this.missingStart} to ${this.missingEnd}` + } +} + +/** + * Union type representing all possible validation errors. + */ +export type ValidationError = + | DuplicateNetworkError + | NetworkCountChangedError + | UnexpectedNetworkError + | MissingPrevHashError + | InvalidPrevHashError + | HashMismatchOnConsecutiveBlocksError + | InvalidReorgError + | GapError + +// ============================================================================= +// Protocol Stream Errors +// ============================================================================= + +/** + * Represents a validation error wrapped for the ProtocolStream service. + */ +export class ProtocolValidationError extends Schema.TaggedError( + "Amp/ProtocolStream/ProtocolValidationError" +)("ProtocolValidationError", { + /** + * The underlying validation error. + */ + cause: Schema.Defect +}) { + override get message(): string { + const cause = this.cause as ValidationError + return `Protocol validation failed: ${cause.message}` + } +} + +/** + * Represents an error from the underlying Arrow Flight stream. + */ +export class ProtocolArrowFlightError extends Schema.TaggedError( + "Amp/ProtocolStream/ProtocolArrowFlightError" +)("ProtocolArrowFlightError", { + /** + * The underlying Arrow Flight error. + */ + cause: Schema.Defect +}) {} + +/** + * Union type representing all possible ProtocolStream errors. + */ +export type ProtocolStreamError = + | ProtocolValidationError + | ProtocolArrowFlightError diff --git a/packages/amp/src/protocol-stream/index.ts b/packages/amp/src/protocol-stream/index.ts new file mode 100644 index 0000000..8ea0e71 --- /dev/null +++ b/packages/amp/src/protocol-stream/index.ts @@ -0,0 +1,115 @@ +/** + * Protocol Stream types for processing Amp streams with reorg detection. + * + * This module provides the types and validation functions used by + * `ArrowFlight.streamProtocol()` for stateless reorg detection. + * + * ## Overview + * + * The protocol stream interprets raw batches from Arrow Flight and emits + * three types of messages: + * + * - **Data**: New records to process, along with the block ranges they cover + * - **Reorg**: Chain reorganization detected, with invalidation ranges + * - **Watermark**: Confirmation that block ranges are complete + * + * ## Usage + * + * ```typescript + * import * as Effect from "effect/Effect" + * import * as Stream from "effect/Stream" + * import { ArrowFlight } from "@edgeandnode/amp" + * + * const program = Effect.gen(function*() { + * const arrowFlight = yield* ArrowFlight.ArrowFlight + * + * yield* arrowFlight.streamProtocol("SELECT * FROM eth.logs").pipe( + * Stream.runForEach((message) => { + * switch (message._tag) { + * case "Data": + * return Effect.log(`Received ${message.data.length} records`) + * case "Reorg": + * return Effect.log(`Reorg: invalidating ${message.invalidation.length} ranges`) + * case "Watermark": + * return Effect.log(`Watermark at block ${message.ranges[0]?.numbers.end}`) + * } + * }) + * ) + * }) + * ``` + * + * ## Reorg Detection + * + * The stream detects chain reorganizations by monitoring block range progression: + * + * 1. **Consecutive blocks**: Normal progression where new blocks build on previous + * 2. **Backwards jump**: Indicates a reorg where the chain has forked + * 3. **Forward gap**: Protocol violation (should never happen) + * + * When a reorg is detected, the stream emits a `Reorg` message containing: + * - The previous known block ranges + * - The new incoming block ranges + * - Specific invalidation ranges describing which blocks are affected + * + * ## Validation + * + * The stream validates protocol invariants: + * - Hash chain integrity (prevHash must match previous block's hash) + * - Network consistency (same networks across all batches) + * - No gaps in block sequences + * + * Validation errors terminate the stream, as they indicate protocol violations + * from the server that cannot be recovered without reconnection. + * + * @module + */ + +// ============================================================================= +// Messages +// ============================================================================= + +export { + InvalidationRange, + ProtocolMessage, + ProtocolMessageData, + ProtocolMessageReorg, + ProtocolMessageWatermark, + data, + invalidates, + makeInvalidationRange, + reorg, + watermark +} from "./messages.ts" + +// ============================================================================= +// Errors +// ============================================================================= + +export { + // Validation errors + DuplicateNetworkError, + GapError, + HashMismatchOnConsecutiveBlocksError, + InvalidPrevHashError, + InvalidReorgError, + MissingPrevHashError, + NetworkCountChangedError, + UnexpectedNetworkError, + type ValidationError, + + // Protocol stream errors + ProtocolArrowFlightError, + ProtocolValidationError, + type ProtocolStreamError +} from "./errors.ts" + +// ============================================================================= +// Validation +// ============================================================================= + +export { + validateAll, + validateConsecutiveness, + validateNetworks, + validatePrevHash +} from "./validation.ts" diff --git a/packages/amp/src/protocol-stream/messages.ts b/packages/amp/src/protocol-stream/messages.ts new file mode 100644 index 0000000..85836ee --- /dev/null +++ b/packages/amp/src/protocol-stream/messages.ts @@ -0,0 +1,235 @@ +/** + * This module contains the protocol message types emitted by the ProtocolStream. + * + * The ProtocolStream transforms raw Arrow Flight responses into three types of + * protocol messages: + * - Data: New data to process + * - Reorg: Chain reorganization detected + * - Watermark: Confirmed completion marker + * + * @module + */ +import * as Schema from "effect/Schema" +import { BlockNumber, BlockRange, Network } from "../models.ts" + +// ============================================================================= +// Invalidation Range +// ============================================================================= + +/** + * Represents a range of blocks that must be invalidated due to a reorg. + * + * When a chain reorganization is detected, this type describes which blocks + * on a specific network need to be considered invalid. + */ +export const InvalidationRange = Schema.Struct({ + /** + * The network where blocks need to be invalidated. + */ + network: Network, + /** + * The start of the invalidation range (inclusive). + */ + start: BlockNumber, + /** + * The end of the invalidation range (inclusive). + */ + end: BlockNumber +}).annotations({ + identifier: "InvalidationRange", + description: "A range of blocks that must be invalidated due to a reorg" +}) +export type InvalidationRange = typeof InvalidationRange.Type + +/** + * Creates an InvalidationRange. + * + * @param network - The network where blocks need to be invalidated. + * @param start - The start of the invalidation range (inclusive). + * @param end - The end of the invalidation range (inclusive). + * @returns An InvalidationRange instance. + */ +export const makeInvalidationRange = ( + network: string, + start: number, + end: number +): InvalidationRange => ({ + network: network as typeof Network.Type, + start: start as typeof BlockNumber.Type, + end: end as typeof BlockNumber.Type +}) + +/** + * Checks if a block range overlaps with an invalidation range. + * + * Returns true if the block range is on the same network and has any + * overlapping block numbers with the invalidation range. + * + * @param invalidation - The invalidation range to check against. + * @param range - The block range to check. + * @returns True if the ranges overlap, false otherwise. + */ +export const invalidates = ( + invalidation: InvalidationRange, + range: typeof BlockRange.Type +): boolean => { + if (invalidation.network !== range.network) { + return false + } + // Check for no overlap: invalidation ends before range starts OR range ends before invalidation starts + const noOverlap = invalidation.end < range.numbers.start || range.numbers.end < invalidation.start + return !noOverlap +} + +// ============================================================================= +// Protocol Messages +// ============================================================================= + +/** + * Represents new data received from the server. + * + * Contains a batch of records and the block ranges they cover. + */ +export const ProtocolMessageData = Schema.TaggedStruct("Data", { + /** + * The decoded record batch data as an array of records. + */ + data: Schema.Array(Schema.Record({ + key: Schema.String, + value: Schema.Unknown + })), + /** + * The block ranges covered by this batch. + */ + ranges: Schema.Array(BlockRange) +}).annotations({ + identifier: "ProtocolMessage.Data", + description: "New data to process from the protocol stream" +}) +export type ProtocolMessageData = typeof ProtocolMessageData.Type + +/** + * Represents a chain reorganization detected by the protocol. + * + * When a reorg is detected, this message contains: + * - The previous block ranges that were known + * - The new incoming block ranges from the server + * - The specific ranges that need to be invalidated + */ +export const ProtocolMessageReorg = Schema.TaggedStruct("Reorg", { + /** + * The previous block ranges that were known before this message. + */ + previous: Schema.Array(BlockRange), + /** + * The new incoming block ranges from the server. + */ + incoming: Schema.Array(BlockRange), + /** + * The ranges that need to be invalidated due to the reorg. + */ + invalidation: Schema.Array(InvalidationRange) +}).annotations({ + identifier: "ProtocolMessage.Reorg", + description: "Chain reorganization detected" +}) +export type ProtocolMessageReorg = typeof ProtocolMessageReorg.Type + +/** + * Represents a watermark indicating ranges are confirmed complete. + * + * Watermarks are emitted when `rangesComplete` is true and the batch + * contains no data. They indicate that the specified block ranges + * have been fully processed and can be considered stable. + */ +export const ProtocolMessageWatermark = Schema.TaggedStruct("Watermark", { + /** + * The block ranges that are confirmed complete. + */ + ranges: Schema.Array(BlockRange) +}).annotations({ + identifier: "ProtocolMessage.Watermark", + description: "Watermark indicating ranges are confirmed complete" +}) +export type ProtocolMessageWatermark = typeof ProtocolMessageWatermark.Type + +/** + * Discriminated union of all protocol message types. + * + * Use pattern matching on the `_tag` field to handle each message type: + * + * ```typescript + * switch (message._tag) { + * case "Data": + * // Process new data + * break; + * case "Reorg": + * // Handle chain reorganization + * break; + * case "Watermark": + * // Checkpoint progress + * break; + * } + * ``` + */ +export const ProtocolMessage = Schema.Union( + ProtocolMessageData, + ProtocolMessageReorg, + ProtocolMessageWatermark +).annotations({ + identifier: "ProtocolMessage", + description: "A message from the protocol stream" +}) +export type ProtocolMessage = typeof ProtocolMessage.Type + +// ============================================================================= +// Constructors +// ============================================================================= + +/** + * Creates a Data protocol message. + * + * @param records - The decoded record batch data. + * @param ranges - The block ranges covered by the data. + * @returns A Data protocol message. + */ +export const data = ( + records: ReadonlyArray>, + ranges: ReadonlyArray +): ProtocolMessageData => ({ + _tag: "Data", + data: records as Array>, + ranges: ranges as Array +}) + +/** + * Creates a Reorg protocol message. + * + * @param previous - The previous known block ranges. + * @param incoming - The new incoming block ranges. + * @param invalidation - The ranges that need to be invalidated. + * @returns A Reorg protocol message. + */ +export const reorg = ( + previous: ReadonlyArray, + incoming: ReadonlyArray, + invalidation: ReadonlyArray +): ProtocolMessageReorg => ({ + _tag: "Reorg", + previous: previous as Array, + incoming: incoming as Array, + invalidation: invalidation as Array +}) + +/** + * Creates a Watermark protocol message. + * + * @param ranges - The block ranges that are confirmed complete. + * @returns A Watermark protocol message. + */ +export const watermark = ( + ranges: ReadonlyArray +): ProtocolMessageWatermark => ({ + _tag: "Watermark", + ranges: ranges as Array +}) diff --git a/packages/amp/src/protocol-stream/validation.ts b/packages/amp/src/protocol-stream/validation.ts new file mode 100644 index 0000000..5375b84 --- /dev/null +++ b/packages/amp/src/protocol-stream/validation.ts @@ -0,0 +1,224 @@ +/** + * This module contains validation functions for the Amp protocol. + * + * These functions validate protocol invariants to ensure the server is + * sending well-formed data. The validation logic is ported from the Rust + * implementation in `.repos/amp/crates/clients/flight/validation.rs`. + * + * Three validations are performed: + * 1. validatePrevHash - Validates prevHash based on block position + * 2. validateNetworks - Validates network consistency across batches + * 3. validateConsecutiveness - Validates block range progression + * + * @module + */ +import * as Effect from "effect/Effect" +import type { BlockNumber, BlockRange } from "../models.ts" +import { + DuplicateNetworkError, + GapError, + HashMismatchOnConsecutiveBlocksError, + InvalidPrevHashError, + InvalidReorgError, + MissingPrevHashError, + NetworkCountChangedError, + UnexpectedNetworkError, + type ValidationError +} from "./errors.ts" + +/** + * The zero hash used to represent "no previous hash" for genesis blocks. + */ +const ZERO_HASH = "0x0000000000000000000000000000000000000000000000000000000000000000" + +/** + * Checks if a hash is a zero hash (all zeros). + */ +const isZeroHash = (hash: string): boolean => hash === ZERO_HASH + +/** + * Validates the prevHash field of a block range. + * + * Rules: + * - Genesis blocks (start = 0): prevHash must be undefined or zero hash + * - Non-genesis blocks (start > 0): prevHash must be defined and non-zero + * + * @param range - The block range to validate. + * @returns An Effect that succeeds if valid, or fails with a validation error. + */ +export const validatePrevHash = ( + range: BlockRange +): Effect.Effect => + Effect.gen(function*() { + const isGenesis = range.numbers.start === 0 + + if (isGenesis) { + // Genesis blocks must have no prevHash or a zero hash + if (range.prevHash !== undefined && !isZeroHash(range.prevHash)) { + return yield* new InvalidPrevHashError({ network: range.network }) + } + } else { + // Non-genesis blocks must have a non-zero prevHash + if (range.prevHash === undefined) { + return yield* new MissingPrevHashError({ + network: range.network, + block: range.numbers.start + }) + } + if (isZeroHash(range.prevHash)) { + return yield* new MissingPrevHashError({ + network: range.network, + block: range.numbers.start + }) + } + } + }) + +/** + * Validates network consistency between previous and incoming batches. + * + * Rules: + * - No duplicate networks within a single batch + * - Network set must remain stable across batches (count and names) + * + * @param previous - The previous batch's block ranges (empty for first batch). + * @param incoming - The incoming batch's block ranges. + * @returns An Effect that succeeds if valid, or fails with a validation error. + */ +export const validateNetworks = ( + previous: ReadonlyArray, + incoming: ReadonlyArray +): Effect.Effect => + Effect.gen(function*() { + // Check for duplicate networks in incoming batch + const incomingNetworks = new Set() + for (const range of incoming) { + if (incomingNetworks.has(range.network)) { + return yield* new DuplicateNetworkError({ network: range.network }) + } + incomingNetworks.add(range.network) + } + + // If this is the first batch, no further validation needed + if (previous.length === 0) { + return + } + + // Check network count matches + if (previous.length !== incoming.length) { + return yield* new NetworkCountChangedError({ + expected: previous.length, + actual: incoming.length + }) + } + + // Check all incoming networks were in previous batch + const previousNetworks = new Set(previous.map((r) => r.network)) + for (const range of incoming) { + if (!previousNetworks.has(range.network)) { + return yield* new UnexpectedNetworkError({ network: range.network }) + } + } + }) + +/** + * Checks if two block ranges are equal. + */ +const blockRangeEquals = (a: BlockRange, b: BlockRange): boolean => + a.network === b.network && + a.numbers.start === b.numbers.start && + a.numbers.end === b.numbers.end && + a.hash === b.hash && + a.prevHash === b.prevHash + +/** + * Validates block range consecutiveness and hash chain integrity. + * + * For each network, validates based on block range position: + * 1. Consecutive (start === prev.end + 1): Hash chain MUST match + * 2. Backwards jump (start < prev.end + 1): Hash chain MUST mismatch (reorg) + * 3. Forward gap (start > prev.end + 1): Always protocol violation + * 4. Identical range: Allowed for watermark repeats + * + * @param previous - The previous batch's block ranges. + * @param incoming - The incoming batch's block ranges. + * @returns An Effect that succeeds if valid, or fails with a validation error. + */ +export const validateConsecutiveness = ( + previous: ReadonlyArray, + incoming: ReadonlyArray +): Effect.Effect => + Effect.gen(function*() { + // If this is the first batch, no consecutiveness check needed + if (previous.length === 0) { + return + } + + for (const incomingRange of incoming) { + // Find the previous range for this network + const prevRange = previous.find((p) => p.network === incomingRange.network) + if (!prevRange) { + // New network - should have been caught by validateNetworks + continue + } + + // Identical ranges are allowed (watermarks can repeat) + if (blockRangeEquals(incomingRange, prevRange)) { + continue + } + + const incomingStart = incomingRange.numbers.start + const prevEnd = prevRange.numbers.end + const expectedNextStart = (prevEnd + 1) as BlockNumber + + if (incomingStart === expectedNextStart) { + // Consecutive blocks: hash chain must match + // incoming.prevHash must equal prev.hash + if (incomingRange.prevHash !== prevRange.hash) { + return yield* new HashMismatchOnConsecutiveBlocksError({ + network: incomingRange.network, + expectedHash: prevRange.hash, + actualPrevHash: incomingRange.prevHash ?? "undefined" + }) + } + } else if (incomingStart < expectedNextStart) { + // Backwards jump: indicates a reorg + // Hash chain MUST mismatch for a valid reorg + if (incomingRange.prevHash === prevRange.hash) { + return yield* new InvalidReorgError({ network: incomingRange.network }) + } + // Hash mismatch is expected for reorg - this is valid + } else { + // Forward gap: always a protocol violation + return yield* new GapError({ + network: incomingRange.network, + missingStart: expectedNextStart, + missingEnd: (incomingStart - 1) as BlockNumber + }) + } + } + }) + +/** + * Runs all validation checks on incoming block ranges. + * + * @param previous - The previous batch's block ranges. + * @param incoming - The incoming batch's block ranges. + * @returns An Effect that succeeds if all validations pass. + */ +export const validateAll = ( + previous: ReadonlyArray, + incoming: ReadonlyArray +): Effect.Effect => + Effect.gen(function*() { + // Validate prevHash for all incoming ranges + for (const range of incoming) { + yield* validatePrevHash(range) + } + + // Validate network consistency + yield* validateNetworks(previous, incoming) + + // Validate consecutiveness + yield* validateConsecutiveness(previous, incoming) + }) diff --git a/packages/amp/test/protocol-stream/validation.test.ts b/packages/amp/test/protocol-stream/validation.test.ts new file mode 100644 index 0000000..a1506b9 --- /dev/null +++ b/packages/amp/test/protocol-stream/validation.test.ts @@ -0,0 +1,293 @@ +/** + * Protocol Stream Validation Tests + * + * Tests for the validation functions used by the ProtocolStream to ensure + * protocol invariants are maintained. + */ +import { + DuplicateNetworkError, + GapError, + HashMismatchOnConsecutiveBlocksError, + InvalidPrevHashError, + InvalidReorgError, + MissingPrevHashError, + NetworkCountChangedError, + UnexpectedNetworkError, + validateAll, + validateConsecutiveness, + validateNetworks, + validatePrevHash +} from "@edgeandnode/amp/protocol-stream/index" +import { describe, it } from "@effect/vitest" +import * as Effect from "effect/Effect" +import * as Either from "effect/Either" +import type { BlockHash, BlockNumber, BlockRange, Network } from "@edgeandnode/amp/models" + +// ============================================================================= +// Test Helpers +// ============================================================================= + +const ZERO_HASH = "0x0000000000000000000000000000000000000000000000000000000000000000" as BlockHash +const HASH_A = "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" as BlockHash +const HASH_B = "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" as BlockHash +const HASH_C = "0xcccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" as BlockHash + +const makeBlockRange = ( + network: string, + start: number, + end: number, + hash: string, + prevHash?: string +): BlockRange => ({ + network: network as Network, + numbers: { + start: start as BlockNumber, + end: end as BlockNumber + }, + hash: hash as BlockHash, + prevHash: prevHash as BlockHash | undefined +}) + +// ============================================================================= +// validatePrevHash Tests +// ============================================================================= + +describe("validatePrevHash", () => { + it.effect("allows genesis block with no prevHash", ({ expect }) => + Effect.gen(function*() { + const range = makeBlockRange("eth", 0, 10, HASH_A) + const result = yield* validatePrevHash(range).pipe(Effect.either) + expect(Either.isRight(result)).toBe(true) + })) + + it.effect("allows genesis block with zero prevHash", ({ expect }) => + Effect.gen(function*() { + const range = makeBlockRange("eth", 0, 10, HASH_A, ZERO_HASH) + const result = yield* validatePrevHash(range).pipe(Effect.either) + expect(Either.isRight(result)).toBe(true) + })) + + it.effect("rejects genesis block with non-zero prevHash", ({ expect }) => + Effect.gen(function*() { + const range = makeBlockRange("eth", 0, 10, HASH_A, HASH_B) + const result = yield* validatePrevHash(range).pipe(Effect.either) + expect(Either.isLeft(result)).toBe(true) + if (Either.isLeft(result)) { + expect(result.left).toBeInstanceOf(InvalidPrevHashError) + } + })) + + it.effect("allows non-genesis block with valid prevHash", ({ expect }) => + Effect.gen(function*() { + const range = makeBlockRange("eth", 100, 110, HASH_A, HASH_B) + const result = yield* validatePrevHash(range).pipe(Effect.either) + expect(Either.isRight(result)).toBe(true) + })) + + it.effect("rejects non-genesis block with no prevHash", ({ expect }) => + Effect.gen(function*() { + const range = makeBlockRange("eth", 100, 110, HASH_A) + const result = yield* validatePrevHash(range).pipe(Effect.either) + expect(Either.isLeft(result)).toBe(true) + if (Either.isLeft(result)) { + expect(result.left).toBeInstanceOf(MissingPrevHashError) + } + })) + + it.effect("rejects non-genesis block with zero prevHash", ({ expect }) => + Effect.gen(function*() { + const range = makeBlockRange("eth", 100, 110, HASH_A, ZERO_HASH) + const result = yield* validatePrevHash(range).pipe(Effect.either) + expect(Either.isLeft(result)).toBe(true) + if (Either.isLeft(result)) { + expect(result.left).toBeInstanceOf(MissingPrevHashError) + } + })) +}) + +// ============================================================================= +// validateNetworks Tests +// ============================================================================= + +describe("validateNetworks", () => { + it.effect("allows first batch with any networks", ({ expect }) => + Effect.gen(function*() { + const incoming = [ + makeBlockRange("eth", 0, 10, HASH_A), + makeBlockRange("polygon", 0, 10, HASH_B) + ] + const result = yield* validateNetworks([], incoming).pipe(Effect.either) + expect(Either.isRight(result)).toBe(true) + })) + + it.effect("allows consistent networks across batches", ({ expect }) => + Effect.gen(function*() { + const previous = [ + makeBlockRange("eth", 0, 10, HASH_A), + makeBlockRange("polygon", 0, 10, HASH_B) + ] + const incoming = [ + makeBlockRange("eth", 11, 20, HASH_B, HASH_A), + makeBlockRange("polygon", 11, 20, HASH_C, HASH_B) + ] + const result = yield* validateNetworks(previous, incoming).pipe(Effect.either) + expect(Either.isRight(result)).toBe(true) + })) + + it.effect("rejects duplicate networks in batch", ({ expect }) => + Effect.gen(function*() { + const incoming = [ + makeBlockRange("eth", 0, 10, HASH_A), + makeBlockRange("eth", 0, 10, HASH_B) + ] + const result = yield* validateNetworks([], incoming).pipe(Effect.either) + expect(Either.isLeft(result)).toBe(true) + if (Either.isLeft(result)) { + expect(result.left).toBeInstanceOf(DuplicateNetworkError) + } + })) + + it.effect("rejects network count change", ({ expect }) => + Effect.gen(function*() { + const previous = [ + makeBlockRange("eth", 0, 10, HASH_A) + ] + const incoming = [ + makeBlockRange("eth", 11, 20, HASH_B, HASH_A), + makeBlockRange("polygon", 0, 10, HASH_C) + ] + const result = yield* validateNetworks(previous, incoming).pipe(Effect.either) + expect(Either.isLeft(result)).toBe(true) + if (Either.isLeft(result)) { + expect(result.left).toBeInstanceOf(NetworkCountChangedError) + } + })) + + it.effect("rejects unexpected network", ({ expect }) => + Effect.gen(function*() { + const previous = [ + makeBlockRange("eth", 0, 10, HASH_A) + ] + const incoming = [ + makeBlockRange("polygon", 0, 10, HASH_B) + ] + const result = yield* validateNetworks(previous, incoming).pipe(Effect.either) + expect(Either.isLeft(result)).toBe(true) + if (Either.isLeft(result)) { + expect(result.left).toBeInstanceOf(UnexpectedNetworkError) + } + })) +}) + +// ============================================================================= +// validateConsecutiveness Tests +// ============================================================================= + +describe("validateConsecutiveness", () => { + it.effect("allows first batch without validation", ({ expect }) => + Effect.gen(function*() { + const incoming = [makeBlockRange("eth", 0, 10, HASH_A)] + const result = yield* validateConsecutiveness([], incoming).pipe(Effect.either) + expect(Either.isRight(result)).toBe(true) + })) + + it.effect("allows consecutive blocks with matching hash chain", ({ expect }) => + Effect.gen(function*() { + const previous = [makeBlockRange("eth", 0, 10, HASH_A)] + const incoming = [makeBlockRange("eth", 11, 20, HASH_B, HASH_A)] + const result = yield* validateConsecutiveness(previous, incoming).pipe(Effect.either) + expect(Either.isRight(result)).toBe(true) + })) + + it.effect("allows identical ranges (watermark repeat)", ({ expect }) => + Effect.gen(function*() { + const range = makeBlockRange("eth", 0, 10, HASH_A) + const result = yield* validateConsecutiveness([range], [range]).pipe(Effect.either) + expect(Either.isRight(result)).toBe(true) + })) + + it.effect("allows backwards jump with hash mismatch (reorg)", ({ expect }) => + Effect.gen(function*() { + const previous = [makeBlockRange("eth", 0, 10, HASH_A)] + // Backwards jump (start=5 < prev.end+1=11) with different hash chain + const incoming = [makeBlockRange("eth", 5, 12, HASH_C, HASH_B)] + const result = yield* validateConsecutiveness(previous, incoming).pipe(Effect.either) + expect(Either.isRight(result)).toBe(true) + })) + + it.effect("rejects consecutive blocks with hash mismatch", ({ expect }) => + Effect.gen(function*() { + const previous = [makeBlockRange("eth", 0, 10, HASH_A)] + // Consecutive (start=11 == prev.end+1=11) but prevHash doesn't match + const incoming = [makeBlockRange("eth", 11, 20, HASH_C, HASH_B)] + const result = yield* validateConsecutiveness(previous, incoming).pipe(Effect.either) + expect(Either.isLeft(result)).toBe(true) + if (Either.isLeft(result)) { + expect(result.left).toBeInstanceOf(HashMismatchOnConsecutiveBlocksError) + } + })) + + it.effect("rejects backwards jump with matching hash (invalid reorg)", ({ expect }) => + Effect.gen(function*() { + const previous = [makeBlockRange("eth", 0, 10, HASH_A)] + // Backwards jump but same hash chain - invalid + const incoming = [makeBlockRange("eth", 5, 12, HASH_B, HASH_A)] + const result = yield* validateConsecutiveness(previous, incoming).pipe(Effect.either) + expect(Either.isLeft(result)).toBe(true) + if (Either.isLeft(result)) { + expect(result.left).toBeInstanceOf(InvalidReorgError) + } + })) + + it.effect("rejects forward gap", ({ expect }) => + Effect.gen(function*() { + const previous = [makeBlockRange("eth", 0, 10, HASH_A)] + // Gap: start=15 > prev.end+1=11 + const incoming = [makeBlockRange("eth", 15, 20, HASH_B, HASH_A)] + const result = yield* validateConsecutiveness(previous, incoming).pipe(Effect.either) + expect(Either.isLeft(result)).toBe(true) + if (Either.isLeft(result)) { + expect(result.left).toBeInstanceOf(GapError) + expect((result.left as GapError).missingStart).toBe(11) + expect((result.left as GapError).missingEnd).toBe(14) + } + })) +}) + +// ============================================================================= +// validateAll Tests +// ============================================================================= + +describe("validateAll", () => { + it.effect("passes all validations for valid consecutive batch", ({ expect }) => + Effect.gen(function*() { + const previous = [makeBlockRange("eth", 100, 110, HASH_A, HASH_B)] + const incoming = [makeBlockRange("eth", 111, 120, HASH_B, HASH_A)] + const result = yield* validateAll(previous, incoming).pipe(Effect.either) + expect(Either.isRight(result)).toBe(true) + })) + + it.effect("fails on prevHash validation", ({ expect }) => + Effect.gen(function*() { + const incoming = [makeBlockRange("eth", 100, 110, HASH_A)] // Missing prevHash + const result = yield* validateAll([], incoming).pipe(Effect.either) + expect(Either.isLeft(result)).toBe(true) + })) + + it.effect("fails on network validation", ({ expect }) => + Effect.gen(function*() { + const previous = [makeBlockRange("eth", 100, 110, HASH_A, HASH_B)] + const incoming = [makeBlockRange("polygon", 100, 110, HASH_A, HASH_B)] + const result = yield* validateAll(previous, incoming).pipe(Effect.either) + expect(Either.isLeft(result)).toBe(true) + })) + + it.effect("fails on consecutiveness validation", ({ expect }) => + Effect.gen(function*() { + const previous = [makeBlockRange("eth", 100, 110, HASH_A, HASH_B)] + // Gap: start=115 > prev.end+1=111 + const incoming = [makeBlockRange("eth", 115, 120, HASH_B, HASH_A)] + const result = yield* validateAll(previous, incoming).pipe(Effect.either) + expect(Either.isLeft(result)).toBe(true) + })) +}) From 8f0c92293926f7bb9e08dcbc6dba8595096a422b Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 5 Feb 2026 14:47:09 -0500 Subject: [PATCH 02/12] Add TransactionalStream with crash recovery and reorg handling Implements a full transactional layer on top of the protocol stream, providing exactly-once semantics for data processing: - Transaction IDs: Monotonically increasing IDs for each event - StateStore: Pluggable persistence via Context.Tag (InMemoryStateStore included) - Watermark Buffer: Tracks watermarks for reorg recovery point calculation - CommitHandle: Explicit commit control with idempotent semantics - Rewind Detection: Detects and invalidates uncommitted transactions on restart - Retention Window: Prunes old watermarks outside configurable block window Also adds protocol-stream reorg tests ported from the Rust implementation. Co-Authored-By: Claude Opus 4.5 --- .../src/transactional-stream/algorithms.ts | 262 +++++++ .../src/transactional-stream/commit-handle.ts | 71 ++ .../amp/src/transactional-stream/errors.ts | 62 ++ .../amp/src/transactional-stream/index.ts | 137 ++++ .../src/transactional-stream/memory-store.ts | 218 ++++++ .../src/transactional-stream/state-actor.ts | 366 +++++++++ .../src/transactional-stream/state-store.ts | 160 ++++ .../amp/src/transactional-stream/stream.ts | 297 ++++++++ .../amp/src/transactional-stream/types.ts | 201 +++++ .../amp/test/protocol-stream/reorg.test.ts | 493 ++++++++++++ .../transactional-stream/algorithms.test.ts | 355 +++++++++ .../transactional-stream/memory-store.test.ts | 321 ++++++++ .../transactional-stream/state-actor.test.ts | 505 +++++++++++++ specs/transactional-stream.md | 702 ++++++++++++++++++ 14 files changed, 4150 insertions(+) create mode 100644 packages/amp/src/transactional-stream/algorithms.ts create mode 100644 packages/amp/src/transactional-stream/commit-handle.ts create mode 100644 packages/amp/src/transactional-stream/errors.ts create mode 100644 packages/amp/src/transactional-stream/index.ts create mode 100644 packages/amp/src/transactional-stream/memory-store.ts create mode 100644 packages/amp/src/transactional-stream/state-actor.ts create mode 100644 packages/amp/src/transactional-stream/state-store.ts create mode 100644 packages/amp/src/transactional-stream/stream.ts create mode 100644 packages/amp/src/transactional-stream/types.ts create mode 100644 packages/amp/test/protocol-stream/reorg.test.ts create mode 100644 packages/amp/test/transactional-stream/algorithms.test.ts create mode 100644 packages/amp/test/transactional-stream/memory-store.test.ts create mode 100644 packages/amp/test/transactional-stream/state-actor.test.ts create mode 100644 specs/transactional-stream.md diff --git a/packages/amp/src/transactional-stream/algorithms.ts b/packages/amp/src/transactional-stream/algorithms.ts new file mode 100644 index 0000000..226979d --- /dev/null +++ b/packages/amp/src/transactional-stream/algorithms.ts @@ -0,0 +1,262 @@ +/** + * Core algorithms for the TransactionalStream. + * + * These are pure functions that implement the buffer management logic + * for recovery point calculation and retention-based pruning. + * + * Ported from Rust implementation in: + * `.repos/amp/crates/clients/flight/src/transactional.rs` + * + * @module + */ +import type { BlockRange } from "../models.ts" +import type { InvalidationRange } from "../protocol-stream/messages.ts" +import type { TransactionId } from "./types.ts" + +// ============================================================================= +// Recovery Point Algorithm +// ============================================================================= + +/** + * Find the recovery point watermark for a reorg. + * + * Walks backwards through watermarks to find the last unaffected watermark. + * This watermark represents the safe recovery point - everything after it + * needs to be invalidated. + * + * A watermark is affected if ANY of its networks have a block range that + * starts at or after the reorg point for that network. + * + * @param buffer - Watermark buffer (oldest to newest) + * @param invalidation - Invalidation ranges from the reorg + * @returns Last unaffected watermark [id, ranges], or undefined if all affected + * + * @example + * ```typescript + * const buffer = [ + * [1, [{ network: "eth", numbers: { start: 0, end: 10 }, ... }]], + * [3, [{ network: "eth", numbers: { start: 11, end: 20 }, ... }]], + * [5, [{ network: "eth", numbers: { start: 21, end: 30 }, ... }]] + * ] + * const invalidation = [{ network: "eth", start: 25, end: 35 }] + * + * // Walks backwards: id=5 is affected (21 >= 25? no, but 25 is within range) + * // Actually: affected if range.start >= reorg_point + * // id=5: start=21 >= 25? No -> not affected by this criteria + * // But the Rust impl checks if range.start >= point, which means + * // "this watermark's data starts after or at the reorg point" + * + * const recovery = findRecoveryPoint(buffer, invalidation) + * // Returns [3, [...]] - last unaffected watermark + * ``` + */ +export const findRecoveryPoint = ( + buffer: ReadonlyArray]>, + invalidation: ReadonlyArray +): readonly [TransactionId, ReadonlyArray] | undefined => { + if (buffer.length === 0) { + return undefined + } + + // Build reorg points map: network -> first invalid block + const points = new Map() + for (const inv of invalidation) { + points.set(inv.network, inv.start) + } + + // Walk backwards through watermarks (newest to oldest) + for (let i = buffer.length - 1; i >= 0; i--) { + const entry = buffer[i]! + const [id, ranges] = entry + + // Check if ANY network in this watermark is affected + // A watermark is affected if its range starts at or after the reorg point + const affected = ranges.some((range) => { + const point = points.get(range.network) + // Only check networks that are in the invalidation list + if (point === undefined) { + return false + } + // Watermark is affected if range.start >= reorg_point + return range.numbers.start >= point + }) + + if (!affected) { + // Found last unaffected watermark + return [id, ranges] + } + } + + // All watermarks are affected + return undefined +} + +// ============================================================================= +// Pruning Point Algorithm +// ============================================================================= + +/** + * Compute the last transaction ID that should be pruned based on retention window. + * + * Walks through watermarks from oldest to newest and identifies the last one + * outside the retention window. A watermark is outside the retention window + * if ALL its networks have ranges that end before the cutoff. + * + * The cutoff for each network is: `latest_start - retention` + * + * @param buffer - Watermark buffer (oldest to newest) + * @param retention - Retention window in blocks + * @returns Last transaction ID to prune (all IDs <= this are removed), or undefined if no pruning needed + * + * @example + * ```typescript + * const buffer = [ + * [1, [{ network: "eth", numbers: { start: 0, end: 10 }, ... }]], + * [3, [{ network: "eth", numbers: { start: 11, end: 20 }, ... }]], + * [5, [{ network: "eth", numbers: { start: 100, end: 110 }, ... }]] + * ] + * const retention = 50 + * + * // Latest start = 100, cutoff = 100 - 50 = 50 + * // id=1: end=10 < 50? Yes -> outside retention + * // id=3: end=20 < 50? Yes -> outside retention + * // id=5: skip (always keep latest) + * + * const prune = findPruningPoint(buffer, retention) + * // Returns 3 - prune all IDs <= 3 + * ``` + */ +export const findPruningPoint = ( + buffer: ReadonlyArray]>, + retention: number +): TransactionId | undefined => { + if (buffer.length === 0) { + return undefined + } + + // Get latest ranges from buffer (last entry) + const latestEntry = buffer[buffer.length - 1]! + const [, latestRanges] = latestEntry + + if (latestRanges.length === 0) { + return undefined + } + + // Calculate cutoff block for each network + // Cutoff = latest_start - retention (minimum 0) + const cutoffs = new Map() + for (const range of latestRanges) { + const cutoff = Math.max(0, range.numbers.start - retention) + cutoffs.set(range.network, cutoff) + } + + let last: TransactionId | undefined = undefined + + // Walk from front (oldest to newest), skipping the last watermark + // We never prune the most recent watermark + for (let i = 0; i < buffer.length - 1; i++) { + const entry = buffer[i]! + const [id, ranges] = entry + + // Check if this watermark is ENTIRELY outside retention window + // ALL networks must have ranges that end before their cutoff + const outside = ranges.every((range) => { + const cutoff = cutoffs.get(range.network) + // If network isn't in latest, consider it outside retention + if (cutoff === undefined) { + return true + } + return range.numbers.end < cutoff + }) + + if (outside) { + last = id + } else { + // First watermark within retention - stop searching + break + } + } + + return last +} + +// ============================================================================= +// Partial Reorg Detection +// ============================================================================= + +/** + * Check if a reorg is partial (unrecoverable). + * + * A partial reorg occurs when the recovery point watermark has a range + * that partially overlaps with the invalidation - meaning the reorg point + * falls within a watermark's range, not at its boundary. + * + * If we were to ignore this, we would end up with a data gap because we're + * telling the consumer to invalidate data that won't be replayed. + * + * @param recoveryRanges - Block ranges from the recovery point watermark + * @param invalidation - Invalidation ranges from the reorg + * @returns The network with partial reorg, or undefined if no partial reorg + */ +export const checkPartialReorg = ( + recoveryRanges: ReadonlyArray, + invalidation: ReadonlyArray +): string | undefined => { + for (const range of recoveryRanges) { + const inv = invalidation.find((i) => i.network === range.network) + if (inv !== undefined) { + const point = inv.start + // Check: recovery.start < reorg_point <= recovery.end + // This means the reorg point falls within the recovery watermark's range + if (range.numbers.start < point && point <= range.numbers.end) { + return range.network + } + } + } + return undefined +} + +// ============================================================================= +// Commit Compression +// ============================================================================= + +/** + * Compress multiple pending commits into a single atomic update. + * + * Takes the maximum pruning point from all pending commits and + * removes any watermarks that would be pruned from the insert list. + * + * @param pendingCommits - Array of [transactionId, { ranges, prune }] tuples + * @returns Compressed commit with combined inserts and maximum prune point + */ +export const compressCommits = ( + pendingCommits: ReadonlyArray< + readonly [TransactionId, { readonly ranges: ReadonlyArray; readonly prune: TransactionId | undefined }] + > +): { insert: Array]>; prune: TransactionId | undefined } => { + const insert: Array]> = [] + let maxPrune: TransactionId | undefined = undefined + + // Collect watermarks and find maximum prune point + for (const [id, commit] of pendingCommits) { + insert.push([id, commit.ranges]) + + // Take the maximum prune point + if (commit.prune !== undefined) { + if (maxPrune === undefined || commit.prune > maxPrune) { + maxPrune = commit.prune + } + } + } + + // Remove any watermarks that were pruned + const filteredInsert = + maxPrune !== undefined + ? insert.filter(([id]) => id > maxPrune!) + : insert + + return { + insert: filteredInsert, + prune: maxPrune + } +} diff --git a/packages/amp/src/transactional-stream/commit-handle.ts b/packages/amp/src/transactional-stream/commit-handle.ts new file mode 100644 index 0000000..d2b5506 --- /dev/null +++ b/packages/amp/src/transactional-stream/commit-handle.ts @@ -0,0 +1,71 @@ +/** + * CommitHandle - handle for committing state changes. + * + * @module + */ +import type * as Effect from "effect/Effect" +import type { StateStoreError } from "./errors.ts" +import type { TransactionId } from "./types.ts" + +// ============================================================================= +// CommitHandle Interface +// ============================================================================= + +/** + * Handle for committing state changes. + * + * Returned with each event from the transactional stream. The user must + * call `commit()` to persist the state change. If the user doesn't commit + * and the process crashes, the event will be replayed via Rewind on restart. + * + * Commits are idempotent - calling `commit()` multiple times is safe. + * Subsequent calls after the first are no-ops. + * + * @example + * ```typescript + * yield* txStream.pipe( + * Stream.runForEach(([event, commitHandle]) => + * Effect.gen(function*() { + * // Process the event + * yield* processEvent(event) + * + * // Commit the state change + * yield* commitHandle.commit() + * }) + * ) + * ) + * ``` + */ +export interface CommitHandle { + /** + * The transaction ID associated with this commit. + */ + readonly id: TransactionId + + /** + * Commit all pending changes up to and including this transaction ID. + * + * Safe to call multiple times - subsequent calls are no-ops if already committed. + * + * Multiple commits can be batched together for efficiency - if you don't commit + * after every event, the next commit will include all previous uncommitted events. + */ + readonly commit: () => Effect.Effect +} + +// ============================================================================= +// Factory +// ============================================================================= + +/** + * Create a CommitHandle. + * + * @internal This is used internally by the StateActor. + */ +export const makeCommitHandle = ( + id: TransactionId, + commitFn: (id: TransactionId) => Effect.Effect +): CommitHandle => ({ + id, + commit: () => commitFn(id) +}) diff --git a/packages/amp/src/transactional-stream/errors.ts b/packages/amp/src/transactional-stream/errors.ts new file mode 100644 index 0000000..a2c5c6b --- /dev/null +++ b/packages/amp/src/transactional-stream/errors.ts @@ -0,0 +1,62 @@ +/** + * Error types for the TransactionalStream. + * + * @module + */ +import * as Schema from "effect/Schema" +import type { ProtocolStreamError } from "../protocol-stream/errors.ts" + +// ============================================================================= +// StateStore Errors +// ============================================================================= + +/** + * Error from StateStore operations. + */ +export class StateStoreError extends Schema.TaggedError( + "Amp/TransactionalStream/StateStoreError" +)("StateStoreError", { + reason: Schema.String, + operation: Schema.Literal("advance", "commit", "truncate", "load"), + cause: Schema.optional(Schema.Defect) +}) {} + +// ============================================================================= +// Reorg Errors +// ============================================================================= + +/** + * Unrecoverable reorg - all buffered watermarks are affected. + * This occurs when a reorg is so deep that there's no valid recovery point. + * The stream must be restarted with fresh state. + */ +export class UnrecoverableReorgError extends Schema.TaggedError( + "Amp/TransactionalStream/UnrecoverableReorgError" +)("UnrecoverableReorgError", { + reason: Schema.String +}) {} + +/** + * Partial reorg - recovery point doesn't align with invalidation boundary. + * This occurs when a reorg point falls in the middle of a watermark's block range. + * The stream must be restarted with fresh state. + */ +export class PartialReorgError extends Schema.TaggedError( + "Amp/TransactionalStream/PartialReorgError" +)("PartialReorgError", { + reason: Schema.String, + network: Schema.String +}) {} + +// ============================================================================= +// Combined Error Type +// ============================================================================= + +/** + * Union of all transactional stream errors. + */ +export type TransactionalStreamError = + | StateStoreError + | UnrecoverableReorgError + | PartialReorgError + | ProtocolStreamError diff --git a/packages/amp/src/transactional-stream/index.ts b/packages/amp/src/transactional-stream/index.ts new file mode 100644 index 0000000..dc7d40e --- /dev/null +++ b/packages/amp/src/transactional-stream/index.ts @@ -0,0 +1,137 @@ +/** + * TransactionalStream module - exactly-once semantics for data processing. + * + * Provides crash recovery, reorg handling, and commit control on top of + * the stateless protocol stream. + * + * @example + * ```typescript + * import { + * TransactionalStream, + * InMemoryStateStore, + * type TransactionEvent, + * type CommitHandle + * } from "@edgeandnode/amp/transactional-stream" + * + * const program = Effect.gen(function*() { + * const txStream = yield* TransactionalStream + * + * yield* txStream.forEach( + * "SELECT * FROM eth.logs", + * { retention: 128 }, + * (event) => Effect.gen(function*() { + * switch (event._tag) { + * case "Data": + * yield* processData(event.data) + * break + * case "Undo": + * yield* rollback(event.invalidate) + * break + * case "Watermark": + * yield* checkpoint(event.ranges) + * break + * } + * }) + * ) + * }) + * + * Effect.runPromise(program.pipe( + * Effect.provide(TransactionalStream.layer), + * Effect.provide(InMemoryStateStore.layer), + * Effect.provide(ArrowFlight.layer), + * Effect.provide(Transport.layer) + * )) + * ``` + * + * @module + */ + +// ============================================================================= +// Types +// ============================================================================= + +export { + type TransactionId, + TransactionId as TransactionIdSchema, + type TransactionIdRange, + TransactionIdRange as TransactionIdRangeSchema, + type TransactionEvent, + TransactionEvent as TransactionEventSchema, + type TransactionEventData, + TransactionEventData as TransactionEventDataSchema, + type TransactionEventUndo, + TransactionEventUndo as TransactionEventUndoSchema, + type TransactionEventWatermark, + TransactionEventWatermark as TransactionEventWatermarkSchema, + type UndoCause, + UndoCause as UndoCauseSchema, + type UndoCauseReorg, + UndoCauseReorg as UndoCauseReorgSchema, + type UndoCauseRewind, + UndoCauseRewind as UndoCauseRewindSchema, + // Constructors + dataEvent, + undoEvent, + watermarkEvent, + reorgCause, + rewindCause +} from "./types.ts" + +// ============================================================================= +// Errors +// ============================================================================= + +export { + StateStoreError, + UnrecoverableReorgError, + PartialReorgError, + type TransactionalStreamError +} from "./errors.ts" + +// ============================================================================= +// StateStore Service +// ============================================================================= + +export { + StateStore, + type StateStoreService, + type StateSnapshot, + type Commit, + emptySnapshot, + emptyCommit, + makeCommit +} from "./state-store.ts" + +// ============================================================================= +// InMemoryStateStore Layer +// ============================================================================= + +export * as InMemoryStateStore from "./memory-store.ts" + +// ============================================================================= +// CommitHandle +// ============================================================================= + +export { type CommitHandle, makeCommitHandle } from "./commit-handle.ts" + +// ============================================================================= +// TransactionalStream Service +// ============================================================================= + +export { + TransactionalStream, + layer, + type TransactionalStreamService, + type TransactionalStreamOptions +} from "./stream.ts" + +// ============================================================================= +// Algorithms (for advanced use cases and testing) +// ============================================================================= + +export { + findRecoveryPoint, + findPruningPoint, + checkPartialReorg, + compressCommits +} from "./algorithms.ts" diff --git a/packages/amp/src/transactional-stream/memory-store.ts b/packages/amp/src/transactional-stream/memory-store.ts new file mode 100644 index 0000000..8c1b1e3 --- /dev/null +++ b/packages/amp/src/transactional-stream/memory-store.ts @@ -0,0 +1,218 @@ +/** + * InMemoryStateStore - reference implementation of StateStore. + * + * Uses Effect Ref for in-memory state management. Not crash-safe but + * suitable for development, testing, and ephemeral use cases. + * + * @module + */ +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" +import * as Ref from "effect/Ref" +import { + StateStore, + type StateSnapshot, + type StateStoreService, + type Commit, + emptySnapshot +} from "./state-store.ts" +import type { TransactionId } from "./types.ts" + +// ============================================================================= +// Implementation +// ============================================================================= + +/** + * Create InMemoryStateStore service implementation. + */ +const makeWithInitialState = (initial: StateSnapshot) => + Effect.gen(function*() { + const stateRef = yield* Ref.make(initial) + + const advance = (next: TransactionId) => + Ref.update(stateRef, (state) => ({ + ...state, + next + })) + + const commit = (commitData: Commit) => + Ref.update(stateRef, (state) => { + let buffer = [...state.buffer] + + // Remove pruned watermarks (all IDs <= prune) + if (commitData.prune !== undefined) { + buffer = buffer.filter(([id]) => id > commitData.prune!) + } + + // Add new watermarks + for (const entry of commitData.insert) { + buffer.push(entry) + } + + return { ...state, buffer } + }) + + const truncate = (from: TransactionId) => + Ref.update(stateRef, (state) => ({ + ...state, + buffer: state.buffer.filter(([id]) => id < from) + })) + + const load = () => Ref.get(stateRef) + + return { + advance, + commit, + truncate, + load + } satisfies StateStoreService + }) + +/** + * Create InMemoryStateStore service with empty initial state. + */ +const make = makeWithInitialState(emptySnapshot) + +// ============================================================================= +// Layers +// ============================================================================= + +/** + * Layer providing InMemoryStateStore with empty initial state. + * + * @example + * ```typescript + * const program = Effect.gen(function*() { + * const store = yield* StateStore + * const snapshot = yield* store.load() + * console.log(snapshot.next) // 0 + * }) + * + * Effect.runPromise(program.pipe(Effect.provide(InMemoryStateStore.layer))) + * ``` + */ +export const layer: Layer.Layer = Layer.effect(StateStore, make) + +/** + * Create a layer with pre-populated initial state. + * + * Useful for testing scenarios that require specific initial conditions. + * + * @example + * ```typescript + * const testState: StateSnapshot = { + * buffer: [[5 as TransactionId, [{ network: "eth", ... }]]], + * next: 10 as TransactionId + * } + * + * const TestLayer = InMemoryStateStore.layerWithState(testState) + * + * Effect.runPromise( + * program.pipe(Effect.provide(TestLayer)) + * ) + * ``` + */ +export const layerWithState = (initial: StateSnapshot): Layer.Layer => + Layer.effect(StateStore, makeWithInitialState(initial)) + +// ============================================================================= +// Testing Utilities +// ============================================================================= + +/** + * Create an InMemoryStateStore service directly (for testing). + * + * Returns both the service and a reference to inspect internal state. + * + * @example + * ```typescript + * const { service, stateRef } = yield* InMemoryStateStore.makeTestable() + * + * yield* service.advance(5 as TransactionId) + * + * const state = yield* Ref.get(stateRef) + * expect(state.next).toBe(5) + * ``` + */ +export const makeTestable = Effect.gen(function*() { + const stateRef = yield* Ref.make(emptySnapshot) + + const advance = (next: TransactionId) => + Ref.update(stateRef, (state) => ({ + ...state, + next + })) + + const commit = (commitData: Commit) => + Ref.update(stateRef, (state) => { + let buffer = [...state.buffer] + + if (commitData.prune !== undefined) { + buffer = buffer.filter(([id]) => id > commitData.prune!) + } + + for (const entry of commitData.insert) { + buffer.push(entry) + } + + return { ...state, buffer } + }) + + const truncate = (from: TransactionId) => + Ref.update(stateRef, (state) => ({ + ...state, + buffer: state.buffer.filter(([id]) => id < from) + })) + + const load = () => Ref.get(stateRef) + + const service: StateStoreService = { + advance, + commit, + truncate, + load + } + + return { service, stateRef } +}) + +/** + * Create a layer that also exposes the internal state ref for testing. + */ +export class TestStateRef extends Effect.Service()("TestStateRef", { + effect: Effect.gen(function*() { + return yield* Ref.make(emptySnapshot) + }) +}) {} + +export const testLayer: Layer.Layer = Layer.effect( + StateStore, + Effect.gen(function*() { + const stateRef = yield* TestStateRef + + const advance = (next: TransactionId) => + Ref.update(stateRef, (state) => ({ ...state, next })) + + const commit = (commitData: Commit) => + Ref.update(stateRef, (state) => { + let buffer = [...state.buffer] + if (commitData.prune !== undefined) { + buffer = buffer.filter(([id]) => id > commitData.prune!) + } + for (const entry of commitData.insert) { + buffer.push(entry) + } + return { ...state, buffer } + }) + + const truncate = (from: TransactionId) => + Ref.update(stateRef, (state) => ({ + ...state, + buffer: state.buffer.filter(([id]) => id < from) + })) + + const load = () => Ref.get(stateRef) + + return { advance, commit, truncate, load } satisfies StateStoreService + }) +).pipe(Layer.provideMerge(TestStateRef.Default)) diff --git a/packages/amp/src/transactional-stream/state-actor.ts b/packages/amp/src/transactional-stream/state-actor.ts new file mode 100644 index 0000000..c914915 --- /dev/null +++ b/packages/amp/src/transactional-stream/state-actor.ts @@ -0,0 +1,366 @@ +/** + * StateActor - internal state management for TransactionalStream. + * + * Wraps a StateStore with a Ref for concurrent-safe in-memory state management. + * Created once per stream instance. Not exported publicly. + * + * @module + * @internal + */ +import * as Effect from "effect/Effect" +import * as Ref from "effect/Ref" +import type { BlockRange } from "../models.ts" +import type { ProtocolMessage } from "../protocol-stream/messages.ts" +import { findRecoveryPoint, findPruningPoint, checkPartialReorg, compressCommits } from "./algorithms.ts" +import { makeCommitHandle, type CommitHandle } from "./commit-handle.ts" +import { PartialReorgError, type StateStoreError, UnrecoverableReorgError } from "./errors.ts" +import type { StateStoreService } from "./state-store.ts" +import { + type TransactionId, + type TransactionEvent, + dataEvent, + undoEvent, + watermarkEvent, + reorgCause, + rewindCause +} from "./types.ts" + +// ============================================================================= +// Types +// ============================================================================= + +/** + * Pending commit waiting for user to call commit handle. + */ +interface PendingCommit { + readonly ranges: ReadonlyArray + readonly prune: TransactionId | undefined +} + +/** + * Internal state container - in-memory copy of persisted state. + */ +interface StateContainer { + /** Next transaction ID to assign */ + next: TransactionId + /** Watermark buffer (oldest to newest) */ + buffer: Array]> + /** Uncommitted watermarks awaiting user commit */ + uncommitted: Array +} + +/** + * Action union for execute(). + */ +export type Action = + | { readonly _tag: "Message"; readonly message: ProtocolMessage } + | { readonly _tag: "Rewind" } + +/** + * StateActor interface - manages transactional stream state. + */ +export interface StateActor { + /** Get last watermark from buffer */ + readonly watermark: () => Effect.Effect] | undefined> + + /** Get next transaction ID without incrementing */ + readonly peek: () => Effect.Effect + + /** Execute an action and return event with commit handle */ + readonly execute: ( + action: Action + ) => Effect.Effect + + /** Commit pending changes up to and including this ID */ + readonly commit: (id: TransactionId) => Effect.Effect +} + +// ============================================================================= +// Factory +// ============================================================================= + +/** + * Create a StateActor from a StateStore service. + * + * @param store - The StateStore service to wrap + * @param retention - Retention window in blocks for pruning + * @returns Effect that creates a StateActor + */ +export const makeStateActor = ( + store: StateStoreService, + retention: number +): Effect.Effect => + Effect.gen(function*() { + // Load initial state from store + const snapshot = yield* store.load() + + // Create mutable state container wrapped in Ref + const containerRef = yield* Ref.make({ + next: snapshot.next, + buffer: [...snapshot.buffer], + uncommitted: [] + }) + + // ========================================================================= + // watermark() + // ========================================================================= + + const watermark = () => + Ref.get(containerRef).pipe( + Effect.map((state) => + state.buffer.length > 0 + ? state.buffer[state.buffer.length - 1] + : undefined + ) + ) + + // ========================================================================= + // peek() + // ========================================================================= + + const peek = () => Ref.get(containerRef).pipe(Effect.map((state) => state.next)) + + // ========================================================================= + // execute() + // ========================================================================= + + const execute = (action: Action) => + Effect.gen(function*() { + // 1. Pre-allocate monotonic ID + const id = yield* Ref.getAndUpdate(containerRef, (state) => ({ + ...state, + next: (state.next + 1) as TransactionId + })).pipe(Effect.map((state) => state.next)) + + const nextId = (id + 1) as TransactionId + + // Persist the new next ID immediately (ensures monotonicity survives crashes) + yield* store.advance(nextId) + + // 2. Execute action based on type + const event: TransactionEvent = yield* ((): Effect.Effect< + TransactionEvent, + StateStoreError | UnrecoverableReorgError | PartialReorgError + > => { + switch (action._tag) { + case "Rewind": + return executeRewind(id, containerRef) + + case "Message": + return executeMessage(id, action.message, containerRef, store, retention) + } + })() + + // 3. Return event with commit handle + const handle = makeCommitHandle(id, commit) + + return [event, handle] as const + }) + + // ========================================================================= + // commit() + // ========================================================================= + + const commit = (id: TransactionId): Effect.Effect => + Effect.gen(function*() { + const state = yield* Ref.get(containerRef) + + // Find position where IDs become > id (all before this are <= id) + const pos = state.uncommitted.findIndex(([currentId]) => currentId > id) + const endIndex = pos === -1 ? state.uncommitted.length : pos + + if (endIndex === 0) { + // Nothing to commit + return + } + + // Collect commits [0..endIndex) + const pending = state.uncommitted.slice(0, endIndex) + + // Compress and persist + const compressed = compressCommits(pending) + + if (compressed.insert.length > 0 || compressed.prune !== undefined) { + // Apply pruning to in-memory buffer + yield* Ref.update(containerRef, (s) => { + let buffer = s.buffer + if (compressed.prune !== undefined) { + buffer = buffer.filter(([bufferId]) => bufferId > compressed.prune!) + } + return { ...s, buffer } + }) + + // Persist to store + yield* store.commit({ + insert: compressed.insert, + prune: compressed.prune + }) + } + + // Remove committed from uncommitted queue + yield* Ref.update(containerRef, (s) => ({ + ...s, + uncommitted: s.uncommitted.slice(endIndex) + })) + }) + + return { watermark, peek, execute, commit } + }) + +// ============================================================================= +// Action Handlers +// ============================================================================= + +/** + * Execute a Rewind action. + */ +const executeRewind = ( + id: TransactionId, + containerRef: Ref.Ref +): Effect.Effect => + Effect.gen(function*() { + const state = yield* Ref.get(containerRef) + + // Compute invalidation range based on buffer state + let invalidateStart: TransactionId + let invalidateEnd: TransactionId + + if (state.buffer.length === 0) { + // Empty buffer (early crash before any watermark): invalidate from the beginning + invalidateStart = 0 as TransactionId + invalidateEnd = Math.max(0, id - 1) as TransactionId + } else { + // Normal rewind: invalidate after last watermark + const lastWatermarkId = state.buffer[state.buffer.length - 1]![0] + invalidateStart = (lastWatermarkId + 1) as TransactionId + invalidateEnd = Math.max(invalidateStart, id - 1) as TransactionId + } + + return undoEvent(id, rewindCause(), { + start: invalidateStart, + end: invalidateEnd + }) + }) + +/** + * Execute a Message action. + */ +const executeMessage = ( + id: TransactionId, + message: ProtocolMessage, + containerRef: Ref.Ref, + store: StateStoreService, + retention: number +): Effect.Effect => + Effect.gen(function*() { + switch (message._tag) { + case "Data": + // Data events just pass through - no buffer mutation + return dataEvent(id, message.data, message.ranges) + + case "Watermark": + return yield* executeWatermark(id, message.ranges, containerRef, retention) + + case "Reorg": + return yield* executeReorg(id, message, containerRef, store) + } + }) + +/** + * Execute a Watermark message. + */ +const executeWatermark = ( + id: TransactionId, + ranges: ReadonlyArray, + containerRef: Ref.Ref, + retention: number +): Effect.Effect => + Effect.gen(function*() { + // Add watermark to buffer + yield* Ref.update(containerRef, (state) => ({ + ...state, + buffer: [...state.buffer, [id, ranges] as const] + })) + + // Compute pruning point based on current buffer state + const state = yield* Ref.get(containerRef) + const prune = findPruningPoint(state.buffer, retention) + + // Record in uncommitted queue + yield* Ref.update(containerRef, (s) => ({ + ...s, + uncommitted: [...s.uncommitted, [id, { ranges, prune }] as const] + })) + + return watermarkEvent(id, ranges, prune ?? null) + }) + +/** + * Execute a Reorg message. + */ +const executeReorg = ( + id: TransactionId, + message: Extract, + containerRef: Ref.Ref, + store: StateStoreService +): Effect.Effect => + Effect.gen(function*() { + const state = yield* Ref.get(containerRef) + const { invalidation } = message + + // 1. Find recovery point + const recovery = findRecoveryPoint(state.buffer, invalidation) + + // 2. Compute invalidation range + let invalidateStart: TransactionId + let invalidateEnd: TransactionId + + if (recovery === undefined) { + if (state.buffer.length === 0) { + // If the buffer is empty, invalidate everything up to before the current event + invalidateStart = 0 as TransactionId + invalidateEnd = Math.max(0, id - 1) as TransactionId + } else { + // No recovery point with a non-empty buffer means all buffered watermarks + // are affected by the reorg. This is not recoverable. + return yield* Effect.fail( + new UnrecoverableReorgError({ + reason: "All buffered watermarks are affected by the reorg" + }) + ) + } + } else { + const [recoveryId, recoveryRanges] = recovery + + // 3. Check for partial reorg + const partialNetwork = checkPartialReorg(recoveryRanges, invalidation) + if (partialNetwork !== undefined) { + return yield* Effect.fail( + new PartialReorgError({ + reason: "Recovery point doesn't align with reorg boundary", + network: partialNetwork + }) + ) + } + + invalidateStart = (recoveryId + 1) as TransactionId + invalidateEnd = Math.max(invalidateStart, id - 1) as TransactionId + } + + // 4. Truncate both in-memory and store + const truncateFrom = recovery !== undefined ? (recovery[0] + 1) as TransactionId : 0 as TransactionId + + yield* Ref.update(containerRef, (s) => ({ + ...s, + buffer: s.buffer.filter(([bufferId]) => bufferId < truncateFrom), + uncommitted: s.uncommitted.filter(([uncommittedId]) => uncommittedId < truncateFrom) + })) + + yield* store.truncate(truncateFrom) + + // 5. Emit Undo with Cause::Reorg + return undoEvent(id, reorgCause(invalidation), { + start: invalidateStart, + end: invalidateEnd + }) + }) diff --git a/packages/amp/src/transactional-stream/state-store.ts b/packages/amp/src/transactional-stream/state-store.ts new file mode 100644 index 0000000..8a05589 --- /dev/null +++ b/packages/amp/src/transactional-stream/state-store.ts @@ -0,0 +1,160 @@ +/** + * StateStore service definition. + * + * The StateStore is defined as an Effect Context.Tag service, allowing different + * implementations to be swapped via Layers. This mirrors the Rust pattern where + * `StateStore` is a trait implemented by `InMemoryStateStore`, `LmdbStateStore`, + * and `PostgresStateStore`. + * + * @module + */ +import * as Context from "effect/Context" +import type * as Effect from "effect/Effect" +import type { BlockRange } from "../models.ts" +import type { StateStoreError } from "./errors.ts" +import type { TransactionId } from "./types.ts" + +// ============================================================================= +// Data Types +// ============================================================================= + +/** + * Persisted state snapshot for crash recovery. + * Loaded once on startup, then maintained in-memory. + */ +export interface StateSnapshot { + /** + * Buffer of watermarks (oldest to newest). + * Each entry is a tuple of [transactionId, blockRanges]. + * Only watermarks are stored, not data events. + */ + readonly buffer: ReadonlyArray]> + + /** + * Next transaction ID to assign. + * Pre-allocated to ensure monotonicity survives crashes. + */ + readonly next: TransactionId +} + +/** + * Atomic state update for persistence. + * Batches multiple watermark inserts with optional pruning. + */ +export interface Commit { + /** + * Watermarks to insert (oldest to newest). + */ + readonly insert: ReadonlyArray]> + + /** + * Last transaction ID to prune (inclusive). + * All watermarks with ID <= this value are removed. + */ + readonly prune: TransactionId | undefined +} + +// ============================================================================= +// StateStore Service Interface +// ============================================================================= + +/** + * StateStore service interface - pluggable persistence for stream state. + * + * Different implementations provide different crash recovery guarantees: + * - InMemoryStateStore: No persistence (development/testing) + * - IndexedDBStateStore: Browser persistence (future) + * - SqliteStateStore: Node.js file-based persistence (future) + * - PostgresStateStore: Distributed persistence (future) + */ +export interface StateStoreService { + /** + * Pre-allocate next transaction ID. + * + * Called immediately after incrementing the in-memory counter. + * Ensures ID monotonicity survives crashes - IDs are never reused. + * + * @param next - The next transaction ID to persist + */ + readonly advance: (next: TransactionId) => Effect.Effect + + /** + * Atomically commit watermarks and apply pruning. + * + * Called when user invokes CommitHandle.commit(). + * Must be idempotent - safe to call multiple times with same data. + * + * @param commit - The commit containing watermarks to insert and optional prune point + */ + readonly commit: (commit: Commit) => Effect.Effect + + /** + * Truncate buffer during reorg handling. + * + * Removes all watermarks with ID >= from. + * Called immediately when reorg is detected (before emitting Undo). + * + * @param from - Remove all watermarks with ID >= this value + */ + readonly truncate: (from: TransactionId) => Effect.Effect + + /** + * Load initial state on startup. + * + * Called once when TransactionalStream is created. + * Returns empty state if no prior state exists. + */ + readonly load: () => Effect.Effect +} + +// ============================================================================= +// StateStore Context.Tag +// ============================================================================= + +/** + * StateStore Context.Tag - use this to depend on StateStore in Effects. + * + * @example + * ```typescript + * const program = Effect.gen(function*() { + * const store = yield* StateStore + * const snapshot = yield* store.load() + * // ... + * }) + * ``` + */ +export class StateStore extends Context.Tag("Amp/TransactionalStream/StateStore")< + StateStore, + StateStoreService +>() {} + +// ============================================================================= +// Helpers +// ============================================================================= + +/** + * Initial empty state for new streams or cleared stores. + */ +export const emptySnapshot: StateSnapshot = { + buffer: [], + next: 0 as TransactionId +} + +/** + * Create an empty commit (no-op). + */ +export const emptyCommit: Commit = { + insert: [], + prune: undefined +} + +/** + * Create a commit with watermarks to insert. + */ +export const makeCommit = ( + insert: ReadonlyArray]>, + prune?: TransactionId +): Commit => ({ + insert, + prune +}) diff --git a/packages/amp/src/transactional-stream/stream.ts b/packages/amp/src/transactional-stream/stream.ts new file mode 100644 index 0000000..0c71517 --- /dev/null +++ b/packages/amp/src/transactional-stream/stream.ts @@ -0,0 +1,297 @@ +/** + * TransactionalStream service - provides exactly-once semantics for data processing. + * + * The TransactionalStream wraps ArrowFlight's protocol stream with: + * - Transaction IDs for each event + * - Crash recovery via persistent state + * - Rewind detection for uncommitted transactions + * - CommitHandle for explicit commit control + * + * @module + */ +import * as Context from "effect/Context" +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" +import * as Stream from "effect/Stream" +import { ArrowFlight, type ProtocolStreamOptions, type QueryOptions } from "../arrow-flight.ts" +import type { BlockRange } from "../models.ts" +import type { ProtocolStreamError } from "../protocol-stream/errors.ts" +import type { CommitHandle } from "./commit-handle.ts" +import type { StateStoreError, TransactionalStreamError, UnrecoverableReorgError, PartialReorgError } from "./errors.ts" +import { type Action, makeStateActor, type StateActor } from "./state-actor.ts" +import { StateStore } from "./state-store.ts" +import type { TransactionEvent, TransactionId } from "./types.ts" + +// ============================================================================= +// Options +// ============================================================================= + +/** + * Options for creating a transactional stream. + * Note: StateStore is NOT passed here - it comes from the Layer context. + */ +export interface TransactionalStreamOptions { + /** + * Retention window in blocks for pruning old watermarks. + * Watermarks older than this will be pruned to save memory. + * @default 128 + */ + readonly retention?: number + + /** + * Optional schema for data validation. + * If provided, data will be validated and decoded using this schema. + */ + readonly schema?: QueryOptions["schema"] +} + +// ============================================================================= +// Service Interface +// ============================================================================= + +/** + * TransactionalStream service interface. + * + * Provides transactional semantics on top of the protocol stream: + * - Each event has a unique, monotonically increasing transaction ID + * - Events can be committed explicitly via CommitHandle + * - Uncommitted events are replayed on restart via Rewind + * - Reorgs emit Undo events with invalidation ranges + */ +export interface TransactionalStreamService { + /** + * Create a transactional stream from a SQL query. + * + * Returns tuples of [event, commitHandle] for manual commit control. + * If you don't call commit() and the process crashes, events will be + * replayed via a Rewind event on restart. + * + * @example + * ```typescript + * const txStream = yield* TransactionalStream + * + * yield* txStream.streamTransactional("SELECT * FROM eth.logs", { retention: 128 }).pipe( + * Stream.runForEach(([event, commitHandle]) => + * Effect.gen(function*() { + * yield* processEvent(event) + * yield* commitHandle.commit() + * }) + * ) + * ) + * ``` + */ + readonly streamTransactional: ( + sql: string, + options?: TransactionalStreamOptions + ) => Stream.Stream< + readonly [TransactionEvent, CommitHandle], + TransactionalStreamError + > + + /** + * High-level consumer: auto-commit after callback succeeds. + * + * If callback fails, stream stops and pending batch remains uncommitted. + * On restart, uncommitted batch triggers Rewind. + * + * @example + * ```typescript + * const txStream = yield* TransactionalStream + * + * yield* txStream.forEach( + * "SELECT * FROM eth.logs", + * { retention: 128 }, + * (event) => Effect.gen(function*() { + * switch (event._tag) { + * case "Data": + * yield* processData(event.data) + * break + * case "Undo": + * yield* rollback(event.invalidate) + * break + * case "Watermark": + * yield* checkpoint(event.ranges) + * break + * } + * }) + * ) + * ``` + */ + readonly forEach: ( + sql: string, + options: TransactionalStreamOptions, + handler: (event: TransactionEvent) => Effect.Effect + ) => Effect.Effect +} + +// ============================================================================= +// Context.Tag +// ============================================================================= + +/** + * TransactionalStream Context.Tag - use this to depend on TransactionalStream in Effects. + * + * @example + * ```typescript + * const program = Effect.gen(function*() { + * const txStream = yield* TransactionalStream + * yield* txStream.forEach("SELECT * FROM eth.logs", {}, processEvent) + * }) + * + * Effect.runPromise(program.pipe( + * Effect.provide(TransactionalStream.layer), + * Effect.provide(InMemoryStateStore.layer), + * Effect.provide(ArrowFlight.layer), + * Effect.provide(Transport.layer) + * )) + * ``` + */ +export class TransactionalStream extends Context.Tag("Amp/TransactionalStream")< + TransactionalStream, + TransactionalStreamService +>() {} + +// ============================================================================= +// Implementation +// ============================================================================= + +const DEFAULT_RETENTION = 128 + +/** + * Check if a rewind is needed based on persisted state. + * + * A rewind is needed when: + * - Buffer is empty but next > 0 (crash before any watermark committed) + * - Buffer has watermarks but next > last_watermark_id + 1 (crash after processing but before commit) + */ +const needsRewind = ( + next: TransactionId, + lastWatermarkId: TransactionId | undefined +): boolean => { + if (lastWatermarkId === undefined) { + // No watermarks - rewind if we've processed anything (next > 0) + return next > 0 + } + // Rewind if next ID is beyond what we've committed + return next > (lastWatermarkId + 1) +} + +/** + * Create TransactionalStream service implementation. + */ +const make = Effect.gen(function*() { + const arrowFlight = yield* ArrowFlight + const storeService = yield* StateStore + + const streamTransactional = ( + sql: string, + options?: TransactionalStreamOptions + ): Stream.Stream< + readonly [TransactionEvent, CommitHandle], + TransactionalStreamError + > => { + const retention = options?.retention ?? DEFAULT_RETENTION + + // Create the stream with proper scoping + return Stream.unwrapScoped( + Effect.gen(function*() { + // 1. Create StateActor + const actor: StateActor = yield* makeStateActor(storeService, retention) + + // 2. Get current watermark and next ID + const watermark = yield* actor.watermark() + const nextId = yield* actor.peek() + + // 3. Determine resume cursor + const resumeWatermark: ReadonlyArray | undefined = watermark !== undefined + ? watermark[1] + : undefined + + // 4. Check if rewind is needed + const lastWatermarkId = watermark?.[0] + const shouldRewind = needsRewind(nextId, lastWatermarkId) + + // 5. Get protocol stream with resume cursor + const protocolOptions: ProtocolStreamOptions = resumeWatermark !== undefined + ? { schema: options?.schema, resumeWatermark } + : { schema: options?.schema } + const protocolStream = arrowFlight.streamProtocol(sql, protocolOptions) + + // 6. Build transactional stream + // First emit Rewind if needed, then map protocol messages through actor + const rewindStream: Stream.Stream< + readonly [TransactionEvent, CommitHandle], + StateStoreError | UnrecoverableReorgError | PartialReorgError + > = shouldRewind + ? Stream.fromEffect( + actor.execute({ _tag: "Rewind" }) + ) + : Stream.empty + + const messageStream: Stream.Stream< + readonly [TransactionEvent, CommitHandle], + ProtocolStreamError | StateStoreError | UnrecoverableReorgError | PartialReorgError + > = protocolStream.pipe( + Stream.mapEffect((message) => + actor.execute({ _tag: "Message", message } as Action) + ) + ) + + return Stream.concat(rewindStream, messageStream) + }) + ).pipe( + Stream.withSpan("TransactionalStream.streamTransactional") + ) + } + + const forEach = ( + sql: string, + options: TransactionalStreamOptions, + handler: (event: TransactionEvent) => Effect.Effect + ): Effect.Effect => + streamTransactional(sql, options).pipe( + Stream.runForEach(([event, commitHandle]) => + Effect.gen(function*() { + // Process the event + yield* handler(event) + // Auto-commit after successful processing + yield* commitHandle.commit() + }) + ), + Effect.withSpan("TransactionalStream.forEach") + ) + + return { + streamTransactional, + forEach + } satisfies TransactionalStreamService +}) + +// ============================================================================= +// Layer +// ============================================================================= + +/** + * Layer providing TransactionalStream. + * + * Requires ArrowFlight and StateStore in context. + * + * @example + * ```typescript + * // Development/Testing with InMemoryStateStore + * const DevLayer = TransactionalStream.layer.pipe( + * Layer.provide(InMemoryStateStore.layer), + * Layer.provide(ArrowFlight.layer), + * Layer.provide(Transport.layer) + * ) + * + * // Production with persistent store (future) + * const ProdLayer = TransactionalStream.layer.pipe( + * Layer.provide(IndexedDBStateStore.layer), + * Layer.provide(ArrowFlight.layer), + * Layer.provide(Transport.layer) + * ) + * ``` + */ +export const layer: Layer.Layer = + Layer.effect(TransactionalStream, make) diff --git a/packages/amp/src/transactional-stream/types.ts b/packages/amp/src/transactional-stream/types.ts new file mode 100644 index 0000000..3c87854 --- /dev/null +++ b/packages/amp/src/transactional-stream/types.ts @@ -0,0 +1,201 @@ +/** + * Core type definitions for the TransactionalStream. + * + * @module + */ +import * as Option from "effect/Option" +import * as Schema from "effect/Schema" +import { BlockRange } from "../models.ts" +import { InvalidationRange } from "../protocol-stream/messages.ts" + +// ============================================================================= +// TransactionId +// ============================================================================= + +/** + * Transaction ID - monotonically increasing identifier for each event. + * Guaranteed to be unique and never reused, even across crashes. + */ +export const TransactionId = Schema.NonNegativeInt.pipe( + Schema.brand("Amp/TransactionalStream/TransactionId") +).annotations({ + identifier: "TransactionId", + description: "Monotonically increasing transaction identifier" +}) +export type TransactionId = typeof TransactionId.Type + +// ============================================================================= +// TransactionIdRange +// ============================================================================= + +/** + * Inclusive range of transaction IDs for invalidation. + */ +export const TransactionIdRange = Schema.Struct({ + start: TransactionId, + end: TransactionId +}).annotations({ + identifier: "TransactionIdRange", + description: "Inclusive range of transaction IDs" +}) +export type TransactionIdRange = typeof TransactionIdRange.Type + +// ============================================================================= +// UndoCause +// ============================================================================= + +/** + * Cause of an Undo event - either a blockchain reorg or a rewind on restart. + */ +export const UndoCauseReorg = Schema.TaggedStruct("Reorg", { + invalidation: Schema.Array(InvalidationRange) +}).annotations({ + identifier: "UndoCause.Reorg", + description: "Undo caused by blockchain reorganization" +}) +export type UndoCauseReorg = typeof UndoCauseReorg.Type + +export const UndoCauseRewind = Schema.TaggedStruct("Rewind", {}).annotations({ + identifier: "UndoCause.Rewind", + description: "Undo caused by rewind on restart (uncommitted transactions)" +}) +export type UndoCauseRewind = typeof UndoCauseRewind.Type + +export const UndoCause = Schema.Union(UndoCauseReorg, UndoCauseRewind).annotations({ + identifier: "UndoCause", + description: "Cause of an Undo event" +}) +export type UndoCause = typeof UndoCause.Type + +// ============================================================================= +// TransactionEvent +// ============================================================================= + +/** + * Data event - new records to process. + */ +export const TransactionEventData = Schema.TaggedStruct("Data", { + /** Transaction ID of this event */ + id: TransactionId, + /** Decoded record batch data */ + data: Schema.Array( + Schema.Record({ + key: Schema.String, + value: Schema.Unknown + }) + ), + /** Block ranges covered by this data */ + ranges: Schema.Array(BlockRange) +}).annotations({ + identifier: "TransactionEvent.Data", + description: "New data to process" +}) +export type TransactionEventData = typeof TransactionEventData.Type + +/** + * Undo event - consumer must delete/rollback data with the invalidated IDs. + */ +export const TransactionEventUndo = Schema.TaggedStruct("Undo", { + /** Transaction ID of this event */ + id: TransactionId, + /** Cause of the undo (reorg or rewind) */ + cause: UndoCause, + /** Range of transaction IDs to invalidate (inclusive) */ + invalidate: TransactionIdRange +}).annotations({ + identifier: "TransactionEvent.Undo", + description: "Undo/rollback previously processed data" +}) +export type TransactionEventUndo = typeof TransactionEventUndo.Type + +/** + * Watermark event - confirms block ranges are complete. + */ +export const TransactionEventWatermark = Schema.TaggedStruct("Watermark", { + /** Transaction ID of this event */ + id: TransactionId, + /** Block ranges confirmed complete */ + ranges: Schema.Array(BlockRange), + /** Last transaction ID pruned at this watermark, if any */ + prune: Schema.OptionFromNullOr(TransactionId) +}).annotations({ + identifier: "TransactionEvent.Watermark", + description: "Watermark confirming block ranges are complete" +}) +export type TransactionEventWatermark = typeof TransactionEventWatermark.Type + +/** + * Union of all transaction event types. + */ +export const TransactionEvent = Schema.Union( + TransactionEventData, + TransactionEventUndo, + TransactionEventWatermark +).annotations({ + identifier: "TransactionEvent", + description: "Event emitted by the transactional stream" +}) +export type TransactionEvent = typeof TransactionEvent.Type + +// ============================================================================= +// Constructors +// ============================================================================= + +/** + * Create a Data transaction event. + */ +export const dataEvent = ( + id: TransactionId, + data: ReadonlyArray>, + ranges: ReadonlyArray +): TransactionEventData => ({ + _tag: "Data", + id, + data: data as Array>, + ranges: ranges as Array +}) + +/** + * Create an Undo transaction event. + */ +export const undoEvent = ( + id: TransactionId, + cause: UndoCause, + invalidate: TransactionIdRange +): TransactionEventUndo => ({ + _tag: "Undo", + id, + cause, + invalidate +}) + +/** + * Create a Watermark transaction event. + */ +export const watermarkEvent = ( + id: TransactionId, + ranges: ReadonlyArray, + prune: TransactionId | null +): TransactionEventWatermark => ({ + _tag: "Watermark", + id, + ranges: ranges as Array, + prune: Option.fromNullable(prune) +}) + +/** + * Create a Reorg cause. + */ +export const reorgCause = ( + invalidation: ReadonlyArray +): UndoCauseReorg => ({ + _tag: "Reorg", + invalidation: invalidation as Array +}) + +/** + * Create a Rewind cause. + */ +export const rewindCause = (): UndoCauseRewind => ({ + _tag: "Rewind" +}) diff --git a/packages/amp/test/protocol-stream/reorg.test.ts b/packages/amp/test/protocol-stream/reorg.test.ts new file mode 100644 index 0000000..fe0b702 --- /dev/null +++ b/packages/amp/test/protocol-stream/reorg.test.ts @@ -0,0 +1,493 @@ +/** + * Blockchain Reorg Detection Tests + * + * Ported from Rust tests in: + * `.repos/amp/crates/clients/flight/src/tests/it_blockchain_reorg_test.rs` + * + * These tests verify the stateless reorg detection logic used by the protocol stream. + * Unlike the Rust implementation which maintains transaction IDs and a buffer, + * the TypeScript version is stateless and detects reorgs purely from consecutive + * batch comparisons. + * + * Key behaviors tested: + * - Reorg detection via backwards jumps with hash mismatches + * - Invalidation range calculation + * - Multi-network partial reorg detection + * - Watermark and data message generation + */ +import type { BlockHash, BlockNumber, BlockRange, Network } from "@edgeandnode/amp/models" +import { data, invalidates, makeInvalidationRange, reorg, watermark } from "@edgeandnode/amp/protocol-stream/index" +import { describe, it } from "@effect/vitest" +import * as Effect from "effect/Effect" +import * as Either from "effect/Either" + +// ============================================================================= +// Test Helpers - Ported from Rust utils/response.rs +// ============================================================================= + +/** + * Standard test hashes for different epochs. + * Epoch 0 = HASH_A, Epoch 1 = HASH_B, etc. + */ +const ZERO_HASH = "0x0000000000000000000000000000000000000000000000000000000000000000" as BlockHash +const HASH_EPOCH_0 = "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" as BlockHash +const HASH_EPOCH_1 = "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" as BlockHash +const HASH_EPOCH_2 = "0xcccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" as BlockHash + +/** + * Generates a deterministic hash for a block based on network, block number, and epoch. + * This mirrors the Rust implementation for consistent hash generation. + */ +const makeHash = (network: string, block: number, epoch: number = 0): BlockHash => { + // Use epoch as the main distinguisher, with network and block encoded + const epochHex = epoch.toString(16).padStart(2, "0") + const networkHash = Array.from(network).reduce((acc, c) => (acc * 31 + c.charCodeAt(0)) & 0xffffffff, 0) + const networkHex = networkHash.toString(16).padStart(8, "0") + const blockHex = block.toString(16).padStart(16, "0") + return `0x${epochHex}${networkHex}${"0".repeat(38)}${blockHex}` as BlockHash +} + +/** + * Creates a BlockRange for testing. + * Automatically generates hashes based on network, block numbers, and epoch. + */ +const makeBlockRange = ( + network: string, + start: number, + end: number, + epoch: number = 0 +): BlockRange => ({ + network: network as Network, + numbers: { + start: start as BlockNumber, + end: end as BlockNumber + }, + hash: makeHash(network, end, epoch), + prevHash: start > 0 ? makeHash(network, start - 1, epoch) : undefined +}) + +/** + * Creates a BlockRange with an explicit prevHash that doesn't match the epoch. + * Used to simulate reorgs where the hash chain doesn't match. + */ +const makeBlockRangeWithReorg = ( + network: string, + start: number, + end: number, + epoch: number, + prevHashEpoch: number +): BlockRange => ({ + network: network as Network, + numbers: { + start: start as BlockNumber, + end: end as BlockNumber + }, + hash: makeHash(network, end, epoch), + prevHash: start > 0 ? makeHash(network, start - 1, prevHashEpoch) : undefined +}) + +// ============================================================================= +// Reorg Detection Helper +// ============================================================================= + +/** + * Detects reorgs by comparing incoming ranges to previous ranges. + * This is a copy of the detection logic from arrow-flight.ts for testing. + */ +const detectReorgs = ( + previous: ReadonlyArray, + incoming: ReadonlyArray +): ReadonlyArray> => { + const invalidations: Array> = [] + + for (const incomingRange of incoming) { + const prevRange = previous.find((p) => p.network === incomingRange.network) + if (!prevRange) continue + + // Skip identical ranges (watermarks can repeat) + if ( + incomingRange.network === prevRange.network && + incomingRange.numbers.start === prevRange.numbers.start && + incomingRange.numbers.end === prevRange.numbers.end && + incomingRange.hash === prevRange.hash && + incomingRange.prevHash === prevRange.prevHash + ) { + continue + } + + const incomingStart = incomingRange.numbers.start + const prevEnd = prevRange.numbers.end + + // Detect backwards jump (reorg indicator) + if (incomingStart < prevEnd + 1) { + invalidations.push( + makeInvalidationRange( + incomingRange.network, + incomingStart, + Math.max(incomingRange.numbers.end, prevEnd) + ) + ) + } + } + + return invalidations +} + +// ============================================================================= +// Reorg Detection Tests - Basic Scenarios +// ============================================================================= + +describe("detectReorgs", () => { + /** + * Tests basic reorg detection when a batch arrives with a backwards jump + * and different hash chain. The reorg should create an invalidation range. + * + * Corresponds to Rust test: reorg_invalidates_affected_batches + */ + it.effect("detects reorg when backwards jump with hash mismatch occurs", ({ expect }) => + Effect.gen(function*() { + // Previous: blocks 0-10 in epoch 0 + const previous = [makeBlockRange("eth", 0, 10, 0)] + + // Incoming: blocks 5-12 in epoch 1 (different hash chain, backwards jump) + const incoming = [makeBlockRangeWithReorg("eth", 5, 12, 1, 1)] + + const invalidations = detectReorgs(previous, incoming) + + expect(invalidations.length).toBe(1) + expect(invalidations[0]!.network).toBe("eth") + expect(invalidations[0]!.start).toBe(5) // Start of reorg + expect(invalidations[0]!.end).toBe(12) // End covers incoming range + })) + + /** + * Tests that consecutive blocks with matching hash chains don't trigger reorg. + */ + it.effect("does not detect reorg for consecutive blocks with matching hashes", ({ expect }) => + Effect.gen(function*() { + // Previous: blocks 0-10 in epoch 0 + const previous = [makeBlockRange("eth", 0, 10, 0)] + + // Incoming: blocks 11-20 in epoch 0 (consecutive, same hash chain) + const incoming = [makeBlockRange("eth", 11, 20, 0)] + + const invalidations = detectReorgs(previous, incoming) + + expect(invalidations.length).toBe(0) + })) + + /** + * Tests that watermarks don't invalidate previous batches when block ranges + * don't overlap. This verifies the "protected by watermark" behavior. + * + * Corresponds to Rust test: reorg_does_not_invalidate_unaffected_batches + */ + it.effect("does not detect reorg for forward progress", ({ expect }) => + Effect.gen(function*() { + // Previous: blocks 0-10 (finalized by watermark) + const previous = [makeBlockRange("eth", 0, 10, 0)] + + // Incoming: blocks 11-20 (consecutive, no overlap) + const incoming = [makeBlockRange("eth", 11, 20, 0)] + + const invalidations = detectReorgs(previous, incoming) + + expect(invalidations.length).toBe(0) + })) + + /** + * Tests identical range handling (watermark repeats). + */ + it.effect("does not detect reorg for identical ranges (watermark repeat)", ({ expect }) => + Effect.gen(function*() { + const range = makeBlockRange("eth", 0, 10, 0) + + const invalidations = detectReorgs([range], [range]) + + expect(invalidations.length).toBe(0) + })) +}) + +// ============================================================================= +// Multi-Network Reorg Tests +// ============================================================================= + +describe("detectReorgs - multi-network", () => { + /** + * Tests partial reorg in multi-network scenarios where only some networks + * experience a reorg while others continue normally. + * + * Corresponds to Rust test: multi_network_reorg_partial_invalidation + */ + it.effect("detects partial reorg when only one network reorgs", ({ expect }) => + Effect.gen(function*() { + // Previous: both networks at blocks 0-10 + const previous = [ + makeBlockRange("eth", 0, 10, 0), + makeBlockRange("polygon", 0, 10, 0) + ] + + // Incoming: eth reorgs back to block 5, polygon continues normally + const incoming = [ + makeBlockRangeWithReorg("eth", 5, 12, 1, 1), // Reorg with different epoch + makeBlockRange("polygon", 11, 20, 0) // Normal continuation + ] + + const invalidations = detectReorgs(previous, incoming) + + // Only eth should have an invalidation + expect(invalidations.length).toBe(1) + expect(invalidations[0]!.network).toBe("eth") + expect(invalidations[0]!.start).toBe(5) + })) + + /** + * Tests multi-network reorg where both networks reorg. + */ + it.effect("detects reorg on both networks when both reorg", ({ expect }) => + Effect.gen(function*() { + // Previous: both networks at blocks 0-10 + const previous = [ + makeBlockRange("eth", 0, 10, 0), + makeBlockRange("polygon", 0, 10, 0) + ] + + // Incoming: both networks reorg + const incoming = [ + makeBlockRangeWithReorg("eth", 5, 12, 1, 1), + makeBlockRangeWithReorg("polygon", 7, 15, 1, 1) + ] + + const invalidations = detectReorgs(previous, incoming) + + expect(invalidations.length).toBe(2) + expect(invalidations.find((i) => i.network === "eth")).toBeDefined() + expect(invalidations.find((i) => i.network === "polygon")).toBeDefined() + })) +}) + +// ============================================================================= +// Invalidation Range Tests +// ============================================================================= + +describe("invalidates", () => { + it.effect("returns true when block range overlaps with invalidation range", ({ expect }) => + Effect.gen(function*() { + const invalidation = makeInvalidationRange("eth", 10, 20) + const range = makeBlockRange("eth", 15, 25, 0) + + expect(invalidates(invalidation, range)).toBe(true) + })) + + it.effect("returns false when block range is before invalidation range", ({ expect }) => + Effect.gen(function*() { + const invalidation = makeInvalidationRange("eth", 20, 30) + const range = makeBlockRange("eth", 5, 15, 0) + + expect(invalidates(invalidation, range)).toBe(false) + })) + + it.effect("returns false when block range is after invalidation range", ({ expect }) => + Effect.gen(function*() { + const invalidation = makeInvalidationRange("eth", 5, 15) + const range = makeBlockRange("eth", 20, 30, 0) + + expect(invalidates(invalidation, range)).toBe(false) + })) + + it.effect("returns false when networks don't match", ({ expect }) => + Effect.gen(function*() { + const invalidation = makeInvalidationRange("eth", 10, 20) + const range = makeBlockRange("polygon", 10, 20, 0) + + expect(invalidates(invalidation, range)).toBe(false) + })) + + it.effect("returns true when ranges are identical", ({ expect }) => + Effect.gen(function*() { + const invalidation = makeInvalidationRange("eth", 10, 20) + const range = makeBlockRange("eth", 10, 20, 0) + + expect(invalidates(invalidation, range)).toBe(true) + })) + + it.effect("returns true when invalidation range contains block range", ({ expect }) => + Effect.gen(function*() { + const invalidation = makeInvalidationRange("eth", 5, 30) + const range = makeBlockRange("eth", 10, 20, 0) + + expect(invalidates(invalidation, range)).toBe(true) + })) +}) + +// ============================================================================= +// Protocol Message Construction Tests +// ============================================================================= + +describe("protocol messages", () => { + it.effect("creates Data message with records and ranges", ({ expect }) => + Effect.gen(function*() { + const records = [{ id: 1 }, { id: 2 }] + const ranges = [makeBlockRange("eth", 0, 10, 0)] + + const message = data(records, ranges) + + expect(message._tag).toBe("Data") + expect(message.data.length).toBe(2) + expect(message.ranges.length).toBe(1) + })) + + it.effect("creates Watermark message with ranges", ({ expect }) => + Effect.gen(function*() { + const ranges = [makeBlockRange("eth", 0, 10, 0)] + + const message = watermark(ranges) + + expect(message._tag).toBe("Watermark") + expect(message.ranges.length).toBe(1) + })) + + it.effect("creates Reorg message with previous, incoming, and invalidation", ({ expect }) => + Effect.gen(function*() { + const previous = [makeBlockRange("eth", 0, 10, 0)] + const incoming = [makeBlockRangeWithReorg("eth", 5, 12, 1, 1)] + const invalidation = [makeInvalidationRange("eth", 5, 12)] + + const message = reorg(previous, incoming, invalidation) + + expect(message._tag).toBe("Reorg") + expect(message.previous.length).toBe(1) + expect(message.incoming.length).toBe(1) + expect(message.invalidation.length).toBe(1) + })) +}) + +// ============================================================================= +// Deep Reorg Scenarios +// ============================================================================= + +describe("deep reorg scenarios", () => { + /** + * Tests that a deep reorg (going back many blocks) correctly identifies + * the invalidation range. In stateless detection, we only see the + * immediate previous batch, so the invalidation range is calculated + * based on that comparison. + * + * Corresponds to Rust test: reorg_invalidates_multiple_consecutive_batches + */ + it.effect("calculates invalidation range for deep reorg", ({ expect }) => + Effect.gen(function*() { + // Simulate state after multiple data batches: + // Previous state shows blocks 31-40 were the last received + const previous = [makeBlockRange("eth", 31, 40, 0)] + + // Incoming: reorg back to block 15 (deep reorg) + const incoming = [makeBlockRangeWithReorg("eth", 15, 25, 1, 1)] + + const invalidations = detectReorgs(previous, incoming) + + expect(invalidations.length).toBe(1) + expect(invalidations[0]!.network).toBe("eth") + expect(invalidations[0]!.start).toBe(15) // Reorg point + // End is max of incoming.end (25) and previous.end (40) + expect(invalidations[0]!.end).toBe(40) + })) + + /** + * Tests consecutive reorgs - multiple reorgs in sequence. + * + * Corresponds to Rust test: consecutive_reorgs_cumulative_invalidation + */ + it.effect("handles consecutive reorgs correctly", ({ expect }) => + Effect.gen(function*() { + // First reorg scenario + let previous = [makeBlockRange("eth", 21, 30, 0)] + let incoming = [makeBlockRangeWithReorg("eth", 21, 25, 1, 1)] + + let invalidations = detectReorgs(previous, incoming) + + expect(invalidations.length).toBe(1) + expect(invalidations[0]!.start).toBe(21) + expect(invalidations[0]!.end).toBe(30) + + // Continue with new batch after first reorg + previous = incoming + incoming = [makeBlockRange("eth", 26, 30, 1)] + + invalidations = detectReorgs(previous, incoming) + expect(invalidations.length).toBe(0) // Normal continuation + + // Second reorg occurs + previous = incoming + incoming = [makeBlockRangeWithReorg("eth", 21, 30, 2, 2)] + + invalidations = detectReorgs(previous, incoming) + + expect(invalidations.length).toBe(1) + expect(invalidations[0]!.start).toBe(21) + expect(invalidations[0]!.end).toBe(30) + })) + + /** + * Tests reorg with backwards jump that succeeds validation. + * + * Corresponds to Rust test: reorg_with_backwards_jump_succeeds + */ + it.effect("detects reorg with backwards jump", ({ expect }) => + Effect.gen(function*() { + // Previous: blocks 11-20 + const previous = [makeBlockRange("eth", 11, 20, 0)] + + // Incoming: reorg back to block 15 (backwards jump from end=20 to start=15) + const incoming = [makeBlockRangeWithReorg("eth", 15, 25, 1, 1)] + + const invalidations = detectReorgs(previous, incoming) + + expect(invalidations.length).toBe(1) + expect(invalidations[0]!.network).toBe("eth") + expect(invalidations[0]!.start).toBe(15) + expect(invalidations[0]!.end).toBe(25) // max(incoming.end, previous.end) + })) +}) + +// ============================================================================= +// Edge Cases +// ============================================================================= + +describe("edge cases", () => { + it.effect("handles empty previous ranges (first batch)", ({ expect }) => + Effect.gen(function*() { + const previous: Array = [] + const incoming = [makeBlockRange("eth", 0, 10, 0)] + + const invalidations = detectReorgs(previous, incoming) + + expect(invalidations.length).toBe(0) + })) + + it.effect("handles new network in incoming (no reorg)", ({ expect }) => + Effect.gen(function*() { + const previous = [makeBlockRange("eth", 0, 10, 0)] + const incoming = [ + makeBlockRange("eth", 11, 20, 0), + makeBlockRange("polygon", 0, 10, 0) // New network + ] + + const invalidations = detectReorgs(previous, incoming) + + // New networks don't cause reorg detection (handled by validateNetworks) + expect(invalidations.length).toBe(0) + })) + + it.effect("handles single block ranges", ({ expect }) => + Effect.gen(function*() { + const previous = [makeBlockRange("eth", 10, 10, 0)] + const incoming = [makeBlockRangeWithReorg("eth", 10, 12, 1, 1)] + + const invalidations = detectReorgs(previous, incoming) + + expect(invalidations.length).toBe(1) + expect(invalidations[0]!.start).toBe(10) + expect(invalidations[0]!.end).toBe(12) + })) +}) diff --git a/packages/amp/test/transactional-stream/algorithms.test.ts b/packages/amp/test/transactional-stream/algorithms.test.ts new file mode 100644 index 0000000..b1916b3 --- /dev/null +++ b/packages/amp/test/transactional-stream/algorithms.test.ts @@ -0,0 +1,355 @@ +/** + * Tests for the TransactionalStream algorithms. + * + * @module + */ +import { describe, expect, it } from "vitest" +import type { BlockRange } from "../../src/models.ts" +import type { InvalidationRange } from "../../src/protocol-stream/messages.ts" +import { + checkPartialReorg, + compressCommits, + findPruningPoint, + findRecoveryPoint +} from "../../src/transactional-stream/algorithms.ts" +import type { TransactionId } from "../../src/transactional-stream/types.ts" + +// ============================================================================= +// Test Helpers +// ============================================================================= + +const makeBlockRange = ( + network: string, + start: number, + end: number, + hash = "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef" +): BlockRange => + ({ + network, + numbers: { start, end }, + hash, + prevHash: undefined + }) as BlockRange + +const makeInvalidation = (network: string, start: number, end: number): InvalidationRange => + ({ + network, + start, + end + }) as InvalidationRange + +const makeWatermark = ( + id: number, + ranges: ReadonlyArray +): readonly [TransactionId, ReadonlyArray] => [id as TransactionId, ranges] + +// ============================================================================= +// findRecoveryPoint Tests +// ============================================================================= + +describe("findRecoveryPoint", () => { + it("returns undefined for empty buffer", () => { + const result = findRecoveryPoint([], [makeInvalidation("eth", 100, 110)]) + expect(result).toBeUndefined() + }) + + it("returns last watermark when no network is affected", () => { + const buffer = [ + makeWatermark(1, [makeBlockRange("eth", 0, 10)]), + makeWatermark(2, [makeBlockRange("eth", 11, 20)]), + makeWatermark(3, [makeBlockRange("eth", 21, 30)]) + ] + + // Invalidation on different network + const result = findRecoveryPoint(buffer, [makeInvalidation("polygon", 100, 110)]) + expect(result).toEqual(buffer[2]) + }) + + it("finds last unaffected watermark", () => { + const buffer = [ + makeWatermark(1, [makeBlockRange("eth", 0, 10)]), + makeWatermark(2, [makeBlockRange("eth", 11, 20)]), + makeWatermark(3, [makeBlockRange("eth", 21, 30)]) + ] + + // Reorg at block 21 - watermarks at id=1,2 are safe, id=3 starts at affected point + const result = findRecoveryPoint(buffer, [makeInvalidation("eth", 21, 35)]) + expect(result?.[0]).toBe(2) + }) + + it("returns undefined when all watermarks affected", () => { + const buffer = [ + makeWatermark(1, [makeBlockRange("eth", 100, 110)]), + makeWatermark(2, [makeBlockRange("eth", 111, 120)]) + ] + + // Reorg affects from block 100 + const result = findRecoveryPoint(buffer, [makeInvalidation("eth", 100, 130)]) + expect(result).toBeUndefined() + }) + + it("handles multi-network scenarios", () => { + const buffer = [ + makeWatermark(1, [makeBlockRange("eth", 0, 10), makeBlockRange("polygon", 0, 100)]), + makeWatermark(2, [makeBlockRange("eth", 11, 20), makeBlockRange("polygon", 101, 200)]), + makeWatermark(3, [makeBlockRange("eth", 21, 30), makeBlockRange("polygon", 201, 300)]) + ] + + // Reorg on eth at block 21, polygon at block 201 + const result = findRecoveryPoint(buffer, [ + makeInvalidation("eth", 21, 35), + makeInvalidation("polygon", 201, 350) + ]) + expect(result?.[0]).toBe(2) + }) + + it("single watermark affected returns undefined", () => { + const buffer = [makeWatermark(1, [makeBlockRange("eth", 50, 60)])] + + const result = findRecoveryPoint(buffer, [makeInvalidation("eth", 50, 70)]) + expect(result).toBeUndefined() + }) + + it("handles invalidation that doesn't affect any watermark", () => { + const buffer = [ + makeWatermark(1, [makeBlockRange("eth", 0, 10)]), + makeWatermark(2, [makeBlockRange("eth", 11, 20)]) + ] + + // Invalidation starts at block 100, well beyond our watermarks + // But watermarks with start >= 100 would be affected + // Since no watermarks start at >= 100, all are safe + const result = findRecoveryPoint(buffer, [makeInvalidation("eth", 100, 110)]) + expect(result?.[0]).toBe(2) // Last watermark is safe + }) +}) + +// ============================================================================= +// findPruningPoint Tests +// ============================================================================= + +describe("findPruningPoint", () => { + it("returns undefined for empty buffer", () => { + const result = findPruningPoint([], 50) + expect(result).toBeUndefined() + }) + + it("returns undefined for single watermark", () => { + const buffer = [makeWatermark(1, [makeBlockRange("eth", 100, 110)])] + const result = findPruningPoint(buffer, 50) + expect(result).toBeUndefined() + }) + + it("returns undefined when all within retention", () => { + const buffer = [ + makeWatermark(1, [makeBlockRange("eth", 90, 95)]), + makeWatermark(2, [makeBlockRange("eth", 96, 100)]) + ] + // Retention 50, latest start is 96, cutoff = 96 - 50 = 46 + // Watermark 1 ends at 95, which is >= 46, so it's within retention + const result = findPruningPoint(buffer, 50) + expect(result).toBeUndefined() + }) + + it("finds watermarks outside retention", () => { + const buffer = [ + makeWatermark(1, [makeBlockRange("eth", 0, 10)]), + makeWatermark(2, [makeBlockRange("eth", 11, 20)]), + makeWatermark(3, [makeBlockRange("eth", 100, 110)]) + ] + // Retention 50, latest start is 100, cutoff = 100 - 50 = 50 + // Watermark 1: end=10 < 50, outside retention + // Watermark 2: end=20 < 50, outside retention + // Never prune latest (id=3) + const result = findPruningPoint(buffer, 50) + expect(result).toBe(2) + }) + + it("stops at first watermark within retention", () => { + const buffer = [ + makeWatermark(1, [makeBlockRange("eth", 0, 10)]), + makeWatermark(2, [makeBlockRange("eth", 45, 55)]), // Within retention + makeWatermark(3, [makeBlockRange("eth", 100, 110)]) + ] + // Retention 50, latest start is 100, cutoff = 50 + // Watermark 1: end=10 < 50, outside retention + // Watermark 2: end=55 >= 50, within retention - stop here + const result = findPruningPoint(buffer, 50) + expect(result).toBe(1) // Only prune up to id=1 + }) + + it("handles multi-network scenarios", () => { + const buffer = [ + makeWatermark(1, [makeBlockRange("eth", 0, 10), makeBlockRange("polygon", 0, 100)]), + makeWatermark(2, [makeBlockRange("eth", 200, 210), makeBlockRange("polygon", 200, 300)]) + ] + // For eth: cutoff = 200 - 50 = 150 + // For polygon: cutoff = 200 - 50 = 150 + // Watermark 1: eth.end=10 < 150, polygon.end=100 < 150 - both outside + const result = findPruningPoint(buffer, 50) + expect(result).toBe(1) + }) + + it("does not prune when watermark has network not in latest", () => { + const buffer = [ + makeWatermark(1, [makeBlockRange("eth", 0, 10), makeBlockRange("polygon", 0, 100)]), + makeWatermark(2, [makeBlockRange("eth", 200, 210)]) // polygon removed + ] + // For eth: cutoff = 200 - 50 = 150 + // Watermark 1: eth.end=10 < 150, polygon not in latest (considered outside) + // So watermark 1 is prunable + const result = findPruningPoint(buffer, 50) + expect(result).toBe(1) + }) +}) + +// ============================================================================= +// checkPartialReorg Tests +// ============================================================================= + +describe("checkPartialReorg", () => { + it("returns undefined when no overlap", () => { + const recoveryRanges = [makeBlockRange("eth", 0, 10)] + const invalidation = [makeInvalidation("eth", 11, 20)] + + const result = checkPartialReorg(recoveryRanges, invalidation) + expect(result).toBeUndefined() + }) + + it("returns undefined when reorg at exact boundary", () => { + const recoveryRanges = [makeBlockRange("eth", 0, 10)] + // Reorg starts exactly at end+1 of recovery range + const invalidation = [makeInvalidation("eth", 11, 20)] + + const result = checkPartialReorg(recoveryRanges, invalidation) + expect(result).toBeUndefined() + }) + + it("detects partial reorg within range", () => { + const recoveryRanges = [makeBlockRange("eth", 0, 10)] + // Reorg point 5 falls within range [0, 10] + const invalidation = [makeInvalidation("eth", 5, 20)] + + const result = checkPartialReorg(recoveryRanges, invalidation) + expect(result).toBe("eth") + }) + + it("detects partial reorg at end of range", () => { + const recoveryRanges = [makeBlockRange("eth", 0, 10)] + // Reorg point 10 equals end of range - this is partial + const invalidation = [makeInvalidation("eth", 10, 20)] + + const result = checkPartialReorg(recoveryRanges, invalidation) + expect(result).toBe("eth") + }) + + it("returns undefined for different networks", () => { + const recoveryRanges = [makeBlockRange("eth", 0, 10)] + const invalidation = [makeInvalidation("polygon", 5, 20)] + + const result = checkPartialReorg(recoveryRanges, invalidation) + expect(result).toBeUndefined() + }) + + it("returns first partial network in multi-network scenario", () => { + const recoveryRanges = [ + makeBlockRange("eth", 0, 10), + makeBlockRange("polygon", 0, 100) + ] + // Both have partial reorg + const invalidation = [ + makeInvalidation("eth", 5, 20), + makeInvalidation("polygon", 50, 150) + ] + + const result = checkPartialReorg(recoveryRanges, invalidation) + // Returns first one found (eth) + expect(result).toBe("eth") + }) + + it("returns undefined when reorg before range", () => { + const recoveryRanges = [makeBlockRange("eth", 10, 20)] + // Reorg point is before the range starts - not partial + const invalidation = [makeInvalidation("eth", 5, 25)] + + // This is actually partial because 5 < 10 but 5 <= 20 + // Wait, the check is: range.start < point && point <= range.end + // 10 < 5? No. So not partial. + const result = checkPartialReorg(recoveryRanges, invalidation) + expect(result).toBeUndefined() + }) +}) + +// ============================================================================= +// compressCommits Tests +// ============================================================================= + +describe("compressCommits", () => { + it("returns empty for empty input", () => { + const result = compressCommits([]) + expect(result.insert).toEqual([]) + expect(result.prune).toBeUndefined() + }) + + it("collects single commit", () => { + const pending = [ + [ + 1 as TransactionId, + { ranges: [makeBlockRange("eth", 0, 10)], prune: undefined } + ] as const + ] + + const result = compressCommits(pending) + expect(result.insert).toHaveLength(1) + expect(result.insert[0]![0]).toBe(1) + expect(result.prune).toBeUndefined() + }) + + it("collects multiple commits", () => { + const pending = [ + [1 as TransactionId, { ranges: [makeBlockRange("eth", 0, 10)], prune: undefined }] as const, + [2 as TransactionId, { ranges: [makeBlockRange("eth", 11, 20)], prune: undefined }] as const + ] + + const result = compressCommits(pending) + expect(result.insert).toHaveLength(2) + expect(result.prune).toBeUndefined() + }) + + it("takes maximum prune point", () => { + const pending = [ + [1 as TransactionId, { ranges: [makeBlockRange("eth", 0, 10)], prune: undefined }] as const, + [2 as TransactionId, { ranges: [makeBlockRange("eth", 11, 20)], prune: 0 as TransactionId }] as const, + [3 as TransactionId, { ranges: [makeBlockRange("eth", 21, 30)], prune: 1 as TransactionId }] as const + ] + + const result = compressCommits(pending) + expect(result.prune).toBe(1) + }) + + it("filters pruned watermarks from inserts", () => { + const pending = [ + [1 as TransactionId, { ranges: [makeBlockRange("eth", 0, 10)], prune: undefined }] as const, + [2 as TransactionId, { ranges: [makeBlockRange("eth", 11, 20)], prune: undefined }] as const, + [3 as TransactionId, { ranges: [makeBlockRange("eth", 21, 30)], prune: 1 as TransactionId }] as const + ] + + const result = compressCommits(pending) + // Should filter out id=1 since prune=1 + expect(result.insert).toHaveLength(2) + expect(result.insert.map(([id]) => id)).toEqual([2, 3]) + expect(result.prune).toBe(1) + }) + + it("handles all watermarks being pruned", () => { + const pending = [ + [1 as TransactionId, { ranges: [makeBlockRange("eth", 0, 10)], prune: undefined }] as const, + [2 as TransactionId, { ranges: [makeBlockRange("eth", 11, 20)], prune: 2 as TransactionId }] as const + ] + + const result = compressCommits(pending) + // All IDs <= 2 are pruned + expect(result.insert).toHaveLength(0) + expect(result.prune).toBe(2) + }) +}) diff --git a/packages/amp/test/transactional-stream/memory-store.test.ts b/packages/amp/test/transactional-stream/memory-store.test.ts new file mode 100644 index 0000000..27adb22 --- /dev/null +++ b/packages/amp/test/transactional-stream/memory-store.test.ts @@ -0,0 +1,321 @@ +/** + * Tests for InMemoryStateStore. + * + * @module + */ +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" +import * as Ref from "effect/Ref" +import { describe, expect, it } from "vitest" +import type { BlockRange } from "../../src/models.ts" +import * as InMemoryStateStore from "../../src/transactional-stream/memory-store.ts" +import { StateStore, emptySnapshot, type StateSnapshot } from "../../src/transactional-stream/state-store.ts" +import type { TransactionId } from "../../src/transactional-stream/types.ts" + +// ============================================================================= +// Test Helpers +// ============================================================================= + +const makeBlockRange = ( + network: string, + start: number, + end: number +): BlockRange => + ({ + network, + numbers: { start, end }, + hash: "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + prevHash: undefined + }) as BlockRange + +const runWithStore = ( + effect: Effect.Effect +): Promise => + Effect.runPromise( + effect.pipe(Effect.provide(InMemoryStateStore.layer)) + ) + +const runWithState = ( + initial: StateSnapshot, + effect: Effect.Effect +): Promise => + Effect.runPromise( + effect.pipe(Effect.provide(InMemoryStateStore.layerWithState(initial))) + ) + +// ============================================================================= +// Layer Tests +// ============================================================================= + +describe("InMemoryStateStore.layer", () => { + it("provides empty initial state", async () => { + const result = await runWithStore( + Effect.gen(function*() { + const store = yield* StateStore + return yield* store.load() + }) + ) + + expect(result).toEqual(emptySnapshot) + }) +}) + +describe("InMemoryStateStore.layerWithState", () => { + it("provides custom initial state", async () => { + const initial: StateSnapshot = { + buffer: [[5 as TransactionId, [makeBlockRange("eth", 0, 10)]]], + next: 10 as TransactionId + } + + const result = await runWithState( + initial, + Effect.gen(function*() { + const store = yield* StateStore + return yield* store.load() + }) + ) + + expect(result).toEqual(initial) + }) +}) + +// ============================================================================= +// StateStore Operations Tests +// ============================================================================= + +describe("StateStore.advance", () => { + it("updates the next transaction ID", async () => { + const result = await runWithStore( + Effect.gen(function*() { + const store = yield* StateStore + + yield* store.advance(5 as TransactionId) + + const snapshot = yield* store.load() + return snapshot.next + }) + ) + + expect(result).toBe(5) + }) + + it("can be called multiple times", async () => { + const result = await runWithStore( + Effect.gen(function*() { + const store = yield* StateStore + + yield* store.advance(1 as TransactionId) + yield* store.advance(2 as TransactionId) + yield* store.advance(10 as TransactionId) + + const snapshot = yield* store.load() + return snapshot.next + }) + ) + + expect(result).toBe(10) + }) +}) + +describe("StateStore.commit", () => { + it("inserts watermarks to buffer", async () => { + const result = await runWithStore( + Effect.gen(function*() { + const store = yield* StateStore + + yield* store.commit({ + insert: [ + [1 as TransactionId, [makeBlockRange("eth", 0, 10)]], + [2 as TransactionId, [makeBlockRange("eth", 11, 20)]] + ], + prune: undefined + }) + + const snapshot = yield* store.load() + return snapshot.buffer + }) + ) + + expect(result).toHaveLength(2) + expect(result[0]![0]).toBe(1) + expect(result[1]![0]).toBe(2) + }) + + it("prunes watermarks with ID <= prune point", async () => { + const initial: StateSnapshot = { + buffer: [ + [1 as TransactionId, [makeBlockRange("eth", 0, 10)]], + [2 as TransactionId, [makeBlockRange("eth", 11, 20)]], + [3 as TransactionId, [makeBlockRange("eth", 21, 30)]] + ], + next: 4 as TransactionId + } + + const result = await runWithState( + initial, + Effect.gen(function*() { + const store = yield* StateStore + + yield* store.commit({ + insert: [], + prune: 2 as TransactionId + }) + + const snapshot = yield* store.load() + return snapshot.buffer + }) + ) + + expect(result).toHaveLength(1) + expect(result[0]![0]).toBe(3) + }) + + it("can insert and prune atomically", async () => { + const initial: StateSnapshot = { + buffer: [ + [1 as TransactionId, [makeBlockRange("eth", 0, 10)]], + [2 as TransactionId, [makeBlockRange("eth", 11, 20)]] + ], + next: 3 as TransactionId + } + + const result = await runWithState( + initial, + Effect.gen(function*() { + const store = yield* StateStore + + yield* store.commit({ + insert: [[3 as TransactionId, [makeBlockRange("eth", 21, 30)]]], + prune: 1 as TransactionId + }) + + const snapshot = yield* store.load() + return snapshot.buffer + }) + ) + + expect(result).toHaveLength(2) + expect(result.map(([id]) => id)).toEqual([2, 3]) + }) +}) + +describe("StateStore.truncate", () => { + it("removes watermarks with ID >= from", async () => { + const initial: StateSnapshot = { + buffer: [ + [1 as TransactionId, [makeBlockRange("eth", 0, 10)]], + [2 as TransactionId, [makeBlockRange("eth", 11, 20)]], + [3 as TransactionId, [makeBlockRange("eth", 21, 30)]] + ], + next: 4 as TransactionId + } + + const result = await runWithState( + initial, + Effect.gen(function*() { + const store = yield* StateStore + + yield* store.truncate(2 as TransactionId) + + const snapshot = yield* store.load() + return snapshot.buffer + }) + ) + + expect(result).toHaveLength(1) + expect(result[0]![0]).toBe(1) + }) + + it("removes all watermarks when truncating from 0", async () => { + const initial: StateSnapshot = { + buffer: [ + [1 as TransactionId, [makeBlockRange("eth", 0, 10)]], + [2 as TransactionId, [makeBlockRange("eth", 11, 20)]] + ], + next: 3 as TransactionId + } + + const result = await runWithState( + initial, + Effect.gen(function*() { + const store = yield* StateStore + + yield* store.truncate(0 as TransactionId) + + const snapshot = yield* store.load() + return snapshot.buffer + }) + ) + + expect(result).toHaveLength(0) + }) + + it("preserves next ID", async () => { + const initial: StateSnapshot = { + buffer: [[1 as TransactionId, [makeBlockRange("eth", 0, 10)]]], + next: 10 as TransactionId + } + + const result = await runWithState( + initial, + Effect.gen(function*() { + const store = yield* StateStore + + yield* store.truncate(1 as TransactionId) + + const snapshot = yield* store.load() + return snapshot.next + }) + ) + + expect(result).toBe(10) + }) +}) + +// ============================================================================= +// makeTestable Tests +// ============================================================================= + +describe("InMemoryStateStore.makeTestable", () => { + it("exposes internal state ref for inspection", async () => { + const result = await Effect.runPromise( + Effect.gen(function*() { + const { service, stateRef } = yield* InMemoryStateStore.makeTestable + + yield* service.advance(5 as TransactionId) + yield* service.commit({ + insert: [[1 as TransactionId, [makeBlockRange("eth", 0, 10)]]], + prune: undefined + }) + + const state = yield* Ref.get(stateRef) + return state + }) + ) + + expect(result.next).toBe(5) + expect(result.buffer).toHaveLength(1) + }) +}) + +// ============================================================================= +// testLayer Tests +// ============================================================================= + +describe("InMemoryStateStore.testLayer", () => { + it("provides both StateStore and TestStateRef", async () => { + const result = await Effect.runPromise( + Effect.gen(function*() { + const store = yield* StateStore + const stateRef = yield* InMemoryStateStore.TestStateRef + + yield* store.advance(5 as TransactionId) + + const state = yield* Ref.get(stateRef) + return state.next + }).pipe(Effect.provide(InMemoryStateStore.testLayer)) + ) + + expect(result).toBe(5) + }) +}) diff --git a/packages/amp/test/transactional-stream/state-actor.test.ts b/packages/amp/test/transactional-stream/state-actor.test.ts new file mode 100644 index 0000000..85a20a2 --- /dev/null +++ b/packages/amp/test/transactional-stream/state-actor.test.ts @@ -0,0 +1,505 @@ +/** + * Tests for StateActor - reorg and rewind handling. + * + * @module + */ +import * as Effect from "effect/Effect" +import * as Ref from "effect/Ref" +import { describe, expect, it } from "vitest" +import type { BlockRange } from "../../src/models.ts" +import type { ProtocolMessage } from "../../src/protocol-stream/messages.ts" +import * as InMemoryStateStore from "../../src/transactional-stream/memory-store.ts" +import { makeStateActor, type Action, type StateActor } from "../../src/transactional-stream/state-actor.ts" +import type { StateSnapshot } from "../../src/transactional-stream/state-store.ts" +import type { TransactionId } from "../../src/transactional-stream/types.ts" + +// ============================================================================= +// Test Helpers +// ============================================================================= + +const makeBlockRange = ( + network: string, + start: number, + end: number, + hash = "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef" +): BlockRange => + ({ + network, + numbers: { start, end }, + hash, + prevHash: undefined + }) as BlockRange + +const dataMessage = ( + data: ReadonlyArray>, + ranges: ReadonlyArray +): ProtocolMessage => ({ + _tag: "Data", + data, + ranges +}) + +const watermarkMessage = (ranges: ReadonlyArray): ProtocolMessage => ({ + _tag: "Watermark", + ranges +}) + +const reorgMessage = ( + previous: ReadonlyArray, + incoming: ReadonlyArray, + invalidation: ReadonlyArray<{ network: string; start: number; end: number }> +): ProtocolMessage => ({ + _tag: "Reorg", + previous, + incoming, + invalidation: invalidation.map((i) => ({ + network: i.network, + start: i.start, + end: i.end + })) as ReadonlyArray<{ network: string; start: number; end: number }> +}) + +const runWithActor = ( + initial: StateSnapshot, + retention: number, + fn: (actor: StateActor) => Effect.Effect +): Promise => + Effect.runPromise( + Effect.gen(function*() { + const { service, stateRef } = yield* InMemoryStateStore.makeTestable + + // Set initial state + yield* Ref.set(stateRef, initial) + + const actor = yield* makeStateActor(service, retention) + return yield* fn(actor) + }) + ) + +// ============================================================================= +// Basic Operation Tests +// ============================================================================= + +describe("StateActor.watermark", () => { + it("returns undefined for empty buffer", async () => { + const result = await runWithActor( + { buffer: [], next: 0 as TransactionId }, + 128, + (actor) => actor.watermark() + ) + + expect(result).toBeUndefined() + }) + + it("returns last watermark from buffer", async () => { + const initial: StateSnapshot = { + buffer: [ + [1 as TransactionId, [makeBlockRange("eth", 0, 10)]], + [2 as TransactionId, [makeBlockRange("eth", 11, 20)]] + ], + next: 3 as TransactionId + } + + const result = await runWithActor(initial, 128, (actor) => actor.watermark()) + + expect(result?.[0]).toBe(2) + }) +}) + +describe("StateActor.peek", () => { + it("returns next transaction ID", async () => { + const result = await runWithActor( + { buffer: [], next: 10 as TransactionId }, + 128, + (actor) => actor.peek() + ) + + expect(result).toBe(10) + }) +}) + +// ============================================================================= +// Data Event Tests +// ============================================================================= + +describe("StateActor.execute - Data", () => { + it("passes through data events with transaction ID", async () => { + const result = await runWithActor( + { buffer: [], next: 5 as TransactionId }, + 128, + (actor) => + Effect.gen(function*() { + const [event] = yield* actor.execute({ + _tag: "Message", + message: dataMessage([{ value: 1 }], [makeBlockRange("eth", 0, 10)]) + }) + return event + }) + ) + + expect(result._tag).toBe("Data") + expect(result.id).toBe(5) + if (result._tag === "Data") { + expect(result.data).toEqual([{ value: 1 }]) + } + }) + + it("increments transaction ID for each event", async () => { + const ids = await runWithActor( + { buffer: [], next: 0 as TransactionId }, + 128, + (actor) => + Effect.gen(function*() { + const [event1] = yield* actor.execute({ + _tag: "Message", + message: dataMessage([{ value: 1 }], [makeBlockRange("eth", 0, 10)]) + }) + const [event2] = yield* actor.execute({ + _tag: "Message", + message: dataMessage([{ value: 2 }], [makeBlockRange("eth", 11, 20)]) + }) + return [event1.id, event2.id] + }) + ) + + expect(ids).toEqual([0, 1]) + }) +}) + +// ============================================================================= +// Watermark Event Tests +// ============================================================================= + +describe("StateActor.execute - Watermark", () => { + it("emits watermark event with transaction ID", async () => { + const result = await runWithActor( + { buffer: [], next: 3 as TransactionId }, + 128, + (actor) => + Effect.gen(function*() { + const [event] = yield* actor.execute({ + _tag: "Message", + message: watermarkMessage([makeBlockRange("eth", 0, 10)]) + }) + return event + }) + ) + + expect(result._tag).toBe("Watermark") + expect(result.id).toBe(3) + }) + + it("adds watermark to internal buffer", async () => { + const watermark = await runWithActor( + { buffer: [], next: 0 as TransactionId }, + 128, + (actor) => + Effect.gen(function*() { + yield* actor.execute({ + _tag: "Message", + message: watermarkMessage([makeBlockRange("eth", 0, 10)]) + }) + return yield* actor.watermark() + }) + ) + + expect(watermark?.[0]).toBe(0) + }) + + it("computes prune point based on retention", async () => { + // Build up a buffer with watermarks far apart + const result = await runWithActor( + { + buffer: [ + [0 as TransactionId, [makeBlockRange("eth", 0, 10)]], + [1 as TransactionId, [makeBlockRange("eth", 11, 20)]] + ], + next: 2 as TransactionId + }, + 50, // Retention of 50 blocks + (actor) => + Effect.gen(function*() { + // Add a watermark at block 200, which should prune old watermarks + const [event] = yield* actor.execute({ + _tag: "Message", + message: watermarkMessage([makeBlockRange("eth", 200, 210)]) + }) + return event + }) + ) + + expect(result._tag).toBe("Watermark") + if (result._tag === "Watermark") { + // Cutoff is 200 - 50 = 150 + // Watermarks 0 and 1 end at 10 and 20, both < 150, so prune up to 1 + expect(result.prune._tag).toBe("Some") + } + }) +}) + +// ============================================================================= +// Rewind Tests +// ============================================================================= + +describe("StateActor.execute - Rewind", () => { + it("emits Undo event with Rewind cause", async () => { + const result = await runWithActor( + { buffer: [], next: 5 as TransactionId }, + 128, + (actor) => + Effect.gen(function*() { + const [event] = yield* actor.execute({ _tag: "Rewind" }) + return event + }) + ) + + expect(result._tag).toBe("Undo") + if (result._tag === "Undo") { + expect(result.cause._tag).toBe("Rewind") + } + }) + + it("computes invalidation range for empty buffer", async () => { + const result = await runWithActor( + { buffer: [], next: 5 as TransactionId }, + 128, + (actor) => + Effect.gen(function*() { + const [event] = yield* actor.execute({ _tag: "Rewind" }) + return event + }) + ) + + if (result._tag === "Undo") { + // Empty buffer, next=5, so invalidate from 0 to id-1=4 + expect(result.invalidate.start).toBe(0) + expect(result.invalidate.end).toBe(4) + } + }) + + it("computes invalidation range from last watermark", async () => { + const result = await runWithActor( + { + buffer: [[3 as TransactionId, [makeBlockRange("eth", 0, 10)]]], + next: 7 as TransactionId + }, + 128, + (actor) => + Effect.gen(function*() { + const [event] = yield* actor.execute({ _tag: "Rewind" }) + return event + }) + ) + + if (result._tag === "Undo") { + // Last watermark at id=3, next=7, new id=7 + // Invalidate from 4 (3+1) to 6 (7-1) + expect(result.invalidate.start).toBe(4) + expect(result.invalidate.end).toBe(6) + } + }) +}) + +// ============================================================================= +// Reorg Tests +// ============================================================================= + +describe("StateActor.execute - Reorg", () => { + it("emits Undo event with Reorg cause", async () => { + const result = await runWithActor( + { + buffer: [[1 as TransactionId, [makeBlockRange("eth", 0, 10)]]], + next: 2 as TransactionId + }, + 128, + (actor) => + Effect.gen(function*() { + const [event] = yield* actor.execute({ + _tag: "Message", + message: reorgMessage( + [makeBlockRange("eth", 0, 10)], + [makeBlockRange("eth", 5, 15)], + [{ network: "eth", start: 11, end: 15 }] + ) + }) + return event + }) + ) + + expect(result._tag).toBe("Undo") + if (result._tag === "Undo") { + expect(result.cause._tag).toBe("Reorg") + } + }) + + it("finds recovery point and truncates buffer", async () => { + const result = await runWithActor( + { + buffer: [ + [1 as TransactionId, [makeBlockRange("eth", 0, 10)]], + [2 as TransactionId, [makeBlockRange("eth", 11, 20)]], + [3 as TransactionId, [makeBlockRange("eth", 21, 30)]] + ], + next: 4 as TransactionId + }, + 128, + (actor) => + Effect.gen(function*() { + // Reorg at block 21 affects watermark id=3 + yield* actor.execute({ + _tag: "Message", + message: reorgMessage( + [makeBlockRange("eth", 21, 30)], + [makeBlockRange("eth", 21, 35)], + [{ network: "eth", start: 21, end: 35 }] + ) + }) + + // Check that buffer is truncated + const watermark = yield* actor.watermark() + return watermark + }) + ) + + // Recovery point is id=2 (last unaffected), buffer truncated to keep only id=1,2 + expect(result?.[0]).toBe(2) + }) + + it("handles reorg with empty buffer", async () => { + const result = await runWithActor( + { buffer: [], next: 5 as TransactionId }, + 128, + (actor) => + Effect.gen(function*() { + const [event] = yield* actor.execute({ + _tag: "Message", + message: reorgMessage( + [makeBlockRange("eth", 0, 10)], + [makeBlockRange("eth", 5, 15)], + [{ network: "eth", start: 5, end: 15 }] + ) + }) + return event + }) + ) + + // Empty buffer case: invalidate from 0 to id-1 + if (result._tag === "Undo") { + expect(result.invalidate.start).toBe(0) + } + }) + + it("fails with UnrecoverableReorgError when all watermarks affected", async () => { + const result = await Effect.runPromiseExit( + Effect.gen(function*() { + const { service, stateRef } = yield* InMemoryStateStore.makeTestable + + yield* Ref.set(stateRef, { + buffer: [ + [1 as TransactionId, [makeBlockRange("eth", 100, 110)]], + [2 as TransactionId, [makeBlockRange("eth", 111, 120)]] + ], + next: 3 as TransactionId + }) + + const actor = yield* makeStateActor(service, 128) + + // Reorg at block 100 affects all watermarks + yield* actor.execute({ + _tag: "Message", + message: reorgMessage( + [makeBlockRange("eth", 100, 120)], + [makeBlockRange("eth", 100, 130)], + [{ network: "eth", start: 100, end: 130 }] + ) + }) + }) + ) + + expect(result._tag).toBe("Failure") + if (result._tag === "Failure") { + const error = result.cause + // Check that it's an UnrecoverableReorgError + expect(error).toBeDefined() + } + }) + + it("fails with PartialReorgError when reorg point falls within watermark range", async () => { + const result = await Effect.runPromiseExit( + Effect.gen(function*() { + const { service, stateRef } = yield* InMemoryStateStore.makeTestable + + yield* Ref.set(stateRef, { + buffer: [[1 as TransactionId, [makeBlockRange("eth", 0, 20)]]], + next: 2 as TransactionId + }) + + const actor = yield* makeStateActor(service, 128) + + // Reorg at block 10 falls within watermark range [0, 20] + yield* actor.execute({ + _tag: "Message", + message: reorgMessage( + [makeBlockRange("eth", 0, 20)], + [makeBlockRange("eth", 10, 30)], + [{ network: "eth", start: 10, end: 30 }] + ) + }) + }) + ) + + expect(result._tag).toBe("Failure") + }) +}) + +// ============================================================================= +// Commit Tests +// ============================================================================= + +describe("StateActor.commit", () => { + it("commits watermarks via commit handle", async () => { + const result = await runWithActor( + { buffer: [], next: 0 as TransactionId }, + 128, + (actor) => + Effect.gen(function*() { + const [, handle] = yield* actor.execute({ + _tag: "Message", + message: watermarkMessage([makeBlockRange("eth", 0, 10)]) + }) + + yield* handle.commit() + + // The watermark should now be committed + return yield* actor.watermark() + }) + ) + + expect(result?.[0]).toBe(0) + }) + + it("batches multiple commits", async () => { + await runWithActor( + { buffer: [], next: 0 as TransactionId }, + 128, + (actor) => + Effect.gen(function*() { + // Execute multiple watermarks without committing + const [, handle1] = yield* actor.execute({ + _tag: "Message", + message: watermarkMessage([makeBlockRange("eth", 0, 10)]) + }) + const [, handle2] = yield* actor.execute({ + _tag: "Message", + message: watermarkMessage([makeBlockRange("eth", 11, 20)]) + }) + + // Commit only the second one - should commit both + yield* handle2.commit() + + const watermark = yield* actor.watermark() + expect(watermark?.[0]).toBe(1) + }) + ) + }) +}) diff --git a/specs/transactional-stream.md b/specs/transactional-stream.md new file mode 100644 index 0000000..b27bcb7 --- /dev/null +++ b/specs/transactional-stream.md @@ -0,0 +1,702 @@ +# TransactionalStream Implementation Spec + +Port Rust `TransactionalStream` from `.repos/amp/crates/clients/flight/src/transactional.rs` to TypeScript with Effect, providing exactly-once semantics, crash recovery, and reorg handling. + +## Overview + +The current TypeScript implementation has stateless reorg detection (`streamProtocol` in `arrow-flight.ts`). This spec adds a full transactional layer matching Rust parity: + +- **Transaction IDs**: Monotonically increasing IDs for each event +- **State Store**: Pluggable persistence with InMemoryStateStore reference implementation +- **Watermark Buffer**: Tracks watermarks for reorg recovery point calculation +- **Commit Handles**: Exactly-once semantics via explicit commit +- **Rewind Detection**: Detects and invalidates uncommitted transactions on restart +- **Retention Window**: Prunes old watermarks outside configurable block window + +## File Structure + +``` +packages/amp/src/transactional-stream/ +├── index.ts # Public exports (see below) +├── types.ts # TransactionEvent, TransactionId, UndoCause, TransactionIdRange +├── errors.ts # StateStoreError, UnrecoverableReorgError, PartialReorgError +├── state-store.ts # StateStore Context.Tag, StateSnapshot, Commit +├── memory-store.ts # InMemoryStateStore Layer implementation +├── algorithms.ts # findRecoveryPoint, findPruningPoint (pure functions) +├── state-actor.ts # StateActor (internal, not exported) +├── commit-handle.ts # CommitHandle interface +└── stream.ts # TransactionalStream Context.Tag and Layer + +packages/amp/test/transactional-stream/ +├── algorithms.test.ts # Recovery/pruning algorithm tests +├── memory-store.test.ts # InMemoryStateStore conformance tests +├── reorg.test.ts # Reorg scenarios (port Rust tests) +├── rewind.test.ts # Crash recovery scenarios +└── integration.test.ts # Full stream integration +``` + +## Public API (index.ts exports) + +```typescript +// ============================================================================= +// Types +// ============================================================================= +export { + TransactionId, + TransactionEvent, + TransactionEventData, + TransactionEventUndo, + TransactionEventWatermark, + UndoCause, + TransactionIdRange +} from "./types.ts" + +// ============================================================================= +// Errors +// ============================================================================= +export { + StateStoreError, + UnrecoverableReorgError, + PartialReorgError, + type TransactionalStreamError +} from "./errors.ts" + +// ============================================================================= +// StateStore Service +// ============================================================================= +export { + StateStore, // Context.Tag for DI + type StateSnapshot, + type Commit, + emptySnapshot +} from "./state-store.ts" + +// ============================================================================= +// InMemoryStateStore Layer +// ============================================================================= +export * as InMemoryStateStore from "./memory-store.ts" + +// ============================================================================= +// CommitHandle +// ============================================================================= +export { type CommitHandle } from "./commit-handle.ts" + +// ============================================================================= +// TransactionalStream Service +// ============================================================================= +export { + TransactionalStream, // Context.Tag for DI + layer, // Layer.Layer + type TransactionalStreamOptions +} from "./stream.ts" +``` + +**Usage from consumer code:** + +```typescript +import { + TransactionalStream, + InMemoryStateStore, + StateStore, + type TransactionEvent, + type CommitHandle +} from "@edgeandnode/amp/transactional-stream" +``` + +## Type Definitions + +### types.ts + +```typescript +// TransactionId - branded non-negative integer +export const TransactionId = Schema.NonNegativeInt.pipe( + Schema.brand("Amp/TransactionalStream/TransactionId") +) + +// UndoCause - discriminated union +export const UndoCause = Schema.Union( + Schema.TaggedStruct("Reorg", { invalidation: Schema.Array(InvalidationRange) }), + Schema.TaggedStruct("Rewind", {}) +) + +// TransactionEvent - three variants matching Rust +export const TransactionEventData = Schema.TaggedStruct("Data", { + id: TransactionId, + data: Schema.Array(Schema.Record({ key: Schema.String, value: Schema.Unknown })), + ranges: Schema.Array(BlockRange) +}) + +export const TransactionEventUndo = Schema.TaggedStruct("Undo", { + id: TransactionId, + cause: UndoCause, + invalidate: Schema.Struct({ start: TransactionId, end: TransactionId }) +}) + +export const TransactionEventWatermark = Schema.TaggedStruct("Watermark", { + id: TransactionId, + ranges: Schema.Array(BlockRange), + prune: Schema.OptionFromNullOr(TransactionId) +}) + +export const TransactionEvent = Schema.Union( + TransactionEventData, + TransactionEventUndo, + TransactionEventWatermark +) +``` + +### state-store.ts + +The StateStore is defined as an Effect **Context.Tag service**, allowing different implementations to be swapped via Layers. This mirrors the Rust pattern where `StateStore` is a trait implemented by `InMemoryStateStore`, `LmdbStateStore`, and `PostgresStateStore`. + +```typescript +import * as Context from "effect/Context" +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" + +// ============================================================================= +// Data Types +// ============================================================================= + +/** + * StateSnapshot - persisted state for crash recovery. + * Loaded once on startup, then maintained in-memory. + */ +export interface StateSnapshot { + readonly buffer: ReadonlyArray]> + readonly next: TransactionId +} + +/** + * Commit - atomic state update for persistence. + * Batches multiple watermark inserts with optional pruning. + */ +export interface Commit { + readonly insert: ReadonlyArray]> + readonly prune: TransactionId | undefined +} + +// ============================================================================= +// StateStore Service (Context.Tag) +// ============================================================================= + +/** + * StateStore service interface - pluggable persistence for stream state. + * + * Different implementations provide crash recovery guarantees: + * - InMemoryStateStore: No persistence (development/testing) + * - IndexedDBStateStore: Browser persistence (future) + * - SqliteStateStore: Node.js file-based persistence (future) + * - PostgresStateStore: Distributed persistence (future) + */ +export interface StateStore { + /** + * Pre-allocate next transaction ID. + * Called immediately after incrementing in-memory counter. + * Ensures ID monotonicity survives crashes. + */ + readonly advance: (next: TransactionId) => Effect.Effect + + /** + * Atomically commit watermarks and apply pruning. + * Called when user invokes CommitHandle.commit(). + * Must be idempotent (safe to call multiple times with same data). + */ + readonly commit: (commit: Commit) => Effect.Effect + + /** + * Truncate buffer during reorg handling. + * Removes all watermarks with ID >= from. + * Called immediately when reorg detected (before emitting Undo). + */ + readonly truncate: (from: TransactionId) => Effect.Effect + + /** + * Load initial state on startup. + * Called once when TransactionalStream is created. + * Returns empty state if no prior state exists. + */ + readonly load: () => Effect.Effect +} + +/** + * StateStore Context.Tag - use this to depend on StateStore in Effects. + */ +export class StateStore extends Context.Tag("Amp/TransactionalStream/StateStore")< + StateStore, + StateStore +>() {} + +// ============================================================================= +// Empty State Helper +// ============================================================================= + +/** + * Initial empty state for new streams or cleared stores. + */ +export const emptySnapshot: StateSnapshot = { + buffer: [], + next: 0 as TransactionId +} +``` + +### memory-store.ts (InMemoryStateStore Layer) + +Reference implementation using Effect Ref. Not crash-safe but suitable for development, testing, and ephemeral use cases. + +```typescript +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" +import * as Ref from "effect/Ref" +import { StateStore, type StateSnapshot, type Commit, emptySnapshot } from "./state-store.ts" + +/** + * Create InMemoryStateStore Layer. + * + * @example + * ```typescript + * const program = Effect.gen(function*() { + * const store = yield* StateStore + * const snapshot = yield* store.load() + * // ... + * }) + * + * Effect.runPromise( + * program.pipe(Effect.provide(InMemoryStateStore.layer)) + * ) + * ``` + */ +const make = Effect.gen(function*() { + const stateRef = yield* Ref.make(emptySnapshot) + + const advance = (next: TransactionId) => + Ref.update(stateRef, (state) => ({ ...state, next })) + + const commit = (commit: Commit) => + Ref.update(stateRef, (state) => { + let buffer = [...state.buffer] + + // Remove pruned watermarks (all IDs <= prune) + if (commit.prune !== undefined) { + buffer = buffer.filter(([id]) => id > commit.prune!) + } + + // Add new watermarks + for (const entry of commit.insert) { + buffer.push(entry) + } + + return { ...state, buffer } + }) + + const truncate = (from: TransactionId) => + Ref.update(stateRef, (state) => ({ + ...state, + buffer: state.buffer.filter(([id]) => id < from) + })) + + const load = () => Ref.get(stateRef) + + return { advance, commit, truncate, load } satisfies StateStore +}) + +/** + * Layer providing InMemoryStateStore. + */ +export const layer: Layer.Layer = Layer.effect(StateStore, make) + +/** + * Create layer with initial state (useful for testing). + */ +export const layerWithState = ( + initial: StateSnapshot +): Layer.Layer => + Layer.effect( + StateStore, + Effect.gen(function*() { + const stateRef = yield* Ref.make(initial) + // ... same implementation as above + }) + ) +``` + +### Future Store Implementations + +Additional StateStore implementations can be added as separate packages or modules: + +```typescript +// packages/amp-store-indexeddb/src/index.ts +export const layer: Layer.Layer = Layer.effect(StateStore, make) + +// packages/amp-store-sqlite/src/index.ts +export const layer: Layer.Layer = Layer.effect(StateStore, make) + +// packages/amp-store-postgres/src/index.ts +export const layer: Layer.Layer = + Layer.effect(StateStore, make) +``` + +## Key Algorithms + +### findRecoveryPoint (algorithms.ts) + +Walk backwards through watermark buffer to find last unaffected watermark: + +```typescript +export const findRecoveryPoint = ( + buffer: ReadonlyArray]>, + invalidation: ReadonlyArray +): readonly [TransactionId, ReadonlyArray] | undefined => { + // Build map: network -> first invalid block + const points = new Map(invalidation.map(inv => [inv.network, inv.start])) + + // Walk backwards (newest to oldest) + for (let i = buffer.length - 1; i >= 0; i--) { + const [id, ranges] = buffer[i]! + const affected = ranges.some(range => { + const point = points.get(range.network) + return point !== undefined && range.numbers.start >= point + }) + if (!affected) return [id, ranges] + } + return undefined +} +``` + +### findPruningPoint (algorithms.ts) + +Find oldest watermark outside retention window: + +```typescript +export const findPruningPoint = ( + buffer: ReadonlyArray]>, + retention: number +): TransactionId | undefined => { + if (buffer.length === 0) return undefined + + const [, latestRanges] = buffer[buffer.length - 1]! + const cutoffs = new Map(latestRanges.map(r => + [r.network, Math.max(0, r.numbers.start - retention)] + )) + + let last: TransactionId | undefined + for (let i = 0; i < buffer.length - 1; i++) { + const [id, ranges] = buffer[i]! + const outside = ranges.every(r => r.numbers.end < cutoffs.get(r.network)!) + if (outside) last = id + else break + } + return last +} +``` + +## StateActor (Internal) + +StateActor is an **internal** component (not exported publicly). It wraps a StateStore instance with a Ref for concurrent-safe in-memory state management. Created once per stream instance. + +```typescript +import * as Ref from "effect/Ref" + +/** + * Internal state container - in-memory copy of persisted state. + * The store is the source of truth; this is a working cache. + */ +interface StateContainer { + readonly store: StateStore // From context + readonly retention: number // Configured retention window + next: TransactionId // Next ID to assign + buffer: Array]> + uncommitted: Array +} + +/** + * StateActor interface - manages transactional stream state. + */ +export interface StateActor { + /** Get last watermark from buffer */ + readonly watermark: () => Effect.Effect<[TransactionId, ReadonlyArray] | undefined> + + /** Get next transaction ID without incrementing */ + readonly peek: () => Effect.Effect + + /** Execute an action and return event with commit handle */ + readonly execute: (action: Action) => Effect.Effect< + [TransactionEvent, CommitHandle], + TransactionalStreamError + > + + /** Commit pending changes up to and including this ID */ + readonly commit: (id: TransactionId) => Effect.Effect +} + +// Action union for execute() +type Action = + | { readonly _tag: "Message"; readonly message: ProtocolMessage } + | { readonly _tag: "Rewind" } + +/** + * Create a StateActor from a StateStore. + * Called internally by TransactionalStream. + */ +export const makeStateActor = ( + store: StateStore, + retention: number +): Effect.Effect => + Effect.gen(function*() { + // Load initial state from store + const snapshot = yield* store.load() + + // Create mutable state container wrapped in Ref + const containerRef = yield* Ref.make({ + store, + retention, + next: snapshot.next, + buffer: [...snapshot.buffer], + uncommitted: [] + }) + + // ... implement watermark, peek, execute, commit + return { watermark, peek, execute, commit } + }) +``` + +**execute() Logic:** + +1. **Pre-allocate monotonic ID**: Increment `next` in Ref, call `store.advance(next)` immediately +2. **Match on action type**: + - **Rewind**: Compute invalidate range from buffer state, emit `Undo(Rewind)` + - **Message(Data)**: Pass through as `TransactionEventData` (no buffer mutation) + - **Message(Watermark)**: Add to buffer, compute prune via `findPruningPoint`, add to uncommitted queue, emit `TransactionEventWatermark` + - **Message(Reorg)**: Find recovery point via `findRecoveryPoint`, check for partial reorg (error if unrecoverable), truncate buffer + call `store.truncate()`, clear uncommitted after recovery point, emit `Undo(Reorg)` +3. **Return** `[event, CommitHandle]` where CommitHandle captures actor + ID + +## TransactionalStream Service + +The TransactionalStream service depends on both `ArrowFlight` and `StateStore` via the Layer system. Users compose layers to provide the desired StateStore implementation. + +```typescript +import * as Context from "effect/Context" +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" +import * as Stream from "effect/Stream" + +/** + * Options for creating a transactional stream. + * Note: StateStore is NOT passed here - it comes from the Layer context. + */ +export interface TransactionalStreamOptions { + /** Retention window in blocks for pruning old watermarks */ + readonly retention: number + /** Optional schema for data validation */ + readonly schema?: Schema.Any +} + +/** + * TransactionalStream service interface. + */ +export interface TransactionalStream { + /** + * Create a transactional stream from a SQL query. + * Returns tuples of [event, commitHandle] for manual commit control. + */ + readonly streamTransactional: ( + sql: string, + options: TransactionalStreamOptions + ) => Stream.Stream + + /** + * High-level consumer: auto-commit after callback succeeds. + * If callback fails, stream stops and pending batch remains uncommitted. + * On restart, uncommitted batch triggers Rewind. + */ + readonly forEach: ( + sql: string, + options: TransactionalStreamOptions, + handler: (event: TransactionEvent) => Effect.Effect + ) => Effect.Effect +} + +/** + * TransactionalStream Context.Tag + */ +export class TransactionalStream extends Context.Tag("Amp/TransactionalStream")< + TransactionalStream, + TransactionalStream +>() {} + +/** + * Layer providing TransactionalStream. + * Requires ArrowFlight and StateStore in context. + */ +export const layer: Layer.Layer = + Layer.effect(TransactionalStream, make) +``` + +**streamTransactional() Logic:** + +1. Get StateStore from context via `yield* StateStore` +2. Create StateActor wrapping the store +3. Get watermark for resume cursor +4. Detect rewind: `next > watermark[0] + 1` or `next > 0` (no watermark) +5. Get protocol stream with resume watermark +6. Emit Rewind if needed, then map protocol messages through actor.execute() + +## Layer Composition + +Users compose layers to wire up the full dependency graph: + +```typescript +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" +import { TransactionalStream } from "@edgeandnode/amp/transactional-stream" +import { InMemoryStateStore } from "@edgeandnode/amp/transactional-stream/memory-store" +import { ArrowFlight, Transport } from "@edgeandnode/amp" + +// ============================================================================= +// Example 1: Development/Testing with InMemoryStateStore +// ============================================================================= + +const DevLayer = TransactionalStream.layer.pipe( + Layer.provide(InMemoryStateStore.layer), + Layer.provide(ArrowFlight.layer), + Layer.provide(MyTransportLayer) +) + +const program = Effect.gen(function*() { + const stream = yield* TransactionalStream + + yield* stream.forEach( + "SELECT * FROM eth.logs", + { retention: 128 }, + (event) => Effect.gen(function*() { + switch (event._tag) { + case "Data": + yield* processData(event.id, event.data, event.ranges) + break + case "Undo": + yield* rollback(event.invalidate) + break + case "Watermark": + yield* checkpoint(event.id, event.ranges) + break + } + }) + ) +}) + +Effect.runPromise(program.pipe(Effect.provide(DevLayer))) + +// ============================================================================= +// Example 2: Production with Persistent StateStore (future) +// ============================================================================= + +import { IndexedDBStateStore } from "@edgeandnode/amp-store-indexeddb" + +const ProdLayer = TransactionalStream.layer.pipe( + Layer.provide(IndexedDBStateStore.layer({ dbName: "amp-state" })), + Layer.provide(ArrowFlight.layer), + Layer.provide(MyTransportLayer) +) + +// ============================================================================= +// Example 3: Testing with Pre-populated State +// ============================================================================= + +const testState: StateSnapshot = { + buffer: [[5 as TransactionId, [{ network: "eth", ... }]]], + next: 10 as TransactionId +} + +const TestLayer = TransactionalStream.layer.pipe( + Layer.provide(InMemoryStateStore.layerWithState(testState)), + Layer.provide(ArrowFlight.layer), + Layer.provide(MockTransportLayer) +) + +// ============================================================================= +// Example 4: Manual Stream Control with CommitHandle +// ============================================================================= + +const manualProgram = Effect.gen(function*() { + const stream = yield* TransactionalStream + + const txStream = stream.streamTransactional( + "SELECT * FROM eth.logs", + { retention: 128 } + ) + + yield* txStream.pipe( + Stream.runForEach(([event, commitHandle]) => + Effect.gen(function*() { + // Process event + yield* processEvent(event) + + // Manually decide when to commit + if (shouldCommit(event)) { + yield* commitHandle.commit() + } + // If we don't commit and crash, this event will be replayed via Rewind + }) + ) + ) +}) +``` + +## Implementation Order + +1. **types.ts** - TransactionId, UndoCause, TransactionEvent schemas +2. **errors.ts** - StateStoreError, UnrecoverableReorgError, PartialReorgError +3. **state-store.ts** - StateStore interface, StateSnapshot, Commit types +4. **algorithms.ts** - findRecoveryPoint, findPruningPoint (with tests) +5. **memory-store.ts** - InMemoryStateStore with Ref +6. **commit-handle.ts** - CommitHandle interface +7. **state-actor.ts** - StateActor service with execute() logic +8. **stream.ts** - TransactionalStream service +9. **index.ts** - Public exports +10. **Tests** - Port Rust test scenarios + +## Test Plan + +### algorithms.test.ts + +- findRecoveryPoint: empty buffer, partial buffer, all affected, multi-network +- findPruningPoint: empty buffer, within retention, outside retention + +### reorg.test.ts (port from Rust) + +- `reorg_invalidates_affected_batches` +- `reorg_invalidates_multiple_consecutive_batches` +- `reorg_does_not_invalidate_unaffected_batches` +- `multi_network_reorg_partial_invalidation` +- `consecutive_reorgs_cumulative_invalidation` +- `reorg_with_backwards_jump_succeeds` + +### rewind.test.ts + +- Uncommitted after watermark triggers rewind +- Multiple uncommitted batches +- No rewind when fully committed +- Rewind with empty buffer (early crash) + +### integration.test.ts + +- Full stream lifecycle with InMemoryStateStore +- Auto-commit via forEach() +- Manual commit via CommitHandle + +## Critical Files to Modify/Reference + +| File | Purpose | +|------|---------| +| `packages/amp/src/arrow-flight.ts` | Pattern for Layer.effect, contains streamProtocol to wrap | +| `packages/amp/src/protocol-stream/messages.ts` | ProtocolMessage, InvalidationRange to consume | +| `packages/amp/src/protocol-stream/errors.ts` | ProtocolStreamError to include in union | +| `packages/amp/src/models.ts` | BlockRange type | +| `.repos/amp/crates/clients/flight/src/transactional.rs` | Rust source to port | + +## Verification + +1. Run `pnpm vitest run packages/amp/test/transactional-stream/` - all tests pass +2. Run `pnpm oxlint packages/amp/src/transactional-stream/` - no lint errors +3. Run `pnpm dprint check` - formatting correct +4. Verify reorg test parity with Rust scenarios From 9205a0fbc2699325391e4fc107bb5802674a8473 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 5 Feb 2026 14:59:34 -0500 Subject: [PATCH 03/12] Extract ProtocolStream into separate service Refactors the protocol stream functionality from ArrowFlight into its own ProtocolStream Context.Tag service: - Add protocol-stream/service.ts with ProtocolStream service definition - Remove streamProtocol, detectReorgs, and related code from arrow-flight.ts - Update TransactionalStream to depend on ProtocolStream instead of ArrowFlight - Update exports in protocol-stream/index.ts This improves separation of concerns and allows the protocol stream logic to be used independently or replaced with alternative implementations. Co-Authored-By: Claude Opus 4.5 --- packages/amp/src/arrow-flight.ts | 227 +------------ packages/amp/src/protocol-stream/index.ts | 30 +- packages/amp/src/protocol-stream/service.ts | 306 ++++++++++++++++++ .../amp/src/transactional-stream/stream.ts | 16 +- 4 files changed, 341 insertions(+), 238 deletions(-) create mode 100644 packages/amp/src/protocol-stream/service.ts diff --git a/packages/amp/src/arrow-flight.ts b/packages/amp/src/arrow-flight.ts index 4c78ed3..54d85fe 100644 --- a/packages/amp/src/arrow-flight.ts +++ b/packages/amp/src/arrow-flight.ts @@ -26,20 +26,6 @@ import { parseRecordBatch } from "./internal/arrow-flight-ipc/RecordBatch.ts" import { type ArrowSchema, getMessageType, MessageHeaderType, parseSchema } from "./internal/arrow-flight-ipc/Schema.ts" import type { AuthInfo, BlockRange, RecordBatchMetadata } from "./models.ts" import { RecordBatchMetadataFromUint8Array } from "./models.ts" -import { - type ProtocolStreamError, - ProtocolArrowFlightError, - ProtocolValidationError -} from "./protocol-stream/errors.ts" -import { - type InvalidationRange, - type ProtocolMessage, - data as protocolData, - makeInvalidationRange, - reorg as protocolReorg, - watermark as protocolWatermark -} from "./protocol-stream/messages.ts" -import { validateAll } from "./protocol-stream/validation.ts" import { FlightDescriptor_DescriptorType, FlightDescriptorSchema, FlightService } from "./protobuf/Flight_pb.ts" import { CommandStatementQuerySchema } from "./protobuf/FlightSql_pb.ts" @@ -284,57 +270,8 @@ export class ArrowFlight extends Context.Tag("Amp/ArrowFlight") Stream.Stream, ArrowFlightError> - - /** - * Executes an Arrow Flight SQL query and returns a stream of protocol messages - * with stateless reorg detection. - * - * Protocol messages include: - * - `Data`: New records to process with block ranges - * - `Reorg`: Chain reorganization detected with invalidation ranges - * - `Watermark`: Confirmation that block ranges are complete - * - * @example - * ```typescript - * const arrowFlight = yield* ArrowFlight - * - * yield* arrowFlight.streamProtocol("SELECT * FROM eth.logs").pipe( - * Stream.runForEach((message) => { - * switch (message._tag) { - * case "Data": - * return Effect.log(`Data: ${message.data.length} records`) - * case "Reorg": - * return Effect.log(`Reorg: ${message.invalidation.length} ranges`) - * case "Watermark": - * return Effect.log(`Watermark confirmed`) - * } - * }) - * ) - * ``` - */ - readonly streamProtocol: ( - sql: string, - options?: ProtocolStreamOptions - ) => Stream.Stream }>() {} -/** - * Options for creating a protocol stream. - */ -export interface ProtocolStreamOptions { - /** - * Schema to validate and decode the record batch data. - * If provided, data will be validated against this schema. - */ - readonly schema?: QueryOptions["schema"] - - /** - * Resume watermark from a previous session. - * Allows resumption of streaming queries from a known position. - */ - readonly resumeWatermark?: ReadonlyArray -} - const make = Effect.gen(function*() { const auth = yield* Effect.serviceOption(Auth) const transport = yield* Transport @@ -471,148 +408,10 @@ const make = Effect.gen(function*() { } ) as any - /** - * Internal state maintained by the protocol stream for reorg detection. - */ - interface ProtocolStreamState { - readonly previous: ReadonlyArray - readonly initialized: boolean - } - - /** - * Detects reorgs by comparing incoming ranges to previous ranges. - */ - const detectReorgs = ( - previous: ReadonlyArray, - incoming: ReadonlyArray - ): ReadonlyArray => { - const invalidations: Array = [] - - for (const incomingRange of incoming) { - const prevRange = previous.find((p) => p.network === incomingRange.network) - if (!prevRange) continue - - // Skip identical ranges (watermarks can repeat) - if ( - incomingRange.network === prevRange.network && - incomingRange.numbers.start === prevRange.numbers.start && - incomingRange.numbers.end === prevRange.numbers.end && - incomingRange.hash === prevRange.hash && - incomingRange.prevHash === prevRange.prevHash - ) { - continue - } - - const incomingStart = incomingRange.numbers.start - const prevEnd = prevRange.numbers.end - - // Detect backwards jump (reorg indicator) - if (incomingStart < prevEnd + 1) { - invalidations.push( - makeInvalidationRange( - incomingRange.network, - incomingStart, - Math.max(incomingRange.numbers.end, prevEnd) - ) - ) - } - } - - return invalidations - } - - const streamProtocol = ( - sql: string, - options?: ProtocolStreamOptions - ): Stream.Stream => { - // Get the underlying Arrow Flight stream - const rawStream = streamQuery(sql, { - schema: options?.schema, - stream: true, - resumeWatermark: options?.resumeWatermark - }) as unknown as Stream.Stream< - QueryResult>>, - ArrowFlightError - > - - const initialState: ProtocolStreamState = { - previous: [], - initialized: false - } - - const ZERO_HASH = "0x0000000000000000000000000000000000000000000000000000000000000000" - - return rawStream.pipe( - // Map Arrow Flight errors to protocol errors - Stream.mapError((error: ArrowFlightError) => new ProtocolArrowFlightError({ cause: error })), - - // Process each batch with state tracking - Stream.mapAccumEffect(initialState, (state, queryResult) => - Effect.gen(function*() { - const batchData = queryResult.data - const metadata = queryResult.metadata - const incoming = metadata.ranges - - // Validate the incoming batch - if (state.initialized) { - yield* validateAll(state.previous, incoming).pipe( - Effect.mapError((error) => new ProtocolValidationError({ cause: error })) - ) - } else { - // Validate prevHash for first batch - for (const range of incoming) { - const isGenesis = range.numbers.start === 0 - if (isGenesis) { - if (range.prevHash !== undefined && range.prevHash !== ZERO_HASH) { - return yield* Effect.fail( - new ProtocolValidationError({ - cause: { _tag: "InvalidPrevHashError", network: range.network } - }) - ) - } - } else { - if (range.prevHash === undefined || range.prevHash === ZERO_HASH) { - return yield* Effect.fail( - new ProtocolValidationError({ - cause: { _tag: "MissingPrevHashError", network: range.network, block: range.numbers.start } - }) - ) - } - } - } - } - - // Detect reorgs - const invalidations = state.initialized ? detectReorgs(state.previous, incoming) : [] - - // Determine message type - let message: ProtocolMessage - - if (invalidations.length > 0) { - message = protocolReorg(state.previous, incoming, invalidations) - } else if (metadata.rangesComplete && batchData.length === 0) { - message = protocolWatermark(incoming) - } else { - message = protocolData(batchData as unknown as ReadonlyArray>, incoming) - } - - const newState: ProtocolStreamState = { - previous: incoming, - initialized: true - } - - return [newState, message] as const - })), - - Stream.withSpan("ArrowFlight.streamProtocol") - ) - } - return { client, query, - streamQuery, - streamProtocol + streamQuery } as const }) @@ -642,27 +441,3 @@ const blockRangesToResumeWatermark = (ranges: ReadonlyArray): string } return JSON.stringify(watermarks) } - -// ============================================================================= -// Protocol Stream Re-exports -// ============================================================================= - -export type { ProtocolStreamError } - -export { - ProtocolArrowFlightError, - ProtocolValidationError -} from "./protocol-stream/errors.ts" - -export { - InvalidationRange, - ProtocolMessage, - ProtocolMessageData, - ProtocolMessageReorg, - ProtocolMessageWatermark, - data as protocolMessageData, - invalidates, - makeInvalidationRange, - reorg as protocolMessageReorg, - watermark as protocolMessageWatermark -} from "./protocol-stream/messages.ts" diff --git a/packages/amp/src/protocol-stream/index.ts b/packages/amp/src/protocol-stream/index.ts index 8ea0e71..7423b8a 100644 --- a/packages/amp/src/protocol-stream/index.ts +++ b/packages/amp/src/protocol-stream/index.ts @@ -1,8 +1,8 @@ /** - * Protocol Stream types for processing Amp streams with reorg detection. + * Protocol Stream - provides protocol-level stream processing with reorg detection. * - * This module provides the types and validation functions used by - * `ArrowFlight.streamProtocol()` for stateless reorg detection. + * This module provides the `ProtocolStream` service and related types for + * processing Amp streams with stateless reorg detection. * * ## Overview * @@ -18,12 +18,13 @@ * ```typescript * import * as Effect from "effect/Effect" * import * as Stream from "effect/Stream" - * import { ArrowFlight } from "@edgeandnode/amp" + * import { ProtocolStream } from "@edgeandnode/amp/protocol-stream" + * import { ArrowFlight, Transport } from "@edgeandnode/amp" * * const program = Effect.gen(function*() { - * const arrowFlight = yield* ArrowFlight.ArrowFlight + * const protocolStream = yield* ProtocolStream * - * yield* arrowFlight.streamProtocol("SELECT * FROM eth.logs").pipe( + * yield* protocolStream.stream("SELECT * FROM eth.logs").pipe( * Stream.runForEach((message) => { * switch (message._tag) { * case "Data": @@ -36,6 +37,12 @@ * }) * ) * }) + * + * Effect.runPromise(program.pipe( + * Effect.provide(ProtocolStream.layer), + * Effect.provide(ArrowFlight.layer), + * Effect.provide(Transport.layer) + * )) * ``` * * ## Reorg Detection @@ -113,3 +120,14 @@ export { validateNetworks, validatePrevHash } from "./validation.ts" + +// ============================================================================= +// Service +// ============================================================================= + +export { + ProtocolStream, + layer, + type ProtocolStreamService, + type ProtocolStreamOptions +} from "./service.ts" diff --git a/packages/amp/src/protocol-stream/service.ts b/packages/amp/src/protocol-stream/service.ts new file mode 100644 index 0000000..ccab06d --- /dev/null +++ b/packages/amp/src/protocol-stream/service.ts @@ -0,0 +1,306 @@ +/** + * ProtocolStream service - provides protocol-level stream processing with reorg detection. + * + * The ProtocolStream wraps ArrowFlight's raw streaming with: + * - Stateless reorg detection via block range progression + * - Protocol validation (hash chains, network consistency) + * - Message categorization (Data, Reorg, Watermark) + * + * @module + */ +import * as Context from "effect/Context" +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" +import * as Stream from "effect/Stream" +import { + ArrowFlight, + type ArrowFlightError, + type QueryOptions, + type QueryResult +} from "../arrow-flight.ts" +import type { BlockRange } from "../models.ts" +import { + ProtocolArrowFlightError, + ProtocolValidationError, + type ProtocolStreamError +} from "./errors.ts" +import { + type InvalidationRange, + type ProtocolMessage, + data as protocolData, + makeInvalidationRange, + reorg as protocolReorg, + watermark as protocolWatermark +} from "./messages.ts" +import { validateAll } from "./validation.ts" + +// ============================================================================= +// Options +// ============================================================================= + +/** + * Options for creating a protocol stream. + */ +export interface ProtocolStreamOptions { + /** + * Schema to validate and decode the record batch data. + * If provided, data will be validated against this schema. + */ + readonly schema?: QueryOptions["schema"] + + /** + * Resume watermark from a previous session. + * Allows resumption of streaming queries from a known position. + */ + readonly resumeWatermark?: ReadonlyArray +} + +// ============================================================================= +// Service Interface +// ============================================================================= + +/** + * ProtocolStream service interface. + * + * Provides protocol-level stream processing on top of raw Arrow Flight streams: + * - Stateless reorg detection by monitoring block range progression + * - Protocol validation (hash chains, network consistency, no gaps) + * - Message categorization into Data, Reorg, and Watermark types + */ +export interface ProtocolStreamService { + /** + * Execute a SQL query and return a stream of protocol messages + * with stateless reorg detection. + * + * Protocol messages include: + * - `Data`: New records to process with block ranges + * - `Reorg`: Chain reorganization detected with invalidation ranges + * - `Watermark`: Confirmation that block ranges are complete + * + * @example + * ```typescript + * const protocolStream = yield* ProtocolStream + * + * yield* protocolStream.stream("SELECT * FROM eth.logs").pipe( + * Stream.runForEach((message) => { + * switch (message._tag) { + * case "Data": + * return Effect.log(`Data: ${message.data.length} records`) + * case "Reorg": + * return Effect.log(`Reorg: ${message.invalidation.length} ranges`) + * case "Watermark": + * return Effect.log(`Watermark confirmed`) + * } + * }) + * ) + * ``` + */ + readonly stream: ( + sql: string, + options?: ProtocolStreamOptions + ) => Stream.Stream +} + +// Re-export ProtocolStreamError from errors.ts for convenience +export type { ProtocolStreamError } + +// ============================================================================= +// Context.Tag +// ============================================================================= + +/** + * ProtocolStream Context.Tag - use this to depend on ProtocolStream in Effects. + * + * @example + * ```typescript + * const program = Effect.gen(function*() { + * const protocolStream = yield* ProtocolStream + * yield* protocolStream.stream("SELECT * FROM eth.logs").pipe( + * Stream.runForEach((message) => Effect.log(message._tag)) + * ) + * }) + * + * Effect.runPromise(program.pipe( + * Effect.provide(ProtocolStream.layer), + * Effect.provide(ArrowFlight.layer), + * Effect.provide(Transport.layer) + * )) + * ``` + */ +export class ProtocolStream extends Context.Tag("Amp/ProtocolStream")< + ProtocolStream, + ProtocolStreamService +>() {} + +// ============================================================================= +// Implementation +// ============================================================================= + +/** + * Internal state maintained by the protocol stream for reorg detection. + */ +interface ProtocolStreamState { + readonly previous: ReadonlyArray + readonly initialized: boolean +} + +/** + * Detects reorgs by comparing incoming ranges to previous ranges. + * + * A reorg is detected when the incoming block range starts before or at + * the previous range's end (a "backwards jump"). + */ +const detectReorgs = ( + previous: ReadonlyArray, + incoming: ReadonlyArray +): ReadonlyArray => { + const invalidations: Array = [] + + for (const incomingRange of incoming) { + const prevRange = previous.find((p) => p.network === incomingRange.network) + if (!prevRange) continue + + // Skip identical ranges (watermarks can repeat) + if ( + incomingRange.network === prevRange.network && + incomingRange.numbers.start === prevRange.numbers.start && + incomingRange.numbers.end === prevRange.numbers.end && + incomingRange.hash === prevRange.hash && + incomingRange.prevHash === prevRange.prevHash + ) { + continue + } + + const incomingStart = incomingRange.numbers.start + const prevEnd = prevRange.numbers.end + + // Detect backwards jump (reorg indicator) + if (incomingStart < prevEnd + 1) { + invalidations.push( + makeInvalidationRange( + incomingRange.network, + incomingStart, + Math.max(incomingRange.numbers.end, prevEnd) + ) + ) + } + } + + return invalidations +} + +const ZERO_HASH = "0x0000000000000000000000000000000000000000000000000000000000000000" + +/** + * Create ProtocolStream service implementation. + */ +const make = Effect.gen(function*() { + const arrowFlight = yield* ArrowFlight + + const stream = ( + sql: string, + options?: ProtocolStreamOptions + ): Stream.Stream => { + // Get the underlying Arrow Flight stream + const rawStream = arrowFlight.streamQuery(sql, { + schema: options?.schema, + stream: true, + resumeWatermark: options?.resumeWatermark + }) as unknown as Stream.Stream< + QueryResult>>, + ArrowFlightError + > + + const initialState: ProtocolStreamState = { + previous: [], + initialized: false + } + + return rawStream.pipe( + // Map Arrow Flight errors to protocol errors + Stream.mapError((error: ArrowFlightError) => new ProtocolArrowFlightError({ cause: error })), + + // Process each batch with state tracking + Stream.mapAccumEffect(initialState, (state, queryResult) => + Effect.gen(function*() { + const batchData = queryResult.data + const metadata = queryResult.metadata + const incoming = metadata.ranges + + // Validate the incoming batch + if (state.initialized) { + yield* validateAll(state.previous, incoming).pipe( + Effect.mapError((error) => new ProtocolValidationError({ cause: error })) + ) + } else { + // Validate prevHash for first batch + for (const range of incoming) { + const isGenesis = range.numbers.start === 0 + if (isGenesis) { + if (range.prevHash !== undefined && range.prevHash !== ZERO_HASH) { + return yield* Effect.fail( + new ProtocolValidationError({ + cause: { _tag: "InvalidPrevHashError", network: range.network } + }) + ) + } + } else { + if (range.prevHash === undefined || range.prevHash === ZERO_HASH) { + return yield* Effect.fail( + new ProtocolValidationError({ + cause: { _tag: "MissingPrevHashError", network: range.network, block: range.numbers.start } + }) + ) + } + } + } + } + + // Detect reorgs + const invalidations = state.initialized ? detectReorgs(state.previous, incoming) : [] + + // Determine message type + let message: ProtocolMessage + + if (invalidations.length > 0) { + message = protocolReorg(state.previous, incoming, invalidations) + } else if (metadata.rangesComplete && batchData.length === 0) { + message = protocolWatermark(incoming) + } else { + message = protocolData(batchData as unknown as ReadonlyArray>, incoming) + } + + const newState: ProtocolStreamState = { + previous: incoming, + initialized: true + } + + return [newState, message] as const + })), + + Stream.withSpan("ProtocolStream.stream") + ) + } + + return { stream } satisfies ProtocolStreamService +}) + +// ============================================================================= +// Layer +// ============================================================================= + +/** + * Layer providing ProtocolStream. + * + * Requires ArrowFlight in context. + * + * @example + * ```typescript + * const AppLayer = ProtocolStream.layer.pipe( + * Layer.provide(ArrowFlight.layer), + * Layer.provide(Transport.layer) + * ) + * ``` + */ +export const layer: Layer.Layer = + Layer.effect(ProtocolStream, make) diff --git a/packages/amp/src/transactional-stream/stream.ts b/packages/amp/src/transactional-stream/stream.ts index 0c71517..243ee95 100644 --- a/packages/amp/src/transactional-stream/stream.ts +++ b/packages/amp/src/transactional-stream/stream.ts @@ -1,7 +1,7 @@ /** * TransactionalStream service - provides exactly-once semantics for data processing. * - * The TransactionalStream wraps ArrowFlight's protocol stream with: + * The TransactionalStream wraps ProtocolStream with: * - Transaction IDs for each event * - Crash recovery via persistent state * - Rewind detection for uncommitted transactions @@ -13,9 +13,10 @@ import * as Context from "effect/Context" import * as Effect from "effect/Effect" import * as Layer from "effect/Layer" import * as Stream from "effect/Stream" -import { ArrowFlight, type ProtocolStreamOptions, type QueryOptions } from "../arrow-flight.ts" +import type { QueryOptions } from "../arrow-flight.ts" import type { BlockRange } from "../models.ts" import type { ProtocolStreamError } from "../protocol-stream/errors.ts" +import { ProtocolStream, type ProtocolStreamOptions } from "../protocol-stream/service.ts" import type { CommitHandle } from "./commit-handle.ts" import type { StateStoreError, TransactionalStreamError, UnrecoverableReorgError, PartialReorgError } from "./errors.ts" import { type Action, makeStateActor, type StateActor } from "./state-actor.ts" @@ -141,6 +142,7 @@ export interface TransactionalStreamService { * Effect.runPromise(program.pipe( * Effect.provide(TransactionalStream.layer), * Effect.provide(InMemoryStateStore.layer), + * Effect.provide(ProtocolStream.layer), * Effect.provide(ArrowFlight.layer), * Effect.provide(Transport.layer) * )) @@ -180,7 +182,7 @@ const needsRewind = ( * Create TransactionalStream service implementation. */ const make = Effect.gen(function*() { - const arrowFlight = yield* ArrowFlight + const protocolStreamService = yield* ProtocolStream const storeService = yield* StateStore const streamTransactional = ( @@ -215,7 +217,7 @@ const make = Effect.gen(function*() { const protocolOptions: ProtocolStreamOptions = resumeWatermark !== undefined ? { schema: options?.schema, resumeWatermark } : { schema: options?.schema } - const protocolStream = arrowFlight.streamProtocol(sql, protocolOptions) + const protocolStream = protocolStreamService.stream(sql, protocolOptions) // 6. Build transactional stream // First emit Rewind if needed, then map protocol messages through actor @@ -274,13 +276,14 @@ const make = Effect.gen(function*() { /** * Layer providing TransactionalStream. * - * Requires ArrowFlight and StateStore in context. + * Requires ProtocolStream and StateStore in context. * * @example * ```typescript * // Development/Testing with InMemoryStateStore * const DevLayer = TransactionalStream.layer.pipe( * Layer.provide(InMemoryStateStore.layer), + * Layer.provide(ProtocolStream.layer), * Layer.provide(ArrowFlight.layer), * Layer.provide(Transport.layer) * ) @@ -288,10 +291,11 @@ const make = Effect.gen(function*() { * // Production with persistent store (future) * const ProdLayer = TransactionalStream.layer.pipe( * Layer.provide(IndexedDBStateStore.layer), + * Layer.provide(ProtocolStream.layer), * Layer.provide(ArrowFlight.layer), * Layer.provide(Transport.layer) * ) * ``` */ -export const layer: Layer.Layer = +export const layer: Layer.Layer = Layer.effect(TransactionalStream, make) From 078ac3675531c207ec4613f390ee12299c0f8abb Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 5 Feb 2026 15:38:23 -0500 Subject: [PATCH 04/12] Move barrel exports to root-level module files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace subdirectory index.ts barrel files with root-level .ts files that serve as the public API for each module: - protocol-stream/index.ts → protocol-stream.ts - transactional-stream/index.ts → transactional-stream.ts The package.json exports field (./* → ./src/*.ts) already supports this pattern, so imports like "@edgeandnode/amp/protocol-stream" resolve to the new root-level files. Co-Authored-By: Claude Opus 4.6 --- packages/amp/src/index.ts | 6 ++---- .../index.ts => protocol-stream.ts} | 8 ++++---- .../index.ts => transactional-stream.ts} | 15 ++++++++------- packages/amp/test/protocol-stream/reorg.test.ts | 2 +- .../amp/test/protocol-stream/validation.test.ts | 2 +- 5 files changed, 16 insertions(+), 17 deletions(-) rename packages/amp/src/{protocol-stream/index.ts => protocol-stream.ts} (96%) rename packages/amp/src/{transactional-stream/index.ts => transactional-stream.ts} (89%) diff --git a/packages/amp/src/index.ts b/packages/amp/src/index.ts index d019dd8..c7bf126 100644 --- a/packages/amp/src/index.ts +++ b/packages/amp/src/index.ts @@ -26,8 +26,6 @@ export * as AdminApi from "./admin/api.ts" export * as RegistryApi from "./registry/api.ts" /** - * Protocol stream types and validation for reorg detection. - * - * These types are used by `ArrowFlight.streamProtocol()`. + * Protocol stream service with reorg detection. */ -export * as ProtocolStream from "./protocol-stream/index.ts" +export * as ProtocolStream from "./protocol-stream.ts" diff --git a/packages/amp/src/protocol-stream/index.ts b/packages/amp/src/protocol-stream.ts similarity index 96% rename from packages/amp/src/protocol-stream/index.ts rename to packages/amp/src/protocol-stream.ts index 7423b8a..3f557e7 100644 --- a/packages/amp/src/protocol-stream/index.ts +++ b/packages/amp/src/protocol-stream.ts @@ -86,7 +86,7 @@ export { makeInvalidationRange, reorg, watermark -} from "./messages.ts" +} from "./protocol-stream/messages.ts" // ============================================================================= // Errors @@ -108,7 +108,7 @@ export { ProtocolArrowFlightError, ProtocolValidationError, type ProtocolStreamError -} from "./errors.ts" +} from "./protocol-stream/errors.ts" // ============================================================================= // Validation @@ -119,7 +119,7 @@ export { validateConsecutiveness, validateNetworks, validatePrevHash -} from "./validation.ts" +} from "./protocol-stream/validation.ts" // ============================================================================= // Service @@ -130,4 +130,4 @@ export { layer, type ProtocolStreamService, type ProtocolStreamOptions -} from "./service.ts" +} from "./protocol-stream/service.ts" diff --git a/packages/amp/src/transactional-stream/index.ts b/packages/amp/src/transactional-stream.ts similarity index 89% rename from packages/amp/src/transactional-stream/index.ts rename to packages/amp/src/transactional-stream.ts index dc7d40e..e825c6f 100644 --- a/packages/amp/src/transactional-stream/index.ts +++ b/packages/amp/src/transactional-stream.ts @@ -38,6 +38,7 @@ * Effect.runPromise(program.pipe( * Effect.provide(TransactionalStream.layer), * Effect.provide(InMemoryStateStore.layer), + * Effect.provide(ProtocolStream.layer), * Effect.provide(ArrowFlight.layer), * Effect.provide(Transport.layer) * )) @@ -75,7 +76,7 @@ export { watermarkEvent, reorgCause, rewindCause -} from "./types.ts" +} from "./transactional-stream/types.ts" // ============================================================================= // Errors @@ -86,7 +87,7 @@ export { UnrecoverableReorgError, PartialReorgError, type TransactionalStreamError -} from "./errors.ts" +} from "./transactional-stream/errors.ts" // ============================================================================= // StateStore Service @@ -100,19 +101,19 @@ export { emptySnapshot, emptyCommit, makeCommit -} from "./state-store.ts" +} from "./transactional-stream/state-store.ts" // ============================================================================= // InMemoryStateStore Layer // ============================================================================= -export * as InMemoryStateStore from "./memory-store.ts" +export * as InMemoryStateStore from "./transactional-stream/memory-store.ts" // ============================================================================= // CommitHandle // ============================================================================= -export { type CommitHandle, makeCommitHandle } from "./commit-handle.ts" +export { type CommitHandle, makeCommitHandle } from "./transactional-stream/commit-handle.ts" // ============================================================================= // TransactionalStream Service @@ -123,7 +124,7 @@ export { layer, type TransactionalStreamService, type TransactionalStreamOptions -} from "./stream.ts" +} from "./transactional-stream/stream.ts" // ============================================================================= // Algorithms (for advanced use cases and testing) @@ -134,4 +135,4 @@ export { findPruningPoint, checkPartialReorg, compressCommits -} from "./algorithms.ts" +} from "./transactional-stream/algorithms.ts" diff --git a/packages/amp/test/protocol-stream/reorg.test.ts b/packages/amp/test/protocol-stream/reorg.test.ts index fe0b702..2aea909 100644 --- a/packages/amp/test/protocol-stream/reorg.test.ts +++ b/packages/amp/test/protocol-stream/reorg.test.ts @@ -16,7 +16,7 @@ * - Watermark and data message generation */ import type { BlockHash, BlockNumber, BlockRange, Network } from "@edgeandnode/amp/models" -import { data, invalidates, makeInvalidationRange, reorg, watermark } from "@edgeandnode/amp/protocol-stream/index" +import { data, invalidates, makeInvalidationRange, reorg, watermark } from "@edgeandnode/amp/protocol-stream" import { describe, it } from "@effect/vitest" import * as Effect from "effect/Effect" import * as Either from "effect/Either" diff --git a/packages/amp/test/protocol-stream/validation.test.ts b/packages/amp/test/protocol-stream/validation.test.ts index a1506b9..8a395fb 100644 --- a/packages/amp/test/protocol-stream/validation.test.ts +++ b/packages/amp/test/protocol-stream/validation.test.ts @@ -17,7 +17,7 @@ import { validateConsecutiveness, validateNetworks, validatePrevHash -} from "@edgeandnode/amp/protocol-stream/index" +} from "@edgeandnode/amp/protocol-stream" import { describe, it } from "@effect/vitest" import * as Effect from "effect/Effect" import * as Either from "effect/Either" From 1c2f688263641f34cdc64421a3aa7cc1550d8c29 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 5 Feb 2026 16:52:16 -0500 Subject: [PATCH 05/12] Refactor transactional-stream tests to @effect/vitest Replace vitest imports with @effect/vitest and convert all tests to it.effect with per-test Effect.provide for isolation. Remove runWithStore, runWithState, and runWithActor helpers. Use Effect.exit instead of Effect.runPromiseExit for failure tests. Co-Authored-By: Claude Opus 4.6 --- .../transactional-stream/memory-store.test.ts | 364 ++++----- .../transactional-stream/state-actor.test.ts | 707 ++++++++---------- 2 files changed, 458 insertions(+), 613 deletions(-) diff --git a/packages/amp/test/transactional-stream/memory-store.test.ts b/packages/amp/test/transactional-stream/memory-store.test.ts index 27adb22..60c46fa 100644 --- a/packages/amp/test/transactional-stream/memory-store.test.ts +++ b/packages/amp/test/transactional-stream/memory-store.test.ts @@ -3,24 +3,22 @@ * * @module */ +import type { BlockRange } from "@edgeandnode/amp/models" +import { + emptySnapshot, + InMemoryStateStore, + type StateSnapshot, + StateStore, + type TransactionId +} from "@edgeandnode/amp/transactional-stream" import * as Effect from "effect/Effect" -import * as Layer from "effect/Layer" -import * as Ref from "effect/Ref" -import { describe, expect, it } from "vitest" -import type { BlockRange } from "../../src/models.ts" -import * as InMemoryStateStore from "../../src/transactional-stream/memory-store.ts" -import { StateStore, emptySnapshot, type StateSnapshot } from "../../src/transactional-stream/state-store.ts" -import type { TransactionId } from "../../src/transactional-stream/types.ts" +import { describe, expect, it } from "@effect/vitest" // ============================================================================= // Test Helpers // ============================================================================= -const makeBlockRange = ( - network: string, - start: number, - end: number -): BlockRange => +const makeBlockRange = (network: string, start: number, end: number): BlockRange => ({ network, numbers: { start, end }, @@ -28,55 +26,34 @@ const makeBlockRange = ( prevHash: undefined }) as BlockRange -const runWithStore = ( - effect: Effect.Effect -): Promise => - Effect.runPromise( - effect.pipe(Effect.provide(InMemoryStateStore.layer)) - ) - -const runWithState = ( - initial: StateSnapshot, - effect: Effect.Effect -): Promise => - Effect.runPromise( - effect.pipe(Effect.provide(InMemoryStateStore.layerWithState(initial))) - ) - // ============================================================================= // Layer Tests // ============================================================================= describe("InMemoryStateStore.layer", () => { - it("provides empty initial state", async () => { - const result = await runWithStore( - Effect.gen(function*() { - const store = yield* StateStore - return yield* store.load() - }) - ) - - expect(result).toEqual(emptySnapshot) - }) + it.effect("provides empty initial state", () => + Effect.gen(function*() { + const store = yield* StateStore + const result = yield* store.load() + expect(result).toEqual(emptySnapshot) + }).pipe(Effect.provide(InMemoryStateStore.layer))) }) describe("InMemoryStateStore.layerWithState", () => { - it("provides custom initial state", async () => { - const initial: StateSnapshot = { + it.effect("provides custom initial state", () => + Effect.gen(function*() { + const initial: StateSnapshot = { + buffer: [[5 as TransactionId, [makeBlockRange("eth", 0, 10)]]], + next: 10 as TransactionId + } + + const store = yield* StateStore + const result = yield* store.load() + expect(result).toEqual(initial) + }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ buffer: [[5 as TransactionId, [makeBlockRange("eth", 0, 10)]]], next: 10 as TransactionId - } - - const result = await runWithState( - initial, - Effect.gen(function*() { - const store = yield* StateStore - return yield* store.load() - }) - ) - - expect(result).toEqual(initial) - }) + })))) }) // ============================================================================= @@ -84,238 +61,173 @@ describe("InMemoryStateStore.layerWithState", () => { // ============================================================================= describe("StateStore.advance", () => { - it("updates the next transaction ID", async () => { - const result = await runWithStore( - Effect.gen(function*() { - const store = yield* StateStore - - yield* store.advance(5 as TransactionId) + it.effect("updates the next transaction ID", () => + Effect.gen(function*() { + const store = yield* StateStore - const snapshot = yield* store.load() - return snapshot.next - }) - ) + yield* store.advance(5 as TransactionId) - expect(result).toBe(5) - }) + const snapshot = yield* store.load() + expect(snapshot.next).toBe(5) + }).pipe(Effect.provide(InMemoryStateStore.layer))) - it("can be called multiple times", async () => { - const result = await runWithStore( - Effect.gen(function*() { - const store = yield* StateStore + it.effect("can be called multiple times", () => + Effect.gen(function*() { + const store = yield* StateStore - yield* store.advance(1 as TransactionId) - yield* store.advance(2 as TransactionId) - yield* store.advance(10 as TransactionId) - - const snapshot = yield* store.load() - return snapshot.next - }) - ) + yield* store.advance(1 as TransactionId) + yield* store.advance(2 as TransactionId) + yield* store.advance(10 as TransactionId) - expect(result).toBe(10) - }) + const snapshot = yield* store.load() + expect(snapshot.next).toBe(10) + }).pipe(Effect.provide(InMemoryStateStore.layer))) }) describe("StateStore.commit", () => { - it("inserts watermarks to buffer", async () => { - const result = await runWithStore( - Effect.gen(function*() { - const store = yield* StateStore - - yield* store.commit({ - insert: [ - [1 as TransactionId, [makeBlockRange("eth", 0, 10)]], - [2 as TransactionId, [makeBlockRange("eth", 11, 20)]] - ], - prune: undefined - }) - - const snapshot = yield* store.load() - return snapshot.buffer + it.effect("inserts watermarks to buffer", () => + Effect.gen(function*() { + const store = yield* StateStore + + yield* store.commit({ + insert: [ + [1 as TransactionId, [makeBlockRange("eth", 0, 10)]], + [2 as TransactionId, [makeBlockRange("eth", 11, 20)]] + ], + prune: undefined }) - ) - expect(result).toHaveLength(2) - expect(result[0]![0]).toBe(1) - expect(result[1]![0]).toBe(2) - }) + const snapshot = yield* store.load() + const result = snapshot.buffer + expect(result).toHaveLength(2) + expect(result[0]![0]).toBe(1) + expect(result[1]![0]).toBe(2) + }).pipe(Effect.provide(InMemoryStateStore.layer))) - it("prunes watermarks with ID <= prune point", async () => { - const initial: StateSnapshot = { + it.effect("prunes watermarks with ID <= prune point", () => + Effect.gen(function*() { + const store = yield* StateStore + + yield* store.commit({ + insert: [], + prune: 2 as TransactionId + }) + + const snapshot = yield* store.load() + const result = snapshot.buffer + expect(result).toHaveLength(1) + expect(result[0]![0]).toBe(3) + }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ buffer: [ [1 as TransactionId, [makeBlockRange("eth", 0, 10)]], [2 as TransactionId, [makeBlockRange("eth", 11, 20)]], [3 as TransactionId, [makeBlockRange("eth", 21, 30)]] ], next: 4 as TransactionId - } + })))) - const result = await runWithState( - initial, - Effect.gen(function*() { - const store = yield* StateStore + it.effect("can insert and prune atomically", () => + Effect.gen(function*() { + const store = yield* StateStore - yield* store.commit({ - insert: [], - prune: 2 as TransactionId - }) - - const snapshot = yield* store.load() - return snapshot.buffer + yield* store.commit({ + insert: [[3 as TransactionId, [makeBlockRange("eth", 21, 30)]]], + prune: 1 as TransactionId }) - ) - - expect(result).toHaveLength(1) - expect(result[0]![0]).toBe(3) - }) - it("can insert and prune atomically", async () => { - const initial: StateSnapshot = { + const snapshot = yield* store.load() + const result = snapshot.buffer + expect(result).toHaveLength(2) + expect(result.map(([id]) => id)).toEqual([2, 3]) + }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ buffer: [ [1 as TransactionId, [makeBlockRange("eth", 0, 10)]], [2 as TransactionId, [makeBlockRange("eth", 11, 20)]] ], next: 3 as TransactionId - } - - const result = await runWithState( - initial, - Effect.gen(function*() { - const store = yield* StateStore - - yield* store.commit({ - insert: [[3 as TransactionId, [makeBlockRange("eth", 21, 30)]]], - prune: 1 as TransactionId - }) - - const snapshot = yield* store.load() - return snapshot.buffer - }) - ) - - expect(result).toHaveLength(2) - expect(result.map(([id]) => id)).toEqual([2, 3]) - }) + })))) }) describe("StateStore.truncate", () => { - it("removes watermarks with ID >= from", async () => { - const initial: StateSnapshot = { + it.effect("removes watermarks with ID >= from", () => + Effect.gen(function*() { + const store = yield* StateStore + + yield* store.truncate(2 as TransactionId) + + const snapshot = yield* store.load() + const result = snapshot.buffer + expect(result).toHaveLength(1) + expect(result[0]![0]).toBe(1) + }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ buffer: [ [1 as TransactionId, [makeBlockRange("eth", 0, 10)]], [2 as TransactionId, [makeBlockRange("eth", 11, 20)]], [3 as TransactionId, [makeBlockRange("eth", 21, 30)]] ], next: 4 as TransactionId - } - - const result = await runWithState( - initial, - Effect.gen(function*() { - const store = yield* StateStore + })))) - yield* store.truncate(2 as TransactionId) - - const snapshot = yield* store.load() - return snapshot.buffer - }) - ) + it.effect("removes all watermarks when truncating from 0", () => + Effect.gen(function*() { + const store = yield* StateStore - expect(result).toHaveLength(1) - expect(result[0]![0]).toBe(1) - }) + yield* store.truncate(0 as TransactionId) - it("removes all watermarks when truncating from 0", async () => { - const initial: StateSnapshot = { + const snapshot = yield* store.load() + const result = snapshot.buffer + expect(result).toHaveLength(0) + }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ buffer: [ [1 as TransactionId, [makeBlockRange("eth", 0, 10)]], [2 as TransactionId, [makeBlockRange("eth", 11, 20)]] ], next: 3 as TransactionId - } + })))) - const result = await runWithState( - initial, - Effect.gen(function*() { - const store = yield* StateStore + it.effect("preserves next ID", () => + Effect.gen(function*() { + const store = yield* StateStore - yield* store.truncate(0 as TransactionId) + yield* store.truncate(1 as TransactionId) - const snapshot = yield* store.load() - return snapshot.buffer - }) - ) - - expect(result).toHaveLength(0) - }) - - it("preserves next ID", async () => { - const initial: StateSnapshot = { + const snapshot = yield* store.load() + expect(snapshot.next).toBe(10) + }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ buffer: [[1 as TransactionId, [makeBlockRange("eth", 0, 10)]]], next: 10 as TransactionId - } - - const result = await runWithState( - initial, - Effect.gen(function*() { - const store = yield* StateStore - - yield* store.truncate(1 as TransactionId) - - const snapshot = yield* store.load() - return snapshot.next - }) - ) - - expect(result).toBe(10) - }) + })))) }) // ============================================================================= -// makeTestable Tests +// layerTest Tests // ============================================================================= -describe("InMemoryStateStore.makeTestable", () => { - it("exposes internal state ref for inspection", async () => { - const result = await Effect.runPromise( - Effect.gen(function*() { - const { service, stateRef } = yield* InMemoryStateStore.makeTestable +describe("InMemoryStateStore.layerTest", () => { + it.effect("exposes internal state via TestState", () => + Effect.gen(function*() { + const store = yield* StateStore + const testState = yield* InMemoryStateStore.TestState - yield* service.advance(5 as TransactionId) - yield* service.commit({ - insert: [[1 as TransactionId, [makeBlockRange("eth", 0, 10)]]], - prune: undefined - }) - - const state = yield* Ref.get(stateRef) - return state + yield* store.advance(5 as TransactionId) + yield* store.commit({ + insert: [[1 as TransactionId, [makeBlockRange("eth", 0, 10)]]], + prune: undefined }) - ) - - expect(result.next).toBe(5) - expect(result.buffer).toHaveLength(1) - }) -}) - -// ============================================================================= -// testLayer Tests -// ============================================================================= -describe("InMemoryStateStore.testLayer", () => { - it("provides both StateStore and TestStateRef", async () => { - const result = await Effect.runPromise( - Effect.gen(function*() { - const store = yield* StateStore - const stateRef = yield* InMemoryStateStore.TestStateRef + const result = yield* testState.get + expect(result.next).toBe(5) + expect(result.buffer).toHaveLength(1) + }).pipe(Effect.provide(InMemoryStateStore.layerTest))) - yield* store.advance(5 as TransactionId) + it.effect("provides StateStore via layer", () => + Effect.gen(function*() { + const store = yield* StateStore + const testState = yield* InMemoryStateStore.TestState - const state = yield* Ref.get(stateRef) - return state.next - }).pipe(Effect.provide(InMemoryStateStore.testLayer)) - ) + yield* store.advance(5 as TransactionId) - expect(result).toBe(5) - }) + const state = yield* testState.get + expect(state.next).toBe(5) + }).pipe(Effect.provide(InMemoryStateStore.layerTest))) }) diff --git a/packages/amp/test/transactional-stream/state-actor.test.ts b/packages/amp/test/transactional-stream/state-actor.test.ts index 85a20a2..11091cf 100644 --- a/packages/amp/test/transactional-stream/state-actor.test.ts +++ b/packages/amp/test/transactional-stream/state-actor.test.ts @@ -3,15 +3,17 @@ * * @module */ +import type { BlockNumber, BlockRange, Network } from "@edgeandnode/amp/models" +import type { ProtocolMessage } from "@edgeandnode/amp/protocol-stream" +import { + InMemoryStateStore, + type StateSnapshot, + StateStore, + type TransactionId +} from "@edgeandnode/amp/transactional-stream" +import { makeStateActor, type StateActor } from "@edgeandnode/amp/transactional-stream/state-actor" import * as Effect from "effect/Effect" -import * as Ref from "effect/Ref" -import { describe, expect, it } from "vitest" -import type { BlockRange } from "../../src/models.ts" -import type { ProtocolMessage } from "../../src/protocol-stream/messages.ts" -import * as InMemoryStateStore from "../../src/transactional-stream/memory-store.ts" -import { makeStateActor, type Action, type StateActor } from "../../src/transactional-stream/state-actor.ts" -import type { StateSnapshot } from "../../src/transactional-stream/state-store.ts" -import type { TransactionId } from "../../src/transactional-stream/types.ts" +import { describe, expect, it } from "@effect/vitest" // ============================================================================= // Test Helpers @@ -53,69 +55,54 @@ const reorgMessage = ( previous, incoming, invalidation: invalidation.map((i) => ({ - network: i.network, - start: i.start, - end: i.end - })) as ReadonlyArray<{ network: string; start: number; end: number }> + network: i.network as Network, + start: i.start as BlockNumber, + end: i.end as BlockNumber + })) }) -const runWithActor = ( - initial: StateSnapshot, - retention: number, - fn: (actor: StateActor) => Effect.Effect -): Promise => - Effect.runPromise( - Effect.gen(function*() { - const { service, stateRef } = yield* InMemoryStateStore.makeTestable - - // Set initial state - yield* Ref.set(stateRef, initial) - - const actor = yield* makeStateActor(service, retention) - return yield* fn(actor) - }) - ) - // ============================================================================= // Basic Operation Tests // ============================================================================= describe("StateActor.watermark", () => { - it("returns undefined for empty buffer", async () => { - const result = await runWithActor( - { buffer: [], next: 0 as TransactionId }, - 128, - (actor) => actor.watermark() - ) - - expect(result).toBeUndefined() - }) - - it("returns last watermark from buffer", async () => { - const initial: StateSnapshot = { + it.effect("returns undefined for empty buffer", () => + Effect.gen(function*() { + const store = yield* StateStore + const actor = yield* makeStateActor(store, 128) + const result = yield* actor.watermark() + expect(result).toBeUndefined() + }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ + buffer: [], + next: 0 as TransactionId + })))) + + it.effect("returns last watermark from buffer", () => + Effect.gen(function*() { + const store = yield* StateStore + const actor = yield* makeStateActor(store, 128) + const result = yield* actor.watermark() + expect(result?.[0]).toBe(2) + }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ buffer: [ [1 as TransactionId, [makeBlockRange("eth", 0, 10)]], [2 as TransactionId, [makeBlockRange("eth", 11, 20)]] ], next: 3 as TransactionId - } - - const result = await runWithActor(initial, 128, (actor) => actor.watermark()) - - expect(result?.[0]).toBe(2) - }) + })))) }) describe("StateActor.peek", () => { - it("returns next transaction ID", async () => { - const result = await runWithActor( - { buffer: [], next: 10 as TransactionId }, - 128, - (actor) => actor.peek() - ) - - expect(result).toBe(10) - }) + it.effect("returns next transaction ID", () => + Effect.gen(function*() { + const store = yield* StateStore + const actor = yield* makeStateActor(store, 128) + const result = yield* actor.peek() + expect(result).toBe(10) + }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ + buffer: [], + next: 10 as TransactionId + })))) }) // ============================================================================= @@ -123,47 +110,41 @@ describe("StateActor.peek", () => { // ============================================================================= describe("StateActor.execute - Data", () => { - it("passes through data events with transaction ID", async () => { - const result = await runWithActor( - { buffer: [], next: 5 as TransactionId }, - 128, - (actor) => - Effect.gen(function*() { - const [event] = yield* actor.execute({ - _tag: "Message", - message: dataMessage([{ value: 1 }], [makeBlockRange("eth", 0, 10)]) - }) - return event - }) - ) - - expect(result._tag).toBe("Data") - expect(result.id).toBe(5) - if (result._tag === "Data") { - expect(result.data).toEqual([{ value: 1 }]) - } - }) - - it("increments transaction ID for each event", async () => { - const ids = await runWithActor( - { buffer: [], next: 0 as TransactionId }, - 128, - (actor) => - Effect.gen(function*() { - const [event1] = yield* actor.execute({ - _tag: "Message", - message: dataMessage([{ value: 1 }], [makeBlockRange("eth", 0, 10)]) - }) - const [event2] = yield* actor.execute({ - _tag: "Message", - message: dataMessage([{ value: 2 }], [makeBlockRange("eth", 11, 20)]) - }) - return [event1.id, event2.id] - }) - ) - - expect(ids).toEqual([0, 1]) - }) + it.effect("passes through data events with transaction ID", () => + Effect.gen(function*() { + const store = yield* StateStore + const actor = yield* makeStateActor(store, 128) + const [event] = yield* actor.execute({ + _tag: "Message", + message: dataMessage([{ value: 1 }], [makeBlockRange("eth", 0, 10)]) + }) + expect(event._tag).toBe("Data") + expect(event.id).toBe(5) + if (event._tag === "Data") { + expect(event.data).toEqual([{ value: 1 }]) + } + }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ + buffer: [], + next: 5 as TransactionId + })))) + + it.effect("increments transaction ID for each event", () => + Effect.gen(function*() { + const store = yield* StateStore + const actor = yield* makeStateActor(store, 128) + const [event1] = yield* actor.execute({ + _tag: "Message", + message: dataMessage([{ value: 1 }], [makeBlockRange("eth", 0, 10)]) + }) + const [event2] = yield* actor.execute({ + _tag: "Message", + message: dataMessage([{ value: 2 }], [makeBlockRange("eth", 11, 20)]) + }) + expect([event1.id, event2.id]).toEqual([0, 1]) + }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ + buffer: [], + next: 0 as TransactionId + })))) }) // ============================================================================= @@ -171,70 +152,58 @@ describe("StateActor.execute - Data", () => { // ============================================================================= describe("StateActor.execute - Watermark", () => { - it("emits watermark event with transaction ID", async () => { - const result = await runWithActor( - { buffer: [], next: 3 as TransactionId }, - 128, - (actor) => - Effect.gen(function*() { - const [event] = yield* actor.execute({ - _tag: "Message", - message: watermarkMessage([makeBlockRange("eth", 0, 10)]) - }) - return event - }) - ) - - expect(result._tag).toBe("Watermark") - expect(result.id).toBe(3) - }) + it.effect("emits watermark event with transaction ID", () => + Effect.gen(function*() { + const store = yield* StateStore + const actor = yield* makeStateActor(store, 128) + const [event] = yield* actor.execute({ + _tag: "Message", + message: watermarkMessage([makeBlockRange("eth", 0, 10)]) + }) + expect(event._tag).toBe("Watermark") + expect(event.id).toBe(3) + }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ + buffer: [], + next: 3 as TransactionId + })))) - it("adds watermark to internal buffer", async () => { - const watermark = await runWithActor( - { buffer: [], next: 0 as TransactionId }, - 128, - (actor) => - Effect.gen(function*() { - yield* actor.execute({ - _tag: "Message", - message: watermarkMessage([makeBlockRange("eth", 0, 10)]) - }) - return yield* actor.watermark() - }) - ) - - expect(watermark?.[0]).toBe(0) - }) - - it("computes prune point based on retention", async () => { - // Build up a buffer with watermarks far apart - const result = await runWithActor( - { - buffer: [ - [0 as TransactionId, [makeBlockRange("eth", 0, 10)]], - [1 as TransactionId, [makeBlockRange("eth", 11, 20)]] - ], - next: 2 as TransactionId - }, - 50, // Retention of 50 blocks - (actor) => - Effect.gen(function*() { - // Add a watermark at block 200, which should prune old watermarks - const [event] = yield* actor.execute({ - _tag: "Message", - message: watermarkMessage([makeBlockRange("eth", 200, 210)]) - }) - return event - }) - ) - - expect(result._tag).toBe("Watermark") - if (result._tag === "Watermark") { - // Cutoff is 200 - 50 = 150 - // Watermarks 0 and 1 end at 10 and 20, both < 150, so prune up to 1 - expect(result.prune._tag).toBe("Some") - } - }) + it.effect("adds watermark to internal buffer", () => + Effect.gen(function*() { + const store = yield* StateStore + const actor = yield* makeStateActor(store, 128) + yield* actor.execute({ + _tag: "Message", + message: watermarkMessage([makeBlockRange("eth", 0, 10)]) + }) + const watermark = yield* actor.watermark() + expect(watermark?.[0]).toBe(0) + }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ + buffer: [], + next: 0 as TransactionId + })))) + + it.effect("computes prune point based on retention", () => + Effect.gen(function*() { + const store = yield* StateStore + const actor = yield* makeStateActor(store, 50) + // Add a watermark at block 200, which should prune old watermarks + const [event] = yield* actor.execute({ + _tag: "Message", + message: watermarkMessage([makeBlockRange("eth", 200, 210)]) + }) + expect(event._tag).toBe("Watermark") + if (event._tag === "Watermark") { + // Cutoff is 200 - 50 = 150 + // Watermarks 0 and 1 end at 10 and 20, both < 150, so prune up to 1 + expect(event.prune._tag).toBe("Some") + } + }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ + buffer: [ + [0 as TransactionId, [makeBlockRange("eth", 0, 10)]], + [1 as TransactionId, [makeBlockRange("eth", 11, 20)]] + ], + next: 2 as TransactionId + })))) }) // ============================================================================= @@ -242,62 +211,50 @@ describe("StateActor.execute - Watermark", () => { // ============================================================================= describe("StateActor.execute - Rewind", () => { - it("emits Undo event with Rewind cause", async () => { - const result = await runWithActor( - { buffer: [], next: 5 as TransactionId }, - 128, - (actor) => - Effect.gen(function*() { - const [event] = yield* actor.execute({ _tag: "Rewind" }) - return event - }) - ) - - expect(result._tag).toBe("Undo") - if (result._tag === "Undo") { - expect(result.cause._tag).toBe("Rewind") - } - }) - - it("computes invalidation range for empty buffer", async () => { - const result = await runWithActor( - { buffer: [], next: 5 as TransactionId }, - 128, - (actor) => - Effect.gen(function*() { - const [event] = yield* actor.execute({ _tag: "Rewind" }) - return event - }) - ) - - if (result._tag === "Undo") { - // Empty buffer, next=5, so invalidate from 0 to id-1=4 - expect(result.invalidate.start).toBe(0) - expect(result.invalidate.end).toBe(4) - } - }) - - it("computes invalidation range from last watermark", async () => { - const result = await runWithActor( - { - buffer: [[3 as TransactionId, [makeBlockRange("eth", 0, 10)]]], - next: 7 as TransactionId - }, - 128, - (actor) => - Effect.gen(function*() { - const [event] = yield* actor.execute({ _tag: "Rewind" }) - return event - }) - ) - - if (result._tag === "Undo") { - // Last watermark at id=3, next=7, new id=7 - // Invalidate from 4 (3+1) to 6 (7-1) - expect(result.invalidate.start).toBe(4) - expect(result.invalidate.end).toBe(6) - } - }) + it.effect("emits Undo event with Rewind cause", () => + Effect.gen(function*() { + const store = yield* StateStore + const actor = yield* makeStateActor(store, 128) + const [event] = yield* actor.execute({ _tag: "Rewind" }) + expect(event._tag).toBe("Undo") + if (event._tag === "Undo") { + expect(event.cause._tag).toBe("Rewind") + } + }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ + buffer: [], + next: 5 as TransactionId + })))) + + it.effect("computes invalidation range for empty buffer", () => + Effect.gen(function*() { + const store = yield* StateStore + const actor = yield* makeStateActor(store, 128) + const [event] = yield* actor.execute({ _tag: "Rewind" }) + if (event._tag === "Undo") { + // Empty buffer, next=5, so invalidate from 0 to id-1=4 + expect(event.invalidate.start).toBe(0) + expect(event.invalidate.end).toBe(4) + } + }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ + buffer: [], + next: 5 as TransactionId + })))) + + it.effect("computes invalidation range from last watermark", () => + Effect.gen(function*() { + const store = yield* StateStore + const actor = yield* makeStateActor(store, 128) + const [event] = yield* actor.execute({ _tag: "Rewind" }) + if (event._tag === "Undo") { + // Last watermark at id=3, next=7, new id=7 + // Invalidate from 4 (3+1) to 6 (7-1) + expect(event.invalidate.start).toBe(4) + expect(event.invalidate.end).toBe(6) + } + }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ + buffer: [[3 as TransactionId, [makeBlockRange("eth", 0, 10)]]], + next: 7 as TransactionId + })))) }) // ============================================================================= @@ -305,151 +262,130 @@ describe("StateActor.execute - Rewind", () => { // ============================================================================= describe("StateActor.execute - Reorg", () => { - it("emits Undo event with Reorg cause", async () => { - const result = await runWithActor( - { - buffer: [[1 as TransactionId, [makeBlockRange("eth", 0, 10)]]], - next: 2 as TransactionId - }, - 128, - (actor) => - Effect.gen(function*() { - const [event] = yield* actor.execute({ - _tag: "Message", - message: reorgMessage( - [makeBlockRange("eth", 0, 10)], - [makeBlockRange("eth", 5, 15)], - [{ network: "eth", start: 11, end: 15 }] - ) - }) - return event - }) - ) - - expect(result._tag).toBe("Undo") - if (result._tag === "Undo") { - expect(result.cause._tag).toBe("Reorg") - } - }) - - it("finds recovery point and truncates buffer", async () => { - const result = await runWithActor( - { - buffer: [ - [1 as TransactionId, [makeBlockRange("eth", 0, 10)]], - [2 as TransactionId, [makeBlockRange("eth", 11, 20)]], - [3 as TransactionId, [makeBlockRange("eth", 21, 30)]] - ], - next: 4 as TransactionId - }, - 128, - (actor) => + it.effect("emits Undo event with Reorg cause", () => + Effect.gen(function*() { + const store = yield* StateStore + const actor = yield* makeStateActor(store, 128) + const [event] = yield* actor.execute({ + _tag: "Message", + message: reorgMessage( + [makeBlockRange("eth", 0, 10)], + [makeBlockRange("eth", 5, 15)], + [{ network: "eth", start: 11, end: 15 }] + ) + }) + expect(event._tag).toBe("Undo") + if (event._tag === "Undo") { + expect(event.cause._tag).toBe("Reorg") + } + }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ + buffer: [[1 as TransactionId, [makeBlockRange("eth", 0, 10)]]], + next: 2 as TransactionId + })))) + + it.effect("finds recovery point and truncates buffer", () => + Effect.gen(function*() { + const store = yield* StateStore + const actor = yield* makeStateActor(store, 128) + // Reorg at block 21 affects watermark id=3 + yield* actor.execute({ + _tag: "Message", + message: reorgMessage( + [makeBlockRange("eth", 21, 30)], + [makeBlockRange("eth", 21, 35)], + [{ network: "eth", start: 21, end: 35 }] + ) + }) + + // Check that buffer is truncated + const result = yield* actor.watermark() + // Recovery point is id=2 (last unaffected), buffer truncated to keep only id=1,2 + expect(result?.[0]).toBe(2) + }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ + buffer: [ + [1 as TransactionId, [makeBlockRange("eth", 0, 10)]], + [2 as TransactionId, [makeBlockRange("eth", 11, 20)]], + [3 as TransactionId, [makeBlockRange("eth", 21, 30)]] + ], + next: 4 as TransactionId + })))) + + it.effect("handles reorg with empty buffer", () => + Effect.gen(function*() { + const store = yield* StateStore + const actor = yield* makeStateActor(store, 128) + const [event] = yield* actor.execute({ + _tag: "Message", + message: reorgMessage( + [makeBlockRange("eth", 0, 10)], + [makeBlockRange("eth", 5, 15)], + [{ network: "eth", start: 5, end: 15 }] + ) + }) + // Empty buffer case: invalidate from 0 to id-1 + if (event._tag === "Undo") { + expect(event.invalidate.start).toBe(0) + } + }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ + buffer: [], + next: 5 as TransactionId + })))) + + it.effect("fails with UnrecoverableReorgError when all watermarks affected", () => + Effect.gen(function*() { + const result = yield* Effect.exit( Effect.gen(function*() { - // Reorg at block 21 affects watermark id=3 + const store = yield* StateStore + const actor = yield* makeStateActor(store, 128) + + // Reorg at block 100 affects all watermarks yield* actor.execute({ _tag: "Message", message: reorgMessage( - [makeBlockRange("eth", 21, 30)], - [makeBlockRange("eth", 21, 35)], - [{ network: "eth", start: 21, end: 35 }] + [makeBlockRange("eth", 100, 120)], + [makeBlockRange("eth", 100, 130)], + [{ network: "eth", start: 100, end: 130 }] ) }) - - // Check that buffer is truncated - const watermark = yield* actor.watermark() - return watermark }) - ) - - // Recovery point is id=2 (last unaffected), buffer truncated to keep only id=1,2 - expect(result?.[0]).toBe(2) - }) + ) + expect(result._tag).toBe("Failure") + if (result._tag === "Failure") { + const error = result.cause + // Check that it's an UnrecoverableReorgError + expect(error).toBeDefined() + } + }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ + buffer: [ + [1 as TransactionId, [makeBlockRange("eth", 100, 110)]], + [2 as TransactionId, [makeBlockRange("eth", 111, 120)]] + ], + next: 3 as TransactionId + })))) - it("handles reorg with empty buffer", async () => { - const result = await runWithActor( - { buffer: [], next: 5 as TransactionId }, - 128, - (actor) => + it.effect("fails with PartialReorgError when reorg point falls within watermark range", () => + Effect.gen(function*() { + const result = yield* Effect.exit( Effect.gen(function*() { - const [event] = yield* actor.execute({ + const store = yield* StateStore + const actor = yield* makeStateActor(store, 128) + + // Reorg at block 10 falls within watermark range [0, 20] + yield* actor.execute({ _tag: "Message", message: reorgMessage( - [makeBlockRange("eth", 0, 10)], - [makeBlockRange("eth", 5, 15)], - [{ network: "eth", start: 5, end: 15 }] + [makeBlockRange("eth", 0, 20)], + [makeBlockRange("eth", 10, 30)], + [{ network: "eth", start: 10, end: 30 }] ) }) - return event }) - ) - - // Empty buffer case: invalidate from 0 to id-1 - if (result._tag === "Undo") { - expect(result.invalidate.start).toBe(0) - } - }) - - it("fails with UnrecoverableReorgError when all watermarks affected", async () => { - const result = await Effect.runPromiseExit( - Effect.gen(function*() { - const { service, stateRef } = yield* InMemoryStateStore.makeTestable - - yield* Ref.set(stateRef, { - buffer: [ - [1 as TransactionId, [makeBlockRange("eth", 100, 110)]], - [2 as TransactionId, [makeBlockRange("eth", 111, 120)]] - ], - next: 3 as TransactionId - }) - - const actor = yield* makeStateActor(service, 128) - - // Reorg at block 100 affects all watermarks - yield* actor.execute({ - _tag: "Message", - message: reorgMessage( - [makeBlockRange("eth", 100, 120)], - [makeBlockRange("eth", 100, 130)], - [{ network: "eth", start: 100, end: 130 }] - ) - }) - }) - ) - - expect(result._tag).toBe("Failure") - if (result._tag === "Failure") { - const error = result.cause - // Check that it's an UnrecoverableReorgError - expect(error).toBeDefined() - } - }) - - it("fails with PartialReorgError when reorg point falls within watermark range", async () => { - const result = await Effect.runPromiseExit( - Effect.gen(function*() { - const { service, stateRef } = yield* InMemoryStateStore.makeTestable - - yield* Ref.set(stateRef, { - buffer: [[1 as TransactionId, [makeBlockRange("eth", 0, 20)]]], - next: 2 as TransactionId - }) - - const actor = yield* makeStateActor(service, 128) - - // Reorg at block 10 falls within watermark range [0, 20] - yield* actor.execute({ - _tag: "Message", - message: reorgMessage( - [makeBlockRange("eth", 0, 20)], - [makeBlockRange("eth", 10, 30)], - [{ network: "eth", start: 10, end: 30 }] - ) - }) - }) - ) - - expect(result._tag).toBe("Failure") - }) + ) + expect(result._tag).toBe("Failure") + }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ + buffer: [[1 as TransactionId, [makeBlockRange("eth", 0, 20)]]], + next: 2 as TransactionId + })))) }) // ============================================================================= @@ -457,49 +393,46 @@ describe("StateActor.execute - Reorg", () => { // ============================================================================= describe("StateActor.commit", () => { - it("commits watermarks via commit handle", async () => { - const result = await runWithActor( - { buffer: [], next: 0 as TransactionId }, - 128, - (actor) => - Effect.gen(function*() { - const [, handle] = yield* actor.execute({ - _tag: "Message", - message: watermarkMessage([makeBlockRange("eth", 0, 10)]) - }) - - yield* handle.commit() + it.effect("commits watermarks via commit handle", () => + Effect.gen(function*() { + const store = yield* StateStore + const actor = yield* makeStateActor(store, 128) + const [, handle] = yield* actor.execute({ + _tag: "Message", + message: watermarkMessage([makeBlockRange("eth", 0, 10)]) + }) - // The watermark should now be committed - return yield* actor.watermark() - }) - ) + yield* handle.commit() - expect(result?.[0]).toBe(0) - }) + // The watermark should now be committed + const result = yield* actor.watermark() + expect(result?.[0]).toBe(0) + }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ + buffer: [], + next: 0 as TransactionId + })))) - it("batches multiple commits", async () => { - await runWithActor( - { buffer: [], next: 0 as TransactionId }, - 128, - (actor) => - Effect.gen(function*() { - // Execute multiple watermarks without committing - const [, handle1] = yield* actor.execute({ - _tag: "Message", - message: watermarkMessage([makeBlockRange("eth", 0, 10)]) - }) - const [, handle2] = yield* actor.execute({ - _tag: "Message", - message: watermarkMessage([makeBlockRange("eth", 11, 20)]) - }) + it.effect("batches multiple commits", () => + Effect.gen(function*() { + const store = yield* StateStore + const actor = yield* makeStateActor(store, 128) + // Execute multiple watermarks without committing + yield* actor.execute({ + _tag: "Message", + message: watermarkMessage([makeBlockRange("eth", 0, 10)]) + }) + const [, handle2] = yield* actor.execute({ + _tag: "Message", + message: watermarkMessage([makeBlockRange("eth", 11, 20)]) + }) - // Commit only the second one - should commit both - yield* handle2.commit() + // Commit only the second one - should commit both + yield* handle2.commit() - const watermark = yield* actor.watermark() - expect(watermark?.[0]).toBe(1) - }) - ) - }) + const watermark = yield* actor.watermark() + expect(watermark?.[0]).toBe(1) + }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ + buffer: [], + next: 0 as TransactionId + })))) }) From 79657eca4a7e5bafc4428555093e0fb50b3adb46 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 5 Feb 2026 16:53:36 -0500 Subject: [PATCH 06/12] Fix import ordering in refactored test files Co-Authored-By: Claude Opus 4.6 --- .../test/transactional-stream/memory-store.test.ts | 2 +- .../amp/test/transactional-stream/state-actor.test.ts | 11 +++-------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/packages/amp/test/transactional-stream/memory-store.test.ts b/packages/amp/test/transactional-stream/memory-store.test.ts index 60c46fa..2309148 100644 --- a/packages/amp/test/transactional-stream/memory-store.test.ts +++ b/packages/amp/test/transactional-stream/memory-store.test.ts @@ -11,8 +11,8 @@ import { StateStore, type TransactionId } from "@edgeandnode/amp/transactional-stream" -import * as Effect from "effect/Effect" import { describe, expect, it } from "@effect/vitest" +import * as Effect from "effect/Effect" // ============================================================================= // Test Helpers diff --git a/packages/amp/test/transactional-stream/state-actor.test.ts b/packages/amp/test/transactional-stream/state-actor.test.ts index 11091cf..49297f8 100644 --- a/packages/amp/test/transactional-stream/state-actor.test.ts +++ b/packages/amp/test/transactional-stream/state-actor.test.ts @@ -5,15 +5,10 @@ */ import type { BlockNumber, BlockRange, Network } from "@edgeandnode/amp/models" import type { ProtocolMessage } from "@edgeandnode/amp/protocol-stream" -import { - InMemoryStateStore, - type StateSnapshot, - StateStore, - type TransactionId -} from "@edgeandnode/amp/transactional-stream" -import { makeStateActor, type StateActor } from "@edgeandnode/amp/transactional-stream/state-actor" -import * as Effect from "effect/Effect" +import { InMemoryStateStore, StateStore, type TransactionId } from "@edgeandnode/amp/transactional-stream" +import { makeStateActor } from "@edgeandnode/amp/transactional-stream/state-actor" import { describe, expect, it } from "@effect/vitest" +import * as Effect from "effect/Effect" // ============================================================================= // Test Helpers From 1f70e772b4370841537085bb320a2b968e43c6e8 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 5 Feb 2026 16:53:58 -0500 Subject: [PATCH 07/12] add rest of stream code --- packages/amp/src/protocol-stream.ts | 30 +- packages/amp/src/protocol-stream/service.ts | 139 ++++--- .../amp/src/protocol-stream/validation.ts | 48 +-- packages/amp/src/transactional-stream.ts | 63 +-- .../src/transactional-stream/algorithms.ts | 7 +- .../src/transactional-stream/commit-handle.ts | 3 +- .../src/transactional-stream/memory-store.ts | 238 +++++------ .../src/transactional-stream/state-actor.ts | 389 +++++++++--------- .../amp/src/transactional-stream/stream.ts | 50 ++- .../test/protocol-stream/validation.test.ts | 2 +- .../transactional-stream/algorithms.test.ts | 12 +- 11 files changed, 474 insertions(+), 507 deletions(-) diff --git a/packages/amp/src/protocol-stream.ts b/packages/amp/src/protocol-stream.ts index 3f557e7..492e25f 100644 --- a/packages/amp/src/protocol-stream.ts +++ b/packages/amp/src/protocol-stream.ts @@ -38,11 +38,12 @@ * ) * }) * - * Effect.runPromise(program.pipe( - * Effect.provide(ProtocolStream.layer), - * Effect.provide(ArrowFlight.layer), - * Effect.provide(Transport.layer) - * )) + * const AppLayer = ProtocolStream.layer.pipe( + * Layer.provide(ArrowFlight.layer), + * Layer.provide(Transport.layer) + * ) + * + * Effect.runPromise(program.pipe(Effect.provide(AppLayer))) * ``` * * ## Reorg Detection @@ -76,14 +77,14 @@ // ============================================================================= export { + data, + invalidates, InvalidationRange, + makeInvalidationRange, ProtocolMessage, ProtocolMessageData, ProtocolMessageReorg, ProtocolMessageWatermark, - data, - invalidates, - makeInvalidationRange, reorg, watermark } from "./protocol-stream/messages.ts" @@ -101,13 +102,12 @@ export { InvalidReorgError, MissingPrevHashError, NetworkCountChangedError, - UnexpectedNetworkError, - type ValidationError, - // Protocol stream errors ProtocolArrowFlightError, + type ProtocolStreamError, ProtocolValidationError, - type ProtocolStreamError + UnexpectedNetworkError, + type ValidationError } from "./protocol-stream/errors.ts" // ============================================================================= @@ -126,8 +126,8 @@ export { // ============================================================================= export { - ProtocolStream, layer, - type ProtocolStreamService, - type ProtocolStreamOptions + ProtocolStream, + type ProtocolStreamOptions, + type ProtocolStreamService } from "./protocol-stream/service.ts" diff --git a/packages/amp/src/protocol-stream/service.ts b/packages/amp/src/protocol-stream/service.ts index ccab06d..362e031 100644 --- a/packages/amp/src/protocol-stream/service.ts +++ b/packages/amp/src/protocol-stream/service.ts @@ -12,23 +12,14 @@ import * as Context from "effect/Context" import * as Effect from "effect/Effect" import * as Layer from "effect/Layer" import * as Stream from "effect/Stream" -import { - ArrowFlight, - type ArrowFlightError, - type QueryOptions, - type QueryResult -} from "../arrow-flight.ts" +import { ArrowFlight, type ArrowFlightError, type QueryOptions, type QueryResult } from "../arrow-flight.ts" import type { BlockRange } from "../models.ts" +import { ProtocolArrowFlightError, type ProtocolStreamError, ProtocolValidationError } from "./errors.ts" import { - ProtocolArrowFlightError, - ProtocolValidationError, - type ProtocolStreamError -} from "./errors.ts" -import { - type InvalidationRange, - type ProtocolMessage, data as protocolData, + type InvalidationRange, makeInvalidationRange, + type ProtocolMessage, reorg as protocolReorg, watermark as protocolWatermark } from "./messages.ts" @@ -120,11 +111,12 @@ export type { ProtocolStreamError } * ) * }) * - * Effect.runPromise(program.pipe( - * Effect.provide(ProtocolStream.layer), - * Effect.provide(ArrowFlight.layer), - * Effect.provide(Transport.layer) - * )) + * const AppLayer = ProtocolStream.layer.pipe( + * Layer.provide(ArrowFlight.layer), + * Layer.provide(Transport.layer) + * ) + * + * Effect.runPromise(program.pipe(Effect.provide(AppLayer))) * ``` */ export class ProtocolStream extends Context.Tag("Amp/ProtocolStream")< @@ -219,65 +211,73 @@ const make = Effect.gen(function*() { return rawStream.pipe( // Map Arrow Flight errors to protocol errors Stream.mapError((error: ArrowFlightError) => new ProtocolArrowFlightError({ cause: error })), - // Process each batch with state tracking - Stream.mapAccumEffect(initialState, (state, queryResult) => - Effect.gen(function*() { - const batchData = queryResult.data - const metadata = queryResult.metadata - const incoming = metadata.ranges - - // Validate the incoming batch - if (state.initialized) { - yield* validateAll(state.previous, incoming).pipe( - Effect.mapError((error) => new ProtocolValidationError({ cause: error })) - ) - } else { - // Validate prevHash for first batch - for (const range of incoming) { - const isGenesis = range.numbers.start === 0 - if (isGenesis) { - if (range.prevHash !== undefined && range.prevHash !== ZERO_HASH) { - return yield* Effect.fail( - new ProtocolValidationError({ - cause: { _tag: "InvalidPrevHashError", network: range.network } - }) - ) - } - } else { - if (range.prevHash === undefined || range.prevHash === ZERO_HASH) { - return yield* Effect.fail( - new ProtocolValidationError({ - cause: { _tag: "MissingPrevHashError", network: range.network, block: range.numbers.start } - }) - ) + Stream.mapAccumEffect( + initialState, + Effect.fnUntraced( + function*( + state: ProtocolStreamState, + queryResult: QueryResult>> + ): Effect.fn.Return< + readonly [ProtocolStreamState, ProtocolMessage], + ProtocolStreamError + > { + const batchData = queryResult.data + const metadata = queryResult.metadata + const incoming = metadata.ranges + + // Validate the incoming batch + if (state.initialized) { + yield* validateAll(state.previous, incoming).pipe( + Effect.mapError((error) => new ProtocolValidationError({ cause: error })) + ) + } else { + // Validate prevHash for first batch + for (const range of incoming) { + const isGenesis = range.numbers.start === 0 + if (isGenesis) { + if (range.prevHash !== undefined && range.prevHash !== ZERO_HASH) { + return yield* Effect.fail( + new ProtocolValidationError({ + cause: { _tag: "InvalidPrevHashError", network: range.network } + }) + ) + } + } else { + if (range.prevHash === undefined || range.prevHash === ZERO_HASH) { + return yield* Effect.fail( + new ProtocolValidationError({ + cause: { _tag: "MissingPrevHashError", network: range.network, block: range.numbers.start } + }) + ) + } } } } - } - - // Detect reorgs - const invalidations = state.initialized ? detectReorgs(state.previous, incoming) : [] - // Determine message type - let message: ProtocolMessage + // Detect reorgs + const invalidations = state.initialized ? detectReorgs(state.previous, incoming) : [] - if (invalidations.length > 0) { - message = protocolReorg(state.previous, incoming, invalidations) - } else if (metadata.rangesComplete && batchData.length === 0) { - message = protocolWatermark(incoming) - } else { - message = protocolData(batchData as unknown as ReadonlyArray>, incoming) - } + // Determine message type + let message: ProtocolMessage - const newState: ProtocolStreamState = { - previous: incoming, - initialized: true - } + if (invalidations.length > 0) { + message = protocolReorg(state.previous, incoming, invalidations) + } else if (metadata.rangesComplete && batchData.length === 0) { + message = protocolWatermark(incoming) + } else { + message = protocolData(batchData as unknown as ReadonlyArray>, incoming) + } - return [newState, message] as const - })), + const newState: ProtocolStreamState = { + previous: incoming, + initialized: true + } + return [newState, message] as const + } + ) + ), Stream.withSpan("ProtocolStream.stream") ) } @@ -302,5 +302,4 @@ const make = Effect.gen(function*() { * ) * ``` */ -export const layer: Layer.Layer = - Layer.effect(ProtocolStream, make) +export const layer: Layer.Layer = Layer.effect(ProtocolStream, make) diff --git a/packages/amp/src/protocol-stream/validation.ts b/packages/amp/src/protocol-stream/validation.ts index 5375b84..dc5005c 100644 --- a/packages/amp/src/protocol-stream/validation.ts +++ b/packages/amp/src/protocol-stream/validation.ts @@ -46,10 +46,8 @@ const isZeroHash = (hash: string): boolean => hash === ZERO_HASH * @param range - The block range to validate. * @returns An Effect that succeeds if valid, or fails with a validation error. */ -export const validatePrevHash = ( - range: BlockRange -): Effect.Effect => - Effect.gen(function*() { +export const validatePrevHash = Effect.fnUntraced( + function*(range: BlockRange): Effect.fn.Return { const isGenesis = range.numbers.start === 0 if (isGenesis) { @@ -72,7 +70,8 @@ export const validatePrevHash = ( }) } } - }) + } +) /** * Validates network consistency between previous and incoming batches. @@ -85,11 +84,11 @@ export const validatePrevHash = ( * @param incoming - The incoming batch's block ranges. * @returns An Effect that succeeds if valid, or fails with a validation error. */ -export const validateNetworks = ( - previous: ReadonlyArray, - incoming: ReadonlyArray -): Effect.Effect => - Effect.gen(function*() { +export const validateNetworks = Effect.fnUntraced( + function*( + previous: ReadonlyArray, + incoming: ReadonlyArray + ): Effect.fn.Return { // Check for duplicate networks in incoming batch const incomingNetworks = new Set() for (const range of incoming) { @@ -119,7 +118,8 @@ export const validateNetworks = ( return yield* new UnexpectedNetworkError({ network: range.network }) } } - }) + } +) /** * Checks if two block ranges are equal. @@ -144,11 +144,11 @@ const blockRangeEquals = (a: BlockRange, b: BlockRange): boolean => * @param incoming - The incoming batch's block ranges. * @returns An Effect that succeeds if valid, or fails with a validation error. */ -export const validateConsecutiveness = ( - previous: ReadonlyArray, - incoming: ReadonlyArray -): Effect.Effect => - Effect.gen(function*() { +export const validateConsecutiveness = Effect.fnUntraced( + function*( + previous: ReadonlyArray, + incoming: ReadonlyArray + ): Effect.fn.Return { // If this is the first batch, no consecutiveness check needed if (previous.length === 0) { return @@ -197,7 +197,8 @@ export const validateConsecutiveness = ( }) } } - }) + } +) /** * Runs all validation checks on incoming block ranges. @@ -206,11 +207,11 @@ export const validateConsecutiveness = ( * @param incoming - The incoming batch's block ranges. * @returns An Effect that succeeds if all validations pass. */ -export const validateAll = ( - previous: ReadonlyArray, - incoming: ReadonlyArray -): Effect.Effect => - Effect.gen(function*() { +export const validateAll = Effect.fnUntraced( + function*( + previous: ReadonlyArray, + incoming: ReadonlyArray + ): Effect.fn.Return { // Validate prevHash for all incoming ranges for (const range of incoming) { yield* validatePrevHash(range) @@ -221,4 +222,5 @@ export const validateAll = ( // Validate consecutiveness yield* validateConsecutiveness(previous, incoming) - }) + } +) diff --git a/packages/amp/src/transactional-stream.ts b/packages/amp/src/transactional-stream.ts index e825c6f..9e0ba4a 100644 --- a/packages/amp/src/transactional-stream.ts +++ b/packages/amp/src/transactional-stream.ts @@ -19,7 +19,7 @@ * yield* txStream.forEach( * "SELECT * FROM eth.logs", * { retention: 128 }, - * (event) => Effect.gen(function*() { + * Effect.fnUntraced(function*(event) { * switch (event._tag) { * case "Data": * yield* processData(event.data) @@ -35,13 +35,14 @@ * ) * }) * - * Effect.runPromise(program.pipe( - * Effect.provide(TransactionalStream.layer), - * Effect.provide(InMemoryStateStore.layer), - * Effect.provide(ProtocolStream.layer), - * Effect.provide(ArrowFlight.layer), - * Effect.provide(Transport.layer) - * )) + * const AppLayer = TransactionalStream.layer.pipe( + * Layer.provide(InMemoryStateStore.layer), + * Layer.provide(ProtocolStream.layer), + * Layer.provide(ArrowFlight.layer), + * Layer.provide(Transport.layer) + * ) + * + * Effect.runPromise(program.pipe(Effect.provide(AppLayer))) * ``` * * @module @@ -52,10 +53,10 @@ // ============================================================================= export { - type TransactionId, - TransactionId as TransactionIdSchema, - type TransactionIdRange, - TransactionIdRange as TransactionIdRangeSchema, + // Constructors + dataEvent, + reorgCause, + rewindCause, type TransactionEvent, TransactionEvent as TransactionEventSchema, type TransactionEventData, @@ -64,18 +65,18 @@ export { TransactionEventUndo as TransactionEventUndoSchema, type TransactionEventWatermark, TransactionEventWatermark as TransactionEventWatermarkSchema, + type TransactionId, + TransactionId as TransactionIdSchema, + type TransactionIdRange, + TransactionIdRange as TransactionIdRangeSchema, type UndoCause, UndoCause as UndoCauseSchema, type UndoCauseReorg, UndoCauseReorg as UndoCauseReorgSchema, type UndoCauseRewind, UndoCauseRewind as UndoCauseRewindSchema, - // Constructors - dataEvent, undoEvent, - watermarkEvent, - reorgCause, - rewindCause + watermarkEvent } from "./transactional-stream/types.ts" // ============================================================================= @@ -83,10 +84,10 @@ export { // ============================================================================= export { - StateStoreError, - UnrecoverableReorgError, PartialReorgError, - type TransactionalStreamError + StateStoreError, + type TransactionalStreamError, + UnrecoverableReorgError } from "./transactional-stream/errors.ts" // ============================================================================= @@ -94,13 +95,13 @@ export { // ============================================================================= export { - StateStore, - type StateStoreService, - type StateSnapshot, type Commit, - emptySnapshot, emptyCommit, - makeCommit + emptySnapshot, + makeCommit, + type StateSnapshot, + StateStore, + type StateStoreService } from "./transactional-stream/state-store.ts" // ============================================================================= @@ -120,10 +121,10 @@ export { type CommitHandle, makeCommitHandle } from "./transactional-stream/comm // ============================================================================= export { - TransactionalStream, layer, - type TransactionalStreamService, - type TransactionalStreamOptions + TransactionalStream, + type TransactionalStreamOptions, + type TransactionalStreamService } from "./transactional-stream/stream.ts" // ============================================================================= @@ -131,8 +132,8 @@ export { // ============================================================================= export { - findRecoveryPoint, - findPruningPoint, checkPartialReorg, - compressCommits + compressCommits, + findPruningPoint, + findRecoveryPoint } from "./transactional-stream/algorithms.ts" diff --git a/packages/amp/src/transactional-stream/algorithms.ts b/packages/amp/src/transactional-stream/algorithms.ts index 226979d..091a0ed 100644 --- a/packages/amp/src/transactional-stream/algorithms.ts +++ b/packages/amp/src/transactional-stream/algorithms.ts @@ -250,10 +250,9 @@ export const compressCommits = ( } // Remove any watermarks that were pruned - const filteredInsert = - maxPrune !== undefined - ? insert.filter(([id]) => id > maxPrune!) - : insert + const filteredInsert = maxPrune !== undefined + ? insert.filter(([id]) => id > maxPrune!) + : insert return { insert: filteredInsert, diff --git a/packages/amp/src/transactional-stream/commit-handle.ts b/packages/amp/src/transactional-stream/commit-handle.ts index d2b5506..08c8be7 100644 --- a/packages/amp/src/transactional-stream/commit-handle.ts +++ b/packages/amp/src/transactional-stream/commit-handle.ts @@ -24,8 +24,7 @@ import type { TransactionId } from "./types.ts" * @example * ```typescript * yield* txStream.pipe( - * Stream.runForEach(([event, commitHandle]) => - * Effect.gen(function*() { + * Stream.runForEach(Effect.fnUntraced(function*([event, commitHandle]) { * // Process the event * yield* processEvent(event) * diff --git a/packages/amp/src/transactional-stream/memory-store.ts b/packages/amp/src/transactional-stream/memory-store.ts index 8c1b1e3..ea6b2f4 100644 --- a/packages/amp/src/transactional-stream/memory-store.ts +++ b/packages/amp/src/transactional-stream/memory-store.ts @@ -6,16 +6,11 @@ * * @module */ +import * as Context from "effect/Context" import * as Effect from "effect/Effect" import * as Layer from "effect/Layer" import * as Ref from "effect/Ref" -import { - StateStore, - type StateSnapshot, - type StateStoreService, - type Commit, - emptySnapshot -} from "./state-store.ts" +import { type Commit, emptySnapshot, type StateSnapshot, StateStore, type StateStoreService } from "./state-store.ts" import type { TransactionId } from "./types.ts" // ============================================================================= @@ -25,48 +20,47 @@ import type { TransactionId } from "./types.ts" /** * Create InMemoryStateStore service implementation. */ -const makeWithInitialState = (initial: StateSnapshot) => - Effect.gen(function*() { - const stateRef = yield* Ref.make(initial) - - const advance = (next: TransactionId) => - Ref.update(stateRef, (state) => ({ - ...state, - next - })) - - const commit = (commitData: Commit) => - Ref.update(stateRef, (state) => { - let buffer = [...state.buffer] - - // Remove pruned watermarks (all IDs <= prune) - if (commitData.prune !== undefined) { - buffer = buffer.filter(([id]) => id > commitData.prune!) - } - - // Add new watermarks - for (const entry of commitData.insert) { - buffer.push(entry) - } - - return { ...state, buffer } - }) - - const truncate = (from: TransactionId) => - Ref.update(stateRef, (state) => ({ - ...state, - buffer: state.buffer.filter(([id]) => id < from) - })) - - const load = () => Ref.get(stateRef) - - return { - advance, - commit, - truncate, - load - } satisfies StateStoreService - }) +const makeWithInitialState = Effect.fnUntraced(function*(initial: StateSnapshot): Effect.fn.Return { + const stateRef = yield* Ref.make(initial) + + const advance = (next: TransactionId) => + Ref.update(stateRef, (state) => ({ + ...state, + next + })) + + const commit = (commitData: Commit) => + Ref.update(stateRef, (state) => { + let buffer = [...state.buffer] + + // Remove pruned watermarks (all IDs <= prune) + if (commitData.prune !== undefined) { + buffer = buffer.filter(([id]) => id > commitData.prune!) + } + + // Add new watermarks + for (const entry of commitData.insert) { + buffer.push(entry) + } + + return { ...state, buffer } + }) + + const truncate = (from: TransactionId) => + Ref.update(stateRef, (state) => ({ + ...state, + buffer: state.buffer.filter(([id]) => id < from) + })) + + const load = () => Ref.get(stateRef) + + return { + advance, + commit, + truncate, + load + } satisfies StateStoreService +}) /** * Create InMemoryStateStore service with empty initial state. @@ -120,99 +114,79 @@ export const layerWithState = (initial: StateSnapshot): Layer.Layer // ============================================================================= /** - * Create an InMemoryStateStore service directly (for testing). + * Service tag exposing the internal state of a test store for inspection. * - * Returns both the service and a reference to inspect internal state. + * Provided alongside `StateStore` by {@link layerTest}. * * @example * ```typescript - * const { service, stateRef } = yield* InMemoryStateStore.makeTestable() + * const testState = yield* InMemoryStateStore.TestState + * const snapshot = yield* testState.get + * expect(snapshot.next).toBe(5) + * ``` + */ +export class TestState extends Context.Tag("Amp/TransactionalStream/TestState")< + TestState, + { readonly get: Effect.Effect } +>() {} + +/** + * Test layer providing both `StateStore` and `TestState`. * - * yield* service.advance(5 as TransactionId) + * Creates a fresh in-memory store and exposes its internal state + * via the `TestState` tag, allowing tests to inspect snapshots + * without needing a raw Ref. * - * const state = yield* Ref.get(stateRef) - * expect(state.next).toBe(5) + * @example + * ```typescript + * const program = Effect.gen(function*() { + * const store = yield* StateStore + * const testState = yield* InMemoryStateStore.TestState + * + * yield* store.advance(5 as TransactionId) + * + * const snapshot = yield* testState.get + * expect(snapshot.next).toBe(5) + * }) + * + * Effect.runPromise(program.pipe(Effect.provide(InMemoryStateStore.layerTest))) * ``` */ -export const makeTestable = Effect.gen(function*() { - const stateRef = yield* Ref.make(emptySnapshot) - - const advance = (next: TransactionId) => - Ref.update(stateRef, (state) => ({ - ...state, - next - })) - - const commit = (commitData: Commit) => - Ref.update(stateRef, (state) => { - let buffer = [...state.buffer] - - if (commitData.prune !== undefined) { - buffer = buffer.filter(([id]) => id > commitData.prune!) - } - - for (const entry of commitData.insert) { - buffer.push(entry) - } +export const layerTest: Layer.Layer = Layer.effectContext( + Effect.gen(function*() { + const ref = yield* Ref.make(emptySnapshot) - return { ...state, buffer } + const state = TestState.of({ + get: Ref.get(ref) }) - const truncate = (from: TransactionId) => - Ref.update(stateRef, (state) => ({ - ...state, - buffer: state.buffer.filter(([id]) => id < from) - })) - - const load = () => Ref.get(stateRef) - - const service: StateStoreService = { - advance, - commit, - truncate, - load - } - - return { service, stateRef } -}) - -/** - * Create a layer that also exposes the internal state ref for testing. - */ -export class TestStateRef extends Effect.Service()("TestStateRef", { - effect: Effect.gen(function*() { - return yield* Ref.make(emptySnapshot) - }) -}) {} + const store = StateStore.of({ + advance: (next) => Ref.update(ref, (state) => ({ ...state, next })), + + commit: (commitData) => + Ref.update(ref, (state) => { + let buffer = [...state.buffer] + if (commitData.prune !== undefined) { + buffer = buffer.filter(([id]) => id > commitData.prune!) + } + for (const entry of commitData.insert) { + buffer.push(entry) + } + return { ...state, buffer } + }), + + truncate: (from) => + Ref.update(ref, (state) => ({ + ...state, + buffer: state.buffer.filter(([id]) => id < from) + })), + + load: () => Ref.get(ref) + }) -export const testLayer: Layer.Layer = Layer.effect( - StateStore, - Effect.gen(function*() { - const stateRef = yield* TestStateRef - - const advance = (next: TransactionId) => - Ref.update(stateRef, (state) => ({ ...state, next })) - - const commit = (commitData: Commit) => - Ref.update(stateRef, (state) => { - let buffer = [...state.buffer] - if (commitData.prune !== undefined) { - buffer = buffer.filter(([id]) => id > commitData.prune!) - } - for (const entry of commitData.insert) { - buffer.push(entry) - } - return { ...state, buffer } - }) - - const truncate = (from: TransactionId) => - Ref.update(stateRef, (state) => ({ - ...state, - buffer: state.buffer.filter(([id]) => id < from) - })) - - const load = () => Ref.get(stateRef) - - return { advance, commit, truncate, load } satisfies StateStoreService + return Context.mergeAll( + Context.make(StateStore, store), + Context.make(TestState, state) + ) }) -).pipe(Layer.provideMerge(TestStateRef.Default)) +) diff --git a/packages/amp/src/transactional-stream/state-actor.ts b/packages/amp/src/transactional-stream/state-actor.ts index c914915..d5b3d63 100644 --- a/packages/amp/src/transactional-stream/state-actor.ts +++ b/packages/amp/src/transactional-stream/state-actor.ts @@ -11,18 +11,18 @@ import * as Effect from "effect/Effect" import * as Ref from "effect/Ref" import type { BlockRange } from "../models.ts" import type { ProtocolMessage } from "../protocol-stream/messages.ts" -import { findRecoveryPoint, findPruningPoint, checkPartialReorg, compressCommits } from "./algorithms.ts" -import { makeCommitHandle, type CommitHandle } from "./commit-handle.ts" +import { checkPartialReorg, compressCommits, findPruningPoint, findRecoveryPoint } from "./algorithms.ts" +import { type CommitHandle, makeCommitHandle } from "./commit-handle.ts" import { PartialReorgError, type StateStoreError, UnrecoverableReorgError } from "./errors.ts" import type { StateStoreService } from "./state-store.ts" import { - type TransactionId, - type TransactionEvent, dataEvent, - undoEvent, - watermarkEvent, reorgCause, - rewindCause + rewindCause, + type TransactionEvent, + type TransactionId, + undoEvent, + watermarkEvent } from "./types.ts" // ============================================================================= @@ -69,7 +69,10 @@ export interface StateActor { /** Execute an action and return event with commit handle */ readonly execute: ( action: Action - ) => Effect.Effect + ) => Effect.Effect< + readonly [TransactionEvent, CommitHandle], + StateStoreError | UnrecoverableReorgError | PartialReorgError + > /** Commit pending changes up to and including this ID */ readonly commit: (id: TransactionId) => Effect.Effect @@ -86,11 +89,8 @@ export interface StateActor { * @param retention - Retention window in blocks for pruning * @returns Effect that creates a StateActor */ -export const makeStateActor = ( - store: StateStoreService, - retention: number -): Effect.Effect => - Effect.gen(function*() { +export const makeStateActor = Effect.fnUntraced( + function*(store: StateStoreService, retention: number): Effect.fn.Return { // Load initial state from store const snapshot = yield* store.load() @@ -124,88 +124,87 @@ export const makeStateActor = ( // execute() // ========================================================================= - const execute = (action: Action) => - Effect.gen(function*() { - // 1. Pre-allocate monotonic ID - const id = yield* Ref.getAndUpdate(containerRef, (state) => ({ - ...state, - next: (state.next + 1) as TransactionId - })).pipe(Effect.map((state) => state.next)) - - const nextId = (id + 1) as TransactionId - - // Persist the new next ID immediately (ensures monotonicity survives crashes) - yield* store.advance(nextId) - - // 2. Execute action based on type - const event: TransactionEvent = yield* ((): Effect.Effect< - TransactionEvent, - StateStoreError | UnrecoverableReorgError | PartialReorgError - > => { - switch (action._tag) { - case "Rewind": - return executeRewind(id, containerRef) - - case "Message": - return executeMessage(id, action.message, containerRef, store, retention) - } - })() + const execute = Effect.fnUntraced(function*(action: Action): Effect.fn.Return< + readonly [TransactionEvent, CommitHandle], + StateStoreError | UnrecoverableReorgError | PartialReorgError + > { + // 1. Pre-allocate monotonic ID + const id = yield* Ref.getAndUpdate(containerRef, (state) => ({ + ...state, + next: (state.next + 1) as TransactionId + })).pipe(Effect.map((state) => state.next)) + + const nextId = (id + 1) as TransactionId + + // Persist the new next ID immediately (ensures monotonicity survives crashes) + yield* store.advance(nextId) + + // 2. Execute action based on type + const event: TransactionEvent = yield* (() => { + switch (action._tag) { + case "Rewind": + return executeRewind(id, containerRef) + + case "Message": + return executeMessage(id, action.message, containerRef, store, retention) + } + })() - // 3. Return event with commit handle - const handle = makeCommitHandle(id, commit) + // 3. Return event with commit handle + const handle = makeCommitHandle(id, commit) - return [event, handle] as const - }) + return [event, handle] as const + }) // ========================================================================= // commit() // ========================================================================= - const commit = (id: TransactionId): Effect.Effect => - Effect.gen(function*() { - const state = yield* Ref.get(containerRef) + const commit = Effect.fnUntraced(function*(id: TransactionId): Effect.fn.Return { + const state = yield* Ref.get(containerRef) - // Find position where IDs become > id (all before this are <= id) - const pos = state.uncommitted.findIndex(([currentId]) => currentId > id) - const endIndex = pos === -1 ? state.uncommitted.length : pos + // Find position where IDs become > id (all before this are <= id) + const pos = state.uncommitted.findIndex(([currentId]) => currentId > id) + const endIndex = pos === -1 ? state.uncommitted.length : pos - if (endIndex === 0) { - // Nothing to commit - return - } + if (endIndex === 0) { + // Nothing to commit + return + } - // Collect commits [0..endIndex) - const pending = state.uncommitted.slice(0, endIndex) - - // Compress and persist - const compressed = compressCommits(pending) - - if (compressed.insert.length > 0 || compressed.prune !== undefined) { - // Apply pruning to in-memory buffer - yield* Ref.update(containerRef, (s) => { - let buffer = s.buffer - if (compressed.prune !== undefined) { - buffer = buffer.filter(([bufferId]) => bufferId > compressed.prune!) - } - return { ...s, buffer } - }) - - // Persist to store - yield* store.commit({ - insert: compressed.insert, - prune: compressed.prune - }) - } + // Collect commits [0..endIndex) + const pending = state.uncommitted.slice(0, endIndex) + + // Compress and persist + const compressed = compressCommits(pending) + + if (compressed.insert.length > 0 || compressed.prune !== undefined) { + // Apply pruning to in-memory buffer + yield* Ref.update(containerRef, (s) => { + let buffer = s.buffer + if (compressed.prune !== undefined) { + buffer = buffer.filter(([bufferId]) => bufferId > compressed.prune!) + } + return { ...s, buffer } + }) + + // Persist to store + yield* store.commit({ + insert: compressed.insert, + prune: compressed.prune + }) + } - // Remove committed from uncommitted queue - yield* Ref.update(containerRef, (s) => ({ - ...s, - uncommitted: s.uncommitted.slice(endIndex) - })) - }) + // Remove committed from uncommitted queue + yield* Ref.update(containerRef, (s) => ({ + ...s, + uncommitted: s.uncommitted.slice(endIndex) + })) + }) return { watermark, peek, execute, commit } - }) + } +) // ============================================================================= // Action Handlers @@ -214,153 +213,149 @@ export const makeStateActor = ( /** * Execute a Rewind action. */ -const executeRewind = ( +const executeRewind = Effect.fnUntraced(function*( id: TransactionId, containerRef: Ref.Ref -): Effect.Effect => - Effect.gen(function*() { - const state = yield* Ref.get(containerRef) - - // Compute invalidation range based on buffer state - let invalidateStart: TransactionId - let invalidateEnd: TransactionId - - if (state.buffer.length === 0) { - // Empty buffer (early crash before any watermark): invalidate from the beginning - invalidateStart = 0 as TransactionId - invalidateEnd = Math.max(0, id - 1) as TransactionId - } else { - // Normal rewind: invalidate after last watermark - const lastWatermarkId = state.buffer[state.buffer.length - 1]![0] - invalidateStart = (lastWatermarkId + 1) as TransactionId - invalidateEnd = Math.max(invalidateStart, id - 1) as TransactionId - } - - return undoEvent(id, rewindCause(), { - start: invalidateStart, - end: invalidateEnd - }) +): Effect.fn.Return { + const state = yield* Ref.get(containerRef) + + // Compute invalidation range based on buffer state + let invalidateStart: TransactionId + let invalidateEnd: TransactionId + + if (state.buffer.length === 0) { + // Empty buffer (early crash before any watermark): invalidate from the beginning + invalidateStart = 0 as TransactionId + invalidateEnd = Math.max(0, id - 1) as TransactionId + } else { + // Normal rewind: invalidate after last watermark + const lastWatermarkId = state.buffer[state.buffer.length - 1]![0] + invalidateStart = (lastWatermarkId + 1) as TransactionId + invalidateEnd = Math.max(invalidateStart, id - 1) as TransactionId + } + + return undoEvent(id, rewindCause(), { + start: invalidateStart, + end: invalidateEnd }) +}) /** * Execute a Message action. */ -const executeMessage = ( +const executeMessage = Effect.fnUntraced(function*( id: TransactionId, message: ProtocolMessage, containerRef: Ref.Ref, store: StateStoreService, retention: number -): Effect.Effect => - Effect.gen(function*() { - switch (message._tag) { - case "Data": - // Data events just pass through - no buffer mutation - return dataEvent(id, message.data, message.ranges) - - case "Watermark": - return yield* executeWatermark(id, message.ranges, containerRef, retention) - - case "Reorg": - return yield* executeReorg(id, message, containerRef, store) - } - }) +): Effect.fn.Return { + switch (message._tag) { + case "Data": + // Data events just pass through - no buffer mutation + return dataEvent(id, message.data, message.ranges) + + case "Watermark": + return yield* executeWatermark(id, message.ranges, containerRef, retention) + + case "Reorg": + return yield* executeReorg(id, message, containerRef, store) + } +}) /** * Execute a Watermark message. */ -const executeWatermark = ( +const executeWatermark = Effect.fnUntraced(function*( id: TransactionId, ranges: ReadonlyArray, containerRef: Ref.Ref, retention: number -): Effect.Effect => - Effect.gen(function*() { - // Add watermark to buffer - yield* Ref.update(containerRef, (state) => ({ - ...state, - buffer: [...state.buffer, [id, ranges] as const] - })) - - // Compute pruning point based on current buffer state - const state = yield* Ref.get(containerRef) - const prune = findPruningPoint(state.buffer, retention) - - // Record in uncommitted queue - yield* Ref.update(containerRef, (s) => ({ - ...s, - uncommitted: [...s.uncommitted, [id, { ranges, prune }] as const] - })) - - return watermarkEvent(id, ranges, prune ?? null) - }) +): Effect.fn.Return { + // Add watermark to buffer + yield* Ref.update(containerRef, (state) => ({ + ...state, + buffer: [...state.buffer, [id, ranges] as const] + })) + + // Compute pruning point based on current buffer state + const state = yield* Ref.get(containerRef) + const prune = findPruningPoint(state.buffer, retention) + + // Record in uncommitted queue + yield* Ref.update(containerRef, (s) => ({ + ...s, + uncommitted: [...s.uncommitted, [id, { ranges, prune }] as const] + })) + + return watermarkEvent(id, ranges, prune ?? null) +}) /** * Execute a Reorg message. */ -const executeReorg = ( +const executeReorg = Effect.fnUntraced(function*( id: TransactionId, message: Extract, containerRef: Ref.Ref, store: StateStoreService -): Effect.Effect => - Effect.gen(function*() { - const state = yield* Ref.get(containerRef) - const { invalidation } = message - - // 1. Find recovery point - const recovery = findRecoveryPoint(state.buffer, invalidation) - - // 2. Compute invalidation range - let invalidateStart: TransactionId - let invalidateEnd: TransactionId - - if (recovery === undefined) { - if (state.buffer.length === 0) { - // If the buffer is empty, invalidate everything up to before the current event - invalidateStart = 0 as TransactionId - invalidateEnd = Math.max(0, id - 1) as TransactionId - } else { - // No recovery point with a non-empty buffer means all buffered watermarks - // are affected by the reorg. This is not recoverable. - return yield* Effect.fail( - new UnrecoverableReorgError({ - reason: "All buffered watermarks are affected by the reorg" - }) - ) - } - } else { - const [recoveryId, recoveryRanges] = recovery - - // 3. Check for partial reorg - const partialNetwork = checkPartialReorg(recoveryRanges, invalidation) - if (partialNetwork !== undefined) { - return yield* Effect.fail( - new PartialReorgError({ - reason: "Recovery point doesn't align with reorg boundary", - network: partialNetwork - }) - ) - } +): Effect.fn.Return { + const state = yield* Ref.get(containerRef) + const { invalidation } = message + + // 1. Find recovery point + const recovery = findRecoveryPoint(state.buffer, invalidation) + + // 2. Compute invalidation range + let invalidateStart: TransactionId + let invalidateEnd: TransactionId - invalidateStart = (recoveryId + 1) as TransactionId - invalidateEnd = Math.max(invalidateStart, id - 1) as TransactionId + if (recovery === undefined) { + if (state.buffer.length === 0) { + // If the buffer is empty, invalidate everything up to before the current event + invalidateStart = 0 as TransactionId + invalidateEnd = Math.max(0, id - 1) as TransactionId + } else { + // No recovery point with a non-empty buffer means all buffered watermarks + // are affected by the reorg. This is not recoverable. + return yield* Effect.fail( + new UnrecoverableReorgError({ + reason: "All buffered watermarks are affected by the reorg" + }) + ) + } + } else { + const [recoveryId, recoveryRanges] = recovery + + // 3. Check for partial reorg + const partialNetwork = checkPartialReorg(recoveryRanges, invalidation) + if (partialNetwork !== undefined) { + return yield* Effect.fail( + new PartialReorgError({ + reason: "Recovery point doesn't align with reorg boundary", + network: partialNetwork + }) + ) } - // 4. Truncate both in-memory and store - const truncateFrom = recovery !== undefined ? (recovery[0] + 1) as TransactionId : 0 as TransactionId + invalidateStart = (recoveryId + 1) as TransactionId + invalidateEnd = Math.max(invalidateStart, id - 1) as TransactionId + } - yield* Ref.update(containerRef, (s) => ({ - ...s, - buffer: s.buffer.filter(([bufferId]) => bufferId < truncateFrom), - uncommitted: s.uncommitted.filter(([uncommittedId]) => uncommittedId < truncateFrom) - })) + // 4. Truncate both in-memory and store + const truncateFrom = recovery !== undefined ? (recovery[0] + 1) as TransactionId : 0 as TransactionId - yield* store.truncate(truncateFrom) + yield* Ref.update(containerRef, (s) => ({ + ...s, + buffer: s.buffer.filter(([bufferId]) => bufferId < truncateFrom), + uncommitted: s.uncommitted.filter(([uncommittedId]) => uncommittedId < truncateFrom) + })) - // 5. Emit Undo with Cause::Reorg - return undoEvent(id, reorgCause(invalidation), { - start: invalidateStart, - end: invalidateEnd - }) + yield* store.truncate(truncateFrom) + + // 5. Emit Undo with Cause::Reorg + return undoEvent(id, reorgCause(invalidation), { + start: invalidateStart, + end: invalidateEnd }) +}) diff --git a/packages/amp/src/transactional-stream/stream.ts b/packages/amp/src/transactional-stream/stream.ts index 243ee95..d43c6b5 100644 --- a/packages/amp/src/transactional-stream/stream.ts +++ b/packages/amp/src/transactional-stream/stream.ts @@ -18,7 +18,7 @@ import type { BlockRange } from "../models.ts" import type { ProtocolStreamError } from "../protocol-stream/errors.ts" import { ProtocolStream, type ProtocolStreamOptions } from "../protocol-stream/service.ts" import type { CommitHandle } from "./commit-handle.ts" -import type { StateStoreError, TransactionalStreamError, UnrecoverableReorgError, PartialReorgError } from "./errors.ts" +import type { PartialReorgError, StateStoreError, TransactionalStreamError, UnrecoverableReorgError } from "./errors.ts" import { type Action, makeStateActor, type StateActor } from "./state-actor.ts" import { StateStore } from "./state-store.ts" import type { TransactionEvent, TransactionId } from "./types.ts" @@ -72,8 +72,7 @@ export interface TransactionalStreamService { * const txStream = yield* TransactionalStream * * yield* txStream.streamTransactional("SELECT * FROM eth.logs", { retention: 128 }).pipe( - * Stream.runForEach(([event, commitHandle]) => - * Effect.gen(function*() { + * Stream.runForEach(Effect.fnUntraced(function*([event, commitHandle]) { * yield* processEvent(event) * yield* commitHandle.commit() * }) @@ -102,7 +101,7 @@ export interface TransactionalStreamService { * yield* txStream.forEach( * "SELECT * FROM eth.logs", * { retention: 128 }, - * (event) => Effect.gen(function*() { + * Effect.fnUntraced(function*(event) { * switch (event._tag) { * case "Data": * yield* processData(event.data) @@ -139,13 +138,14 @@ export interface TransactionalStreamService { * yield* txStream.forEach("SELECT * FROM eth.logs", {}, processEvent) * }) * - * Effect.runPromise(program.pipe( - * Effect.provide(TransactionalStream.layer), - * Effect.provide(InMemoryStateStore.layer), - * Effect.provide(ProtocolStream.layer), - * Effect.provide(ArrowFlight.layer), - * Effect.provide(Transport.layer) - * )) + * const AppLayer = TransactionalStream.layer.pipe( + * Layer.provide(InMemoryStateStore.layer), + * Layer.provide(ProtocolStream.layer), + * Layer.provide(ArrowFlight.layer), + * Layer.provide(Transport.layer) + * ) + * + * Effect.runPromise(program.pipe(Effect.provide(AppLayer))) * ``` */ export class TransactionalStream extends Context.Tag("Amp/TransactionalStream")< @@ -226,17 +226,15 @@ const make = Effect.gen(function*() { StateStoreError | UnrecoverableReorgError | PartialReorgError > = shouldRewind ? Stream.fromEffect( - actor.execute({ _tag: "Rewind" }) - ) + actor.execute({ _tag: "Rewind" }) + ) : Stream.empty const messageStream: Stream.Stream< readonly [TransactionEvent, CommitHandle], ProtocolStreamError | StateStoreError | UnrecoverableReorgError | PartialReorgError > = protocolStream.pipe( - Stream.mapEffect((message) => - actor.execute({ _tag: "Message", message } as Action) - ) + Stream.mapEffect((message) => actor.execute({ _tag: "Message", message } as Action)) ) return Stream.concat(rewindStream, messageStream) @@ -252,14 +250,12 @@ const make = Effect.gen(function*() { handler: (event: TransactionEvent) => Effect.Effect ): Effect.Effect => streamTransactional(sql, options).pipe( - Stream.runForEach(([event, commitHandle]) => - Effect.gen(function*() { - // Process the event - yield* handler(event) - // Auto-commit after successful processing - yield* commitHandle.commit() - }) - ), + Stream.runForEach(Effect.fnUntraced(function*([event, commitHandle]: readonly [TransactionEvent, CommitHandle]) { + // Process the event + yield* handler(event) + // Auto-commit after successful processing + yield* commitHandle.commit() + })), Effect.withSpan("TransactionalStream.forEach") ) @@ -297,5 +293,7 @@ const make = Effect.gen(function*() { * ) * ``` */ -export const layer: Layer.Layer = - Layer.effect(TransactionalStream, make) +export const layer: Layer.Layer = Layer.effect( + TransactionalStream, + make +) diff --git a/packages/amp/test/protocol-stream/validation.test.ts b/packages/amp/test/protocol-stream/validation.test.ts index 8a395fb..fa1279e 100644 --- a/packages/amp/test/protocol-stream/validation.test.ts +++ b/packages/amp/test/protocol-stream/validation.test.ts @@ -4,6 +4,7 @@ * Tests for the validation functions used by the ProtocolStream to ensure * protocol invariants are maintained. */ +import type { BlockHash, BlockNumber, BlockRange, Network } from "@edgeandnode/amp/models" import { DuplicateNetworkError, GapError, @@ -21,7 +22,6 @@ import { import { describe, it } from "@effect/vitest" import * as Effect from "effect/Effect" import * as Either from "effect/Either" -import type { BlockHash, BlockNumber, BlockRange, Network } from "@edgeandnode/amp/models" // ============================================================================= // Test Helpers diff --git a/packages/amp/test/transactional-stream/algorithms.test.ts b/packages/amp/test/transactional-stream/algorithms.test.ts index b1916b3..638c9ec 100644 --- a/packages/amp/test/transactional-stream/algorithms.test.ts +++ b/packages/amp/test/transactional-stream/algorithms.test.ts @@ -3,16 +3,16 @@ * * @module */ -import { describe, expect, it } from "vitest" -import type { BlockRange } from "../../src/models.ts" -import type { InvalidationRange } from "../../src/protocol-stream/messages.ts" +import type { BlockRange } from "@edgeandnode/amp/models" +import type { InvalidationRange } from "@edgeandnode/amp/protocol-stream" import { checkPartialReorg, compressCommits, findPruningPoint, - findRecoveryPoint -} from "../../src/transactional-stream/algorithms.ts" -import type { TransactionId } from "../../src/transactional-stream/types.ts" + findRecoveryPoint, + type TransactionId +} from "@edgeandnode/amp/transactional-stream" +import { describe, expect, it } from "vitest" // ============================================================================= // Test Helpers From a6066b733455eb2345b76046a7488865e2aa7d1e Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 6 Feb 2026 16:08:01 -0500 Subject: [PATCH 08/12] Remove unnecessary thunks from service interfaces Effect is already lazy, so wrapping service methods in `() =>` thunks is redundant. Convert all zero-arg thunk methods to plain Effect values across StateStore, StateActor, CommitHandle, and AdminApi interfaces. Also remove unused imports/variables in reorg.test.ts. Co-Authored-By: Claude Opus 4.6 --- packages/amp/src/admin/service.ts | 36 +++--- .../src/transactional-stream/commit-handle.ts | 6 +- .../src/transactional-stream/memory-store.ts | 62 ++++------ .../src/transactional-stream/state-actor.ts | 107 +++++++++--------- .../src/transactional-stream/state-store.ts | 4 +- .../amp/src/transactional-stream/stream.ts | 6 +- .../amp/test/protocol-stream/reorg.test.ts | 5 - .../transactional-stream/memory-store.test.ts | 20 ++-- .../transactional-stream/state-actor.test.ts | 18 +-- 9 files changed, 116 insertions(+), 148 deletions(-) diff --git a/packages/amp/src/admin/service.ts b/packages/amp/src/admin/service.ts index a15d4a9..e7b6e58 100644 --- a/packages/amp/src/admin/service.ts +++ b/packages/amp/src/admin/service.ts @@ -66,7 +66,7 @@ export class AdminApi extends Context.Tag("Amp/AdminApi") Effect.Effect< + readonly getDatasets: Effect.Effect< Domain.GetDatasetsResponse, HttpError | Api.GetDatasetsError > @@ -208,7 +208,7 @@ export class AdminApi extends Context.Tag("Amp/AdminApi") Effect.Effect< + readonly getWorkers: Effect.Effect< Domain.GetWorkersResponse, HttpError | Api.GetWorkersError > @@ -218,7 +218,7 @@ export class AdminApi extends Context.Tag("Amp/AdminApi") Effect.Effect + readonly getProviders: Effect.Effect /** * Register a manifest. @@ -300,11 +300,11 @@ const make = Effect.fnUntraced(function*(options: MakeOptions) { Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) ) - const getDatasets: Service["getDatasets"] = Effect.fn("AdminApi.getDatasets")( - function*() { - return yield* client.dataset.getDatasets({}) - }, - Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) + const getDatasets: Service["getDatasets"] = Effect.gen(function*() { + return yield* client.dataset.getDatasets({}) + }).pipe( + Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die), + Effect.withSpan("AdminApi.getDatasets") ) const getDatasetVersion: Service["getDatasetVersion"] = Effect.fn("AdminApi.getDatasetVersion")( @@ -387,20 +387,20 @@ const make = Effect.fnUntraced(function*(options: MakeOptions) { // Worker Operations - const getWorkers: Service["getWorkers"] = Effect.fn("AdminApi.getWorkers")( - function*() { - return yield* client.worker.getWorkers({}) - }, - Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) + const getWorkers: Service["getWorkers"] = Effect.gen(function*() { + return yield* client.worker.getWorkers({}) + }).pipe( + Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die), + Effect.withSpan("AdminApi.getWorkers") ) // Provider Operations - const getProviders: Service["getProviders"] = Effect.fn("AdminApi.getProviders")( - function*() { - return yield* client.provider.getProviders({}) - }, - Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) + const getProviders: Service["getProviders"] = Effect.gen(function*() { + return yield* client.provider.getProviders({}) + }).pipe( + Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die), + Effect.withSpan("AdminApi.getProviders") ) // Manifest Operations diff --git a/packages/amp/src/transactional-stream/commit-handle.ts b/packages/amp/src/transactional-stream/commit-handle.ts index 08c8be7..1f3a1e7 100644 --- a/packages/amp/src/transactional-stream/commit-handle.ts +++ b/packages/amp/src/transactional-stream/commit-handle.ts @@ -29,7 +29,7 @@ import type { TransactionId } from "./types.ts" * yield* processEvent(event) * * // Commit the state change - * yield* commitHandle.commit() + * yield* commitHandle.commit * }) * ) * ) @@ -49,7 +49,7 @@ export interface CommitHandle { * Multiple commits can be batched together for efficiency - if you don't commit * after every event, the next commit will include all previous uncommitted events. */ - readonly commit: () => Effect.Effect + readonly commit: Effect.Effect } // ============================================================================= @@ -66,5 +66,5 @@ export const makeCommitHandle = ( commitFn: (id: TransactionId) => Effect.Effect ): CommitHandle => ({ id, - commit: () => commitFn(id) + commit: commitFn(id) }) diff --git a/packages/amp/src/transactional-stream/memory-store.ts b/packages/amp/src/transactional-stream/memory-store.ts index ea6b2f4..b402d75 100644 --- a/packages/amp/src/transactional-stream/memory-store.ts +++ b/packages/amp/src/transactional-stream/memory-store.ts @@ -10,26 +10,23 @@ import * as Context from "effect/Context" import * as Effect from "effect/Effect" import * as Layer from "effect/Layer" import * as Ref from "effect/Ref" -import { type Commit, emptySnapshot, type StateSnapshot, StateStore, type StateStoreService } from "./state-store.ts" -import type { TransactionId } from "./types.ts" +import { emptySnapshot, type StateSnapshot, StateStore, type StateStoreService } from "./state-store.ts" // ============================================================================= // Implementation // ============================================================================= /** - * Create InMemoryStateStore service implementation. + * Build a StateStoreService backed by a single Ref. */ -const makeWithInitialState = Effect.fnUntraced(function*(initial: StateSnapshot): Effect.fn.Return { - const stateRef = yield* Ref.make(initial) - - const advance = (next: TransactionId) => +const makeServiceFromRef = (stateRef: Ref.Ref): StateStoreService => ({ + advance: (next) => Ref.update(stateRef, (state) => ({ ...state, next - })) + })), - const commit = (commitData: Commit) => + commit: (commitData) => Ref.update(stateRef, (state) => { let buffer = [...state.buffer] @@ -44,22 +41,23 @@ const makeWithInitialState = Effect.fnUntraced(function*(initial: StateSnapshot) } return { ...state, buffer } - }) + }), - const truncate = (from: TransactionId) => + truncate: (from) => Ref.update(stateRef, (state) => ({ ...state, buffer: state.buffer.filter(([id]) => id < from) - })) + })), - const load = () => Ref.get(stateRef) + load: Ref.get(stateRef) +}) - return { - advance, - commit, - truncate, - load - } satisfies StateStoreService +/** + * Create InMemoryStateStore service implementation. + */ +const makeWithInitialState = Effect.fnUntraced(function*(initial: StateSnapshot): Effect.fn.Return { + const stateRef = yield* Ref.make(initial) + return makeServiceFromRef(stateRef) }) /** @@ -78,7 +76,7 @@ const make = makeWithInitialState(emptySnapshot) * ```typescript * const program = Effect.gen(function*() { * const store = yield* StateStore - * const snapshot = yield* store.load() + * const snapshot = yield* store.load * console.log(snapshot.next) // 0 * }) * @@ -160,29 +158,7 @@ export const layerTest: Layer.Layer = Layer.effectContex get: Ref.get(ref) }) - const store = StateStore.of({ - advance: (next) => Ref.update(ref, (state) => ({ ...state, next })), - - commit: (commitData) => - Ref.update(ref, (state) => { - let buffer = [...state.buffer] - if (commitData.prune !== undefined) { - buffer = buffer.filter(([id]) => id > commitData.prune!) - } - for (const entry of commitData.insert) { - buffer.push(entry) - } - return { ...state, buffer } - }), - - truncate: (from) => - Ref.update(ref, (state) => ({ - ...state, - buffer: state.buffer.filter(([id]) => id < from) - })), - - load: () => Ref.get(ref) - }) + const store = StateStore.of(makeServiceFromRef(ref)) return Context.mergeAll( Context.make(StateStore, store), diff --git a/packages/amp/src/transactional-stream/state-actor.ts b/packages/amp/src/transactional-stream/state-actor.ts index d5b3d63..d244253 100644 --- a/packages/amp/src/transactional-stream/state-actor.ts +++ b/packages/amp/src/transactional-stream/state-actor.ts @@ -11,7 +11,13 @@ import * as Effect from "effect/Effect" import * as Ref from "effect/Ref" import type { BlockRange } from "../models.ts" import type { ProtocolMessage } from "../protocol-stream/messages.ts" -import { checkPartialReorg, compressCommits, findPruningPoint, findRecoveryPoint } from "./algorithms.ts" +import { + checkPartialReorg, + compressCommits, + findPruningPoint, + findRecoveryPoint, + type PendingCommit +} from "./algorithms.ts" import { type CommitHandle, makeCommitHandle } from "./commit-handle.ts" import { PartialReorgError, type StateStoreError, UnrecoverableReorgError } from "./errors.ts" import type { StateStoreService } from "./state-store.ts" @@ -21,6 +27,7 @@ import { rewindCause, type TransactionEvent, type TransactionId, + type TransactionIdRange, undoEvent, watermarkEvent } from "./types.ts" @@ -29,14 +36,6 @@ import { // Types // ============================================================================= -/** - * Pending commit waiting for user to call commit handle. - */ -interface PendingCommit { - readonly ranges: ReadonlyArray - readonly prune: TransactionId | undefined -} - /** * Internal state container - in-memory copy of persisted state. */ @@ -61,10 +60,10 @@ export type Action = */ export interface StateActor { /** Get last watermark from buffer */ - readonly watermark: () => Effect.Effect] | undefined> + readonly watermark: Effect.Effect] | undefined> /** Get next transaction ID without incrementing */ - readonly peek: () => Effect.Effect + readonly peek: Effect.Effect /** Execute an action and return event with commit handle */ readonly execute: ( @@ -92,7 +91,7 @@ export interface StateActor { export const makeStateActor = Effect.fnUntraced( function*(store: StateStoreService, retention: number): Effect.fn.Return { // Load initial state from store - const snapshot = yield* store.load() + const snapshot = yield* store.load // Create mutable state container wrapped in Ref const containerRef = yield* Ref.make({ @@ -105,20 +104,19 @@ export const makeStateActor = Effect.fnUntraced( // watermark() // ========================================================================= - const watermark = () => - Ref.get(containerRef).pipe( - Effect.map((state) => - state.buffer.length > 0 - ? state.buffer[state.buffer.length - 1] - : undefined - ) + const watermark = Ref.get(containerRef).pipe( + Effect.map((state) => + state.buffer.length > 0 + ? state.buffer[state.buffer.length - 1] + : undefined ) + ) // ========================================================================= // peek() // ========================================================================= - const peek = () => Ref.get(containerRef).pipe(Effect.map((state) => state.next)) + const peek = Ref.get(containerRef).pipe(Effect.map((state) => state.next)) // ========================================================================= // execute() @@ -206,6 +204,25 @@ export const makeStateActor = Effect.fnUntraced( } ) +// ============================================================================= +// Helpers +// ============================================================================= + +/** + * Compute the invalidation range for an undo event. + * + * @param afterId - The anchor transaction ID (invalidation starts at afterId + 1), or undefined to start from 0 + * @param currentId - The current transaction ID being assigned to the undo event + */ +const computeInvalidationRange = ( + afterId: TransactionId | undefined, + currentId: TransactionId +): TransactionIdRange => { + const start = (afterId !== undefined ? afterId + 1 : 0) as TransactionId + const end = Math.max(start, currentId - 1) as TransactionId + return { start, end } +} + // ============================================================================= // Action Handlers // ============================================================================= @@ -219,25 +236,11 @@ const executeRewind = Effect.fnUntraced(function*( ): Effect.fn.Return { const state = yield* Ref.get(containerRef) - // Compute invalidation range based on buffer state - let invalidateStart: TransactionId - let invalidateEnd: TransactionId - - if (state.buffer.length === 0) { - // Empty buffer (early crash before any watermark): invalidate from the beginning - invalidateStart = 0 as TransactionId - invalidateEnd = Math.max(0, id - 1) as TransactionId - } else { - // Normal rewind: invalidate after last watermark - const lastWatermarkId = state.buffer[state.buffer.length - 1]![0] - invalidateStart = (lastWatermarkId + 1) as TransactionId - invalidateEnd = Math.max(invalidateStart, id - 1) as TransactionId - } + const anchor = state.buffer.length > 0 + ? state.buffer[state.buffer.length - 1]![0] + : undefined - return undoEvent(id, rewindCause(), { - start: invalidateStart, - end: invalidateEnd - }) + return undoEvent(id, rewindCause(), computeInvalidationRange(anchor, id)) }) /** @@ -306,15 +309,13 @@ const executeReorg = Effect.fnUntraced(function*( // 1. Find recovery point const recovery = findRecoveryPoint(state.buffer, invalidation) - // 2. Compute invalidation range - let invalidateStart: TransactionId - let invalidateEnd: TransactionId + // 2. Determine anchor ID for invalidation range + let anchor: TransactionId | undefined if (recovery === undefined) { if (state.buffer.length === 0) { - // If the buffer is empty, invalidate everything up to before the current event - invalidateStart = 0 as TransactionId - invalidateEnd = Math.max(0, id - 1) as TransactionId + // Empty buffer: invalidate everything from 0 + anchor = undefined } else { // No recovery point with a non-empty buffer means all buffered watermarks // are affected by the reorg. This is not recoverable. @@ -338,24 +339,20 @@ const executeReorg = Effect.fnUntraced(function*( ) } - invalidateStart = (recoveryId + 1) as TransactionId - invalidateEnd = Math.max(invalidateStart, id - 1) as TransactionId + anchor = recoveryId } - // 4. Truncate both in-memory and store - const truncateFrom = recovery !== undefined ? (recovery[0] + 1) as TransactionId : 0 as TransactionId + // 4. Compute invalidation range and truncate both in-memory and store + const range = computeInvalidationRange(anchor, id) yield* Ref.update(containerRef, (s) => ({ ...s, - buffer: s.buffer.filter(([bufferId]) => bufferId < truncateFrom), - uncommitted: s.uncommitted.filter(([uncommittedId]) => uncommittedId < truncateFrom) + buffer: s.buffer.filter(([bufferId]) => bufferId < range.start), + uncommitted: s.uncommitted.filter(([uncommittedId]) => uncommittedId < range.start) })) - yield* store.truncate(truncateFrom) + yield* store.truncate(range.start) // 5. Emit Undo with Cause::Reorg - return undoEvent(id, reorgCause(invalidation), { - start: invalidateStart, - end: invalidateEnd - }) + return undoEvent(id, reorgCause(invalidation), range) }) diff --git a/packages/amp/src/transactional-stream/state-store.ts b/packages/amp/src/transactional-stream/state-store.ts index 8a05589..3b46a87 100644 --- a/packages/amp/src/transactional-stream/state-store.ts +++ b/packages/amp/src/transactional-stream/state-store.ts @@ -104,7 +104,7 @@ export interface StateStoreService { * Called once when TransactionalStream is created. * Returns empty state if no prior state exists. */ - readonly load: () => Effect.Effect + readonly load: Effect.Effect } // ============================================================================= @@ -118,7 +118,7 @@ export interface StateStoreService { * ```typescript * const program = Effect.gen(function*() { * const store = yield* StateStore - * const snapshot = yield* store.load() + * const snapshot = yield* store.load * // ... * }) * ``` diff --git a/packages/amp/src/transactional-stream/stream.ts b/packages/amp/src/transactional-stream/stream.ts index d43c6b5..85b2048 100644 --- a/packages/amp/src/transactional-stream/stream.ts +++ b/packages/amp/src/transactional-stream/stream.ts @@ -201,8 +201,8 @@ const make = Effect.gen(function*() { const actor: StateActor = yield* makeStateActor(storeService, retention) // 2. Get current watermark and next ID - const watermark = yield* actor.watermark() - const nextId = yield* actor.peek() + const watermark = yield* actor.watermark + const nextId = yield* actor.peek // 3. Determine resume cursor const resumeWatermark: ReadonlyArray | undefined = watermark !== undefined @@ -254,7 +254,7 @@ const make = Effect.gen(function*() { // Process the event yield* handler(event) // Auto-commit after successful processing - yield* commitHandle.commit() + yield* commitHandle.commit })), Effect.withSpan("TransactionalStream.forEach") ) diff --git a/packages/amp/test/protocol-stream/reorg.test.ts b/packages/amp/test/protocol-stream/reorg.test.ts index 2aea909..dacd6de 100644 --- a/packages/amp/test/protocol-stream/reorg.test.ts +++ b/packages/amp/test/protocol-stream/reorg.test.ts @@ -19,7 +19,6 @@ import type { BlockHash, BlockNumber, BlockRange, Network } from "@edgeandnode/a import { data, invalidates, makeInvalidationRange, reorg, watermark } from "@edgeandnode/amp/protocol-stream" import { describe, it } from "@effect/vitest" import * as Effect from "effect/Effect" -import * as Either from "effect/Either" // ============================================================================= // Test Helpers - Ported from Rust utils/response.rs @@ -29,10 +28,6 @@ import * as Either from "effect/Either" * Standard test hashes for different epochs. * Epoch 0 = HASH_A, Epoch 1 = HASH_B, etc. */ -const ZERO_HASH = "0x0000000000000000000000000000000000000000000000000000000000000000" as BlockHash -const HASH_EPOCH_0 = "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" as BlockHash -const HASH_EPOCH_1 = "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" as BlockHash -const HASH_EPOCH_2 = "0xcccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" as BlockHash /** * Generates a deterministic hash for a block based on network, block number, and epoch. diff --git a/packages/amp/test/transactional-stream/memory-store.test.ts b/packages/amp/test/transactional-stream/memory-store.test.ts index 2309148..a4a6732 100644 --- a/packages/amp/test/transactional-stream/memory-store.test.ts +++ b/packages/amp/test/transactional-stream/memory-store.test.ts @@ -34,7 +34,7 @@ describe("InMemoryStateStore.layer", () => { it.effect("provides empty initial state", () => Effect.gen(function*() { const store = yield* StateStore - const result = yield* store.load() + const result = yield* store.load expect(result).toEqual(emptySnapshot) }).pipe(Effect.provide(InMemoryStateStore.layer))) }) @@ -48,7 +48,7 @@ describe("InMemoryStateStore.layerWithState", () => { } const store = yield* StateStore - const result = yield* store.load() + const result = yield* store.load expect(result).toEqual(initial) }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ buffer: [[5 as TransactionId, [makeBlockRange("eth", 0, 10)]]], @@ -67,7 +67,7 @@ describe("StateStore.advance", () => { yield* store.advance(5 as TransactionId) - const snapshot = yield* store.load() + const snapshot = yield* store.load expect(snapshot.next).toBe(5) }).pipe(Effect.provide(InMemoryStateStore.layer))) @@ -79,7 +79,7 @@ describe("StateStore.advance", () => { yield* store.advance(2 as TransactionId) yield* store.advance(10 as TransactionId) - const snapshot = yield* store.load() + const snapshot = yield* store.load expect(snapshot.next).toBe(10) }).pipe(Effect.provide(InMemoryStateStore.layer))) }) @@ -97,7 +97,7 @@ describe("StateStore.commit", () => { prune: undefined }) - const snapshot = yield* store.load() + const snapshot = yield* store.load const result = snapshot.buffer expect(result).toHaveLength(2) expect(result[0]![0]).toBe(1) @@ -113,7 +113,7 @@ describe("StateStore.commit", () => { prune: 2 as TransactionId }) - const snapshot = yield* store.load() + const snapshot = yield* store.load const result = snapshot.buffer expect(result).toHaveLength(1) expect(result[0]![0]).toBe(3) @@ -135,7 +135,7 @@ describe("StateStore.commit", () => { prune: 1 as TransactionId }) - const snapshot = yield* store.load() + const snapshot = yield* store.load const result = snapshot.buffer expect(result).toHaveLength(2) expect(result.map(([id]) => id)).toEqual([2, 3]) @@ -155,7 +155,7 @@ describe("StateStore.truncate", () => { yield* store.truncate(2 as TransactionId) - const snapshot = yield* store.load() + const snapshot = yield* store.load const result = snapshot.buffer expect(result).toHaveLength(1) expect(result[0]![0]).toBe(1) @@ -174,7 +174,7 @@ describe("StateStore.truncate", () => { yield* store.truncate(0 as TransactionId) - const snapshot = yield* store.load() + const snapshot = yield* store.load const result = snapshot.buffer expect(result).toHaveLength(0) }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ @@ -191,7 +191,7 @@ describe("StateStore.truncate", () => { yield* store.truncate(1 as TransactionId) - const snapshot = yield* store.load() + const snapshot = yield* store.load expect(snapshot.next).toBe(10) }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ buffer: [[1 as TransactionId, [makeBlockRange("eth", 0, 10)]]], diff --git a/packages/amp/test/transactional-stream/state-actor.test.ts b/packages/amp/test/transactional-stream/state-actor.test.ts index 49297f8..cfe2ae7 100644 --- a/packages/amp/test/transactional-stream/state-actor.test.ts +++ b/packages/amp/test/transactional-stream/state-actor.test.ts @@ -65,7 +65,7 @@ describe("StateActor.watermark", () => { Effect.gen(function*() { const store = yield* StateStore const actor = yield* makeStateActor(store, 128) - const result = yield* actor.watermark() + const result = yield* actor.watermark expect(result).toBeUndefined() }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ buffer: [], @@ -76,7 +76,7 @@ describe("StateActor.watermark", () => { Effect.gen(function*() { const store = yield* StateStore const actor = yield* makeStateActor(store, 128) - const result = yield* actor.watermark() + const result = yield* actor.watermark expect(result?.[0]).toBe(2) }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ buffer: [ @@ -92,7 +92,7 @@ describe("StateActor.peek", () => { Effect.gen(function*() { const store = yield* StateStore const actor = yield* makeStateActor(store, 128) - const result = yield* actor.peek() + const result = yield* actor.peek expect(result).toBe(10) }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ buffer: [], @@ -170,7 +170,7 @@ describe("StateActor.execute - Watermark", () => { _tag: "Message", message: watermarkMessage([makeBlockRange("eth", 0, 10)]) }) - const watermark = yield* actor.watermark() + const watermark = yield* actor.watermark expect(watermark?.[0]).toBe(0) }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ buffer: [], @@ -293,7 +293,7 @@ describe("StateActor.execute - Reorg", () => { }) // Check that buffer is truncated - const result = yield* actor.watermark() + const result = yield* actor.watermark // Recovery point is id=2 (last unaffected), buffer truncated to keep only id=1,2 expect(result?.[0]).toBe(2) }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ @@ -397,10 +397,10 @@ describe("StateActor.commit", () => { message: watermarkMessage([makeBlockRange("eth", 0, 10)]) }) - yield* handle.commit() + yield* handle.commit // The watermark should now be committed - const result = yield* actor.watermark() + const result = yield* actor.watermark expect(result?.[0]).toBe(0) }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ buffer: [], @@ -422,9 +422,9 @@ describe("StateActor.commit", () => { }) // Commit only the second one - should commit both - yield* handle2.commit() + yield* handle2.commit - const watermark = yield* actor.watermark() + const watermark = yield* actor.watermark expect(watermark?.[0]).toBe(1) }).pipe(Effect.provide(InMemoryStateStore.layerWithState({ buffer: [], From 7967de71caa30e09e05cd3b7bbc529dee0f20570 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 6 Feb 2026 20:36:40 -0500 Subject: [PATCH 09/12] Refactor models into core/domain module and split arrow-flight into subdirectory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move all domain models from src/models.ts to src/core/domain.ts with a src/core.ts barrel, and split the monolithic src/arrow-flight.ts into errors.ts, types.ts, transport.ts, and service.ts submodules — matching the existing protocol-stream/transactional-stream directory pattern. Co-Authored-By: Claude Opus 4.6 --- packages/amp/src/admin/api.ts | 2 +- packages/amp/src/admin/domain.ts | 2 +- packages/amp/src/admin/service.ts | 2 +- packages/amp/src/arrow-flight.ts | 445 +----------------- packages/amp/src/arrow-flight/errors.ts | 114 +++++ packages/amp/src/arrow-flight/node.ts | 2 +- packages/amp/src/arrow-flight/service.ts | 238 ++++++++++ packages/amp/src/arrow-flight/transport.ts | 68 +++ packages/amp/src/arrow-flight/types.ts | 40 ++ packages/amp/src/auth/service.ts | 2 +- packages/amp/src/config.ts | 295 ------------ packages/amp/src/core.ts | 9 + .../amp/src/{models.ts => core/domain.ts} | 0 packages/amp/src/manifest-builder/service.ts | 2 +- packages/amp/src/protocol-stream/messages.ts | 59 +-- packages/amp/src/protocol-stream/service.ts | 76 +-- .../amp/src/protocol-stream/validation.ts | 4 +- packages/amp/src/registry/api.ts | 2 +- packages/amp/src/registry/domain.ts | 2 +- .../src/transactional-stream/algorithms.ts | 18 +- .../src/transactional-stream/state-actor.ts | 2 +- .../src/transactional-stream/state-store.ts | 2 +- .../amp/src/transactional-stream/stream.ts | 2 +- .../amp/src/transactional-stream/types.ts | 2 +- .../amp/test/protocol-stream/reorg.test.ts | 2 +- .../test/protocol-stream/validation.test.ts | 2 +- .../transactional-stream/algorithms.test.ts | 2 +- .../transactional-stream/memory-store.test.ts | 2 +- .../transactional-stream/state-actor.test.ts | 2 +- packages/cli/src/commands/auth/token.ts | 2 +- 30 files changed, 572 insertions(+), 830 deletions(-) create mode 100644 packages/amp/src/arrow-flight/errors.ts create mode 100644 packages/amp/src/arrow-flight/service.ts create mode 100644 packages/amp/src/arrow-flight/transport.ts create mode 100644 packages/amp/src/arrow-flight/types.ts delete mode 100644 packages/amp/src/config.ts create mode 100644 packages/amp/src/core.ts rename packages/amp/src/{models.ts => core/domain.ts} (100%) diff --git a/packages/amp/src/admin/api.ts b/packages/amp/src/admin/api.ts index fa25b03..44d582e 100644 --- a/packages/amp/src/admin/api.ts +++ b/packages/amp/src/admin/api.ts @@ -15,7 +15,7 @@ import * as HttpApiError from "@effect/platform/HttpApiError" import * as HttpApiGroup from "@effect/platform/HttpApiGroup" import * as HttpApiSchema from "@effect/platform/HttpApiSchema" import * as Schema from "effect/Schema" -import * as Models from "../models.ts" +import * as Models from "../core/domain.ts" import * as Domain from "./domain.ts" import * as Error from "./error.ts" diff --git a/packages/amp/src/admin/domain.ts b/packages/amp/src/admin/domain.ts index c7516bb..f6c5edb 100644 --- a/packages/amp/src/admin/domain.ts +++ b/packages/amp/src/admin/domain.ts @@ -3,7 +3,7 @@ * and responses. */ import * as Schema from "effect/Schema" -import * as Models from "../models.ts" +import * as Models from "../core/domain.ts" // ============================================================================= // Dataset Request/Response Schemas diff --git a/packages/amp/src/admin/service.ts b/packages/amp/src/admin/service.ts index e7b6e58..0fecdcd 100644 --- a/packages/amp/src/admin/service.ts +++ b/packages/amp/src/admin/service.ts @@ -21,7 +21,7 @@ import { constUndefined } from "effect/Function" import * as Layer from "effect/Layer" import * as Option from "effect/Option" import * as Auth from "../auth/service.ts" -import type * as Models from "../models.ts" +import type * as Models from "../core/domain.ts" import * as Api from "./api.ts" import type * as Domain from "./domain.ts" diff --git a/packages/amp/src/arrow-flight.ts b/packages/amp/src/arrow-flight.ts index 54d85fe..7db8e0c 100644 --- a/packages/amp/src/arrow-flight.ts +++ b/packages/amp/src/arrow-flight.ts @@ -1,443 +1,42 @@ -import { create, toBinary } from "@bufbuild/protobuf" -import { anyPack, AnySchema } from "@bufbuild/protobuf/wkt" -import { - type Client, - createClient, - createContextKey, - createContextValues, - type Interceptor, - type Transport as ConnectTransport -} from "@connectrpc/connect" -import * as Arr from "effect/Array" -import * as Cause from "effect/Cause" -import * as Context from "effect/Context" -import * as Effect from "effect/Effect" -import { identity } from "effect/Function" -import * as Layer from "effect/Layer" -import * as Option from "effect/Option" -import * as Predicate from "effect/Predicate" -import * as Redacted from "effect/Redacted" -import * as Schema from "effect/Schema" -import * as Stream from "effect/Stream" -import { Auth } from "./auth/service.ts" -import { decodeRecordBatch, DictionaryRegistry } from "./internal/arrow-flight-ipc/Decoder.ts" -import { recordBatchToJson } from "./internal/arrow-flight-ipc/Json.ts" -import { parseRecordBatch } from "./internal/arrow-flight-ipc/RecordBatch.ts" -import { type ArrowSchema, getMessageType, MessageHeaderType, parseSchema } from "./internal/arrow-flight-ipc/Schema.ts" -import type { AuthInfo, BlockRange, RecordBatchMetadata } from "./models.ts" -import { RecordBatchMetadataFromUint8Array } from "./models.ts" -import { FlightDescriptor_DescriptorType, FlightDescriptorSchema, FlightService } from "./protobuf/Flight_pb.ts" -import { CommandStatementQuerySchema } from "./protobuf/FlightSql_pb.ts" - -// ============================================================================= -// Connect RPC Transport -// ============================================================================= - -/** - * A service which abstracts the underlying transport for a given client. - * - * A transport implements a protocol, such as Connect or gRPC-web, and allows - * for the concrete clients to be independent of the protocol. - */ -export class Transport extends Context.Tag("@edgeandnode/amp/Transport")< - Transport, - ConnectTransport ->() {} - /** - * A service which abstracts the set of interceptors that are passed to a given - * transport. + * An implementation of the Arrow Flight protocol. * - * An interceptor can add logic to clients or servers, similar to the decorators - * or middleware you may have seen in other libraries. Interceptors may - * mutate the request and response, catch errors and retry/recover, emit - * logs, or do nearly everything else. - */ -export class Interceptors extends Context.Reference()( - "Amp/ArrowFlight/ConnectRPC/Interceptors", - { defaultValue: () => Arr.empty() } -) {} - -const AuthInfoContextKey = createContextKey( - undefined, - { description: "Authentication information obtained from the Amp auth server" } -) - -/** - * A layer which will add an interceptor to the configured set of `Interceptors` - * which attempts to read authentication information from the Connect context - * values. + * Provides the `ArrowFlight` service for executing SQL queries against an + * Arrow Flight API, along with transport abstractions, error types, and + * interceptor layers for authentication. * - * If authentication information is found, the interceptor will add an - * `"Authorization"` header to the request containing a bearer token with the - * value of the authentication information access token. + * @module */ -export const layerInterceptorBearerAuth = Layer.effectContext( - Effect.gen(function*() { - const interceptors = yield* Interceptors - - const interceptor: Interceptor = (next) => (request) => { - const authInfo = request.contextValues.get(AuthInfoContextKey) - - if (authInfo !== undefined) { - const accessToken = Redacted.value(authInfo.accessToken) - request.header.append("Authorization", `Bearer ${accessToken}`) - } - return next(request) - } - - return Context.make(Interceptors, Arr.append(interceptors, interceptor)) - }) -) // ============================================================================= -// Errors +// Transport // ============================================================================= -// TODO: improve the error model -/** - * Represents the possible errors that can occur when executing an Arrow Flight - * query. - */ -export type ArrowFlightError = - | RpcError - | NoEndpointsError - | MultipleEndpointsError - | TicketNotFoundError - | ParseRecordBatchError - | ParseDictionaryBatchError - | ParseSchemaError - -/** - * Represents an Arrow Flight RPC request that failed. - */ -export class RpcError extends Schema.TaggedError( - "Amp/RpcError" -)("RpcError", { - method: Schema.String, - /** - * The underlying reason for the failed RPC request. - */ - cause: Schema.Defect -}) {} - -/** - * Represents an error that occurred as a result of a `FlightInfo` request - * returning an empty list of endpoints from which data can be acquired. - */ -export class NoEndpointsError extends Schema.TaggedError( - "Amp/NoEndpointsError" -)("NoEndpointsError", { - /** - * The SQL query that was requested. - */ - query: Schema.String -}) {} - -// TODO: determine if this is _really_ a logical error case -/** - * Represents an error that occured as a result of a `FlightInfo` request - * returning multiple endpoints from which data can be acquired. - * - * For Amp queries, there should only ever be **one** authoritative source - * of data. - */ -export class MultipleEndpointsError extends Schema.TaggedError( - "Amp/MultipleEndpointsError" -)("MultipleEndpointsError", { - /** - * The SQL query that was requested. - */ - query: Schema.String -}) {} - -/** - * Represents an error that occurred as a result of a `FlightInfo` request - * whose endpoint did not have a ticket. - */ -export class TicketNotFoundError extends Schema.TaggedError( - "Amp/TicketNotFoundError" -)("TicketNotFoundError", { - /** - * The SQL query that was requested. - */ - query: Schema.String -}) {} - -/** - * Represents an error that occurred as a result of failing to parse an Apache - * Arrow RecordBatch. - */ -export class ParseRecordBatchError extends Schema.TaggedError( - "Amp/ParseRecordBatchError" -)("ParseRecordBatchError", { - /** - * The underlying reason for the failure to parse a record batch. - */ - cause: Schema.Defect -}) {} - -/** - * Represents an error that occurred as a result of failing to parse an Apache - * Arrow DictionaryBatch. - */ -export class ParseDictionaryBatchError extends Schema.TaggedError( - "Amp/ParseDictionaryBatchError" -)("ParseDictionaryBatchError", { - /** - * The underlying reason for the failure to parse a dictionary batch. - */ - cause: Schema.Defect -}) {} - -/** - * Represents an error that occurred as a result of failing to parse an Apache - * Arrow Schema. - */ -export class ParseSchemaError extends Schema.TaggedError( - "Amp/ParseSchemaError" -)("ParseSchemaError", { - /** - * The underlying reason for the failure to parse a schema. - */ - cause: Schema.Defect -}) {} +export { Interceptors, layerInterceptorBearerAuth, Transport } from "./arrow-flight/transport.ts" // ============================================================================= -// Types +// Errors // ============================================================================= -/** - * Represents the result received from the `ArrowFlight` service when a query - * is successfully executed. - */ -export interface QueryResult { - readonly data: ReadonlyArray - readonly metadata: RecordBatchMetadata -} - -/** - * Represents options that can be passed to `ArrowFlight.query` to control how - * the query is executed. - */ -export interface QueryOptions { - readonly schema?: Schema.Any | undefined - /** - * Sets the `stream` Amp query setting to `true`. - */ - readonly stream?: boolean | undefined - /** - * A set of block ranges which will be converted into a resume watermark - * header and sent with the query. This allows resumption of streaming queries. - */ - readonly resumeWatermark?: ReadonlyArray | undefined -} - -/** - * A utility type to extract the result type for a query. - */ -export type ExtractQueryResult = Options extends { - readonly schema: Schema.Schema -} ? QueryResult<_A> - : QueryResult> +export { + type ArrowFlightError, + MultipleEndpointsError, + NoEndpointsError, + ParseDictionaryBatchError, + ParseRecordBatchError, + ParseSchemaError, + RpcError, + TicketNotFoundError +} from "./arrow-flight/errors.ts" // ============================================================================= -// Arrow Flight Service +// Types // ============================================================================= -// TODO: cleanup service interface (just implemented as is for testing right now) -/** - * A service which can be used to execute queries against an Arrow Flight API. - */ -export class ArrowFlight extends Context.Tag("Amp/ArrowFlight") - - /** - * Executes an Arrow Flight SQL query and returns a all results as an array. - */ - readonly query: ( - sql: string, - options?: Options - ) => Effect.Effect>, ArrowFlightError> - - /** - * Executes an Arrow Flight SQL query and returns a stream of results. - */ - readonly streamQuery: ( - sql: string, - options?: Options - ) => Stream.Stream, ArrowFlightError> -}>() {} - -const make = Effect.gen(function*() { - const auth = yield* Effect.serviceOption(Auth) - const transport = yield* Transport - const client = createClient(FlightService, transport) - - const decodeRecordBatchMetadata = Schema.decode(RecordBatchMetadataFromUint8Array) - - /** - * Execute a SQL query and return a stream of rows. - */ - const streamQuery = (query: string, options?: QueryOptions) => - Effect.gen(function*() { - const contextValues = createContextValues() - const authInfo = Option.isSome(auth) - ? yield* auth.value.getCachedAuthInfo - : Option.none() - - // Setup the query context with authentication information, if available - if (Option.isSome(authInfo)) { - contextValues.set(AuthInfoContextKey, authInfo.value) - } - - const cmd = create(CommandStatementQuerySchema, { query }) - const any = anyPack(CommandStatementQuerySchema, cmd) - const desc = create(FlightDescriptorSchema, { - type: FlightDescriptor_DescriptorType.CMD, - cmd: toBinary(AnySchema, any) - }) - - // Setup the query headers - const headers = new Headers() - if (Predicate.isNotUndefined(options?.stream)) { - headers.set("amp-stream", "true") - } - if (Predicate.isNotUndefined(options?.resumeWatermark)) { - headers.set("amp-resume", blockRangesToResumeWatermark(options.resumeWatermark)) - } - - const flightInfo = yield* Effect.tryPromise({ - try: (signal) => client.getFlightInfo(desc, { contextValues, headers, signal }), - catch: (cause) => new RpcError({ cause, method: "getFlightInfo" }) - }) - - if (flightInfo.endpoint.length !== 1) { - return yield* flightInfo.endpoint.length <= 0 - ? new NoEndpointsError({ query }) - : new MultipleEndpointsError({ query }) - } - - const { ticket } = flightInfo.endpoint[0]! - - if (ticket === undefined) { - return yield* new TicketNotFoundError({ query }) - } - - const flightDataStream = Stream.unwrapScoped(Effect.gen(function*() { - const controller = yield* Effect.acquireRelease( - Effect.sync(() => new AbortController()), - (controller) => Effect.sync(() => controller.abort()) - ) - return Stream.fromAsyncIterable( - client.doGet(ticket, { signal: controller.signal, contextValues }), - (cause) => new RpcError({ cause, method: "doGet" }) - ) - })) - - let schema: ArrowSchema | undefined - const dictionaryRegistry = new DictionaryRegistry() - const dataSchema: Schema.Array$< - Schema.Record$< - typeof Schema.String, - typeof Schema.Unknown - > - > = Schema.Array( - options?.schema ?? Schema.Record({ - key: Schema.String, - value: Schema.Unknown - }) as any - ) - const decodeRecordBatchData = Schema.decode(dataSchema) - - // Convert FlightData stream to a stream of rows - return flightDataStream.pipe( - Stream.mapEffect(Effect.fnUntraced(function*(flightData): Effect.fn.Return< - Option.Option>, - ArrowFlightError - > { - const messageType = yield* Effect.orDie(getMessageType(flightData)) - - switch (messageType) { - case MessageHeaderType.SCHEMA: { - schema = yield* parseSchema(flightData).pipe( - Effect.mapError((cause) => new ParseSchemaError({ cause })) - ) - return Option.none>() - } - case MessageHeaderType.DICTIONARY_BATCH: { - // TODO: figure out what to do (if anything) with dictionary batches - // const dictionaryBatch = yield* parseDictionaryBatch(flightData).pipe( - // Effect.mapError((cause) => new ParseDictionaryBatchError({ cause })) - // ) - // decodeDictionaryBatch(dictionaryBatch, flightData.dataBody, schema!, dictionaryRegistry, readColumnValues) - return Option.none>() - } - case MessageHeaderType.RECORD_BATCH: { - const metadata = yield* decodeRecordBatchMetadata(flightData.appMetadata).pipe( - Effect.mapError((cause) => new ParseRecordBatchError({ cause })) - ) - const recordBatch = yield* parseRecordBatch(flightData).pipe( - Effect.mapError((cause) => new ParseRecordBatchError({ cause })) - ) - const decodedRecordBatch = decodeRecordBatch(recordBatch, flightData.dataBody, schema!) - const json = recordBatchToJson(decodedRecordBatch, { dictionaryRegistry }) - const data = yield* decodeRecordBatchData(json).pipe( - Effect.mapError((cause) => new ParseRecordBatchError({ cause })) - ) - return Option.some({ data, metadata }) - } - } - - return yield* Effect.die(new Cause.RuntimeException(`Invalid message type received: ${messageType}`)) - })), - Stream.filterMap(identity) - ) - }).pipe( - Stream.unwrap, - Stream.withSpan("ArrowFlight.stream") - ) as any - - const query = Effect.fn("ArrowFlight.query")( - function*(query: string, options?: QueryOptions) { - const chunk = yield* Stream.runCollect(streamQuery(query, options)) - return Array.from(chunk) - } - ) as any - - return { - client, - query, - streamQuery - } as const -}) - -/** - * A layer which constructs a concrete implementation of an `ArrowFlight` - * service and depends upon some implementation of a `Transport`. - */ -export const layer: Layer.Layer = Layer.effect(ArrowFlight, make) +export { type ExtractQueryResult, type QueryOptions, type QueryResult } from "./arrow-flight/types.ts" // ============================================================================= -// Internal Utilities +// Service // ============================================================================= -/** - * Converts a list of block ranges into a resume watermark string. - * - * @param ranges - The block ranges to convert. - * @returns A resume watermark string. - */ -const blockRangesToResumeWatermark = (ranges: ReadonlyArray): string => { - const watermarks: Record = {} - for (const range of ranges) { - watermarks[range.network] = { - number: range.numbers.end, - hash: range.hash - } - } - return JSON.stringify(watermarks) -} +export { ArrowFlight, layer } from "./arrow-flight/service.ts" diff --git a/packages/amp/src/arrow-flight/errors.ts b/packages/amp/src/arrow-flight/errors.ts new file mode 100644 index 0000000..ad42d6e --- /dev/null +++ b/packages/amp/src/arrow-flight/errors.ts @@ -0,0 +1,114 @@ +import * as Schema from "effect/Schema" + +// ============================================================================= +// Errors +// ============================================================================= + +// TODO: improve the error model +/** + * Represents the possible errors that can occur when executing an Arrow Flight + * query. + */ +export type ArrowFlightError = + | RpcError + | NoEndpointsError + | MultipleEndpointsError + | TicketNotFoundError + | ParseRecordBatchError + | ParseDictionaryBatchError + | ParseSchemaError + +/** + * Represents an Arrow Flight RPC request that failed. + */ +export class RpcError extends Schema.TaggedError( + "Amp/RpcError" +)("RpcError", { + method: Schema.String, + /** + * The underlying reason for the failed RPC request. + */ + cause: Schema.Defect +}) {} + +/** + * Represents an error that occurred as a result of a `FlightInfo` request + * returning an empty list of endpoints from which data can be acquired. + */ +export class NoEndpointsError extends Schema.TaggedError( + "Amp/NoEndpointsError" +)("NoEndpointsError", { + /** + * The SQL query that was requested. + */ + query: Schema.String +}) {} + +// TODO: determine if this is _really_ a logical error case +/** + * Represents an error that occured as a result of a `FlightInfo` request + * returning multiple endpoints from which data can be acquired. + * + * For Amp queries, there should only ever be **one** authoritative source + * of data. + */ +export class MultipleEndpointsError extends Schema.TaggedError( + "Amp/MultipleEndpointsError" +)("MultipleEndpointsError", { + /** + * The SQL query that was requested. + */ + query: Schema.String +}) {} + +/** + * Represents an error that occurred as a result of a `FlightInfo` request + * whose endpoint did not have a ticket. + */ +export class TicketNotFoundError extends Schema.TaggedError( + "Amp/TicketNotFoundError" +)("TicketNotFoundError", { + /** + * The SQL query that was requested. + */ + query: Schema.String +}) {} + +/** + * Represents an error that occurred as a result of failing to parse an Apache + * Arrow RecordBatch. + */ +export class ParseRecordBatchError extends Schema.TaggedError( + "Amp/ParseRecordBatchError" +)("ParseRecordBatchError", { + /** + * The underlying reason for the failure to parse a record batch. + */ + cause: Schema.Defect +}) {} + +/** + * Represents an error that occurred as a result of failing to parse an Apache + * Arrow DictionaryBatch. + */ +export class ParseDictionaryBatchError extends Schema.TaggedError( + "Amp/ParseDictionaryBatchError" +)("ParseDictionaryBatchError", { + /** + * The underlying reason for the failure to parse a dictionary batch. + */ + cause: Schema.Defect +}) {} + +/** + * Represents an error that occurred as a result of failing to parse an Apache + * Arrow Schema. + */ +export class ParseSchemaError extends Schema.TaggedError( + "Amp/ParseSchemaError" +)("ParseSchemaError", { + /** + * The underlying reason for the failure to parse a schema. + */ + cause: Schema.Defect +}) {} diff --git a/packages/amp/src/arrow-flight/node.ts b/packages/amp/src/arrow-flight/node.ts index 2379ab3..ce6e18a 100644 --- a/packages/amp/src/arrow-flight/node.ts +++ b/packages/amp/src/arrow-flight/node.ts @@ -1,7 +1,7 @@ import { createGrpcTransport, type GrpcTransportOptions } from "@connectrpc/connect-node" import * as Effect from "effect/Effect" import * as Layer from "effect/Layer" -import { Interceptors, Transport } from "../arrow-flight.ts" +import { Interceptors, Transport } from "./transport.ts" /** * Create a `Transport` for the gRPC protocol using the Node.js `http2` module. diff --git a/packages/amp/src/arrow-flight/service.ts b/packages/amp/src/arrow-flight/service.ts new file mode 100644 index 0000000..34124c0 --- /dev/null +++ b/packages/amp/src/arrow-flight/service.ts @@ -0,0 +1,238 @@ +import { create, toBinary } from "@bufbuild/protobuf" +import { anyPack, AnySchema } from "@bufbuild/protobuf/wkt" +import { type Client, createClient, createContextValues } from "@connectrpc/connect" +import * as Cause from "effect/Cause" +import * as Context from "effect/Context" +import * as Effect from "effect/Effect" +import { identity } from "effect/Function" +import * as Layer from "effect/Layer" +import * as Option from "effect/Option" +import * as Predicate from "effect/Predicate" +import * as Schema from "effect/Schema" +import * as Stream from "effect/Stream" +import { Auth } from "../auth/service.ts" +import type { AuthInfo, BlockRange } from "../core/domain.ts" +import { RecordBatchMetadataFromUint8Array } from "../core/domain.ts" +import { decodeRecordBatch, DictionaryRegistry } from "../internal/arrow-flight-ipc/Decoder.ts" +import { recordBatchToJson } from "../internal/arrow-flight-ipc/Json.ts" +import { parseRecordBatch } from "../internal/arrow-flight-ipc/RecordBatch.ts" +import { + type ArrowSchema, + getMessageType, + MessageHeaderType, + parseSchema +} from "../internal/arrow-flight-ipc/Schema.ts" +import { FlightDescriptor_DescriptorType, FlightDescriptorSchema, FlightService } from "../protobuf/Flight_pb.ts" +import { CommandStatementQuerySchema } from "../protobuf/FlightSql_pb.ts" +import { + type ArrowFlightError, + MultipleEndpointsError, + NoEndpointsError, + ParseRecordBatchError, + ParseSchemaError, + RpcError, + TicketNotFoundError +} from "./errors.ts" +import { AuthInfoContextKey, Transport } from "./transport.ts" +import type { ExtractQueryResult, QueryOptions, QueryResult } from "./types.ts" + +// ============================================================================= +// Arrow Flight Service +// ============================================================================= + +// TODO: cleanup service interface (just implemented as is for testing right now) +/** + * A service which can be used to execute queries against an Arrow Flight API. + */ +export class ArrowFlight extends Context.Tag("Amp/ArrowFlight") + + /** + * Executes an Arrow Flight SQL query and returns a all results as an array. + */ + readonly query: ( + sql: string, + options?: Options + ) => Effect.Effect>, ArrowFlightError> + + /** + * Executes an Arrow Flight SQL query and returns a stream of results. + */ + readonly streamQuery: ( + sql: string, + options?: Options + ) => Stream.Stream, ArrowFlightError> +}>() {} + +const make = Effect.gen(function*() { + const auth = yield* Effect.serviceOption(Auth) + const transport = yield* Transport + const client = createClient(FlightService, transport) + + const decodeRecordBatchMetadata = Schema.decode(RecordBatchMetadataFromUint8Array) + + /** + * Execute a SQL query and return a stream of rows. + */ + const streamQuery = (query: string, options?: QueryOptions) => + Effect.gen(function*() { + const contextValues = createContextValues() + const authInfo = Option.isSome(auth) + ? yield* auth.value.getCachedAuthInfo + : Option.none() + + // Setup the query context with authentication information, if available + if (Option.isSome(authInfo)) { + contextValues.set(AuthInfoContextKey, authInfo.value) + } + + const cmd = create(CommandStatementQuerySchema, { query }) + const any = anyPack(CommandStatementQuerySchema, cmd) + const desc = create(FlightDescriptorSchema, { + type: FlightDescriptor_DescriptorType.CMD, + cmd: toBinary(AnySchema, any) + }) + + // Setup the query headers + const headers = new Headers() + if (Predicate.isNotUndefined(options?.stream)) { + headers.set("amp-stream", "true") + } + if (Predicate.isNotUndefined(options?.resumeWatermark)) { + headers.set("amp-resume", blockRangesToResumeWatermark(options.resumeWatermark)) + } + + const flightInfo = yield* Effect.tryPromise({ + try: (signal) => client.getFlightInfo(desc, { contextValues, headers, signal }), + catch: (cause) => new RpcError({ cause, method: "getFlightInfo" }) + }) + + if (flightInfo.endpoint.length !== 1) { + return yield* flightInfo.endpoint.length <= 0 + ? new NoEndpointsError({ query }) + : new MultipleEndpointsError({ query }) + } + + const { ticket } = flightInfo.endpoint[0]! + + if (ticket === undefined) { + return yield* new TicketNotFoundError({ query }) + } + + const flightDataStream = Stream.unwrapScoped(Effect.gen(function*() { + const controller = yield* Effect.acquireRelease( + Effect.sync(() => new AbortController()), + (controller) => Effect.sync(() => controller.abort()) + ) + return Stream.fromAsyncIterable( + client.doGet(ticket, { signal: controller.signal, contextValues }), + (cause) => new RpcError({ cause, method: "doGet" }) + ) + })) + + let schema: ArrowSchema | undefined + const dictionaryRegistry = new DictionaryRegistry() + const dataSchema: Schema.Array$< + Schema.Record$< + typeof Schema.String, + typeof Schema.Unknown + > + > = Schema.Array( + options?.schema ?? Schema.Record({ + key: Schema.String, + value: Schema.Unknown + }) as any + ) + const decodeRecordBatchData = Schema.decode(dataSchema) + + // Convert FlightData stream to a stream of rows + return flightDataStream.pipe( + Stream.mapEffect(Effect.fnUntraced(function*(flightData): Effect.fn.Return< + Option.Option>, + ArrowFlightError + > { + const messageType = yield* Effect.orDie(getMessageType(flightData)) + + switch (messageType) { + case MessageHeaderType.SCHEMA: { + schema = yield* parseSchema(flightData).pipe( + Effect.mapError((cause) => new ParseSchemaError({ cause })) + ) + return Option.none>() + } + case MessageHeaderType.DICTIONARY_BATCH: { + // TODO: figure out what to do (if anything) with dictionary batches + // const dictionaryBatch = yield* parseDictionaryBatch(flightData).pipe( + // Effect.mapError((cause) => new ParseDictionaryBatchError({ cause })) + // ) + // decodeDictionaryBatch(dictionaryBatch, flightData.dataBody, schema!, dictionaryRegistry, readColumnValues) + return Option.none>() + } + case MessageHeaderType.RECORD_BATCH: { + const metadata = yield* decodeRecordBatchMetadata(flightData.appMetadata).pipe( + Effect.mapError((cause) => new ParseRecordBatchError({ cause })) + ) + const recordBatch = yield* parseRecordBatch(flightData).pipe( + Effect.mapError((cause) => new ParseRecordBatchError({ cause })) + ) + const decodedRecordBatch = decodeRecordBatch(recordBatch, flightData.dataBody, schema!) + const json = recordBatchToJson(decodedRecordBatch, { dictionaryRegistry }) + const data = yield* decodeRecordBatchData(json).pipe( + Effect.mapError((cause) => new ParseRecordBatchError({ cause })) + ) + return Option.some({ data, metadata }) + } + } + + return yield* Effect.die(new Cause.RuntimeException(`Invalid message type received: ${messageType}`)) + })), + Stream.filterMap(identity) + ) + }).pipe( + Stream.unwrap, + Stream.withSpan("ArrowFlight.stream") + ) as any + + const query = Effect.fn("ArrowFlight.query")( + function*(query: string, options?: QueryOptions) { + const chunk = yield* Stream.runCollect(streamQuery(query, options)) + return Array.from(chunk) + } + ) as any + + return { + client, + query, + streamQuery + } as const +}) + +/** + * A layer which constructs a concrete implementation of an `ArrowFlight` + * service and depends upon some implementation of a `Transport`. + */ +export const layer: Layer.Layer = Layer.effect(ArrowFlight, make) + +// ============================================================================= +// Internal Utilities +// ============================================================================= + +/** + * Converts a list of block ranges into a resume watermark string. + * + * @param ranges - The block ranges to convert. + * @returns A resume watermark string. + */ +const blockRangesToResumeWatermark = (ranges: ReadonlyArray): string => { + const watermarks: Record = {} + for (const range of ranges) { + watermarks[range.network] = { + number: range.numbers.end, + hash: range.hash + } + } + return JSON.stringify(watermarks) +} diff --git a/packages/amp/src/arrow-flight/transport.ts b/packages/amp/src/arrow-flight/transport.ts new file mode 100644 index 0000000..dd92b39 --- /dev/null +++ b/packages/amp/src/arrow-flight/transport.ts @@ -0,0 +1,68 @@ +import { createContextKey, type Interceptor, type Transport as ConnectTransport } from "@connectrpc/connect" +import * as Arr from "effect/Array" +import * as Context from "effect/Context" +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" +import * as Redacted from "effect/Redacted" +import type { AuthInfo } from "../core/domain.ts" + +// ============================================================================= +// Connect RPC Transport +// ============================================================================= + +/** + * A service which abstracts the underlying transport for a given client. + * + * A transport implements a protocol, such as Connect or gRPC-web, and allows + * for the concrete clients to be independent of the protocol. + */ +export class Transport extends Context.Tag("@edgeandnode/amp/Transport")< + Transport, + ConnectTransport +>() {} + +/** + * A service which abstracts the set of interceptors that are passed to a given + * transport. + * + * An interceptor can add logic to clients or servers, similar to the decorators + * or middleware you may have seen in other libraries. Interceptors may + * mutate the request and response, catch errors and retry/recover, emit + * logs, or do nearly everything else. + */ +export class Interceptors extends Context.Reference()( + "Amp/ArrowFlight/ConnectRPC/Interceptors", + { defaultValue: () => Arr.empty() } +) {} + +export const AuthInfoContextKey = createContextKey( + undefined, + { description: "Authentication information obtained from the Amp auth server" } +) + +/** + * A layer which will add an interceptor to the configured set of `Interceptors` + * which attempts to read authentication information from the Connect context + * values. + * + * If authentication information is found, the interceptor will add an + * `"Authorization"` header to the request containing a bearer token with the + * value of the authentication information access token. + */ +export const layerInterceptorBearerAuth = Layer.effectContext( + Effect.gen(function*() { + const interceptors = yield* Interceptors + + const interceptor: Interceptor = (next) => (request) => { + const authInfo = request.contextValues.get(AuthInfoContextKey) + + if (authInfo !== undefined) { + const accessToken = Redacted.value(authInfo.accessToken) + request.header.append("Authorization", `Bearer ${accessToken}`) + } + return next(request) + } + + return Context.make(Interceptors, Arr.append(interceptors, interceptor)) + }) +) diff --git a/packages/amp/src/arrow-flight/types.ts b/packages/amp/src/arrow-flight/types.ts new file mode 100644 index 0000000..13c1010 --- /dev/null +++ b/packages/amp/src/arrow-flight/types.ts @@ -0,0 +1,40 @@ +import type * as Schema from "effect/Schema" +import type { BlockRange, RecordBatchMetadata } from "../core/domain.ts" + +// ============================================================================= +// Types +// ============================================================================= + +/** + * Represents the result received from the `ArrowFlight` service when a query + * is successfully executed. + */ +export interface QueryResult { + readonly data: ReadonlyArray + readonly metadata: RecordBatchMetadata +} + +/** + * Represents options that can be passed to `ArrowFlight.query` to control how + * the query is executed. + */ +export interface QueryOptions { + readonly schema?: Schema.Any | undefined + /** + * Sets the `stream` Amp query setting to `true`. + */ + readonly stream?: boolean | undefined + /** + * A set of block ranges which will be converted into a resume watermark + * header and sent with the query. This allows resumption of streaming queries. + */ + readonly resumeWatermark?: ReadonlyArray | undefined +} + +/** + * A utility type to extract the result type for a query. + */ +export type ExtractQueryResult = Options extends { + readonly schema: Schema.Schema +} ? QueryResult<_A> + : QueryResult> diff --git a/packages/amp/src/auth/service.ts b/packages/amp/src/auth/service.ts index 48a71b2..c166344 100644 --- a/packages/amp/src/auth/service.ts +++ b/packages/amp/src/auth/service.ts @@ -17,8 +17,8 @@ import * as Predicate from "effect/Predicate" import * as Redacted from "effect/Redacted" import * as Schema from "effect/Schema" import * as Jose from "jose" +import { AccessToken, Address, AuthInfo, RefreshToken, TokenDuration, UserId } from "../core/domain.ts" import { pkceChallenge } from "../internal/pkce.ts" -import { AccessToken, Address, AuthInfo, RefreshToken, TokenDuration, UserId } from "../models.ts" import { AuthCacheError, AuthDeviceFlowError, diff --git a/packages/amp/src/config.ts b/packages/amp/src/config.ts deleted file mode 100644 index b963fa4..0000000 --- a/packages/amp/src/config.ts +++ /dev/null @@ -1,295 +0,0 @@ -import * as FileSystem from "@effect/platform/FileSystem" -import * as Path from "@effect/platform/Path" -import type * as Cause from "effect/Cause" -import * as Context from "effect/Context" -import * as Data from "effect/Data" -import * as Effect from "effect/Effect" -import * as Either from "effect/Either" -import { identity } from "effect/Function" -import * as Layer from "effect/Layer" -import * as Match from "effect/Match" -import type * as Option from "effect/Option" -import * as Predicate from "effect/Predicate" -import * as Schema from "effect/Schema" -import * as Stream from "effect/Stream" -import * as fs from "node:fs" -import * as path from "node:path" -import * as ManifestBuilder from "./manifest-builder/service.ts" -import * as Models from "./models.ts" - -export class ModuleContext { - public definitionPath: string - - constructor(definitionPath: string) { - this.definitionPath = definitionPath - } - - /** - * Reads a file relative to the directory of the dataset definition. - */ - functionSource(relativePath: string): Models.FunctionSource { - const baseDir = path.dirname(path.resolve(this.definitionPath)) - const fullPath = path.resolve(baseDir, relativePath) - if (!fullPath.startsWith(baseDir + path.sep)) { - throw new Error(`Invalid path: directory traversal not allowed`) - } - - let source: string - try { - source = fs.readFileSync(fullPath, "utf8") - } catch (err: any) { - throw new Error( - `Failed to read function source at ${fullPath}: ${err.message}`, - { cause: err } - ) - } - - const func = Models.FunctionSource.make({ - source, - filename: path.basename(fullPath) - }) - return func - } -} - -export class ConfigLoaderError extends Data.TaggedError("ConfigLoaderError")<{ - readonly cause?: unknown - readonly message?: string -}> {} - -export class ConfigLoader extends Context.Tag("Amp/ConfigLoader") Effect.Effect - - /** - * Finds a config file in the given directory by checking for known config file names. - */ - readonly find: (cwd?: string) => Effect.Effect, ConfigLoaderError> - - /** - * Loads and builds a dataset configuration from a file. - */ - readonly build: (file: string) => Effect.Effect - - /** - * Watches a config file for changes and emits built manifests. - */ - readonly watch: (file: string, options?: { - readonly onError?: (cause: Cause.Cause) => Effect.Effect - }) => Stream.Stream -}>() {} - -const make = Effect.gen(function*() { - const path = yield* Path.Path - const fs = yield* FileSystem.FileSystem - const builder = yield* ManifestBuilder.ManifestBuilder - - const decodeDatasetConfig = Schema.decodeUnknown(Models.DatasetConfig) - - const jiti = yield* Effect.tryPromise({ - try: () => - import("jiti").then(({ createJiti }) => - createJiti(import.meta.url, { - moduleCache: false, - tryNative: false - }) - ), - catch: (cause) => new ConfigLoaderError({ cause }) - }).pipe(Effect.cached) - - const loadTypeScript = Effect.fnUntraced(function*(file: string) { - return yield* Effect.tryMapPromise(jiti, { - try: (jiti) => - jiti.import<(context: ModuleContext) => Models.DatasetConfig>(file, { - default: true - }), - catch: identity - }).pipe( - Effect.map((callback) => callback(new ModuleContext(file))), - Effect.flatMap(decodeDatasetConfig), - Effect.mapError((cause) => - new ConfigLoaderError({ - cause, - message: `Failed to load config file ${file}` - }) - ) - ) - }) - - const loadJavaScript = Effect.fnUntraced(function*(file: string) { - return yield* Effect.tryPromise({ - try: () => - import(file).then( - (module) => module.default as (context: ModuleContext) => Models.DatasetConfig - ), - catch: identity - }).pipe( - Effect.map((callback) => callback(new ModuleContext(file))), - Effect.flatMap(decodeDatasetConfig), - Effect.mapError((cause) => - new ConfigLoaderError({ - cause, - message: `Failed to load config file ${file}` - }) - ) - ) - }) - - const loadJson = Effect.fnUntraced(function*(file: string) { - return yield* Effect.tryMap(fs.readFileString(file), { - try: (content) => JSON.parse(content), - catch: identity - }).pipe( - Effect.flatMap(decodeDatasetConfig), - Effect.mapError((cause) => - new ConfigLoaderError({ - cause, - message: `Failed to load config file ${file}` - }) - ) - ) - }) - - const fileMatcher = Match.type().pipe( - Match.when( - (_) => /\.(ts|mts|cts)$/.test(path.extname(_)), - (_) => loadTypeScript(_) - ), - Match.when( - (_) => /\.(js|mjs|cjs)$/.test(path.extname(_)), - (_) => loadJavaScript(_) - ), - Match.when( - (_) => /\.(json)$/.test(path.extname(_)), - (_) => loadJson(_) - ), - Match.orElse((_) => - new ConfigLoaderError({ - message: `Unsupported file extension ${path.extname(_)}` - }) - ) - ) - - const load = Effect.fnUntraced(function*(file: string) { - const resolved = path.resolve(file) - return yield* fileMatcher(resolved) - }) - - const build = Effect.fnUntraced(function*(file: string) { - const config = yield* load(file) - return yield* builder.build(config).pipe( - Effect.mapError( - (cause) => - new ConfigLoaderError({ - cause, - message: `Failed to build config file ${file}` - }) - ) - ) - }) - - const CANDIDATE_CONFIG_FILES = [ - "amp.config.ts", - "amp.config.mts", - "amp.config.cts", - "amp.config.js", - "amp.config.mjs", - "amp.config.cjs", - "amp.config.json" - ] - - const find = Effect.fnUntraced(function*(cwd: string = ".") { - const baseCwd = path.resolve(".") - const resolvedCwd = path.resolve(cwd) - if (resolvedCwd !== baseCwd && !resolvedCwd.startsWith(baseCwd + path.sep)) { - return yield* new ConfigLoaderError({ - message: "Invalid directory path: directory traversal not allowed" - }) - } - const candidates = CANDIDATE_CONFIG_FILES.map((fileName) => { - const filePath = path.resolve(cwd, fileName) - return fs.exists(filePath).pipe( - Effect.flatMap((exists) => exists ? Effect.succeed(filePath) : Effect.fail("not found")) - ) - }) - return yield* Effect.firstSuccessOf(candidates).pipe(Effect.option) - }) - - const watch = (file: string, options?: { - readonly onError?: ( - cause: Cause.Cause - ) => Effect.Effect - }): Stream.Stream< - ManifestBuilder.ManifestBuildResult, - ConfigLoaderError | E, - R - > => { - const baseCwd = path.resolve(".") - const resolved = path.resolve(file) - if (resolved !== baseCwd && !resolved.startsWith(baseCwd + path.sep)) { - return Stream.fail( - new ConfigLoaderError({ - message: "Invalid file path: directory traversal not allowed" - }) - ) - } - const open = load(resolved).pipe( - Effect.tapErrorCause(options?.onError ?? (() => Effect.void)), - Effect.either - ) - - const updates = fs.watch(resolved).pipe( - Stream.buffer({ capacity: 1, strategy: "sliding" }), - Stream.mapError( - (cause) => - new ConfigLoaderError({ - cause, - message: "Failed to watch config file" - }) - ), - Stream.filter(Predicate.isTagged("Update")), - Stream.mapEffect(() => open) - ) - - const build = (config: Models.DatasetConfig) => - builder.build(config).pipe( - Effect.mapError( - (cause) => - new ConfigLoaderError({ - cause, - message: `Failed to build config file ${file}` - }) - ), - Effect.tapErrorCause(options?.onError ?? (() => Effect.void)), - Effect.either - ) - - return Stream.fromEffect(open).pipe( - Stream.concat(updates), - Stream.filterMap(Either.getRight), - Stream.changesWith(DatasetConfigEquivalence), - Stream.mapEffect(build), - Stream.filterMap(Either.getRight), - Stream.changesWith((a, b) => - DatasetDerivedEquivalence(a.manifest, b.manifest) && - DatasetMetadataEquivalence(a.metadata, b.metadata) - ) - ) as Stream.Stream< - ManifestBuilder.ManifestBuildResult, - ConfigLoaderError | E, - R - > - } - - return { load, find, watch, build } -}) - -export const layer = Layer.effect(ConfigLoader, make).pipe( - Layer.provide(ManifestBuilder.layer) -) - -const DatasetConfigEquivalence = Schema.equivalence(Models.DatasetConfig) -const DatasetDerivedEquivalence = Schema.equivalence(Models.DatasetDerived) -const DatasetMetadataEquivalence = Schema.equivalence(Models.DatasetMetadata) diff --git a/packages/amp/src/core.ts b/packages/amp/src/core.ts new file mode 100644 index 0000000..c30bc7a --- /dev/null +++ b/packages/amp/src/core.ts @@ -0,0 +1,9 @@ +/** + * Core domain models for the Amp SDK. + * + * Includes branded types, schemas, and domain primitives used throughout + * the SDK — block ranges, dataset references, auth tokens, and more. + * + * @module + */ +export * from "./core/domain.ts" diff --git a/packages/amp/src/models.ts b/packages/amp/src/core/domain.ts similarity index 100% rename from packages/amp/src/models.ts rename to packages/amp/src/core/domain.ts diff --git a/packages/amp/src/manifest-builder/service.ts b/packages/amp/src/manifest-builder/service.ts index e68d8b6..7d0aa3a 100644 --- a/packages/amp/src/manifest-builder/service.ts +++ b/packages/amp/src/manifest-builder/service.ts @@ -5,7 +5,7 @@ import * as Layer from "effect/Layer" import * as Predicate from "effect/Predicate" import * as Schema from "effect/Schema" import * as AdminApi from "../admin/service.ts" -import * as Models from "../models.ts" +import * as Models from "../core/domain.ts" export const ManifestBuildResult = Schema.Struct({ metadata: Models.DatasetMetadata, diff --git a/packages/amp/src/protocol-stream/messages.ts b/packages/amp/src/protocol-stream/messages.ts index 85836ee..0cf94bc 100644 --- a/packages/amp/src/protocol-stream/messages.ts +++ b/packages/amp/src/protocol-stream/messages.ts @@ -10,7 +10,7 @@ * @module */ import * as Schema from "effect/Schema" -import { BlockNumber, BlockRange, Network } from "../models.ts" +import { BlockNumber, BlockRange, Network } from "../core/domain.ts" // ============================================================================= // Invalidation Range @@ -49,15 +49,12 @@ export type InvalidationRange = typeof InvalidationRange.Type * @param end - The end of the invalidation range (inclusive). * @returns An InvalidationRange instance. */ -export const makeInvalidationRange = ( - network: string, - start: number, - end: number -): InvalidationRange => ({ - network: network as typeof Network.Type, - start: start as typeof BlockNumber.Type, - end: end as typeof BlockNumber.Type -}) +export const makeInvalidationRange = (network: string, start: number, end: number): InvalidationRange => + InvalidationRange.make({ + network: Network.make(network), + start: BlockNumber.make(start), + end: BlockNumber.make(end) + }) /** * Checks if a block range overlaps with an invalidation range. @@ -69,15 +66,14 @@ export const makeInvalidationRange = ( * @param range - The block range to check. * @returns True if the ranges overlap, false otherwise. */ -export const invalidates = ( - invalidation: InvalidationRange, - range: typeof BlockRange.Type -): boolean => { +export const invalidates = (invalidation: InvalidationRange, range: BlockRange): boolean => { if (invalidation.network !== range.network) { return false } + // Check for no overlap: invalidation ends before range starts OR range ends before invalidation starts const noOverlap = invalidation.end < range.numbers.start || range.numbers.end < invalidation.start + return !noOverlap } @@ -195,12 +191,12 @@ export type ProtocolMessage = typeof ProtocolMessage.Type */ export const data = ( records: ReadonlyArray>, - ranges: ReadonlyArray -): ProtocolMessageData => ({ - _tag: "Data", - data: records as Array>, - ranges: ranges as Array -}) + ranges: ReadonlyArray +): ProtocolMessageData => + ProtocolMessageData.make({ + data: records, + ranges: ranges + }) /** * Creates a Reorg protocol message. @@ -211,15 +207,15 @@ export const data = ( * @returns A Reorg protocol message. */ export const reorg = ( - previous: ReadonlyArray, - incoming: ReadonlyArray, + previous: ReadonlyArray, + incoming: ReadonlyArray, invalidation: ReadonlyArray -): ProtocolMessageReorg => ({ - _tag: "Reorg", - previous: previous as Array, - incoming: incoming as Array, - invalidation: invalidation as Array -}) +): ProtocolMessageReorg => + ProtocolMessageReorg.make({ + previous: previous, + incoming: incoming, + invalidation: invalidation + }) /** * Creates a Watermark protocol message. @@ -228,8 +224,5 @@ export const reorg = ( * @returns A Watermark protocol message. */ export const watermark = ( - ranges: ReadonlyArray -): ProtocolMessageWatermark => ({ - _tag: "Watermark", - ranges: ranges as Array -}) + ranges: ReadonlyArray +): ProtocolMessageWatermark => ProtocolMessageWatermark.make({ ranges: ranges }) diff --git a/packages/amp/src/protocol-stream/service.ts b/packages/amp/src/protocol-stream/service.ts index 362e031..c0bc489 100644 --- a/packages/amp/src/protocol-stream/service.ts +++ b/packages/amp/src/protocol-stream/service.ts @@ -13,7 +13,7 @@ import * as Effect from "effect/Effect" import * as Layer from "effect/Layer" import * as Stream from "effect/Stream" import { ArrowFlight, type ArrowFlightError, type QueryOptions, type QueryResult } from "../arrow-flight.ts" -import type { BlockRange } from "../models.ts" +import type { BlockRange } from "../core/domain.ts" import { ProtocolArrowFlightError, type ProtocolStreamError, ProtocolValidationError } from "./errors.ts" import { data as protocolData, @@ -23,7 +23,7 @@ import { reorg as protocolReorg, watermark as protocolWatermark } from "./messages.ts" -import { validateAll } from "./validation.ts" +import { blockRangeEquals, validateAll } from "./validation.ts" // ============================================================================= // Options @@ -92,9 +92,6 @@ export interface ProtocolStreamService { ) => Stream.Stream } -// Re-export ProtocolStreamError from errors.ts for convenience -export type { ProtocolStreamError } - // ============================================================================= // Context.Tag // ============================================================================= @@ -149,17 +146,14 @@ const detectReorgs = ( const invalidations: Array = [] for (const incomingRange of incoming) { - const prevRange = previous.find((p) => p.network === incomingRange.network) - if (!prevRange) continue + const prevRange = previous.find((_) => _.network === incomingRange.network) + + if (!prevRange) { + continue + } // Skip identical ranges (watermarks can repeat) - if ( - incomingRange.network === prevRange.network && - incomingRange.numbers.start === prevRange.numbers.start && - incomingRange.numbers.end === prevRange.numbers.end && - incomingRange.hash === prevRange.hash && - incomingRange.prevHash === prevRange.prevHash - ) { + if (blockRangeEquals(incomingRange, prevRange)) { continue } @@ -181,27 +175,21 @@ const detectReorgs = ( return invalidations } -const ZERO_HASH = "0x0000000000000000000000000000000000000000000000000000000000000000" - /** * Create ProtocolStream service implementation. */ const make = Effect.gen(function*() { const arrowFlight = yield* ArrowFlight - const stream = ( - sql: string, - options?: ProtocolStreamOptions - ): Stream.Stream => { - // Get the underlying Arrow Flight stream + const stream = (sql: string, options?: ProtocolStreamOptions): Stream.Stream< + ProtocolMessage, + ProtocolStreamError + > => { const rawStream = arrowFlight.streamQuery(sql, { schema: options?.schema, stream: true, resumeWatermark: options?.resumeWatermark - }) as unknown as Stream.Stream< - QueryResult>>, - ArrowFlightError - > + }) const initialState: ProtocolStreamState = { previous: [], @@ -217,7 +205,7 @@ const make = Effect.gen(function*() { Effect.fnUntraced( function*( state: ProtocolStreamState, - queryResult: QueryResult>> + queryResult: QueryResult> ): Effect.fn.Return< readonly [ProtocolStreamState, ProtocolMessage], ProtocolStreamError @@ -227,33 +215,9 @@ const make = Effect.gen(function*() { const incoming = metadata.ranges // Validate the incoming batch - if (state.initialized) { - yield* validateAll(state.previous, incoming).pipe( - Effect.mapError((error) => new ProtocolValidationError({ cause: error })) - ) - } else { - // Validate prevHash for first batch - for (const range of incoming) { - const isGenesis = range.numbers.start === 0 - if (isGenesis) { - if (range.prevHash !== undefined && range.prevHash !== ZERO_HASH) { - return yield* Effect.fail( - new ProtocolValidationError({ - cause: { _tag: "InvalidPrevHashError", network: range.network } - }) - ) - } - } else { - if (range.prevHash === undefined || range.prevHash === ZERO_HASH) { - return yield* Effect.fail( - new ProtocolValidationError({ - cause: { _tag: "MissingPrevHashError", network: range.network, block: range.numbers.start } - }) - ) - } - } - } - } + yield* validateAll(state.previous, incoming).pipe( + Effect.mapError((error) => new ProtocolValidationError({ cause: error })) + ) // Detect reorgs const invalidations = state.initialized ? detectReorgs(state.previous, incoming) : [] @@ -266,7 +230,7 @@ const make = Effect.gen(function*() { } else if (metadata.rangesComplete && batchData.length === 0) { message = protocolWatermark(incoming) } else { - message = protocolData(batchData as unknown as ReadonlyArray>, incoming) + message = protocolData(batchData, incoming) } const newState: ProtocolStreamState = { @@ -282,7 +246,9 @@ const make = Effect.gen(function*() { ) } - return { stream } satisfies ProtocolStreamService + return ProtocolStream.of({ + stream + }) }) // ============================================================================= diff --git a/packages/amp/src/protocol-stream/validation.ts b/packages/amp/src/protocol-stream/validation.ts index dc5005c..e68403f 100644 --- a/packages/amp/src/protocol-stream/validation.ts +++ b/packages/amp/src/protocol-stream/validation.ts @@ -13,7 +13,7 @@ * @module */ import * as Effect from "effect/Effect" -import type { BlockNumber, BlockRange } from "../models.ts" +import type { BlockNumber, BlockRange } from "../core/domain.ts" import { DuplicateNetworkError, GapError, @@ -124,7 +124,7 @@ export const validateNetworks = Effect.fnUntraced( /** * Checks if two block ranges are equal. */ -const blockRangeEquals = (a: BlockRange, b: BlockRange): boolean => +export const blockRangeEquals = (a: BlockRange, b: BlockRange): boolean => a.network === b.network && a.numbers.start === b.numbers.start && a.numbers.end === b.numbers.end && diff --git a/packages/amp/src/registry/api.ts b/packages/amp/src/registry/api.ts index 5fb8861..f79e13f 100644 --- a/packages/amp/src/registry/api.ts +++ b/packages/amp/src/registry/api.ts @@ -4,7 +4,7 @@ import * as HttpApiError from "@effect/platform/HttpApiError" import * as HttpApiGroup from "@effect/platform/HttpApiGroup" import * as HttpApiSchema from "@effect/platform/HttpApiSchema" import * as Schema from "effect/Schema" -import * as Models from "../models.ts" +import * as Models from "../core/domain.ts" import * as Domain from "./domain.ts" import * as Errors from "./error.ts" diff --git a/packages/amp/src/registry/domain.ts b/packages/amp/src/registry/domain.ts index 392b092..df0cae2 100644 --- a/packages/amp/src/registry/domain.ts +++ b/packages/amp/src/registry/domain.ts @@ -1,5 +1,5 @@ import * as Schema from "effect/Schema" -import * as Models from "../models.ts" +import * as Models from "../core/domain.ts" // TODO(Chris/Max/Sebastian): Should we consider moving these "general" schemas // to the top-level models module (Which I'm considering moving into a /domain diff --git a/packages/amp/src/transactional-stream/algorithms.ts b/packages/amp/src/transactional-stream/algorithms.ts index 091a0ed..76dbdd4 100644 --- a/packages/amp/src/transactional-stream/algorithms.ts +++ b/packages/amp/src/transactional-stream/algorithms.ts @@ -9,10 +9,22 @@ * * @module */ -import type { BlockRange } from "../models.ts" +import type { BlockRange } from "../core/domain.ts" import type { InvalidationRange } from "../protocol-stream/messages.ts" import type { TransactionId } from "./types.ts" +// ============================================================================= +// Types +// ============================================================================= + +/** + * Pending commit waiting for user to call commit handle. + */ +export interface PendingCommit { + readonly ranges: ReadonlyArray + readonly prune: TransactionId | undefined +} + // ============================================================================= // Recovery Point Algorithm // ============================================================================= @@ -230,9 +242,7 @@ export const checkPartialReorg = ( * @returns Compressed commit with combined inserts and maximum prune point */ export const compressCommits = ( - pendingCommits: ReadonlyArray< - readonly [TransactionId, { readonly ranges: ReadonlyArray; readonly prune: TransactionId | undefined }] - > + pendingCommits: ReadonlyArray ): { insert: Array]>; prune: TransactionId | undefined } => { const insert: Array]> = [] let maxPrune: TransactionId | undefined = undefined diff --git a/packages/amp/src/transactional-stream/state-actor.ts b/packages/amp/src/transactional-stream/state-actor.ts index d244253..c47d63f 100644 --- a/packages/amp/src/transactional-stream/state-actor.ts +++ b/packages/amp/src/transactional-stream/state-actor.ts @@ -9,7 +9,7 @@ */ import * as Effect from "effect/Effect" import * as Ref from "effect/Ref" -import type { BlockRange } from "../models.ts" +import type { BlockRange } from "../core/domain.ts" import type { ProtocolMessage } from "../protocol-stream/messages.ts" import { checkPartialReorg, diff --git a/packages/amp/src/transactional-stream/state-store.ts b/packages/amp/src/transactional-stream/state-store.ts index 3b46a87..d8c309c 100644 --- a/packages/amp/src/transactional-stream/state-store.ts +++ b/packages/amp/src/transactional-stream/state-store.ts @@ -10,7 +10,7 @@ */ import * as Context from "effect/Context" import type * as Effect from "effect/Effect" -import type { BlockRange } from "../models.ts" +import type { BlockRange } from "../core/domain.ts" import type { StateStoreError } from "./errors.ts" import type { TransactionId } from "./types.ts" diff --git a/packages/amp/src/transactional-stream/stream.ts b/packages/amp/src/transactional-stream/stream.ts index 85b2048..0f3c590 100644 --- a/packages/amp/src/transactional-stream/stream.ts +++ b/packages/amp/src/transactional-stream/stream.ts @@ -14,7 +14,7 @@ import * as Effect from "effect/Effect" import * as Layer from "effect/Layer" import * as Stream from "effect/Stream" import type { QueryOptions } from "../arrow-flight.ts" -import type { BlockRange } from "../models.ts" +import type { BlockRange } from "../core/domain.ts" import type { ProtocolStreamError } from "../protocol-stream/errors.ts" import { ProtocolStream, type ProtocolStreamOptions } from "../protocol-stream/service.ts" import type { CommitHandle } from "./commit-handle.ts" diff --git a/packages/amp/src/transactional-stream/types.ts b/packages/amp/src/transactional-stream/types.ts index 3c87854..d9a2b0a 100644 --- a/packages/amp/src/transactional-stream/types.ts +++ b/packages/amp/src/transactional-stream/types.ts @@ -5,7 +5,7 @@ */ import * as Option from "effect/Option" import * as Schema from "effect/Schema" -import { BlockRange } from "../models.ts" +import { BlockRange } from "../core/domain.ts" import { InvalidationRange } from "../protocol-stream/messages.ts" // ============================================================================= diff --git a/packages/amp/test/protocol-stream/reorg.test.ts b/packages/amp/test/protocol-stream/reorg.test.ts index dacd6de..f9c8479 100644 --- a/packages/amp/test/protocol-stream/reorg.test.ts +++ b/packages/amp/test/protocol-stream/reorg.test.ts @@ -15,7 +15,7 @@ * - Multi-network partial reorg detection * - Watermark and data message generation */ -import type { BlockHash, BlockNumber, BlockRange, Network } from "@edgeandnode/amp/models" +import type { BlockHash, BlockNumber, BlockRange, Network } from "@edgeandnode/amp/core" import { data, invalidates, makeInvalidationRange, reorg, watermark } from "@edgeandnode/amp/protocol-stream" import { describe, it } from "@effect/vitest" import * as Effect from "effect/Effect" diff --git a/packages/amp/test/protocol-stream/validation.test.ts b/packages/amp/test/protocol-stream/validation.test.ts index fa1279e..b09a7a8 100644 --- a/packages/amp/test/protocol-stream/validation.test.ts +++ b/packages/amp/test/protocol-stream/validation.test.ts @@ -4,7 +4,7 @@ * Tests for the validation functions used by the ProtocolStream to ensure * protocol invariants are maintained. */ -import type { BlockHash, BlockNumber, BlockRange, Network } from "@edgeandnode/amp/models" +import type { BlockHash, BlockNumber, BlockRange, Network } from "@edgeandnode/amp/core" import { DuplicateNetworkError, GapError, diff --git a/packages/amp/test/transactional-stream/algorithms.test.ts b/packages/amp/test/transactional-stream/algorithms.test.ts index 638c9ec..eb355ff 100644 --- a/packages/amp/test/transactional-stream/algorithms.test.ts +++ b/packages/amp/test/transactional-stream/algorithms.test.ts @@ -3,7 +3,7 @@ * * @module */ -import type { BlockRange } from "@edgeandnode/amp/models" +import type { BlockRange } from "@edgeandnode/amp/core" import type { InvalidationRange } from "@edgeandnode/amp/protocol-stream" import { checkPartialReorg, diff --git a/packages/amp/test/transactional-stream/memory-store.test.ts b/packages/amp/test/transactional-stream/memory-store.test.ts index a4a6732..3795928 100644 --- a/packages/amp/test/transactional-stream/memory-store.test.ts +++ b/packages/amp/test/transactional-stream/memory-store.test.ts @@ -3,7 +3,7 @@ * * @module */ -import type { BlockRange } from "@edgeandnode/amp/models" +import type { BlockRange } from "@edgeandnode/amp/core" import { emptySnapshot, InMemoryStateStore, diff --git a/packages/amp/test/transactional-stream/state-actor.test.ts b/packages/amp/test/transactional-stream/state-actor.test.ts index cfe2ae7..ac447a7 100644 --- a/packages/amp/test/transactional-stream/state-actor.test.ts +++ b/packages/amp/test/transactional-stream/state-actor.test.ts @@ -3,7 +3,7 @@ * * @module */ -import type { BlockNumber, BlockRange, Network } from "@edgeandnode/amp/models" +import type { BlockNumber, BlockRange, Network } from "@edgeandnode/amp/core" import type { ProtocolMessage } from "@edgeandnode/amp/protocol-stream" import { InMemoryStateStore, StateStore, type TransactionId } from "@edgeandnode/amp/transactional-stream" import { makeStateActor } from "@edgeandnode/amp/transactional-stream/state-actor" diff --git a/packages/cli/src/commands/auth/token.ts b/packages/cli/src/commands/auth/token.ts index ff17f64..e1d65a9 100644 --- a/packages/cli/src/commands/auth/token.ts +++ b/packages/cli/src/commands/auth/token.ts @@ -1,5 +1,5 @@ import * as Auth from "@edgeandnode/amp/auth/service" -import * as Models from "@edgeandnode/amp/models" +import * as Models from "@edgeandnode/amp/core" import * as Args from "@effect/cli/Args" import * as Command from "@effect/cli/Command" import * as Options from "@effect/cli/Options" From 3634f321f648424599d3d4b750701412a02d673a Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 6 Feb 2026 20:53:08 -0500 Subject: [PATCH 10/12] add rulesync for managing coding agent settigs --- .gitignore | 80 + .rulesync/.aiignore | 1 + .rulesync/commands/review-pr.md | 17 + .rulesync/rules/effect.md | 134 ++ .rulesync/rules/overview.md | 64 + .rulesync/skills/project-context/SKILL.md | 9 + .rulesync/subagents/planner.md | 16 + package.json | 1 + pnpm-lock.yaml | 1829 ++++++++++++++++++++- rulesync.jsonc | 25 + specs/transactional-stream.md | 77 +- 11 files changed, 2188 insertions(+), 65 deletions(-) create mode 100644 .rulesync/.aiignore create mode 100644 .rulesync/commands/review-pr.md create mode 100644 .rulesync/rules/effect.md create mode 100644 .rulesync/rules/overview.md create mode 100644 .rulesync/skills/project-context/SKILL.md create mode 100644 .rulesync/subagents/planner.md create mode 100644 rulesync.jsonc diff --git a/.gitignore b/.gitignore index bdaa44a..8c572a9 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,83 @@ cache/ /infra/amp/providers/*.toml /infra/amp/anvil.json !/infra/amp/providers/anvil.toml + +# Generated by Rulesync +**/AGENTS.md +**/.agents/ +**/.agent/rules/ +**/.agent/skills/ +**/.agent/workflows/ +**/.augmentignore +**/.augment/rules/ +**/.augment-guidelines +**/CLAUDE.md +**/CLAUDE.local.md +**/.claude/CLAUDE.md +**/.claude/CLAUDE.local.md +**/.claude/memories/ +**/.claude/rules/ +**/.claude/commands/ +**/.claude/agents/ +**/.claude/skills/ +**/.claude/settings.local.json +**/.mcp.json +**/.clinerules/ +**/.clinerules/workflows/ +**/.clineignore +**/.cline/mcp.json +**/.codexignore +**/.codex/memories/ +**/.codex/skills/ +**/.codex/subagents/ +**/.cursor/ +**/.cursorignore +**/.factorydroid/AGENTS.md +**/.factorydroid/memories/ +**/.factorydroid/commands/ +**/.factorydroid/droids/ +**/.factorydroid/skills/ +**/.factorydroid/mcp.json +**/GEMINI.md +**/.gemini/memories/ +**/.gemini/commands/ +**/.gemini/subagents/ +**/.gemini/skills/ +**/.geminiignore +**/.github/copilot-instructions.md +**/.github/instructions/ +**/.github/prompts/ +**/.github/agents/ +**/.github/skills/ +**/.vscode/mcp.json +**/.junie/guidelines.md +**/.junie/mcp.json +**/.kilocode/rules/ +**/.kilocode/skills/ +**/.kilocode/workflows/ +**/.kilocode/mcp.json +**/.kilocodeignore +**/.kiro/steering/ +**/.kiro/prompts/ +**/.kiro/skills/ +**/.kiro/agents/ +**/.kiro/settings/mcp.json +**/.aiignore +**/.opencode/memories/ +**/.opencode/command/ +**/.opencode/agent/ +**/.opencode/skill/ +**/QWEN.md +**/.qwen/memories/ +**/replit.md +**/.roo/rules/ +**/.roo/skills/ +**/.rooignore +**/.roo/mcp.json +**/.roo/subagents/ +**/.warp/ +**/WARP.md +**/modular-mcp.json +.rulesync/rules/*.local.md +rulesync.local.jsonc +!.rulesync/.aiignore diff --git a/.rulesync/.aiignore b/.rulesync/.aiignore new file mode 100644 index 0000000..aa29d08 --- /dev/null +++ b/.rulesync/.aiignore @@ -0,0 +1 @@ +credentials/ diff --git a/.rulesync/commands/review-pr.md b/.rulesync/commands/review-pr.md new file mode 100644 index 0000000..a4950cf --- /dev/null +++ b/.rulesync/commands/review-pr.md @@ -0,0 +1,17 @@ +--- +description: 'Review a pull request' +targets: ["*"] +--- + +target_pr = $ARGUMENTS + +If target_pr is not provided, use the PR of the current branch. + +Execute the following in parallel: + +1. Check code quality and style consistency +2. Review test coverage +3. Verify documentation updates +4. Check for potential bugs or security issues + +Then provide a summary of findings and suggestions for improvement. diff --git a/.rulesync/rules/effect.md b/.rulesync/rules/effect.md new file mode 100644 index 0000000..c122d2d --- /dev/null +++ b/.rulesync/rules/effect.md @@ -0,0 +1,134 @@ +--- +root: false +targets: ["*"] +description: "When you write any code using Effect, you must follow these standards." +globs: ["**/*.ts"] +--- + +# Effect Standards + +This codebase uses the [Effect](https://effect.website) library throughout. All new code must follow these patterns. + +## Imports + +Always import Effect modules as namespaces from subpaths: + +```typescript +// Good +import * as Effect from "effect/Effect" +import * as Stream from "effect/Stream" +import * as Schema from "effect/Schema" + +// Bad — never import from the barrel +import { Effect, Stream, Schema } from "effect" +``` + +## Services + +Define services with `Context.Tag`: + +```typescript +class MyService extends Context.Tag("Amp/MyService") Effect.Effect +}>() {} +``` + +Access services in generators with `yield*`: + +```typescript +Effect.gen(function*() { + const svc = yield* MyService + return yield* svc.doSomething("input") +}) +``` + +## Layers + +### Naming: exported layers must start with `layer` + +```typescript +// Good +export const layer: Layer.Layer = Layer.effect(MyService, make) +export const layerWithState = (initial: StateSnapshot): Layer.Layer => ... +export const layerTest = (options: TestOptions): Layer.Layer => ... + +// Bad +export const live: Layer.Layer = ... +export const testLayer: Layer.Layer = ... +``` + +### Compose layers ahead of time — never chain `Effect.provide` + +Multiple `Effect.provide` calls each wrap the effect in another layer of indirection. Compose into a single layer and provide once: + +```typescript +// Bad — repeated provide calls +Effect.runPromise(program.pipe( + Effect.provide(TransactionalStream.layer), + Effect.provide(InMemoryStateStore.layer), + Effect.provide(ProtocolStream.layer), + Effect.provide(ArrowFlight.layer), + Effect.provide(Transport.layer) +)) + +// Good — compose first, provide once +const AppLayer = TransactionalStream.layer.pipe( + Layer.provide(InMemoryStateStore.layer), + Layer.provide(ProtocolStream.layer), + Layer.provide(ArrowFlight.layer), + Layer.provide(Transport.layer) +) + +Effect.runPromise(program.pipe(Effect.provide(AppLayer))) +``` + +## Use `Effect.fn` instead of arrow functions returning `Effect.gen` + +Wrapping `Effect.gen` in an arrow function allocates a new generator on every call. `Effect.fn` avoids this overhead and automatically adds a tracing span. + +```typescript +// Bad — allocates a generator per invocation +const query = (sql: string) => + Effect.gen(function*() { + const svc = yield* ArrowFlight + return yield* svc.query(sql) + }) + +// Good +const query = Effect.fn("query")(function*(sql: string) { + const svc = yield* ArrowFlight + return yield* svc.query(sql) +}) + +// With explicit return type annotation +const query = Effect.fn("query")(function*(sql: string): Effect.fn.Return< + QueryResult, + ArrowFlightError +> { + const svc = yield* ArrowFlight + return yield* svc.query(sql) +}) +``` + +## Schemas and Branded Types + +Use `Schema.brand` for domain primitives: + +```typescript +export const Network = Schema.Lowercase.pipe( + Schema.brand("Amp/Models/Network") +).annotations({ identifier: "Network" }) +export type Network = typeof Network.Type +``` + +## Tagged Errors + +Use `Schema.TaggedError` for typed, serializable errors: + +```typescript +export class MyError extends Schema.TaggedError( + "Amp/MyError" +)("MyError", { + cause: Schema.Defect +}) {} +``` diff --git a/.rulesync/rules/overview.md b/.rulesync/rules/overview.md new file mode 100644 index 0000000..b5669b4 --- /dev/null +++ b/.rulesync/rules/overview.md @@ -0,0 +1,64 @@ +--- +root: true +targets: ["*"] +description: "Project overview, architecture, and development guidelines" +globs: ["**/*"] +--- + +# Amp TypeScript SDK + +## What Is This + +A pnpm workspace monorepo for the **Amp TypeScript SDK** — a toolkit for building and managing blockchain datasets, built on the [Effect](https://effect.website) library. + +| Package | Path | Description | +| ---------------------- | --------------------- | ------------------------------------------------- | +| `@edgeandnode/amp` | `packages/amp/` | Core SDK | +| `@edgeandnode/amp-cli` | `packages/cli/` | CLI tool (`amp` command) via `@effect/cli` | +| `@amp/oxc` | `packages/tools/oxc/` | Custom oxlint rules for Effect import conventions | + +## Core SDK Module Map (`packages/amp/src/`) + +``` +src/ +├── index.ts # Package entrypoint (namespace re-exports) +├── core.ts → core/ # Domain models: branded types, schemas (BlockRange, Network, AuthInfo, Dataset*, etc.) +├── arrow-flight.ts → arrow-flight/ # Arrow Flight SQL client (Transport, ArrowFlight service, errors) +├── protocol-stream.ts → protocol-stream/ # Stateless reorg detection on top of ArrowFlight streams +├── transactional-stream.ts → transactional-stream/ # Exactly-once semantics, crash recovery, commit control +├── auth/ # OAuth2 auth (device flow, token refresh, caching) +├── admin/ # Admin API (datasets, jobs, workers, providers) +├── registry/ # Registry API (dataset discovery) +├── manifest-builder/ # Dataset manifest construction +├── protobuf/ # Generated protobuf (Flight, FlightSql) +└── internal/ # Private: Arrow IPC parsing, PKCE +``` + +Each public module has a **root-level barrel `.ts` file** in `src/` that re-exports from its subdirectory. No `index.ts` barrel files in subdirectories. `src/internal/` is blocked from external import. + +## Commands + +```bash +pnpm check # Type check root project +pnpm check:recursive # Type check all packages +pnpm lint # Check with oxlint + dprint +pnpm lint:fix # Auto-fix with oxlint + dprint +pnpm test # Run all tests (vitest) +pnpm vitest run # Single test file +pnpm build # Full build (tsc + babel) +``` + +## Code Style + +- **Formatter**: dprint — no semicolons (ASI), double quotes, no trailing commas, 120 char line width +- **Linter**: oxlint with custom `@amp/oxc` plugin +- **Array syntax**: `Array` and `ReadonlyArray`, never `T[]` (enforced by `typescript/array-type`) +- **Effect imports**: Always namespace imports from subpaths — `import * as Effect from "effect/Effect"`, never `from "effect"` +- **Type imports**: Inline — `import { type Foo, bar } from "..."` (enforced by `typescript/consistent-type-imports`) +- **TypeScript strictness**: `strict`, `exactOptionalPropertyTypes`, `noUnusedLocals`, `verbatimModuleSyntax` +- **File extensions**: `.ts` in all imports (rewritten to `.js` at build time) +- **Unused variables**: Prefix with `_` + +## Agent Rules + +All coding agent rules are managed via [rulesync](https://github.com/dyoshikawa/rulesync) in `.rulesync/rules/`. Edit rules there, then run `pnpm exec rulesync generate` to regenerate agent-specific config files. Do not edit generated files (e.g. `.claude/rules/`, `.opencode/rules/`) directly. diff --git a/.rulesync/skills/project-context/SKILL.md b/.rulesync/skills/project-context/SKILL.md new file mode 100644 index 0000000..7be0077 --- /dev/null +++ b/.rulesync/skills/project-context/SKILL.md @@ -0,0 +1,9 @@ +--- +name: project-context +description: "Summarize the project context and key constraints" +targets: ["*"] +--- + +Summarize the project goals, core constraints, and relevant dependencies. +Call out any architecture decisions, shared conventions, and validation steps. +Keep the summary concise and ready to reuse in future tasks. \ No newline at end of file diff --git a/.rulesync/subagents/planner.md b/.rulesync/subagents/planner.md new file mode 100644 index 0000000..1200ba1 --- /dev/null +++ b/.rulesync/subagents/planner.md @@ -0,0 +1,16 @@ +--- +name: planner +targets: ["*"] +description: >- + This is the general-purpose planner. The user asks the agent to plan to + suggest a specification, implement a new feature, refactor the codebase, or + fix a bug. This agent can be called by the user explicitly only. +claudecode: + model: inherit +--- + +You are the planner for any tasks. + +Based on the user's instruction, create a plan while analyzing the related files. Then, report the plan in detail. You can output files to @tmp/ if needed. + +Attention, again, you are just the planner, so though you can read any files and run any commands for analysis, please don't write any code. diff --git a/package.json b/package.json index f96f7cf..2dc9f27 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "globals": "^17.2.0", "madge": "^8.0.0", "oxlint": "^1.42.0", + "rulesync": "^6.7.0", "ts-patch": "^3.3.0", "typescript": "^5.9.3", "vite-tsconfig-paths": "^6.0.5", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 65ae904..fb67c45 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -71,6 +71,9 @@ importers: oxlint: specifier: ^1.42.0 version: 1.42.0 + rulesync: + specifier: ^6.7.0 + version: 6.7.0(valibot@1.2.0(typescript@5.9.3)) ts-patch: specifier: ^3.3.0 version: 3.3.0 @@ -97,7 +100,7 @@ importers: version: 6.1.3 viem: specifier: ^2.45.1 - version: 2.45.1(typescript@5.9.3) + version: 2.45.1(typescript@5.9.3)(zod@4.3.6) devDependencies: '@bufbuild/buf': specifier: ^1.64.0 @@ -260,6 +263,9 @@ packages: resolution: {integrity: sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==} engines: {node: '>=18'} + '@borewit/text-codec@0.2.1': + resolution: {integrity: sha512-k7vvKPbf7J2fZ5klGRD9AeKfUvojuZIQ3BT5u7Jfv+puwXkUBUT5PVyMDfJZpy30CBDXGMgw7fguK/lpOMBvgw==} + '@bufbuild/buf-darwin-arm64@1.64.0': resolution: {integrity: sha512-wv4WJ/ho9Jt4b/3SuwrzCa7r68yvOSMhnpztPivu75CMF+btZijWYjc/Wt6JkOYbGmsXAlSzyANXZaQM9KCm8g==} engines: {node: '>=12'} @@ -651,6 +657,12 @@ packages: cpu: [x64] os: [win32] + '@hono/node-server@1.19.9': + resolution: {integrity: sha512-vHL6w3ecZsky+8P5MD+eFfaGTyCeOHUIFYMGpQGbrBTSmNNoxv0if69rEZ5giu36weC5saFuznL411gRX7bJDw==} + engines: {node: '>=18.14.1'} + peerDependencies: + hono: ^4 + '@isaacs/balanced-match@4.0.1': resolution: {integrity: sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==} engines: {node: 20 || >=22} @@ -675,6 +687,16 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@modelcontextprotocol/sdk@1.26.0': + resolution: {integrity: sha512-Y5RmPncpiDtTXDbLKswIJzTqu2hyBKxTNsgKqKclDbhIgg1wgtf1fRuvxgTnRfcnxtvvgbIEcqUOzZrJ6iSReg==} + engines: {node: '>=18'} + peerDependencies: + '@cfworker/json-schema': ^4.1.1 + zod: ^3.25 || ^4.0 + peerDependenciesMeta: + '@cfworker/json-schema': + optional: true + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': resolution: {integrity: sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==} cpu: [arm64] @@ -720,6 +742,70 @@ packages: resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==} engines: {node: ^14.21.3 || >=16} + '@nodelib/fs.scandir@2.1.5': + resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} + engines: {node: '>= 8'} + + '@nodelib/fs.stat@2.0.5': + resolution: {integrity: sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==} + engines: {node: '>= 8'} + + '@nodelib/fs.walk@1.2.8': + resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} + engines: {node: '>= 8'} + + '@octokit/auth-token@6.0.0': + resolution: {integrity: sha512-P4YJBPdPSpWTQ1NU4XYdvHvXJJDxM6YwpS0FZHRgP7YFkdVxsWcpWGy/NVqlAA7PcPCnMacXlRm1y2PFZRWL/w==} + engines: {node: '>= 20'} + + '@octokit/core@7.0.6': + resolution: {integrity: sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q==} + engines: {node: '>= 20'} + + '@octokit/endpoint@11.0.2': + resolution: {integrity: sha512-4zCpzP1fWc7QlqunZ5bSEjxc6yLAlRTnDwKtgXfcI/FxxGoqedDG8V2+xJ60bV2kODqcGB+nATdtap/XYq2NZQ==} + engines: {node: '>= 20'} + + '@octokit/graphql@9.0.3': + resolution: {integrity: sha512-grAEuupr/C1rALFnXTv6ZQhFuL1D8G5y8CN04RgrO4FIPMrtm+mcZzFG7dcBm+nq+1ppNixu+Jd78aeJOYxlGA==} + engines: {node: '>= 20'} + + '@octokit/openapi-types@27.0.0': + resolution: {integrity: sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==} + + '@octokit/plugin-paginate-rest@14.0.0': + resolution: {integrity: sha512-fNVRE7ufJiAA3XUrha2omTA39M6IXIc6GIZLvlbsm8QOQCYvpq/LkMNGyFlB1d8hTDzsAXa3OKtybdMAYsV/fw==} + engines: {node: '>= 20'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/plugin-request-log@6.0.0': + resolution: {integrity: sha512-UkOzeEN3W91/eBq9sPZNQ7sUBvYCqYbrrD8gTbBuGtHEuycE4/awMXcYvx6sVYo7LypPhmQwwpUe4Yyu4QZN5Q==} + engines: {node: '>= 20'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/plugin-rest-endpoint-methods@17.0.0': + resolution: {integrity: sha512-B5yCyIlOJFPqUUeiD0cnBJwWJO8lkJs5d8+ze9QDP6SvfiXSz1BF+91+0MeI1d2yxgOhU/O+CvtiZ9jSkHhFAw==} + engines: {node: '>= 20'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/request-error@7.1.0': + resolution: {integrity: sha512-KMQIfq5sOPpkQYajXHwnhjCC0slzCNScLHs9JafXc4RAJI+9f+jNDlBNaIMTvazOPLgb4BnlhGJOTbnN0wIjPw==} + engines: {node: '>= 20'} + + '@octokit/request@10.0.7': + resolution: {integrity: sha512-v93h0i1yu4idj8qFPZwjehoJx4j3Ntn+JhXsdJrG9pYaX6j/XRz2RmasMUHtNgQD39nrv/VwTWSqK0RNXR8upA==} + engines: {node: '>= 20'} + + '@octokit/rest@22.0.1': + resolution: {integrity: sha512-Jzbhzl3CEexhnivb1iQ0KJ7s5vvjMWcmRtq5aUsKmKDrRW6z3r84ngmiFKFvpZjpiU/9/S6ITPFRpn5s/3uQJw==} + engines: {node: '>= 20'} + + '@octokit/types@16.0.0': + resolution: {integrity: sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==} + '@oxlint/darwin-arm64@1.42.0': resolution: {integrity: sha512-ui5CdAcDsXPQwZQEXOOSWsilJWhgj9jqHCvYBm2tDE8zfwZZuF9q58+hGKH1x5y0SV4sRlyobB2Quq6uU6EgeA==} cpu: [arm64] @@ -979,9 +1065,26 @@ packages: '@scure/bip39@1.6.0': resolution: {integrity: sha512-+lF0BbLiJNwVlev4eKelw1WWLaiKXw7sSl8T6FvBlWkdX+94aGJ4o8XjUdlyhTCjd8c+B3KT3JfS8P0bLRNU6A==} + '@sec-ant/readable-stream@0.4.1': + resolution: {integrity: sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==} + + '@sindresorhus/merge-streams@4.0.0': + resolution: {integrity: sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==} + engines: {node: '>=18'} + '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} + '@tokenizer/inflate@0.4.1': + resolution: {integrity: sha512-2mAv+8pkG6GIZiF1kNg1jAjh27IDxEPKwdGul3snfztFerfPGI1LjDezZp3i7BElXompqEtPmoPx6c2wgtWsOA==} + engines: {node: '>=18'} + + '@tokenizer/token@0.3.0': + resolution: {integrity: sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==} + + '@toon-format/toon@2.1.0': + resolution: {integrity: sha512-JwWptdF5eOA0HaQxbKAzkpQtR4wSWTEfDlEy/y3/4okmOAX1qwnpLZMmtEWr+ncAhTTY1raCKH0kteHhSXnQqg==} + '@ts-graphviz/adapter@2.0.6': resolution: {integrity: sha512-kJ10lIMSWMJkLkkCG5gt927SnGZcBuG0s0HHswGzcHTgvtUe7yk5/3zTEr0bafzsodsOq5Gi6FhQeV775nC35Q==} engines: {node: '>=18'} @@ -1113,6 +1216,11 @@ packages: peerDependencies: typescript: ^5.9.3 + '@valibot/to-json-schema@1.5.0': + resolution: {integrity: sha512-GE7DmSr1C2UCWPiV0upRH6mv0cCPsqYGs819fb6srCS1tWhyXrkGGe+zxUiwzn/L1BOfADH4sNjY/YHCuP8phQ==} + peerDependencies: + valibot: ^1.2.0 + '@vitest/coverage-v8@4.0.18': resolution: {integrity: sha512-7i+N2i0+ME+2JFZhfuz7Tg/FqKtilHjGyGvoHYQ6iLV0zahbsJ9sljC9OcFcPDbhYKCet+sG8SsVqlyGvPflZg==} peerDependencies: @@ -1182,14 +1290,41 @@ packages: zod: optional: true + accepts@1.3.8: + resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} + engines: {node: '>= 0.6'} + + accepts@2.0.0: + resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} + engines: {node: '>= 0.6'} + + ajv-formats@3.0.1: + resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==} + peerDependencies: + ajv: ^8.0.0 + peerDependenciesMeta: + ajv: + optional: true + + ajv@8.17.1: + resolution: {integrity: sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==} + ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} + ansi-regex@6.2.2: + resolution: {integrity: sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==} + engines: {node: '>=12'} + ansi-styles@4.3.0: resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} engines: {node: '>=8'} + ansi-styles@6.2.3: + resolution: {integrity: sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==} + engines: {node: '>=12'} + any-promise@1.3.0: resolution: {integrity: sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==} @@ -1200,6 +1335,12 @@ packages: app-module-path@2.2.0: resolution: {integrity: sha512-gkco+qxENJV+8vFcDiiFhuoSvRXb2a/QPqpSoWhVz829VNJfOTnELbBmPmNKFxf3xdNnw4DWCkzkDaavcX/1YQ==} + argparse@1.0.10: + resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} + + argparse@2.0.1: + resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + assertion-error@2.0.1: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} @@ -1211,6 +1352,12 @@ packages: ast-v8-to-istanbul@0.3.10: resolution: {integrity: sha512-p4K7vMz2ZSk3wN8l5o3y2bJAoZXT3VuJI5OLTATY/01CYWumWvwkUw0SqDBnNq6IiTO3qDa1eSQDibAV8g7XOQ==} + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + + axios@1.13.4: + resolution: {integrity: sha512-1wVkUaAO6WyaYtCkcYCOx12ZgpGf9Zif+qXa4n+oYzK558YryKqiL6UWwd5DqiH3VRW0GYhTZQ/vlgJrCoNQlg==} + babel-plugin-annotate-pure-calls@0.5.0: resolution: {integrity: sha512-bWlaZl2qsJKHv9BJgF1g6bQ04wK/7Topq9g59I795W9jpx9lNs9N5LK5NjlEBCDhHncrO0vcoAMuHJR3oWnTeQ==} engines: {node: '>=18'} @@ -1227,6 +1374,9 @@ packages: resolution: {integrity: sha512-agD0MgJFUP/4nvjqzIB29zRPUuCF7Ge6mEv9s8dHrtYD7QWXRcx75rOADE/d5ah1NI+0vkDl0yorDd5U852IQQ==} hasBin: true + before-after-hook@4.0.0: + resolution: {integrity: sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ==} + binary-extensions@2.3.0: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} engines: {node: '>=8'} @@ -1234,6 +1384,10 @@ packages: bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} + body-parser@2.2.2: + resolution: {integrity: sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==} + engines: {node: '>=18'} + brace-expansion@1.1.12: resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==} @@ -1256,6 +1410,18 @@ packages: resolution: {integrity: sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==} engines: {node: '>=18'} + bytes@3.1.2: + resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} + engines: {node: '>= 0.8'} + + call-bind-apply-helpers@1.0.2: + resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} + engines: {node: '>= 0.4'} + + call-bound@1.0.4: + resolution: {integrity: sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==} + engines: {node: '>= 0.4'} + caniuse-lite@1.0.30001766: resolution: {integrity: sha512-4C0lfJ0/YPjJQHagaE9x2Elb69CIqEPZeG0anQt9SIvIoOH4a4uaRl73IavyO+0qZh6MDLH//DrXThEYKHkmYA==} @@ -1279,6 +1445,10 @@ packages: resolution: {integrity: sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==} engines: {node: '>=6'} + cliui@9.0.1: + resolution: {integrity: sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==} + engines: {node: '>=20'} + clone@1.0.4: resolution: {integrity: sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==} engines: {node: '>=0.8'} @@ -1290,10 +1460,18 @@ packages: color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + commander@12.1.0: resolution: {integrity: sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==} engines: {node: '>=18'} + commander@14.0.3: + resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==} + engines: {node: '>=20'} + commander@6.2.1: resolution: {integrity: sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==} engines: {node: '>= 6'} @@ -1308,9 +1486,45 @@ packages: concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} + consola@3.4.2: + resolution: {integrity: sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==} + engines: {node: ^14.18.0 || >=16.10.0} + + content-disposition@0.5.4: + resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} + engines: {node: '>= 0.6'} + + content-disposition@1.0.1: + resolution: {integrity: sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q==} + engines: {node: '>=18'} + + content-type@1.0.5: + resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} + engines: {node: '>= 0.6'} + convert-source-map@2.0.0: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + cookie-signature@1.2.2: + resolution: {integrity: sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==} + engines: {node: '>=6.6.0'} + + cookie@0.7.2: + resolution: {integrity: sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==} + engines: {node: '>= 0.6'} + + cookies@0.9.1: + resolution: {integrity: sha512-TG2hpqe4ELx54QER/S3HQ9SRVnQnGBtKUz5bLQWtYAQ+o6GpgMs6sYUvaiJjVxb+UXwhRhAEP3m7LbsIZ77Hmw==} + engines: {node: '>= 0.8'} + + cors@2.8.6: + resolution: {integrity: sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==} + engines: {node: '>= 0.10'} + + cross-spawn@7.0.6: + resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} + engines: {node: '>= 8'} + debug@4.4.3: resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} engines: {node: '>=6.0'} @@ -1320,6 +1534,9 @@ packages: supports-color: optional: true + deep-equal@1.0.1: + resolution: {integrity: sha512-bHtC0iYvWhyaTzvV3CZgPeZQqCOBGyGsVV7v4eevpdkLHfiSrXUdBG+qAuSz4RI70sszvjQ1QSZ98An1yNwpSw==} + deep-extend@0.6.0: resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} engines: {node: '>=4.0.0'} @@ -1339,11 +1556,30 @@ packages: resolution: {integrity: sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==} engines: {node: '>=12'} + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + + delegates@1.0.0: + resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} + + depd@1.1.2: + resolution: {integrity: sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==} + engines: {node: '>= 0.6'} + + depd@2.0.0: + resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} + engines: {node: '>= 0.8'} + dependency-tree@11.2.0: resolution: {integrity: sha512-+C1H3mXhcvMCeu5i2Jpg9dc0N29TWTuT6vJD7mHLAfVmAbo9zW8NlkvQ1tYd3PDMab0IRQM0ccoyX68EZtx9xw==} engines: {node: '>=18'} hasBin: true + destroy@1.2.0: + resolution: {integrity: sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==} + engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + detect-libc@2.1.2: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} @@ -1395,12 +1631,32 @@ packages: resolution: {integrity: sha512-CEx+wYARxLAe9o7RCZ77GKae6DF7qjn5Rd98xbWdA3hB4PFBr+kHwLANmNHscNumBAIrCg5ZJj/Kz+OYbJ+GBA==} hasBin: true + dunder-proto@1.0.1: + resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} + engines: {node: '>= 0.4'} + + ee-first@1.1.1: + resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} + effect@3.19.15: resolution: {integrity: sha512-vzMmgfZKLcojmUjBdlQx+uaKryO7yULlRxjpDnHdnvcp1NPHxJyoM6IOXBLlzz2I/uPtZpGKavt5hBv7IvGZkA==} + effect@3.19.16: + resolution: {integrity: sha512-7+XC3vGrbAhCHd8LTFHvnZjRpZKZ8YHRZqJTkpNoxcJ2mCyNs2SwI+6VkV/ij8Y3YW7wfBN4EbU06/F5+m/wkQ==} + electron-to-chromium@1.5.278: resolution: {integrity: sha512-dQ0tM1svDRQOwxnXxm+twlGTjr9Upvt8UFWAgmLsxEzFQxhbti4VwxmMjsDxVC51Zo84swW7FVCXEV+VAkhuPw==} + emoji-regex@10.6.0: + resolution: {integrity: sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==} + + encodeurl@2.0.0: + resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} + engines: {node: '>= 0.8'} + + end-of-stream@1.4.5: + resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} + enhanced-resolve@5.18.4: resolution: {integrity: sha512-LgQMM4WXU3QI+SYgEc2liRgznaD5ojbmY3sb8LxyguVkIg5FxdpTkvk72te2R38/TGKxH634oLxXRGY6d7AP+Q==} engines: {node: '>=10.13.0'} @@ -1409,9 +1665,28 @@ packages: resolution: {integrity: sha512-FDWG5cmEYf2Z00IkYRhbFrwIwvdFKH07uV8dvNy0omp/Qb1xcyCWp2UDtcwJF4QZZvk0sLudP6/hAu42TaqVhQ==} engines: {node: '>=0.12'} + es-define-property@1.0.1: + resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} + engines: {node: '>= 0.4'} + + es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + es-module-lexer@1.7.0: resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==} + es-object-atoms@1.1.1: + resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} + engines: {node: '>= 0.4'} + + es-set-tostringtag@2.1.0: + resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} + engines: {node: '>= 0.4'} + + es-toolkit@1.44.0: + resolution: {integrity: sha512-6penXeZalaV88MM3cGkFZZfOoLGWshWWfdy0tWw/RlVVyhvMaWSBTOvXNeiW3e5FwdS5ePW0LGEu17zT139ktg==} + esbuild@0.27.2: resolution: {integrity: sha512-HyNQImnsOC7X9PMNaCIeAm4ISCQXs5a5YasTXVliKv4uuBo1dKrG0A+uQS8M5eXjVMnLg3WgXaKvprHlFJQffw==} engines: {node: '>=18'} @@ -1421,6 +1696,9 @@ packages: resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} engines: {node: '>=6'} + escape-html@1.0.3: + resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} + escodegen@2.1.0: resolution: {integrity: sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==} engines: {node: '>=6.0'} @@ -1449,17 +1727,72 @@ packages: resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} engines: {node: '>=0.10.0'} + etag@1.8.1: + resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} + engines: {node: '>= 0.6'} + eventemitter3@5.0.1: resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==} + eventsource-parser@3.0.6: + resolution: {integrity: sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==} + engines: {node: '>=18.0.0'} + + eventsource@3.0.7: + resolution: {integrity: sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==} + engines: {node: '>=18.0.0'} + + execa@9.6.1: + resolution: {integrity: sha512-9Be3ZoN4LmYR90tUoVu2te2BsbzHfhJyfEiAVfz7N5/zv+jduIfLrV2xdQXOHbaD6KgpGdO9PRPM1Y4Q9QkPkA==} + engines: {node: ^18.19.0 || >=20.5.0} + expect-type@1.3.0: resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} engines: {node: '>=12.0.0'} + express-rate-limit@8.2.1: + resolution: {integrity: sha512-PCZEIEIxqwhzw4KF0n7QF4QqruVTcF73O5kFKUnGOyjbCCgizBBiFaYpd/fnBLUMPw/BWw9OsiN7GgrNYr7j6g==} + engines: {node: '>= 16'} + peerDependencies: + express: '>= 4.11' + + express@5.2.1: + resolution: {integrity: sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==} + engines: {node: '>= 18'} + + extend-shallow@2.0.1: + resolution: {integrity: sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==} + engines: {node: '>=0.10.0'} + fast-check@3.23.2: resolution: {integrity: sha512-h5+1OzzfCC3Ef7VbtKdcv7zsstUQwUDlYpUTvjeUsJAssPgLn7QzbboPtL5ro04Mq0rPOsMzl7q5hIbRs2wD1A==} engines: {node: '>=8.0.0'} + fast-content-type-parse@3.0.0: + resolution: {integrity: sha512-ZvLdcY8P+N8mGQJahJV5G4U88CSvT1rP8ApL6uETe88MBXrBHAkZlSEySdUlyztF7ccb+Znos3TFqaepHxdhBg==} + + fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + + fast-glob@3.3.3: + resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==} + engines: {node: '>=8.6.0'} + + fast-uri@3.1.0: + resolution: {integrity: sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==} + + fastmcp@3.31.0: + resolution: {integrity: sha512-e2LlF8EmPnwjyAg+JMyXnsMq2ametV3PuSWuhVONP2yZitDXkmDo40TiPQ0qUvU9j7TvdPm5YrjCMsxkkU1XXw==} + hasBin: true + peerDependencies: + jose: ^5.0.0 + peerDependenciesMeta: + jose: + optional: true + + fastq@1.20.1: + resolution: {integrity: sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==} + fdir@6.5.0: resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} engines: {node: '>=12.0.0'} @@ -1472,6 +1805,14 @@ packages: fflate@0.8.2: resolution: {integrity: sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==} + figures@6.1.0: + resolution: {integrity: sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg==} + engines: {node: '>=18'} + + file-type@21.3.0: + resolution: {integrity: sha512-8kPJMIGz1Yt/aPEwOsrR97ZyZaD1Iqm8PClb1nYFclUCkBi0Ma5IsYNQzvSFS9ib51lWyIw5mIT9rWzI/xjpzA==} + engines: {node: '>=20'} + filing-cabinet@5.0.3: resolution: {integrity: sha512-PlPcMwVWg60NQkhvfoxZs4wEHjhlOO/y7OAm4sKM60o1Z9nttRY4mcdQxp/iZ+kg/Vv6Hw1OAaTbYVM9DA9pYg==} engines: {node: '>=18'} @@ -1481,12 +1822,41 @@ packages: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} + finalhandler@2.1.1: + resolution: {integrity: sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==} + engines: {node: '>= 18.0.0'} + find-my-way-ts@0.1.6: resolution: {integrity: sha512-a85L9ZoXtNAey3Y6Z+eBWW658kO/MwR7zIafkIUPUMf3isZG0NCs2pjW2wtjxAKuJPxMAsHUIP4ZPGv0o5gyTA==} flatted@3.3.3: resolution: {integrity: sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==} + follow-redirects@1.15.11: + resolution: {integrity: sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + + form-data@4.0.5: + resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} + engines: {node: '>= 6'} + + forwarded@0.2.0: + resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} + engines: {node: '>= 0.6'} + + fresh@0.5.2: + resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} + engines: {node: '>= 0.6'} + + fresh@2.0.0: + resolution: {integrity: sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==} + engines: {node: '>= 0.8'} + fs-readdir-recursive@1.1.0: resolution: {integrity: sha512-GNanXlVr2pf02+sPN40XN8HG+ePaNcvM0q5mZBd668Obwb0yD5GiUbZOFgwn8kGMY6I3mdyDJzieUy3PTYyTRA==} @@ -1501,6 +1871,10 @@ packages: function-bind@1.1.2: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + fuse.js@7.1.0: + resolution: {integrity: sha512-trLf4SzuuUxfusZADLINj+dE8clK1frKdmqiJNb1Es75fmI5oY6X2mxLVUciLLjxqw/xr72Dhy+lER6dGd02FQ==} + engines: {node: '>=10'} + gensync@1.0.0-beta.2: resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} engines: {node: '>=6.9.0'} @@ -1509,9 +1883,29 @@ packages: resolution: {integrity: sha512-MtjsmYiCXcYDDrGqtNbeIYdAl85n+5mSv2r3FbzER/YV3ZILw4HNNIw34HuV5pyl0jzs6GFYU1VHVEefhgcNHQ==} engines: {node: '>=18'} + get-caller-file@2.0.5: + resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} + engines: {node: 6.* || 8.* || >= 10.*} + + get-east-asian-width@1.4.0: + resolution: {integrity: sha512-QZjmEOC+IT1uk6Rx0sX22V6uHWVwbdbxf1faPqJ1QhLdGgsRGCZoyaQBm/piRdJy/D2um6hM1UP7ZEeQ4EkP+Q==} + engines: {node: '>=18'} + + get-intrinsic@1.3.0: + resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} + engines: {node: '>= 0.4'} + get-own-enumerable-property-symbols@3.0.2: resolution: {integrity: sha512-I0UBV/XOz1XkIJHEUDMZAbzCThU/H8DxmSfmdGcKPnVhu2VfFqr34jr9777IyaTYvxjedWhqVIilEDsCdP5G6g==} + get-proto@1.0.1: + resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} + engines: {node: '>= 0.4'} + + get-stream@9.0.1: + resolution: {integrity: sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==} + engines: {node: '>=18'} + glob-parent@5.1.2: resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} engines: {node: '>= 6'} @@ -1522,7 +1916,7 @@ packages: glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} - deprecated: Glob versions prior to v9 are no longer supported + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me global-prefix@4.0.0: resolution: {integrity: sha512-w0Uf9Y9/nyHinEk5vMJKRie+wa4kR5hmDbEhGGds/kG1PwGLLHKRoNMeJOyCQjjBkANlnScqgzcFwGHgmgLkVA==} @@ -1532,6 +1926,10 @@ packages: resolution: {integrity: sha512-tovnCz/fEq+Ripoq+p/gN1u7l6A7wwkoBT9pRCzTHzsD/LvADIzXZdjmRymh5Ztf0DYC3Rwg5cZRYjxzBmzbWg==} engines: {node: '>=18'} + globby@16.1.0: + resolution: {integrity: sha512-+A4Hq7m7Ze592k9gZRy4gJ27DrXRNnC1vPjxTt1qQxEY8RxagBkBxivkCwg7FxSTG0iLLEMaUx13oOr0R2/qcQ==} + engines: {node: '>=20'} + globrex@0.1.2: resolution: {integrity: sha512-uHJgbwAMwNFf5mLst7IWLNg14x1CkeqglJb/K3doi4dw6q2IvAAmM/Y81kevy83wP+Sst+nutFTYOGg3d1lsxg==} @@ -1540,23 +1938,70 @@ packages: engines: {node: '>=0.6.0'} hasBin: true + gopd@1.2.0: + resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} + engines: {node: '>= 0.4'} + graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + gray-matter@4.0.3: + resolution: {integrity: sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==} + engines: {node: '>=6.0'} + has-flag@4.0.0: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} + has-symbols@1.1.0: + resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} + engines: {node: '>= 0.4'} + + has-tostringtag@1.0.2: + resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==} + engines: {node: '>= 0.4'} + hasown@2.0.2: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} + hono@4.11.8: + resolution: {integrity: sha512-eVkB/CYCCei7K2WElZW9yYQFWssG0DhaDhVvr7wy5jJ22K+ck8fWW0EsLpB0sITUTvPnc97+rrbQqIr5iqiy9Q==} + engines: {node: '>=16.9.0'} + html-escaper@2.0.2: resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==} + http-assert@1.5.0: + resolution: {integrity: sha512-uPpH7OKX4H25hBmU6G1jWNaqJGpTXxey+YOUizJUAgu0AjLUeC8D73hTrhvDS5D+GJN1DN1+hhc/eF/wpxtp0w==} + engines: {node: '>= 0.8'} + + http-errors@1.8.1: + resolution: {integrity: sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==} + engines: {node: '>= 0.6'} + + http-errors@2.0.1: + resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} + engines: {node: '>= 0.8'} + + human-readable-ids@1.0.4: + resolution: {integrity: sha512-h1zwThTims8A/SpqFGWyTx+jG1+WRMJaEeZgbtPGrIpj2AZjsOgy8Y+iNzJ0yAyN669Q6F02EK66WMWcst+2FA==} + + human-signals@8.0.1: + resolution: {integrity: sha512-eKCa6bwnJhvxj14kZk5NCPc6Hb6BdsU9DZcOnmQKSnO1VKrfV0zCvtttPZUsBvjmNDn8rpcJfpwSYnHBjc95MQ==} + engines: {node: '>=18.18.0'} + + iconv-lite@0.7.2: + resolution: {integrity: sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==} + engines: {node: '>=0.10.0'} + ieee754@1.2.1: resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + ignore@7.0.5: + resolution: {integrity: sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==} + engines: {node: '>= 4'} + inflight@1.0.6: resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} deprecated: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful. @@ -1571,6 +2016,14 @@ packages: resolution: {integrity: sha512-X7rqawQBvfdjS10YU1y1YVreA3SsLrW9dX2CewP2EbBJM4ypVNLDkO5y04gejPwKIY9lR+7r9gn3rFPt/kmWFg==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} + ip-address@10.0.1: + resolution: {integrity: sha512-NWv9YLW4PoW2B7xtzaS3NCot75m6nK7Icdv0o3lfMceJVRfSoQwqD4wEH5rLwoKJwUiZ/rfpiVBhnaF0FK4HoA==} + engines: {node: '>= 12'} + + ipaddr.js@1.9.1: + resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} + engines: {node: '>= 0.10'} + is-binary-path@2.1.0: resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==} engines: {node: '>=8'} @@ -1584,6 +2037,10 @@ packages: engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} hasBin: true + is-extendable@0.1.1: + resolution: {integrity: sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==} + engines: {node: '>=0.10.0'} + is-extglob@2.1.1: resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} engines: {node: '>=0.10.0'} @@ -1613,14 +2070,33 @@ packages: resolution: {integrity: sha512-l4RyHgRqGN4Y3+9JHVrNqO+tN0rV5My76uW5/nuO4K1b6vw5G8d/cmFjP9tRfEsdhZNt0IFdZuK/c2Vr4Nb+Qg==} engines: {node: '>=0.10.0'} + is-path-inside@4.0.0: + resolution: {integrity: sha512-lJJV/5dYS+RcL8uQdBDW9c9uWFLLBNRyFhnAKXw5tVqLlKZ4RMGZKv+YQ/IA3OhD+RpbJa1LLFM1FQPGyIXvOA==} + engines: {node: '>=12'} + + is-plain-obj@4.1.0: + resolution: {integrity: sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==} + engines: {node: '>=12'} + + is-promise@4.0.0: + resolution: {integrity: sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==} + is-regexp@1.0.0: resolution: {integrity: sha512-7zjFAPO4/gwyQAAgRRmqeEeyIICSdmCqa3tsVHMdBzaXXRiqopZL4Cyghg/XulGWrtABTpbnYYzzIRffLkP4oA==} engines: {node: '>=0.10.0'} + is-stream@4.0.1: + resolution: {integrity: sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==} + engines: {node: '>=18'} + is-unicode-supported@0.1.0: resolution: {integrity: sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==} engines: {node: '>=10'} + is-unicode-supported@2.1.0: + resolution: {integrity: sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ==} + engines: {node: '>=18'} + is-url-superb@4.0.0: resolution: {integrity: sha512-GI+WjezhPPcbM+tqE9LnmsY5qqjwHzTvjJ36wxYX5ujNXefSUJ/T17r5bqDV8yLhcgB59KTPNOc9O9cmHTPWsA==} engines: {node: '>=10'} @@ -1632,6 +2108,9 @@ packages: resolution: {integrity: sha512-UcVfVfaK4Sc4m7X3dUSoHoozQGBEFeDC+zVo06t98xe8CzHSZZBekNXH+tu0NalHolcJ/QAGqS46Hef7QXBIMw==} engines: {node: '>=16'} + isexe@2.0.0: + resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + isexe@3.1.1: resolution: {integrity: sha512-LpB/54B+/2J5hqQ7imZHfdU31OlgQqx7ZicVlkm9kzg9/w8GKLEcFfJl/t7DCEDueOyBAD6zCCwTO6Fzs0NoEQ==} engines: {node: '>=16'} @@ -1666,20 +2145,56 @@ packages: js-tokens@9.0.1: resolution: {integrity: sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==} + js-yaml@3.14.2: + resolution: {integrity: sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==} + hasBin: true + + js-yaml@4.1.1: + resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==} + hasBin: true + jsesc@3.1.0: resolution: {integrity: sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==} engines: {node: '>=6'} hasBin: true + json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + + json-schema-typed@8.0.2: + resolution: {integrity: sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==} + json5@2.2.3: resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} engines: {node: '>=6'} hasBin: true + jsonc-parser@3.3.1: + resolution: {integrity: sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==} + + keygrip@1.1.0: + resolution: {integrity: sha512-iYSchDJ+liQ8iwbSI2QqsQOvqv58eJCEanyJPJi+Khyu8smkcKSFUCbPwzFcL7YVtZ6eONjqRX/38caJ7QjRAQ==} + engines: {node: '>= 0.6'} + kind-of@6.0.3: resolution: {integrity: sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==} engines: {node: '>=0.10.0'} + knuth-shuffle@1.0.8: + resolution: {integrity: sha512-IdC4Hpp+mx53zTt6VAGsAtbGM0g4BV9fP8tTcviCosSwocHcRDw9uG5Rnv6wLWckF4r72qeXFoK9NkvV1gUJCQ==} + + koa-compose@4.1.0: + resolution: {integrity: sha512-8ODW8TrDuMYvXRwra/Kh7/rJo9BtOfPc6qO8eAfC80CnCvSjSl0bkRM24X6/XBBEyj0v1nRUQ1LyOy3dbqOWXw==} + + koa-router@14.0.0: + resolution: {integrity: sha512-Ue8f/PRsLLNm6b7y+eS6xkqvsG2xH11d2VB1HPcfdfW6p5736kCHf2pXaq8q9XPQ01x0Dk7V/P5Il9pe+tGTxA==} + engines: {node: '>= 20'} + deprecated: 'Please use @koa/router instead, starting from v9! ' + + koa@3.1.1: + resolution: {integrity: sha512-KDDuvpfqSK0ZKEO2gCPedNjl5wYpfj+HNiuVRlbhd1A88S3M0ySkdf2V/EJ4NWt5dwh5PXCdcenrKK2IQJAxsg==} + engines: {node: '>= 18'} + kubernetes-types@1.30.0: resolution: {integrity: sha512-Dew1okvhM/SQcIa2rcgujNndZwU8VnSapDgdxlYoB84ZlpAD43U6KLAFqYo17ykSFGHNPrg0qry0bP+GJd9v7Q==} @@ -1718,6 +2233,46 @@ packages: resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} engines: {node: '>=10'} + math-intrinsics@1.1.0: + resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} + engines: {node: '>= 0.4'} + + mcp-proxy@6.4.0: + resolution: {integrity: sha512-GHDgP1MCzDeWddGEPu/5lxTi/BiD+/3R9xg4gzK65GHSbeiWZEEWTsr0N6mhH3+5A0Gj0M7bDpvPsBPJzz+vnQ==} + hasBin: true + + media-typer@1.1.0: + resolution: {integrity: sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==} + engines: {node: '>= 0.8'} + + merge-descriptors@2.0.0: + resolution: {integrity: sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==} + engines: {node: '>=18'} + + merge2@1.4.1: + resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} + engines: {node: '>= 8'} + + micromatch@4.0.8: + resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} + engines: {node: '>=8.6'} + + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + + mime-db@1.54.0: + resolution: {integrity: sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==} + engines: {node: '>= 0.6'} + + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + + mime-types@3.0.2: + resolution: {integrity: sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==} + engines: {node: '>=18'} + mime@3.0.0: resolution: {integrity: sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==} engines: {node: '>=10.0.0'} @@ -1777,6 +2332,14 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true + negotiator@0.6.3: + resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} + engines: {node: '>= 0.6'} + + negotiator@1.0.0: + resolution: {integrity: sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==} + engines: {node: '>= 0.6'} + node-addon-api@7.1.1: resolution: {integrity: sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==} @@ -1795,9 +2358,25 @@ packages: resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} engines: {node: '>=0.10.0'} + npm-run-path@6.0.0: + resolution: {integrity: sha512-9qny7Z9DsQU8Ou39ERsPU4OZQlSTP47ShQzuKZ6PRXpYLtIFgl/DEBYEXKlvcEa+9tHVcK8CF81Y2V72qaZhWA==} + engines: {node: '>=18'} + + object-assign@4.1.1: + resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} + engines: {node: '>=0.10.0'} + + object-inspect@1.13.4: + resolution: {integrity: sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==} + engines: {node: '>= 0.4'} + obug@2.1.1: resolution: {integrity: sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==} + on-finished@2.4.1: + resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} + engines: {node: '>= 0.8'} + once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} @@ -1835,10 +2414,26 @@ packages: resolution: {integrity: sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA==} engines: {node: '>=6'} + parse-ms@4.0.0: + resolution: {integrity: sha512-TXfryirbmq34y8QBwgqCVLi+8oA3oWx2eAnSn62ITyEhEYaWRlVZ2DvMM9eZbMs/RfxPu/PK/aBLyGj4IrqMHw==} + engines: {node: '>=18'} + + parseurl@1.3.3: + resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} + engines: {node: '>= 0.8'} + path-is-absolute@1.0.1: resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} engines: {node: '>=0.10.0'} + path-key@3.1.1: + resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} + engines: {node: '>=8'} + + path-key@4.0.0: + resolution: {integrity: sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==} + engines: {node: '>=12'} + path-parse@1.0.7: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} @@ -1846,6 +2441,9 @@ packages: resolution: {integrity: sha512-oWyT4gICAu+kaA7QWk/jvCHWarMKNs6pXOGWKDTr7cw4IGcUbW+PeTfbaQiLGheFRpjo6O9J0PmyMfQPjH71oA==} engines: {node: 20 || >=22} + path-to-regexp@8.3.0: + resolution: {integrity: sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA==} + pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} @@ -1864,6 +2462,15 @@ packages: resolution: {integrity: sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==} engines: {node: '>=6'} + pipenet@1.4.0: + resolution: {integrity: sha512-Uc3EH2i8hnJUD0Eupj9z2jaZPjjAbooaiHGh0iFdExbE8/BDt6Lf0919Dtwx5VM83elHNWFzCOsvzsViTD5YZg==} + engines: {node: '>=22.0.0'} + hasBin: true + + pkce-challenge@5.0.1: + resolution: {integrity: sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==} + engines: {node: '>=16.20.0'} + pluralize@8.0.0: resolution: {integrity: sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==} engines: {node: '>=4'} @@ -1891,12 +2498,45 @@ packages: resolution: {integrity: sha512-973driJZvxiGOQ5ONsFhOF/DtzPMOMtgC11kCpUrPGMTgqp2q/1gwzCquocrN33is0VZ5GFHXZYMM9l6h67v2Q==} engines: {node: '>=10'} + pretty-ms@9.3.0: + resolution: {integrity: sha512-gjVS5hOP+M3wMm5nmNOucbIrqudzs9v/57bWRHQWLYklXqoXKrVfYW2W9+glfGsqtPgpiz5WwyEEB+ksXIx3gQ==} + engines: {node: '>=18'} + + proxy-addr@2.0.7: + resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} + engines: {node: '>= 0.10'} + + proxy-from-env@1.1.0: + resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + + pump@3.0.3: + resolution: {integrity: sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==} + + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + pure-rand@6.1.0: resolution: {integrity: sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==} + qs@6.14.1: + resolution: {integrity: sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ==} + engines: {node: '>=0.6'} + + queue-microtask@1.2.3: + resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} + quote-unquote@1.0.0: resolution: {integrity: sha512-twwRO/ilhlG/FIgYeKGFqyHhoEhqgnKVkcmqMKi2r524gz3ZbDTcyFt38E9xjJI2vT+KbRNHVbnJ/e0I25Azwg==} + range-parser@1.2.1: + resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==} + engines: {node: '>= 0.6'} + + raw-body@3.0.2: + resolution: {integrity: sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==} + engines: {node: '>= 0.10'} + rc@1.2.8: resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} hasBin: true @@ -1909,6 +2549,10 @@ packages: resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} engines: {node: '>=8.10.0'} + require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + requirejs-config-file@4.0.0: resolution: {integrity: sha512-jnIre8cbWOyvr8a5F2KuqBnY+SDA4NXr/hzEZJG79Mxm2WiFQz2dzhC8ibtPJS7zkmBEl1mxSwp5HhC1W4qpxw==} engines: {node: '>=10.13.0'} @@ -1931,23 +2575,46 @@ packages: resolution: {integrity: sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==} engines: {node: '>=8'} + reusify@1.1.0: + resolution: {integrity: sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==} + engines: {iojs: '>=1.0.0', node: '>=0.10.0'} + rollup@4.56.0: resolution: {integrity: sha512-9FwVqlgUHzbXtDg9RCMgodF3Ua4Na6Gau+Sdt9vyCN4RhHfVKX2DCHy3BjMLTDd47ITDhYAnTwGulWTblJSDLg==} engines: {node: '>=18.0.0', npm: '>=8.0.0'} hasBin: true + router@2.2.0: + resolution: {integrity: sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==} + engines: {node: '>= 18'} + + rulesync@6.7.0: + resolution: {integrity: sha512-o6s+oddKkqPUXVcy2Ppt7obvyKqS8UeuM3vlW1nMGIkQTSjGz7mc10UXY5RPJFJfF5kSRO+CUU7fX731l0J1SQ==} + engines: {node: '>=22.0.0'} + hasBin: true + run-applescript@7.1.0: resolution: {integrity: sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q==} engines: {node: '>=18'} + run-parallel@1.2.0: + resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} + safe-buffer@5.2.1: resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + safer-buffer@2.1.2: + resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + sass-lookup@6.1.0: resolution: {integrity: sha512-Zx+lVyoWqXZxHuYWlTA17Z5sczJ6braNT2C7rmClw+c4E7r/n911Zwss3h1uHI9reR5AgHZyNHF7c2+VIp5AUA==} engines: {node: '>=18'} hasBin: true + section-matter@1.0.0: + resolution: {integrity: sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==} + engines: {node: '>=4'} + semver@5.7.2: resolution: {integrity: sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==} hasBin: true @@ -1961,20 +2628,67 @@ packages: engines: {node: '>=10'} hasBin: true - siginfo@2.0.0: - resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + send@1.2.1: + resolution: {integrity: sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==} + engines: {node: '>= 18'} - signal-exit@3.0.7: - resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} + serve-static@2.2.1: + resolution: {integrity: sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==} + engines: {node: '>= 18'} - sirv@3.0.2: - resolution: {integrity: sha512-2wcC/oGxHis/BoHkkPwldgiPSYcpZK3JU28WoMVv55yHJgcZ8rlXvuG9iZggz+sU1d4bRgIGASwyWqjxu3FM0g==} - engines: {node: '>=18'} + setprototypeof@1.2.0: + resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + + shebang-command@2.0.0: + resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} + engines: {node: '>=8'} + + shebang-regex@3.0.0: + resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} + engines: {node: '>=8'} + + side-channel-list@1.0.0: + resolution: {integrity: sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==} + engines: {node: '>= 0.4'} + + side-channel-map@1.0.1: + resolution: {integrity: sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==} + engines: {node: '>= 0.4'} + + side-channel-weakmap@1.0.2: + resolution: {integrity: sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==} + engines: {node: '>= 0.4'} + + side-channel@1.1.0: + resolution: {integrity: sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==} + engines: {node: '>= 0.4'} + + siginfo@2.0.0: + resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + + signal-exit@3.0.7: + resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} + + signal-exit@4.1.0: + resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} + engines: {node: '>=14'} + + sirv@3.0.2: + resolution: {integrity: sha512-2wcC/oGxHis/BoHkkPwldgiPSYcpZK3JU28WoMVv55yHJgcZ8rlXvuG9iZggz+sU1d4bRgIGASwyWqjxu3FM0g==} + engines: {node: '>=18'} slash@2.0.0: resolution: {integrity: sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==} engines: {node: '>=6'} + slash@5.1.0: + resolution: {integrity: sha512-ZA6oR3T/pEyuqwMgAKT0/hAv8oAXckzbkmR0UkUosQ+Mc4RxGoJkRmwHgHufaenlyAgE1Mxgpdcrf75y6XcnDg==} + engines: {node: '>=14.16'} + + smol-toml@1.6.0: + resolution: {integrity: sha512-4zemZi0HvTnYwLfrpk/CF9LOd9Lt87kAt50GnqhMpyF9U3poDAP2+iukq2bZsO/ufegbYehBkqINbsWxj4l4cw==} + engines: {node: '>= 18'} + source-map-js@1.2.1: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} @@ -1983,15 +2697,33 @@ packages: resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} engines: {node: '>=0.10.0'} + sprintf-js@1.0.3: + resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} + stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + statuses@1.5.0: + resolution: {integrity: sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==} + engines: {node: '>= 0.6'} + + statuses@2.0.2: + resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} + engines: {node: '>= 0.8'} + std-env@3.10.0: resolution: {integrity: sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==} stream-to-array@2.3.0: resolution: {integrity: sha512-UsZtOYEn4tWU2RGLOXr/o/xjRBftZRlG3dEWoaHr8j4GuypJ3isitGbVyjQKAuMu+xbiop8q224TjiZWc4XTZA==} + strict-event-emitter-types@2.0.0: + resolution: {integrity: sha512-Nk/brWYpD85WlOgzw5h173aci0Teyv8YdIAEtV+N88nDB0dLlazZyJMIsN6eo1/AR61l+p6CJTG1JIyFaoNEEA==} + + string-width@7.2.0: + resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==} + engines: {node: '>=18'} + string_decoder@1.3.0: resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} @@ -2003,14 +2735,30 @@ packages: resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} engines: {node: '>=8'} + strip-ansi@7.1.2: + resolution: {integrity: sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==} + engines: {node: '>=12'} + + strip-bom-string@1.0.0: + resolution: {integrity: sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g==} + engines: {node: '>=0.10.0'} + strip-bom@3.0.0: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'} + strip-final-newline@4.0.0: + resolution: {integrity: sha512-aulFJcD6YK8V1G7iRB5tigAP4TsHBZZrOV8pjV++zdUwmeV8uzbY7yn6h9MswN62adStNZFuCIx4haBnRuMDaw==} + engines: {node: '>=18'} + strip-json-comments@2.0.1: resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} engines: {node: '>=0.10.0'} + strtok3@10.3.4: + resolution: {integrity: sha512-KIy5nylvC5le1OdaaoCJ07L+8iQzJHGH6pWDuzS+d07Cu7n1MZ2x26P8ZKIWfbK02+XIL8Mp4RkWeqdUCrDMfg==} + engines: {node: '>=18'} + stylus-lookup@6.1.0: resolution: {integrity: sha512-5QSwgxAzXPMN+yugy61C60PhoANdItfdjSEZR8siFwz7yL9jTmV0UBKDCfn3K8GkGB4g0Y9py7vTCX8rFu4/pQ==} engines: {node: '>=18'} @@ -2024,6 +2772,14 @@ packages: resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==} engines: {node: '>= 0.4'} + sury@11.0.0-alpha.4: + resolution: {integrity: sha512-oeG/GJWZvQCKtGPpLbu0yCZudfr5LxycDo5kh7SJmKHDPCsEPJssIZL2Eb4Tl7g9aPEvIDuRrkS+L0pybsMEMA==} + peerDependencies: + rescript: 12.x + peerDependenciesMeta: + rescript: + optional: true + tapable@2.3.0: resolution: {integrity: sha512-g9ljZiwki/LfxmQADO3dEY1CbpmXT5Hm2fJ+QaGKwSXUylMybePR7/67YW7jOrrvjEgL1Fmz5kzyAjWVWLlucg==} engines: {node: '>=6'} @@ -2043,10 +2799,22 @@ packages: resolution: {integrity: sha512-PSkbLUoxOFRzJYjjxHJt9xro7D+iilgMX/C9lawzVuYiIdcihh9DXmVibBe8lmcFrRi/VzlPjBxbN7rH24q8/Q==} engines: {node: '>=14.0.0'} + tldjs@2.3.2: + resolution: {integrity: sha512-EORDwFMSZKrHPUVDhejCMDeAovRS5d8jZKiqALFiPp3cjKjEldPkxBY39ZSx3c45awz3RpKwJD1cCgGxEfy8/A==} + engines: {node: '>= 20'} + to-regex-range@5.0.1: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} engines: {node: '>=8.0'} + toidentifier@1.0.1: + resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} + engines: {node: '>=0.6'} + + token-types@6.1.2: + resolution: {integrity: sha512-dRXchy+C0IgK8WPC6xvCHFRIWYUbqqdEIKPaKo/AcTUNzwLTK6AH7RjdLWsEZcAN/TBdtfUw3PYEgPr5VPr6ww==} + engines: {node: '>=14.16'} + toml@3.0.0: resolution: {integrity: sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w==} @@ -2082,11 +2850,23 @@ packages: resolution: {integrity: sha512-NoZ4roiN7LnbKn9QqE1amc9DJfzvZXxF4xDavcOWt1BPkdx+m+0gJuPM+S0vCe7zTJMYUP0R8pO2XMr+Y8oLIg==} engines: {node: '>=6'} + tsscmp@1.0.6: + resolution: {integrity: sha512-LxhtAkPDTkVCMQjt2h6eBVY28KCjikZqZfMcC15YBeNjkgUpdCfBu5HoiOTDu86v6smE8yOjyEktJ8hlbANHQA==} + engines: {node: '>=0.6.x'} + + type-is@2.0.1: + resolution: {integrity: sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==} + engines: {node: '>= 0.6'} + typescript@5.9.3: resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} engines: {node: '>=14.17'} hasBin: true + uint8array-extras@1.5.0: + resolution: {integrity: sha512-rvKSBiC5zqCCiDZ9kAOszZcDvdAHwwIKJG33Ykj43OKcWsnmcBRL09YTU4nOeHZ8Y2a7l1MgTd08SBe9A8Qj6A==} + engines: {node: '>=18'} + undici-types@7.16.0: resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} @@ -2094,12 +2874,30 @@ packages: resolution: {integrity: sha512-Heho1hJD81YChi+uS2RkSjcVO+EQLmLSyUlHyp7Y/wFbxQaGb4WXVKD073JytrjXJVkSZVzoE2MCSOKugFGtOQ==} engines: {node: '>=20.18.1'} + unicorn-magic@0.3.0: + resolution: {integrity: sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA==} + engines: {node: '>=18'} + + unicorn-magic@0.4.0: + resolution: {integrity: sha512-wH590V9VNgYH9g3lH9wWjTrUoKsjLF6sGLjhR4sH1LWpLmCOH0Zf7PukhDA8BiS7KHe4oPNkcTHqYkj7SOGUOw==} + engines: {node: '>=20'} + + universal-user-agent@7.0.3: + resolution: {integrity: sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A==} + + unpipe@1.0.0: + resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} + engines: {node: '>= 0.8'} + update-browserslist-db@1.2.3: resolution: {integrity: sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==} hasBin: true peerDependencies: browserslist: '>= 4.21.0' + uri-templates@0.2.0: + resolution: {integrity: sha512-EWkjYEN0L6KOfEoOH6Wj4ghQqU7eBZMJqRHQnxQAq+dSEzRPClkWjf8557HkWQXF6BrAUoLSAyy9i3RVTliaNg==} + util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} @@ -2107,6 +2905,18 @@ packages: resolution: {integrity: sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==} hasBin: true + valibot@1.2.0: + resolution: {integrity: sha512-mm1rxUsmOxzrwnX5arGS+U4T25RdvpPjPN4yR0u9pUBov9+zGVtO84tif1eY4r6zWxVxu3KzIyknJy3rxfRZZg==} + peerDependencies: + typescript: ^5.9.3 + peerDependenciesMeta: + typescript: + optional: true + + vary@1.1.2: + resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} + engines: {node: '>= 0.8'} + viem@2.45.1: resolution: {integrity: sha512-LN6Pp7vSfv50LgwhkfSbIXftAM5J89lP9x8TeDa8QM7o41IxlHrDh0F9X+FfnCWtsz11pEVV5sn+yBUoOHNqYA==} peerDependencies: @@ -2204,6 +3014,11 @@ packages: wcwidth@1.0.1: resolution: {integrity: sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==} + which@2.0.2: + resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} + engines: {node: '>= 8'} + hasBin: true + which@4.0.0: resolution: {integrity: sha512-GlaYyEb07DPxYCKhKzplCWBJtvxZcZMrL+4UkrTSJHHPyZU4mYYTv3qaOe77H7EODLSSopAUFAc6W8U4yqvscg==} engines: {node: ^16.13.0 || >=18.0.0} @@ -2214,6 +3029,10 @@ packages: engines: {node: '>=8'} hasBin: true + wrap-ansi@9.0.2: + resolution: {integrity: sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==} + engines: {node: '>=18'} + wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} @@ -2245,6 +3064,33 @@ packages: resolution: {integrity: sha512-g/eziiSUNBSsdDJtCLB8bdYEUMj4jR7AGeUo96p/3dTafgjHhpF4RiCFPiRILwjQoDXx5MqkBr4fwWtR3Ky4Wg==} engines: {node: '>=20'} + xsschema@0.4.0-beta.5: + resolution: {integrity: sha512-73pYwf1hMy++7SnOkghJdgdPaGi+Y5I0SaO6rIlxb1ouV6tEyDbEcXP82kyr32KQVTlUbFj6qewi9eUVEiXm+g==} + peerDependencies: + '@valibot/to-json-schema': ^1.0.0 + arktype: ^2.1.20 + effect: ^3.16.0 + sury: ^10.0.0 + zod: ^3.25.0 || ^4.0.0 + zod-to-json-schema: ^3.24.5 + peerDependenciesMeta: + '@valibot/to-json-schema': + optional: true + arktype: + optional: true + effect: + optional: true + sury: + optional: true + zod: + optional: true + zod-to-json-schema: + optional: true + + y18n@5.0.8: + resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} + engines: {node: '>=10'} + yallist@3.1.1: resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} @@ -2253,6 +3099,26 @@ packages: engines: {node: '>= 14.6'} hasBin: true + yargs-parser@22.0.0: + resolution: {integrity: sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==} + engines: {node: ^20.19.0 || ^22.12.0 || >=23} + + yargs@18.0.0: + resolution: {integrity: sha512-4UEqdc2RYGHZc7Doyqkrqiln3p9X2DZVxaGbwhn2pi7MrRagKaOcIKe8L3OxYcbhXLgLFUS3zAYuQjKBQgmuNg==} + engines: {node: ^20.19.0 || ^22.12.0 || >=23} + + yoctocolors@2.1.2: + resolution: {integrity: sha512-CzhO+pFNo8ajLM2d2IW/R93ipy99LWjtwblvC1RsoSUMZgyLbYFr221TnSNT7GjGdYui6P459mw9JH/g/zW2ug==} + engines: {node: '>=18'} + + zod-to-json-schema@3.25.1: + resolution: {integrity: sha512-pM/SU9d3YAggzi6MtR4h7ruuQlqKtad8e9S0fmxcMi+ueAK5Korys/aWcV9LIIHTVbj01NdzxcnXSN+O74ZIVA==} + peerDependencies: + zod: ^3.25 || ^4 + + zod@4.3.6: + resolution: {integrity: sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==} + snapshots: '@adraffy/ens-normalize@1.11.1': {} @@ -2388,6 +3254,8 @@ snapshots: '@bcoe/v8-coverage@1.0.2': {} + '@borewit/text-codec@0.2.1': {} + '@bufbuild/buf-darwin-arm64@1.64.0': optional: true @@ -2665,6 +3533,10 @@ snapshots: '@esbuild/win32-x64@0.27.2': optional: true + '@hono/node-server@1.19.9(hono@4.11.8)': + dependencies: + hono: 4.11.8 + '@isaacs/balanced-match@4.0.1': {} '@isaacs/brace-expansion@5.0.0': @@ -2690,6 +3562,28 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@modelcontextprotocol/sdk@1.26.0(zod@4.3.6)': + dependencies: + '@hono/node-server': 1.19.9(hono@4.11.8) + ajv: 8.17.1 + ajv-formats: 3.0.1(ajv@8.17.1) + content-type: 1.0.5 + cors: 2.8.6 + cross-spawn: 7.0.6 + eventsource: 3.0.7 + eventsource-parser: 3.0.6 + express: 5.2.1 + express-rate-limit: 8.2.1(express@5.2.1) + hono: 4.11.8 + jose: 6.1.3 + json-schema-typed: 8.0.2 + pkce-challenge: 5.0.1 + raw-body: 3.0.2 + zod: 4.3.6 + zod-to-json-schema: 3.25.1(zod@4.3.6) + transitivePeerDependencies: + - supports-color + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': optional: true @@ -2719,6 +3613,80 @@ snapshots: '@noble/hashes@1.8.0': {} + '@nodelib/fs.scandir@2.1.5': + dependencies: + '@nodelib/fs.stat': 2.0.5 + run-parallel: 1.2.0 + + '@nodelib/fs.stat@2.0.5': {} + + '@nodelib/fs.walk@1.2.8': + dependencies: + '@nodelib/fs.scandir': 2.1.5 + fastq: 1.20.1 + + '@octokit/auth-token@6.0.0': {} + + '@octokit/core@7.0.6': + dependencies: + '@octokit/auth-token': 6.0.0 + '@octokit/graphql': 9.0.3 + '@octokit/request': 10.0.7 + '@octokit/request-error': 7.1.0 + '@octokit/types': 16.0.0 + before-after-hook: 4.0.0 + universal-user-agent: 7.0.3 + + '@octokit/endpoint@11.0.2': + dependencies: + '@octokit/types': 16.0.0 + universal-user-agent: 7.0.3 + + '@octokit/graphql@9.0.3': + dependencies: + '@octokit/request': 10.0.7 + '@octokit/types': 16.0.0 + universal-user-agent: 7.0.3 + + '@octokit/openapi-types@27.0.0': {} + + '@octokit/plugin-paginate-rest@14.0.0(@octokit/core@7.0.6)': + dependencies: + '@octokit/core': 7.0.6 + '@octokit/types': 16.0.0 + + '@octokit/plugin-request-log@6.0.0(@octokit/core@7.0.6)': + dependencies: + '@octokit/core': 7.0.6 + + '@octokit/plugin-rest-endpoint-methods@17.0.0(@octokit/core@7.0.6)': + dependencies: + '@octokit/core': 7.0.6 + '@octokit/types': 16.0.0 + + '@octokit/request-error@7.1.0': + dependencies: + '@octokit/types': 16.0.0 + + '@octokit/request@10.0.7': + dependencies: + '@octokit/endpoint': 11.0.2 + '@octokit/request-error': 7.1.0 + '@octokit/types': 16.0.0 + fast-content-type-parse: 3.0.0 + universal-user-agent: 7.0.3 + + '@octokit/rest@22.0.1': + dependencies: + '@octokit/core': 7.0.6 + '@octokit/plugin-paginate-rest': 14.0.0(@octokit/core@7.0.6) + '@octokit/plugin-request-log': 6.0.0(@octokit/core@7.0.6) + '@octokit/plugin-rest-endpoint-methods': 17.0.0(@octokit/core@7.0.6) + + '@octokit/types@16.0.0': + dependencies: + '@octokit/openapi-types': 27.0.0 + '@oxlint/darwin-arm64@1.42.0': optional: true @@ -2893,8 +3861,23 @@ snapshots: '@noble/hashes': 1.8.0 '@scure/base': 1.2.6 + '@sec-ant/readable-stream@0.4.1': {} + + '@sindresorhus/merge-streams@4.0.0': {} + '@standard-schema/spec@1.1.0': {} + '@tokenizer/inflate@0.4.1': + dependencies: + debug: 4.4.3 + token-types: 6.1.2 + transitivePeerDependencies: + - supports-color + + '@tokenizer/token@0.3.0': {} + + '@toon-format/toon@2.1.0': {} + '@ts-graphviz/adapter@2.0.6': dependencies: '@ts-graphviz/common': 2.1.5 @@ -3042,6 +4025,10 @@ snapshots: transitivePeerDependencies: - supports-color + '@valibot/to-json-schema@1.5.0(valibot@1.2.0(typescript@5.9.3))': + dependencies: + valibot: 1.2.0(typescript@5.9.3) + '@vitest/coverage-v8@4.0.18(vitest@4.0.18)': dependencies: '@bcoe/v8-coverage': 1.0.2 @@ -3138,16 +4125,42 @@ snapshots: '@vue/shared@3.5.26': {} - abitype@1.2.3(typescript@5.9.3): + abitype@1.2.3(typescript@5.9.3)(zod@4.3.6): optionalDependencies: typescript: 5.9.3 + zod: 4.3.6 + + accepts@1.3.8: + dependencies: + mime-types: 2.1.35 + negotiator: 0.6.3 + + accepts@2.0.0: + dependencies: + mime-types: 3.0.2 + negotiator: 1.0.0 + + ajv-formats@3.0.1(ajv@8.17.1): + optionalDependencies: + ajv: 8.17.1 + + ajv@8.17.1: + dependencies: + fast-deep-equal: 3.1.3 + fast-uri: 3.1.0 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 ansi-regex@5.0.1: {} + ansi-regex@6.2.2: {} + ansi-styles@4.3.0: dependencies: color-convert: 2.0.1 + ansi-styles@6.2.3: {} + any-promise@1.3.0: {} anymatch@3.1.3: @@ -3158,6 +4171,12 @@ snapshots: app-module-path@2.2.0: {} + argparse@1.0.10: + dependencies: + sprintf-js: 1.0.3 + + argparse@2.0.1: {} + assertion-error@2.0.1: {} ast-module-types@6.0.1: {} @@ -3168,6 +4187,16 @@ snapshots: estree-walker: 3.0.3 js-tokens: 9.0.1 + asynckit@0.4.0: {} + + axios@1.13.4(debug@4.4.3): + dependencies: + follow-redirects: 1.15.11(debug@4.4.3) + form-data: 4.0.5 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + babel-plugin-annotate-pure-calls@0.5.0(@babel/core@7.28.6): dependencies: '@babel/core': 7.28.6 @@ -3178,6 +4207,8 @@ snapshots: baseline-browser-mapping@2.9.17: {} + before-after-hook@4.0.0: {} + binary-extensions@2.3.0: optional: true @@ -3187,6 +4218,20 @@ snapshots: inherits: 2.0.4 readable-stream: 3.6.2 + body-parser@2.2.2: + dependencies: + bytes: 3.1.2 + content-type: 1.0.5 + debug: 4.4.3 + http-errors: 2.0.1 + iconv-lite: 0.7.2 + on-finished: 2.4.1 + qs: 6.14.1 + raw-body: 3.0.2 + type-is: 2.0.1 + transitivePeerDependencies: + - supports-color + brace-expansion@1.1.12: dependencies: balanced-match: 1.0.2 @@ -3199,7 +4244,6 @@ snapshots: braces@3.0.3: dependencies: fill-range: 7.1.1 - optional: true browserslist@4.28.1: dependencies: @@ -3218,6 +4262,18 @@ snapshots: dependencies: run-applescript: 7.1.0 + bytes@3.1.2: {} + + call-bind-apply-helpers@1.0.2: + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + + call-bound@1.0.4: + dependencies: + call-bind-apply-helpers: 1.0.2 + get-intrinsic: 1.3.0 + caniuse-lite@1.0.30001766: {} chai@6.2.2: {} @@ -3246,6 +4302,12 @@ snapshots: cli-spinners@2.9.2: {} + cliui@9.0.1: + dependencies: + string-width: 7.2.0 + strip-ansi: 7.1.2 + wrap-ansi: 9.0.2 + clone@1.0.4: {} color-convert@2.0.1: @@ -3254,8 +4316,14 @@ snapshots: color-name@1.1.4: {} + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + commander@12.1.0: {} + commander@14.0.3: {} + commander@6.2.1: {} commander@7.2.0: {} @@ -3264,12 +4332,44 @@ snapshots: concat-map@0.0.1: {} + consola@3.4.2: {} + + content-disposition@0.5.4: + dependencies: + safe-buffer: 5.2.1 + + content-disposition@1.0.1: {} + + content-type@1.0.5: {} + convert-source-map@2.0.0: {} + cookie-signature@1.2.2: {} + + cookie@0.7.2: {} + + cookies@0.9.1: + dependencies: + depd: 2.0.0 + keygrip: 1.1.0 + + cors@2.8.6: + dependencies: + object-assign: 4.1.1 + vary: 1.1.2 + + cross-spawn@7.0.6: + dependencies: + path-key: 3.1.1 + shebang-command: 2.0.0 + which: 2.0.2 + debug@4.4.3: dependencies: ms: 2.1.3 + deep-equal@1.0.1: {} + deep-extend@0.6.0: {} default-browser-id@5.0.1: {} @@ -3285,6 +4385,14 @@ snapshots: define-lazy-prop@3.0.0: {} + delayed-stream@1.0.0: {} + + delegates@1.0.0: {} + + depd@1.1.2: {} + + depd@2.0.0: {} + dependency-tree@11.2.0: dependencies: commander: 12.1.0 @@ -3294,6 +4402,8 @@ snapshots: transitivePeerDependencies: - supports-color + destroy@1.2.0: {} + detect-libc@2.1.2: {} detective-amd@6.0.1: @@ -3366,13 +4476,34 @@ snapshots: '@dprint/win32-arm64': 0.51.1 '@dprint/win32-x64': 0.51.1 + dunder-proto@1.0.1: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-errors: 1.3.0 + gopd: 1.2.0 + + ee-first@1.1.1: {} + effect@3.19.15: dependencies: '@standard-schema/spec': 1.1.0 fast-check: 3.23.2 + effect@3.19.16: + dependencies: + '@standard-schema/spec': 1.1.0 + fast-check: 3.23.2 + electron-to-chromium@1.5.278: {} + emoji-regex@10.6.0: {} + + encodeurl@2.0.0: {} + + end-of-stream@1.4.5: + dependencies: + once: 1.4.0 + enhanced-resolve@5.18.4: dependencies: graceful-fs: 4.2.11 @@ -3380,8 +4511,25 @@ snapshots: entities@7.0.0: {} + es-define-property@1.0.1: {} + + es-errors@1.3.0: {} + es-module-lexer@1.7.0: {} + es-object-atoms@1.1.1: + dependencies: + es-errors: 1.3.0 + + es-set-tostringtag@2.1.0: + dependencies: + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + has-tostringtag: 1.0.2 + hasown: 2.0.2 + + es-toolkit@1.44.0: {} + esbuild@0.27.2: optionalDependencies: '@esbuild/aix-ppc64': 0.27.2 @@ -3413,6 +4561,8 @@ snapshots: escalade@3.2.0: {} + escape-html@1.0.3: {} + escodegen@2.1.0: dependencies: esprima: 4.0.1 @@ -3435,20 +4585,140 @@ snapshots: esutils@2.0.3: {} + etag@1.8.1: {} + eventemitter3@5.0.1: {} + eventsource-parser@3.0.6: {} + + eventsource@3.0.7: + dependencies: + eventsource-parser: 3.0.6 + + execa@9.6.1: + dependencies: + '@sindresorhus/merge-streams': 4.0.0 + cross-spawn: 7.0.6 + figures: 6.1.0 + get-stream: 9.0.1 + human-signals: 8.0.1 + is-plain-obj: 4.1.0 + is-stream: 4.0.1 + npm-run-path: 6.0.0 + pretty-ms: 9.3.0 + signal-exit: 4.1.0 + strip-final-newline: 4.0.0 + yoctocolors: 2.1.2 + expect-type@1.3.0: {} + express-rate-limit@8.2.1(express@5.2.1): + dependencies: + express: 5.2.1 + ip-address: 10.0.1 + + express@5.2.1: + dependencies: + accepts: 2.0.0 + body-parser: 2.2.2 + content-disposition: 1.0.1 + content-type: 1.0.5 + cookie: 0.7.2 + cookie-signature: 1.2.2 + debug: 4.4.3 + depd: 2.0.0 + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + finalhandler: 2.1.1 + fresh: 2.0.0 + http-errors: 2.0.1 + merge-descriptors: 2.0.0 + mime-types: 3.0.2 + on-finished: 2.4.1 + once: 1.4.0 + parseurl: 1.3.3 + proxy-addr: 2.0.7 + qs: 6.14.1 + range-parser: 1.2.1 + router: 2.2.0 + send: 1.2.1 + serve-static: 2.2.1 + statuses: 2.0.2 + type-is: 2.0.1 + vary: 1.1.2 + transitivePeerDependencies: + - supports-color + + extend-shallow@2.0.1: + dependencies: + is-extendable: 0.1.1 + fast-check@3.23.2: dependencies: pure-rand: 6.1.0 + fast-content-type-parse@3.0.0: {} + + fast-deep-equal@3.1.3: {} + + fast-glob@3.3.3: + dependencies: + '@nodelib/fs.stat': 2.0.5 + '@nodelib/fs.walk': 1.2.8 + glob-parent: 5.1.2 + merge2: 1.4.1 + micromatch: 4.0.8 + + fast-uri@3.1.0: {} + + fastmcp@3.31.0(@valibot/to-json-schema@1.5.0(valibot@1.2.0(typescript@5.9.3)))(effect@3.19.16)(sury@11.0.0-alpha.4): + dependencies: + '@modelcontextprotocol/sdk': 1.26.0(zod@4.3.6) + '@standard-schema/spec': 1.1.0 + execa: 9.6.1 + file-type: 21.3.0 + fuse.js: 7.1.0 + hono: 4.11.8 + mcp-proxy: 6.4.0 + strict-event-emitter-types: 2.0.0 + undici: 7.19.0 + uri-templates: 0.2.0 + xsschema: 0.4.0-beta.5(@valibot/to-json-schema@1.5.0(valibot@1.2.0(typescript@5.9.3)))(effect@3.19.16)(sury@11.0.0-alpha.4)(zod-to-json-schema@3.25.1(zod@4.3.6))(zod@4.3.6) + yargs: 18.0.0 + zod: 4.3.6 + zod-to-json-schema: 3.25.1(zod@4.3.6) + transitivePeerDependencies: + - '@cfworker/json-schema' + - '@valibot/to-json-schema' + - arktype + - effect + - supports-color + - sury + + fastq@1.20.1: + dependencies: + reusify: 1.1.0 + fdir@6.5.0(picomatch@4.0.3): optionalDependencies: picomatch: 4.0.3 fflate@0.8.2: {} + figures@6.1.0: + dependencies: + is-unicode-supported: 2.1.0 + + file-type@21.3.0: + dependencies: + '@tokenizer/inflate': 0.4.1 + strtok3: 10.3.4 + token-types: 6.1.2 + uint8array-extras: 1.5.0 + transitivePeerDependencies: + - supports-color + filing-cabinet@5.0.3: dependencies: app-module-path: 2.2.0 @@ -3466,12 +4736,40 @@ snapshots: fill-range@7.1.1: dependencies: to-regex-range: 5.0.1 - optional: true + + finalhandler@2.1.1: + dependencies: + debug: 4.4.3 + encodeurl: 2.0.0 + escape-html: 1.0.3 + on-finished: 2.4.1 + parseurl: 1.3.3 + statuses: 2.0.2 + transitivePeerDependencies: + - supports-color find-my-way-ts@0.1.6: {} flatted@3.3.3: {} + follow-redirects@1.15.11(debug@4.4.3): + optionalDependencies: + debug: 4.4.3 + + form-data@4.0.5: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + hasown: 2.0.2 + mime-types: 2.1.35 + + forwarded@0.2.0: {} + + fresh@0.5.2: {} + + fresh@2.0.0: {} + fs-readdir-recursive@1.1.0: {} fs.realpath@1.0.0: {} @@ -3481,6 +4779,8 @@ snapshots: function-bind@1.1.2: {} + fuse.js@7.1.0: {} + gensync@1.0.0-beta.2: {} get-amd-module-type@6.0.1: @@ -3488,12 +4788,38 @@ snapshots: ast-module-types: 6.0.1 node-source-walk: 7.0.1 + get-caller-file@2.0.5: {} + + get-east-asian-width@1.4.0: {} + + get-intrinsic@1.3.0: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-define-property: 1.0.1 + es-errors: 1.3.0 + es-object-atoms: 1.1.1 + function-bind: 1.1.2 + get-proto: 1.0.1 + gopd: 1.2.0 + has-symbols: 1.1.0 + hasown: 2.0.2 + math-intrinsics: 1.1.0 + get-own-enumerable-property-symbols@3.0.2: {} + get-proto@1.0.1: + dependencies: + dunder-proto: 1.0.1 + es-object-atoms: 1.1.1 + + get-stream@9.0.1: + dependencies: + '@sec-ant/readable-stream': 0.4.1 + is-stream: 4.0.1 + glob-parent@5.1.2: dependencies: is-glob: 4.0.3 - optional: true glob@13.0.0: dependencies: @@ -3518,24 +4844,83 @@ snapshots: globals@17.2.0: {} + globby@16.1.0: + dependencies: + '@sindresorhus/merge-streams': 4.0.0 + fast-glob: 3.3.3 + ignore: 7.0.5 + is-path-inside: 4.0.0 + slash: 5.1.0 + unicorn-magic: 0.4.0 + globrex@0.1.2: {} gonzales-pe@4.3.0: dependencies: minimist: 1.2.8 + gopd@1.2.0: {} + graceful-fs@4.2.11: {} + gray-matter@4.0.3: + dependencies: + js-yaml: 3.14.2 + kind-of: 6.0.3 + section-matter: 1.0.0 + strip-bom-string: 1.0.0 + has-flag@4.0.0: {} + has-symbols@1.1.0: {} + + has-tostringtag@1.0.2: + dependencies: + has-symbols: 1.1.0 + hasown@2.0.2: dependencies: function-bind: 1.1.2 + hono@4.11.8: {} + html-escaper@2.0.2: {} + http-assert@1.5.0: + dependencies: + deep-equal: 1.0.1 + http-errors: 1.8.1 + + http-errors@1.8.1: + dependencies: + depd: 1.1.2 + inherits: 2.0.4 + setprototypeof: 1.2.0 + statuses: 1.5.0 + toidentifier: 1.0.1 + + http-errors@2.0.1: + dependencies: + depd: 2.0.0 + inherits: 2.0.4 + setprototypeof: 1.2.0 + statuses: 2.0.2 + toidentifier: 1.0.1 + + human-readable-ids@1.0.4: + dependencies: + knuth-shuffle: 1.0.8 + + human-signals@8.0.1: {} + + iconv-lite@0.7.2: + dependencies: + safer-buffer: 2.1.2 + ieee754@1.2.1: {} + ignore@7.0.5: {} + inflight@1.0.6: dependencies: once: 1.4.0 @@ -3547,6 +4932,10 @@ snapshots: ini@4.1.3: {} + ip-address@10.0.1: {} + + ipaddr.js@1.9.1: {} + is-binary-path@2.1.0: dependencies: binary-extensions: 2.3.0 @@ -3558,6 +4947,8 @@ snapshots: is-docker@3.0.0: {} + is-extendable@0.1.1: {} + is-extglob@2.1.1: {} is-glob@4.0.3: @@ -3572,15 +4963,24 @@ snapshots: is-interactive@1.0.0: {} - is-number@7.0.0: - optional: true + is-number@7.0.0: {} is-obj@1.0.1: {} + is-path-inside@4.0.0: {} + + is-plain-obj@4.1.0: {} + + is-promise@4.0.0: {} + is-regexp@1.0.0: {} + is-stream@4.0.1: {} + is-unicode-supported@0.1.0: {} + is-unicode-supported@2.1.0: {} + is-url-superb@4.0.0: {} is-url@1.2.4: {} @@ -3589,6 +4989,8 @@ snapshots: dependencies: is-inside-container: 1.0.0 + isexe@2.0.0: {} + isexe@3.1.1: {} isows@1.0.7(ws@8.18.3): @@ -3616,12 +5018,65 @@ snapshots: js-tokens@9.0.1: {} + js-yaml@3.14.2: + dependencies: + argparse: 1.0.10 + esprima: 4.0.1 + + js-yaml@4.1.1: + dependencies: + argparse: 2.0.1 + jsesc@3.1.0: {} + json-schema-traverse@1.0.0: {} + + json-schema-typed@8.0.2: {} + json5@2.2.3: {} + jsonc-parser@3.3.1: {} + + keygrip@1.1.0: + dependencies: + tsscmp: 1.0.6 + kind-of@6.0.3: {} + knuth-shuffle@1.0.8: {} + + koa-compose@4.1.0: {} + + koa-router@14.0.0: + dependencies: + debug: 4.4.3 + http-errors: 2.0.1 + koa-compose: 4.1.0 + path-to-regexp: 8.3.0 + transitivePeerDependencies: + - supports-color + + koa@3.1.1: + dependencies: + accepts: 1.3.8 + content-disposition: 0.5.4 + content-type: 1.0.5 + cookies: 0.9.1 + delegates: 1.0.0 + destroy: 1.2.0 + encodeurl: 2.0.0 + escape-html: 1.0.3 + fresh: 0.5.2 + http-assert: 1.5.0 + http-errors: 2.0.1 + koa-compose: 4.1.0 + mime-types: 3.0.2 + on-finished: 2.4.1 + parseurl: 1.3.3 + statuses: 2.0.2 + type-is: 2.0.1 + vary: 1.1.2 + kubernetes-types@1.30.0: {} log-symbols@4.1.0: @@ -3673,6 +5128,37 @@ snapshots: dependencies: semver: 7.7.3 + math-intrinsics@1.1.0: {} + + mcp-proxy@6.4.0: + dependencies: + pipenet: 1.4.0 + transitivePeerDependencies: + - supports-color + + media-typer@1.1.0: {} + + merge-descriptors@2.0.0: {} + + merge2@1.4.1: {} + + micromatch@4.0.8: + dependencies: + braces: 3.0.3 + picomatch: 2.3.1 + + mime-db@1.52.0: {} + + mime-db@1.54.0: {} + + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + + mime-types@3.0.2: + dependencies: + mime-db: 1.54.0 + mime@3.0.0: {} mimic-fn@2.1.0: {} @@ -3729,6 +5215,10 @@ snapshots: nanoid@3.3.11: {} + negotiator@0.6.3: {} + + negotiator@1.0.0: {} + node-addon-api@7.1.1: {} node-gyp-build-optional-packages@5.2.2: @@ -3745,8 +5235,21 @@ snapshots: normalize-path@3.0.0: optional: true + npm-run-path@6.0.0: + dependencies: + path-key: 4.0.0 + unicorn-magic: 0.3.0 + + object-assign@4.1.1: {} + + object-inspect@1.13.4: {} + obug@2.1.1: {} + on-finished@2.4.1: + dependencies: + ee-first: 1.1.1 + once@1.4.0: dependencies: wrappy: 1.0.2 @@ -3776,7 +5279,7 @@ snapshots: strip-ansi: 6.0.1 wcwidth: 1.0.1 - ox@0.11.3(typescript@5.9.3): + ox@0.11.3(typescript@5.9.3)(zod@4.3.6): dependencies: '@adraffy/ens-normalize': 1.11.1 '@noble/ciphers': 1.3.0 @@ -3784,7 +5287,7 @@ snapshots: '@noble/hashes': 1.8.0 '@scure/bip32': 1.7.0 '@scure/bip39': 1.6.0 - abitype: 1.2.3(typescript@5.9.3) + abitype: 1.2.3(typescript@5.9.3)(zod@4.3.6) eventemitter3: 5.0.1 optionalDependencies: typescript: 5.9.3 @@ -3804,8 +5307,16 @@ snapshots: parse-ms@2.1.0: {} + parse-ms@4.0.0: {} + + parseurl@1.3.3: {} + path-is-absolute@1.0.1: {} + path-key@3.1.1: {} + + path-key@4.0.0: {} + path-parse@1.0.7: {} path-scurry@2.0.1: @@ -3813,17 +5324,33 @@ snapshots: lru-cache: 11.2.4 minipass: 7.1.2 + path-to-regexp@8.3.0: {} + pathe@2.0.3: {} picocolors@1.1.1: {} - picomatch@2.3.1: - optional: true + picomatch@2.3.1: {} picomatch@4.0.3: {} pify@4.0.1: {} + pipenet@1.4.0: + dependencies: + axios: 1.13.4(debug@4.4.3) + debug: 4.4.3 + human-readable-ids: 1.0.4 + koa: 3.1.1 + koa-router: 14.0.0 + pump: 3.0.3 + tldjs: 2.3.2 + yargs: 18.0.0 + transitivePeerDependencies: + - supports-color + + pkce-challenge@5.0.1: {} + pluralize@8.0.0: {} postcss-values-parser@6.0.2(postcss@8.5.6): @@ -3865,10 +5392,43 @@ snapshots: dependencies: parse-ms: 2.1.0 + pretty-ms@9.3.0: + dependencies: + parse-ms: 4.0.0 + + proxy-addr@2.0.7: + dependencies: + forwarded: 0.2.0 + ipaddr.js: 1.9.1 + + proxy-from-env@1.1.0: {} + + pump@3.0.3: + dependencies: + end-of-stream: 1.4.5 + once: 1.4.0 + + punycode@2.3.1: {} + pure-rand@6.1.0: {} + qs@6.14.1: + dependencies: + side-channel: 1.1.0 + + queue-microtask@1.2.3: {} + quote-unquote@1.0.0: {} + range-parser@1.2.1: {} + + raw-body@3.0.2: + dependencies: + bytes: 3.1.2 + http-errors: 2.0.1 + iconv-lite: 0.7.2 + unpipe: 1.0.0 + rc@1.2.8: dependencies: deep-extend: 0.6.0 @@ -3887,6 +5447,8 @@ snapshots: picomatch: 2.3.1 optional: true + require-from-string@2.0.2: {} + requirejs-config-file@4.0.0: dependencies: esprima: 4.0.1 @@ -3907,6 +5469,8 @@ snapshots: onetime: 5.1.2 signal-exit: 3.0.7 + reusify@1.1.0: {} + rollup@4.56.0: dependencies: '@types/estree': 1.0.8 @@ -3938,25 +5502,136 @@ snapshots: '@rollup/rollup-win32-x64-msvc': 4.56.0 fsevents: 2.3.3 + router@2.2.0: + dependencies: + debug: 4.4.3 + depd: 2.0.0 + is-promise: 4.0.0 + parseurl: 1.3.3 + path-to-regexp: 8.3.0 + transitivePeerDependencies: + - supports-color + + rulesync@6.7.0(valibot@1.2.0(typescript@5.9.3)): + dependencies: + '@modelcontextprotocol/sdk': 1.26.0(zod@4.3.6) + '@octokit/request-error': 7.1.0 + '@octokit/rest': 22.0.1 + '@toon-format/toon': 2.1.0 + '@valibot/to-json-schema': 1.5.0(valibot@1.2.0(typescript@5.9.3)) + commander: 14.0.3 + consola: 3.4.2 + effect: 3.19.16 + es-toolkit: 1.44.0 + fastmcp: 3.31.0(@valibot/to-json-schema@1.5.0(valibot@1.2.0(typescript@5.9.3)))(effect@3.19.16)(sury@11.0.0-alpha.4) + globby: 16.1.0 + gray-matter: 4.0.3 + js-yaml: 4.1.1 + jsonc-parser: 3.3.1 + smol-toml: 1.6.0 + sury: 11.0.0-alpha.4 + zod: 4.3.6 + transitivePeerDependencies: + - '@cfworker/json-schema' + - arktype + - jose + - rescript + - supports-color + - valibot + run-applescript@7.1.0: {} + run-parallel@1.2.0: + dependencies: + queue-microtask: 1.2.3 + safe-buffer@5.2.1: {} + safer-buffer@2.1.2: {} + sass-lookup@6.1.0: dependencies: commander: 12.1.0 enhanced-resolve: 5.18.4 + section-matter@1.0.0: + dependencies: + extend-shallow: 2.0.1 + kind-of: 6.0.3 + semver@5.7.2: {} semver@6.3.1: {} semver@7.7.3: {} + send@1.2.1: + dependencies: + debug: 4.4.3 + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + fresh: 2.0.0 + http-errors: 2.0.1 + mime-types: 3.0.2 + ms: 2.1.3 + on-finished: 2.4.1 + range-parser: 1.2.1 + statuses: 2.0.2 + transitivePeerDependencies: + - supports-color + + serve-static@2.2.1: + dependencies: + encodeurl: 2.0.0 + escape-html: 1.0.3 + parseurl: 1.3.3 + send: 1.2.1 + transitivePeerDependencies: + - supports-color + + setprototypeof@1.2.0: {} + + shebang-command@2.0.0: + dependencies: + shebang-regex: 3.0.0 + + shebang-regex@3.0.0: {} + + side-channel-list@1.0.0: + dependencies: + es-errors: 1.3.0 + object-inspect: 1.13.4 + + side-channel-map@1.0.1: + dependencies: + call-bound: 1.0.4 + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + object-inspect: 1.13.4 + + side-channel-weakmap@1.0.2: + dependencies: + call-bound: 1.0.4 + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + object-inspect: 1.13.4 + side-channel-map: 1.0.1 + + side-channel@1.1.0: + dependencies: + es-errors: 1.3.0 + object-inspect: 1.13.4 + side-channel-list: 1.0.0 + side-channel-map: 1.0.1 + side-channel-weakmap: 1.0.2 + siginfo@2.0.0: {} signal-exit@3.0.7: {} + signal-exit@4.1.0: {} + sirv@3.0.2: dependencies: '@polka/url': 1.0.0-next.29 @@ -3965,19 +5640,37 @@ snapshots: slash@2.0.0: {} + slash@5.1.0: {} + + smol-toml@1.6.0: {} + source-map-js@1.2.1: {} source-map@0.6.1: optional: true + sprintf-js@1.0.3: {} + stackback@0.0.2: {} + statuses@1.5.0: {} + + statuses@2.0.2: {} + std-env@3.10.0: {} stream-to-array@2.3.0: dependencies: any-promise: 1.3.0 + strict-event-emitter-types@2.0.0: {} + + string-width@7.2.0: + dependencies: + emoji-regex: 10.6.0 + get-east-asian-width: 1.4.0 + strip-ansi: 7.1.2 + string_decoder@1.3.0: dependencies: safe-buffer: 5.2.1 @@ -3992,10 +5685,22 @@ snapshots: dependencies: ansi-regex: 5.0.1 + strip-ansi@7.1.2: + dependencies: + ansi-regex: 6.2.2 + + strip-bom-string@1.0.0: {} + strip-bom@3.0.0: {} + strip-final-newline@4.0.0: {} + strip-json-comments@2.0.1: {} + strtok3@10.3.4: + dependencies: + '@tokenizer/token': 0.3.0 + stylus-lookup@6.1.0: dependencies: commander: 12.1.0 @@ -4006,6 +5711,8 @@ snapshots: supports-preserve-symlinks-flag@1.0.0: {} + sury@11.0.0-alpha.4: {} + tapable@2.3.0: {} tinybench@2.9.0: {} @@ -4019,10 +5726,21 @@ snapshots: tinyrainbow@3.0.3: {} + tldjs@2.3.2: + dependencies: + punycode: 2.3.1 + to-regex-range@5.0.1: dependencies: is-number: 7.0.0 - optional: true + + toidentifier@1.0.1: {} + + token-types@6.1.2: + dependencies: + '@borewit/text-codec': 0.2.1 + '@tokenizer/token': 0.3.0 + ieee754: 1.2.1 toml@3.0.0: {} @@ -4058,31 +5776,57 @@ snapshots: minimist: 1.2.8 strip-bom: 3.0.0 + tsscmp@1.0.6: {} + + type-is@2.0.1: + dependencies: + content-type: 1.0.5 + media-typer: 1.1.0 + mime-types: 3.0.2 + typescript@5.9.3: {} + uint8array-extras@1.5.0: {} + undici-types@7.16.0: {} undici@7.19.0: {} + unicorn-magic@0.3.0: {} + + unicorn-magic@0.4.0: {} + + universal-user-agent@7.0.3: {} + + unpipe@1.0.0: {} + update-browserslist-db@1.2.3(browserslist@4.28.1): dependencies: browserslist: 4.28.1 escalade: 3.2.0 picocolors: 1.1.1 + uri-templates@0.2.0: {} + util-deprecate@1.0.2: {} uuid@11.1.0: {} - viem@2.45.1(typescript@5.9.3): + valibot@1.2.0(typescript@5.9.3): + optionalDependencies: + typescript: 5.9.3 + + vary@1.1.2: {} + + viem@2.45.1(typescript@5.9.3)(zod@4.3.6): dependencies: '@noble/curves': 1.9.1 '@noble/hashes': 1.8.0 '@scure/bip32': 1.7.0 '@scure/bip39': 1.6.0 - abitype: 1.2.3(typescript@5.9.3) + abitype: 1.2.3(typescript@5.9.3)(zod@4.3.6) isows: 1.0.7(ws@8.18.3) - ox: 0.11.3(typescript@5.9.3) + ox: 0.11.3(typescript@5.9.3)(zod@4.3.6) ws: 8.18.3 optionalDependencies: typescript: 5.9.3 @@ -4163,6 +5907,10 @@ snapshots: dependencies: defaults: 1.0.4 + which@2.0.2: + dependencies: + isexe: 2.0.0 + which@4.0.0: dependencies: isexe: 3.1.1 @@ -4172,6 +5920,12 @@ snapshots: siginfo: 2.0.0 stackback: 0.0.2 + wrap-ansi@9.0.2: + dependencies: + ansi-styles: 6.2.3 + string-width: 7.2.0 + strip-ansi: 7.1.2 + wrappy@1.0.2: {} ws@8.18.3: {} @@ -4183,6 +5937,35 @@ snapshots: is-wsl: 3.1.0 powershell-utils: 0.1.0 + xsschema@0.4.0-beta.5(@valibot/to-json-schema@1.5.0(valibot@1.2.0(typescript@5.9.3)))(effect@3.19.16)(sury@11.0.0-alpha.4)(zod-to-json-schema@3.25.1(zod@4.3.6))(zod@4.3.6): + optionalDependencies: + '@valibot/to-json-schema': 1.5.0(valibot@1.2.0(typescript@5.9.3)) + effect: 3.19.16 + sury: 11.0.0-alpha.4 + zod: 4.3.6 + zod-to-json-schema: 3.25.1(zod@4.3.6) + + y18n@5.0.8: {} + yallist@3.1.1: {} yaml@2.8.2: {} + + yargs-parser@22.0.0: {} + + yargs@18.0.0: + dependencies: + cliui: 9.0.1 + escalade: 3.2.0 + get-caller-file: 2.0.5 + string-width: 7.2.0 + y18n: 5.0.8 + yargs-parser: 22.0.0 + + yoctocolors@2.1.2: {} + + zod-to-json-schema@3.25.1(zod@4.3.6): + dependencies: + zod: 4.3.6 + + zod@4.3.6: {} diff --git a/rulesync.jsonc b/rulesync.jsonc new file mode 100644 index 0000000..b5a3930 --- /dev/null +++ b/rulesync.jsonc @@ -0,0 +1,25 @@ +{ + "$schema": "https://raw.githubusercontent.com/dyoshikawa/rulesync/refs/heads/main/config-schema.json", + "targets": [ + "claudecode", + "opencode" + ], + "features": [ + "rules", + "ignore", + "commands", + "subagents", + "skills" + ], + "baseDirs": [ + "." + ], + "delete": true, + "verbose": false, + "silent": false, + "global": false, + "simulateCommands": false, + "simulateSubagents": false, + "simulateSkills": false, + "modularMcp": false +} diff --git a/specs/transactional-stream.md b/specs/transactional-stream.md index b27bcb7..af25efa 100644 --- a/specs/transactional-stream.md +++ b/specs/transactional-stream.md @@ -42,33 +42,28 @@ packages/amp/test/transactional-stream/ // Types // ============================================================================= export { - TransactionId, TransactionEvent, TransactionEventData, TransactionEventUndo, TransactionEventWatermark, - UndoCause, - TransactionIdRange + TransactionId, + TransactionIdRange, + UndoCause } from "./types.ts" // ============================================================================= // Errors // ============================================================================= -export { - StateStoreError, - UnrecoverableReorgError, - PartialReorgError, - type TransactionalStreamError -} from "./errors.ts" +export { PartialReorgError, StateStoreError, type TransactionalStreamError, UnrecoverableReorgError } from "./errors.ts" // ============================================================================= // StateStore Service // ============================================================================= export { - StateStore, // Context.Tag for DI - type StateSnapshot, type Commit, - emptySnapshot + emptySnapshot, + type StateSnapshot, + StateStore // Context.Tag for DI } from "./state-store.ts" // ============================================================================= @@ -85,8 +80,8 @@ export { type CommitHandle } from "./commit-handle.ts" // TransactionalStream Service // ============================================================================= export { - TransactionalStream, // Context.Tag for DI - layer, // Layer.Layer + layer, // Layer.Layer + TransactionalStream, // Context.Tag for DI type TransactionalStreamOptions } from "./stream.ts" ``` @@ -95,11 +90,11 @@ export { ```typescript import { - TransactionalStream, + type CommitHandle, InMemoryStateStore, StateStore, - type TransactionEvent, - type CommitHandle + TransactionalStream, + type TransactionEvent } from "@edgeandnode/amp/transactional-stream" ``` @@ -244,11 +239,11 @@ export const emptySnapshot: StateSnapshot = { Reference implementation using Effect Ref. Not crash-safe but suitable for development, testing, and ephemeral use cases. -```typescript +````typescript import * as Effect from "effect/Effect" import * as Layer from "effect/Layer" import * as Ref from "effect/Ref" -import { StateStore, type StateSnapshot, type Commit, emptySnapshot } from "./state-store.ts" +import { type Commit, emptySnapshot, type StateSnapshot, StateStore } from "./state-store.ts" /** * Create InMemoryStateStore Layer. @@ -269,8 +264,7 @@ import { StateStore, type StateSnapshot, type Commit, emptySnapshot } from "./st const make = Effect.gen(function*() { const stateRef = yield* Ref.make(emptySnapshot) - const advance = (next: TransactionId) => - Ref.update(stateRef, (state) => ({ ...state, next })) + const advance = (next: TransactionId) => Ref.update(stateRef, (state) => ({ ...state, next })) const commit = (commit: Commit) => Ref.update(stateRef, (state) => { @@ -318,7 +312,7 @@ export const layerWithState = ( // ... same implementation as above }) ) -``` +```` ### Future Store Implementations @@ -332,8 +326,7 @@ export const layer: Layer.Layer = Layer.effect(StateStore, make) export const layer: Layer.Layer = Layer.effect(StateStore, make) // packages/amp-store-postgres/src/index.ts -export const layer: Layer.Layer = - Layer.effect(StateStore, make) +export const layer: Layer.Layer = Layer.effect(StateStore, make) ``` ## Key Algorithms @@ -348,12 +341,12 @@ export const findRecoveryPoint = ( invalidation: ReadonlyArray ): readonly [TransactionId, ReadonlyArray] | undefined => { // Build map: network -> first invalid block - const points = new Map(invalidation.map(inv => [inv.network, inv.start])) + const points = new Map(invalidation.map((inv) => [inv.network, inv.start])) // Walk backwards (newest to oldest) for (let i = buffer.length - 1; i >= 0; i--) { const [id, ranges] = buffer[i]! - const affected = ranges.some(range => { + const affected = ranges.some((range) => { const point = points.get(range.network) return point !== undefined && range.numbers.start >= point }) @@ -375,14 +368,12 @@ export const findPruningPoint = ( if (buffer.length === 0) return undefined const [, latestRanges] = buffer[buffer.length - 1]! - const cutoffs = new Map(latestRanges.map(r => - [r.network, Math.max(0, r.numbers.start - retention)] - )) + const cutoffs = new Map(latestRanges.map((r) => [r.network, Math.max(0, r.numbers.start - retention)])) let last: TransactionId | undefined for (let i = 0; i < buffer.length - 1; i++) { const [id, ranges] = buffer[i]! - const outside = ranges.every(r => r.numbers.end < cutoffs.get(r.network)!) + const outside = ranges.every((r) => r.numbers.end < cutoffs.get(r.network)!) if (outside) last = id else break } @@ -402,9 +393,9 @@ import * as Ref from "effect/Ref" * The store is the source of truth; this is a working cache. */ interface StateContainer { - readonly store: StateStore // From context - readonly retention: number // Configured retention window - next: TransactionId // Next ID to assign + readonly store: StateStore // From context + readonly retention: number // Configured retention window + next: TransactionId // Next ID to assign buffer: Array]> uncommitted: Array } @@ -528,8 +519,10 @@ export class TransactionalStream extends Context.Tag("Amp/TransactionalStream")< * Layer providing TransactionalStream. * Requires ArrowFlight and StateStore in context. */ -export const layer: Layer.Layer = - Layer.effect(TransactionalStream, make) +export const layer: Layer.Layer = Layer.effect( + TransactionalStream, + make +) ``` **streamTransactional() Logic:** @@ -686,13 +679,13 @@ const manualProgram = Effect.gen(function*() { ## Critical Files to Modify/Reference -| File | Purpose | -|------|---------| -| `packages/amp/src/arrow-flight.ts` | Pattern for Layer.effect, contains streamProtocol to wrap | -| `packages/amp/src/protocol-stream/messages.ts` | ProtocolMessage, InvalidationRange to consume | -| `packages/amp/src/protocol-stream/errors.ts` | ProtocolStreamError to include in union | -| `packages/amp/src/models.ts` | BlockRange type | -| `.repos/amp/crates/clients/flight/src/transactional.rs` | Rust source to port | +| File | Purpose | +| ------------------------------------------------------- | --------------------------------------------------------- | +| `packages/amp/src/arrow-flight.ts` | Pattern for Layer.effect, contains streamProtocol to wrap | +| `packages/amp/src/protocol-stream/messages.ts` | ProtocolMessage, InvalidationRange to consume | +| `packages/amp/src/protocol-stream/errors.ts` | ProtocolStreamError to include in union | +| `packages/amp/src/models.ts` | BlockRange type | +| `.repos/amp/crates/clients/flight/src/transactional.rs` | Rust source to port | ## Verification From 8429a4c2063fd19c4fc866c4c292f216f68de29d Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 6 Feb 2026 20:53:20 -0500 Subject: [PATCH 11/12] lint --- .rulesync/rules/effect.md | 4 ++-- .rulesync/skills/project-context/SKILL.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.rulesync/rules/effect.md b/.rulesync/rules/effect.md index c122d2d..6fe4996 100644 --- a/.rulesync/rules/effect.md +++ b/.rulesync/rules/effect.md @@ -16,11 +16,11 @@ Always import Effect modules as namespaces from subpaths: ```typescript // Good import * as Effect from "effect/Effect" -import * as Stream from "effect/Stream" import * as Schema from "effect/Schema" +import * as Stream from "effect/Stream" // Bad — never import from the barrel -import { Effect, Stream, Schema } from "effect" +import { Effect, Schema, Stream } from "effect" ``` ## Services diff --git a/.rulesync/skills/project-context/SKILL.md b/.rulesync/skills/project-context/SKILL.md index 7be0077..36167d3 100644 --- a/.rulesync/skills/project-context/SKILL.md +++ b/.rulesync/skills/project-context/SKILL.md @@ -6,4 +6,4 @@ targets: ["*"] Summarize the project goals, core constraints, and relevant dependencies. Call out any architecture decisions, shared conventions, and validation steps. -Keep the summary concise and ready to reuse in future tasks. \ No newline at end of file +Keep the summary concise and ready to reuse in future tasks. From 8e868a9ca66fdc7809a0ec3d61c41e42fa08f202 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Sat, 7 Feb 2026 09:06:38 -0500 Subject: [PATCH 12/12] remove manifest builder --- .rulesync/rules/overview.md | 1 - packages/amp/src/manifest-builder/service.ts | 156 ------------------- 2 files changed, 157 deletions(-) delete mode 100644 packages/amp/src/manifest-builder/service.ts diff --git a/.rulesync/rules/overview.md b/.rulesync/rules/overview.md index b5669b4..3adf144 100644 --- a/.rulesync/rules/overview.md +++ b/.rulesync/rules/overview.md @@ -29,7 +29,6 @@ src/ ├── auth/ # OAuth2 auth (device flow, token refresh, caching) ├── admin/ # Admin API (datasets, jobs, workers, providers) ├── registry/ # Registry API (dataset discovery) -├── manifest-builder/ # Dataset manifest construction ├── protobuf/ # Generated protobuf (Flight, FlightSql) └── internal/ # Private: Arrow IPC parsing, PKCE ``` diff --git a/packages/amp/src/manifest-builder/service.ts b/packages/amp/src/manifest-builder/service.ts deleted file mode 100644 index 7d0aa3a..0000000 --- a/packages/amp/src/manifest-builder/service.ts +++ /dev/null @@ -1,156 +0,0 @@ -import * as Context from "effect/Context" -import * as Data from "effect/Data" -import * as Effect from "effect/Effect" -import * as Layer from "effect/Layer" -import * as Predicate from "effect/Predicate" -import * as Schema from "effect/Schema" -import * as AdminApi from "../admin/service.ts" -import * as Models from "../core/domain.ts" - -export const ManifestBuildResult = Schema.Struct({ - metadata: Models.DatasetMetadata, - manifest: Models.DatasetDerived -}) -export type ManifestBuildResult = typeof ManifestBuildResult.Type - -export class ManifestBuilderError extends Data.TaggedError("ManifestBuilderError")<{ - readonly cause: unknown - readonly message: string - readonly table: string -}> {} - -export class ManifestBuilder extends Context.Tag("Amp/ManifestBuilder") Effect.Effect -}>() {} - -const make = Effect.gen(function*() { - const admin = yield* AdminApi.AdminApi - - const build = Effect.fn("ManifestBuilder.build")( - function*(config: Models.DatasetConfig) { - // Extract metadata - const metadata = Models.DatasetMetadata.make({ - namespace: config.namespace ?? Models.DatasetNamespace.make("_"), - name: config.name, - readme: config.readme, - repository: config.repository, - description: config.description, - keywords: config.keywords, - license: config.license, - visibility: config.private ? "private" : "public", - sources: config.sources - }) - - // Build manifest tables - send all tables in one request - const tables = yield* Effect.gen(function*() { - const configTables = config.tables ?? {} - const configFunctions = config.functions ?? {} - - // Build function definitions map from config - const functionsMap: Record = {} - for (const [name, func] of Object.entries(configFunctions)) { - functionsMap[name] = Models.FunctionDefinition.make({ - source: func.source, - inputTypes: func.inputTypes, - outputType: func.outputType - }) - } - - // If no tables and no functions, skip schema request entirely - if (Object.keys(configTables).length === 0 && Object.keys(functionsMap).length === 0) { - return [] - } - - // If no tables but we have functions, still skip schema request - // (when functions-only validation happens server-side, returns empty schema) - if (Object.keys(configTables).length === 0) { - return [] - } - - // Prepare all table SQL queries - const tableSqlMap: Record = {} - for (const [name, table] of Object.entries(configTables)) { - tableSqlMap[name] = table.sql - } - - // Call schema endpoint with all tables and functions at once - const response = yield* admin.getOutputSchema({ - tables: tableSqlMap, - dependencies: config.dependencies, - functions: Object.keys(functionsMap).length > 0 ? functionsMap : undefined - }).pipe(Effect.catchAll((cause) => - new ManifestBuilderError({ - cause, - message: "Failed to get schemas", - table: "(all tables)" - }) - )) - - // Process each table's schema - const tables: Array<[name: string, table: Models.Table]> = [] - for (const [name, table] of Object.entries(configTables)) { - const tableSchema = response.schemas[name] - - if (Predicate.isUndefined(tableSchema)) { - return yield* new ManifestBuilderError({ - message: `No schema returned for table ${name}`, - table: name, - cause: undefined - }) - } - - if (tableSchema.networks.length !== 1) { - return yield* new ManifestBuilderError({ - cause: undefined, - message: `Expected 1 network for SQL query, got ${tableSchema.networks}`, - table: name - }) - } - - const network = Models.Network.make(tableSchema.networks[0]) - const input = Models.TableInput.make({ sql: table.sql }) - const output = Models.Table.make({ input, schema: tableSchema.schema, network }) - - tables.push([name, output]) - } - - return tables - }) - - // Build manifest functions - const functions: Array<[name: string, manifest: Models.FunctionManifest]> = [] - for (const [name, func] of Object.entries(config.functions ?? {})) { - const { inputTypes, outputType, source } = func - const manifest = Models.FunctionManifest.make({ name, source, inputTypes, outputType }) - - functions.push([name, manifest]) - } - - const manifest = Models.DatasetDerived.make({ - kind: "manifest", - startBlock: config.startBlock, - dependencies: config.dependencies, - tables: Object.fromEntries(tables), - functions: Object.fromEntries(functions) - }) - - return ManifestBuildResult.make({ - metadata, - manifest - }) - } - ) - - return { - build - } as const -}) - -/** - * Layer for creating a `ManifestBuilder`. - */ -export const layer: Layer.Layer< - ManifestBuilder, - never, - AdminApi.AdminApi -> = Layer.effect(ManifestBuilder, make)