diff --git a/dist/cjs/Sharding.js b/dist/cjs/Sharding.js index ffd9f07626d9efbf77b8186ce7a8330bf62566eb..4573677ba76c821d7062295c7c606904ab20e2af 100644 --- a/dist/cjs/Sharding.js +++ b/dist/cjs/Sharding.js @@ -125,7 +125,18 @@ const make = /*#__PURE__*/Effect.gen(function* () { fiber: "releaseShard", runner: selfAddress, shardId - }))), Effect.eventually, FiberMap.run(releaseShardsMap, shardId, { + }))), // PATCHED (voltro): was `Effect.eventually` — retry until success with NO + // delay. A shard that cannot be released (storage unreachable, DB down) + // therefore spun a fiber at full speed, logging at DEBUG level where nobody + // reads it. Same semantics — keep trying until it succeeds — with backoff. + // + // `union` takes the MINIMUM of the two delays, so this CAPS at 5s rather + // than compounding: 100ms, 200ms, 400ms … 5s, 5s, 5s. Both schedules recur + // forever, and a union continues while either does, so the retry is still + // unbounded in ATTEMPTS — which is what shard release needs. The defect was + // the missing delay, not the persistence. + Effect.retry(Schedule.exponential(100).pipe(Schedule.union(Schedule.spaced(5000)))), + FiberMap.run(releaseShardsMap, shardId, { onlyIfMissing: true }))); const releaseShards = Effect.gen(function* () { diff --git a/dist/cjs/SqlMessageStorage.js b/dist/cjs/SqlMessageStorage.js index 77ebad1d2e940b73a494bf9ab0ad7e008daddf82..9fa66974a5f7977449778304c48cb7b1b1e72ee9 100644 --- a/dist/cjs/SqlMessageStorage.js +++ b/dist/cjs/SqlMessageStorage.js @@ -41,7 +41,11 @@ const make = exports.make = /*#__PURE__*/Effect.fnUntraced(function* (options) { const messagesTableSql = sql(messagesTable); const repliesTable = table("replies"); const repliesTableSql = sql(repliesTable); - const envelopeToRow = (envelope, message_id, deliver_at) => { + const envelopeToRow = (envelope, message_id, deliver_at_in) => { + // Cast deliver_at (millisecond epoch) to BigInt so mssql's tedious + // driver binds it as BIGINT instead of inferring INT (which overflows + // for any post-2001 timestamp). ONLY mssql — sqlite runs safeIntegers(true) and hangs on a bigint. + const deliver_at = deliver_at_in == null ? null : sql.onDialectOrElse({ mssql: () => BigInt(deliver_at_in), orElse: () => deliver_at_in }); switch (envelope._tag) { case "Request": return { @@ -203,37 +207,16 @@ const make = exports.make = /*#__PURE__*/Effect.fnUntraced(function* (options) { ON target.message_id = source.message_id WHEN NOT MATCHED THEN INSERT ${sql.insert(row)} - OUTPUT - inserted.id, - CASE - WHEN inserted.id IS NULL THEN ( - SELECT r.id, r.kind, r.payload - FROM ${repliesTableSql} r - WHERE r.id = target.last_reply_id - ) - END as reply_id, - CASE - WHEN inserted.id IS NULL THEN ( - SELECT r.kind - FROM ${repliesTableSql} r - WHERE r.id = target.last_reply_id - ) - END as reply_kind, - CASE - WHEN inserted.id IS NULL THEN ( - SELECT r.payload - FROM ${repliesTableSql} r - WHERE r.id = target.last_reply_id - ) - END as reply_payload, - CASE - WHEN inserted.id IS NULL THEN ( - SELECT r.sequence - FROM ${repliesTableSql} r - WHERE r.id = target.last_reply_id - ) - END as reply_sequence; - `, + OUTPUT inserted.id; + `.unprepared.pipe(Effect.flatMap((rows) => { + if (rows.length > 0) return Effect.succeed([]); + return sql` + SELECT m.id, r.id as reply_id, r.kind as reply_kind, r.payload as reply_payload, r.sequence as reply_sequence + FROM ${messagesTableSql} m + LEFT JOIN ${repliesTableSql} r ON r.id = m.last_reply_id + WHERE m.message_id = ${message_id} + `.unprepared; + }), sql.withTransaction), orElse: () => (row, message_id) => sql` SELECT m.id, r.id as reply_id, r.kind as reply_kind, r.payload as reply_payload, r.sequence as reply_sequence FROM ${messagesTableSql} m @@ -262,6 +245,7 @@ const make = exports.make = /*#__PURE__*/Effect.fnUntraced(function* (options) { }); const forUpdate = sql.onDialectOrElse({ sqlite: () => sql.literal(""), + mssql: () => sql.literal(""), orElse: () => sql.literal("FOR UPDATE") }); const getUnprocessedMessages = sql.onDialectOrElse({ diff --git a/dist/cjs/SqlRunnerStorage.js b/dist/cjs/SqlRunnerStorage.js index 9c6defd9723112761eb49ca1e189fd6cc4187ae7..e82351709c76d206df68a8aa46c4740b43caef8d 100644 --- a/dist/cjs/SqlRunnerStorage.js +++ b/dist/cjs/SqlRunnerStorage.js @@ -302,7 +302,7 @@ const make = exports.make = /*#__PURE__*/Effect.fnUntraced(function* (options) { const values = shardIds.map(shardId => sql`(${stringLiteral(shardId)}, ${stringLiteral(address)}, ${sqlNow})`); return sql` MERGE ${locksTableSql} WITH (HOLDLOCK) AS target - USING (SELECT * FROM (VALUES ${sql.csv(values)})) AS source (shard_id, address, acquired_at) + USING (VALUES ${sql.csv(values)}) AS source (shard_id, address, acquired_at) ON target.shard_id = source.shard_id WHEN MATCHED AND (target.address = source.address OR DATEDIFF(SECOND, target.acquired_at, ${sqlNow}) > ${expiresSeconds}) THEN UPDATE SET address = source.address, acquired_at = source.acquired_at diff --git a/dist/esm/Sharding.js b/dist/esm/Sharding.js index c973ed72595ddc9a90229ede2a14a3f571a54af5..0ff7d7347c63ab270520410375ca07709457ace3 100644 --- a/dist/esm/Sharding.js +++ b/dist/esm/Sharding.js @@ -117,7 +117,18 @@ const make = /*#__PURE__*/Effect.gen(function* () { fiber: "releaseShard", runner: selfAddress, shardId - }))), Effect.eventually, FiberMap.run(releaseShardsMap, shardId, { + }))), // PATCHED (voltro): was `Effect.eventually` — retry until success with NO + // delay. A shard that cannot be released (storage unreachable, DB down) + // therefore spun a fiber at full speed, logging at DEBUG level where nobody + // reads it. Same semantics — keep trying until it succeeds — with backoff. + // + // `union` takes the MINIMUM of the two delays, so this CAPS at 5s rather + // than compounding: 100ms, 200ms, 400ms … 5s, 5s, 5s. Both schedules recur + // forever, and a union continues while either does, so the retry is still + // unbounded in ATTEMPTS — which is what shard release needs. The defect was + // the missing delay, not the persistence. + Effect.retry(Schedule.exponential(100).pipe(Schedule.union(Schedule.spaced(5000)))), + FiberMap.run(releaseShardsMap, shardId, { onlyIfMissing: true }))); const releaseShards = Effect.gen(function* () { diff --git a/dist/esm/SqlMessageStorage.js b/dist/esm/SqlMessageStorage.js index e41d91033992bb890bebb1a16a5ac9e6657a9c41..2e0c02d7bda6f2040894d4636f5deea94ae15843 100644 --- a/dist/esm/SqlMessageStorage.js +++ b/dist/esm/SqlMessageStorage.js @@ -33,7 +33,11 @@ export const make = /*#__PURE__*/Effect.fnUntraced(function* (options) { const messagesTableSql = sql(messagesTable); const repliesTable = table("replies"); const repliesTableSql = sql(repliesTable); - const envelopeToRow = (envelope, message_id, deliver_at) => { + const envelopeToRow = (envelope, message_id, deliver_at_in) => { + // Cast deliver_at (millisecond epoch) to BigInt so mssql's tedious + // driver binds it as BIGINT instead of inferring INT (which overflows + // for any post-2001 timestamp). ONLY mssql — sqlite runs safeIntegers(true) and hangs on a bigint. + const deliver_at = deliver_at_in == null ? null : sql.onDialectOrElse({ mssql: () => BigInt(deliver_at_in), orElse: () => deliver_at_in }); switch (envelope._tag) { case "Request": return { @@ -195,37 +199,16 @@ export const make = /*#__PURE__*/Effect.fnUntraced(function* (options) { ON target.message_id = source.message_id WHEN NOT MATCHED THEN INSERT ${sql.insert(row)} - OUTPUT - inserted.id, - CASE - WHEN inserted.id IS NULL THEN ( - SELECT r.id, r.kind, r.payload - FROM ${repliesTableSql} r - WHERE r.id = target.last_reply_id - ) - END as reply_id, - CASE - WHEN inserted.id IS NULL THEN ( - SELECT r.kind - FROM ${repliesTableSql} r - WHERE r.id = target.last_reply_id - ) - END as reply_kind, - CASE - WHEN inserted.id IS NULL THEN ( - SELECT r.payload - FROM ${repliesTableSql} r - WHERE r.id = target.last_reply_id - ) - END as reply_payload, - CASE - WHEN inserted.id IS NULL THEN ( - SELECT r.sequence - FROM ${repliesTableSql} r - WHERE r.id = target.last_reply_id - ) - END as reply_sequence; - `, + OUTPUT inserted.id; + `.unprepared.pipe(Effect.flatMap((rows) => { + if (rows.length > 0) return Effect.succeed([]); + return sql` + SELECT m.id, r.id as reply_id, r.kind as reply_kind, r.payload as reply_payload, r.sequence as reply_sequence + FROM ${messagesTableSql} m + LEFT JOIN ${repliesTableSql} r ON r.id = m.last_reply_id + WHERE m.message_id = ${message_id} + `.unprepared; + }), sql.withTransaction), orElse: () => (row, message_id) => sql` SELECT m.id, r.id as reply_id, r.kind as reply_kind, r.payload as reply_payload, r.sequence as reply_sequence FROM ${messagesTableSql} m @@ -254,6 +237,7 @@ export const make = /*#__PURE__*/Effect.fnUntraced(function* (options) { }); const forUpdate = sql.onDialectOrElse({ sqlite: () => sql.literal(""), + mssql: () => sql.literal(""), orElse: () => sql.literal("FOR UPDATE") }); const getUnprocessedMessages = sql.onDialectOrElse({ diff --git a/dist/esm/SqlRunnerStorage.js b/dist/esm/SqlRunnerStorage.js index d4c94fe72c8ff589c0a8f12ff80bac41614a2096..8f6ee6b12f7892916cbd17d18e66d4be1239d118 100644 --- a/dist/esm/SqlRunnerStorage.js +++ b/dist/esm/SqlRunnerStorage.js @@ -294,7 +294,7 @@ export const make = /*#__PURE__*/Effect.fnUntraced(function* (options) { const values = shardIds.map(shardId => sql`(${stringLiteral(shardId)}, ${stringLiteral(address)}, ${sqlNow})`); return sql` MERGE ${locksTableSql} WITH (HOLDLOCK) AS target - USING (SELECT * FROM (VALUES ${sql.csv(values)})) AS source (shard_id, address, acquired_at) + USING (VALUES ${sql.csv(values)}) AS source (shard_id, address, acquired_at) ON target.shard_id = source.shard_id WHEN MATCHED AND (target.address = source.address OR DATEDIFF(SECOND, target.acquired_at, ${sqlNow}) > ${expiresSeconds}) THEN UPDATE SET address = source.address, acquired_at = source.acquired_at diff --git a/src/SqlMessageStorage.ts b/src/SqlMessageStorage.ts index 4326c3832b425548dac135493a4a6cbec0856f63..4f83bf02f24e8e8021954353ae28b82c987f8483 100644 --- a/src/SqlMessageStorage.ts +++ b/src/SqlMessageStorage.ts @@ -52,8 +52,12 @@ export const make = Effect.fnUntraced(function*(options?: { const envelopeToRow = ( envelope: Envelope.Envelope.Encoded, message_id: string | null, - deliver_at: number | null + deliver_at_in: number | null ): MessageRow => { + // Cast deliver_at (millisecond epoch) to BigInt so mssql's tedious + // driver binds it as BIGINT instead of inferring INT (which overflows + // for any post-2001 timestamp). ONLY mssql — sqlite runs safeIntegers(true) and hangs on a bigint. + const deliver_at: bigint | number | null = deliver_at_in == null ? null : sql.onDialectOrElse({ mssql: () => BigInt(deliver_at_in), orElse: () => deliver_at_in }) switch (envelope._tag) { case "Request": return { @@ -240,37 +244,19 @@ export const make = Effect.fnUntraced(function*(options?: { ON target.message_id = source.message_id WHEN NOT MATCHED THEN INSERT ${sql.insert(row)} - OUTPUT - inserted.id, - CASE - WHEN inserted.id IS NULL THEN ( - SELECT r.id, r.kind, r.payload - FROM ${repliesTableSql} r - WHERE r.id = target.last_reply_id - ) - END as reply_id, - CASE - WHEN inserted.id IS NULL THEN ( - SELECT r.kind - FROM ${repliesTableSql} r - WHERE r.id = target.last_reply_id - ) - END as reply_kind, - CASE - WHEN inserted.id IS NULL THEN ( - SELECT r.payload - FROM ${repliesTableSql} r - WHERE r.id = target.last_reply_id - ) - END as reply_payload, - CASE - WHEN inserted.id IS NULL THEN ( - SELECT r.sequence - FROM ${repliesTableSql} r - WHERE r.id = target.last_reply_id - ) - END as reply_sequence; - `, + OUTPUT inserted.id; + `.unprepared.pipe( + Effect.flatMap((rows) => { + if (rows.length > 0) return Effect.succeed([] as any) + return sql` + SELECT m.id, r.id as reply_id, r.kind as reply_kind, r.payload as reply_payload, r.sequence as reply_sequence + FROM ${messagesTableSql} m + LEFT JOIN ${repliesTableSql} r ON r.id = m.last_reply_id + WHERE m.message_id = ${message_id} + `.unprepared + }), + sql.withTransaction + ), orElse: () => (row, message_id) => sql` SELECT m.id, r.id as reply_id, r.kind as reply_kind, r.payload as reply_payload, r.sequence as reply_sequence @@ -304,6 +290,7 @@ export const make = Effect.fnUntraced(function*(options?: { }) const forUpdate = sql.onDialectOrElse({ sqlite: () => sql.literal(""), + mssql: () => sql.literal(""), orElse: () => sql.literal("FOR UPDATE") }) diff --git a/src/SqlRunnerStorage.ts b/src/SqlRunnerStorage.ts index f291bcb8d22238e0d195a3bb49c91457363aa075..430e25412b4af7faf1f75e7a74fad5b1b8f8c5e0 100644 --- a/src/SqlRunnerStorage.ts +++ b/src/SqlRunnerStorage.ts @@ -362,7 +362,7 @@ export const make = Effect.fnUntraced(function*(options: { const values = shardIds.map((shardId) => sql`(${stringLiteral(shardId)}, ${stringLiteral(address)}, ${sqlNow})`) return sql` MERGE ${locksTableSql} WITH (HOLDLOCK) AS target - USING (SELECT * FROM (VALUES ${sql.csv(values)})) AS source (shard_id, address, acquired_at) + USING (VALUES ${sql.csv(values)}) AS source (shard_id, address, acquired_at) ON target.shard_id = source.shard_id WHEN MATCHED AND (target.address = source.address OR DATEDIFF(SECOND, target.acquired_at, ${sqlNow}) > ${expiresSeconds}) THEN UPDATE SET address = source.address, acquired_at = source.acquired_at