import { CompatibilityContext, CompatibilityEdition, CompatibilityOption, TimeValuePrecision } from '@powersync/service-sync-rules'; import { Binary, BSON, BSONRegExp, BSONSymbol, Code, DBRef, Decimal128, Double, Int32, Long, MaxKey, MinKey, ObjectId, Timestamp } from 'bson'; import { describe, expect, test } from 'vitest'; import { JsonBufferWriter } from '@module/replication/JsonBufferWriter.js'; import { DirectSourceRowConverter, LegacySourceRowConverter, SourceRowConverter } from '@module/replication/SourceRowConverter.js'; type Placement = 'top' | 'array' | 'nested'; type ExpectedPlacements = Record; type RowCapture = { ok: true; row: unknown } | { ok: false; message: string }; type OutputCapture = { ok: true; output: unknown } | { ok: false; message: string }; type ConverterCase = { name: string; buildBuffer: (placement: Placement) => Buffer; expected: ExpectedPlacements; context?: CompatibilityContext; expectedId?: (placement: Placement) => string; }; // Serialization differs in cases between top-level values, values in arrays and values in nested documents. // We test each one. const PLACEMENTS: Placement[] = ['top', 'array', 'nested']; const CONTEXT = CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY; const legacyConverter = new LegacySourceRowConverter(CONTEXT); const directConverter = new DirectSourceRowConverter(CONTEXT); const normalDate = new Date('2023-03-06T13:47:00.000Z'); const positiveExtendedDate = new Date(253402300800000); const negativeExtendedDate = new Date(-62167219200001); const objectId = new ObjectId('66e834cc91d805df11fa0ecb'); const uuidBytes = Buffer.from('00112233445566778899aabbccddeeff', 'hex'); const depth21Expected = '{"nested":{"nested":{"nested":{"nested":{"nested":{"nested":{"nested":{"nested":{"nested":{"nested":{"nested":{"nested":{"nested":{"nested":{"nested":{"nested":{"nested":{"nested":{"nested":{"nested":{"nested":1}}}}}}}}}}}}}}}}}}}}}'; describe('JsonBufferWriter', () => { test('writeQuotedUtf8Slice escapes large control-heavy payloads without corruption', () => { const writer = new JsonBufferWriter(32); const payload = Buffer.alloc(256, 0x01); writer.writeQuotedUtf8Slice(payload, 0, payload.length); const expected = JSON.stringify(payload.toString('latin1')); expect(writer.toString()).toBe(expected); }); test('writeQuotedUtf8Slice escapes large control-heavy payloads without corruption (2)', () => { const writer = new JsonBufferWriter(32); const payload1 = Buffer.alloc(512); const payload2 = Buffer.alloc(256, 0x01); // This makes sure reset() properly clears the previous data writer.writeQuotedUtf8Slice(payload1, 0, payload1.length); writer.reset(); writer.writeQuotedUtf8Slice(payload2, 0, payload2.length); const expected = JSON.stringify(payload2.toString('latin1')); expect(writer.toString()).toBe(expected); }); test('truncate clears discarded bytes', () => { const writer = new JsonBufferWriter(32); writer.writeAscii('discarded'); writer.truncate(0); const { buffer } = writer as unknown as { buffer: Buffer }; expect(buffer.subarray(0, 'discarded'.length)).toEqual(Buffer.alloc('discarded'.length)); }); }); const testCases: ConverterCase[] = [ serializableCase('double', new Double(1.25), placements(1.25, '[1.25]', '{"nested":1.25}')), serializableCase('double:nan', new Double(NaN), placements(NaN, '[null]', '{"nested":null}')), serializableCase('double:infinity', new Double(Infinity), placements(Infinity, '[null]', '{"nested":null}')), serializableCase( 'double:negativeInfinity', new Double(-Infinity), placements(-Infinity, '[null]', '{"nested":null}') ), serializableCase( 'string', 'line 1\nline "2" \\ snowman ☃ \u0001', jsonStringPlacements('line 1\nline "2" \\ snowman ☃ \u0001') ), serializableCase( 'string:surrogate-pair', 'emoji: \ud83d\ude00 rocket: \ud83d\ude80', jsonStringPlacements('emoji: \ud83d\ude00 rocket: \ud83d\ude80') ), // This one is not quite valid utf-8 - it becomes U+FFFD. serializableCase('string:half-surrogate-pair', 'foo: \ud83d"', jsonStringPlacements('foo: \ufffd"')), serializableCase( 'document', { alpha: 1, bravo: 'two', charlie: true, delta: null }, jsonTextPlacements('{"alpha":1,"bravo":"two","charlie":1,"delta":null}') ), serializableCase('array', [1, 'two', false, null, { deep: 3 }], jsonTextPlacements('[1,"two",0,null,{"deep":3}]')), serializableCase('objectId', objectId, jsonStringPlacements('66e834cc91d805df11fa0ecb')), serializableCase('bool', true, placements(1n, '[1]', '{"nested":1}')), { name: 'bool (fixed json)', buildBuffer: (placement) => serializeCaseDocument(`bool (fixed json):${placement}`, placement, false), expected: placements(0n, '[false]', '{"nested":false}'), context: new CompatibilityContext({ edition: CompatibilityEdition.COMPILED_STREAMS, overrides: new Map([[CompatibilityOption.fixedBooleanInJson, true]]) }) } satisfies ConverterCase, serializableCase('date', normalDate, jsonStringPlacements('2023-03-06 13:47:00.000Z')), serializableCase('date:+010000', positiveExtendedDate, jsonStringPlacements('+010000-01-01 00:00:00.000Z')), serializableCase('date:-000001', negativeExtendedDate, jsonStringPlacements('-000001-12-31 23:59:59.999Z')), serializableCase('null', null, placements(null, '[null]', '{"nested":null}')), serializableCase('regex:flags:i', new BSONRegExp('a', 'i'), jsonTextPlacements('{"pattern":"a","options":"i"}')), serializableCase('regex:flags:m', new BSONRegExp('a', 'm'), jsonTextPlacements('{"pattern":"a","options":"m"}')), rawCase('undefined', 0x06, Buffer.alloc(0), placements(null, '[null]', '{}')), serializableCase('code', new Code('return 1;'), jsonTextPlacements('{"code":"return 1;","scope":null}')), serializableCase('symbol', new BSONSymbol('sym'), jsonStringPlacements('sym')), rawCase( 'dbpointer', 0x0c, Buffer.concat([bsonString('mycollection'), Buffer.from('66e834cc91d805df11fa0ecb', 'hex')]), jsonTextPlacements('{"collection":"mycollection","oid":"66e834cc91d805df11fa0ecb","fields":{}}') ), rawCase( 'dbpointer:empty-collection', 0x0c, Buffer.concat([bsonString(''), Buffer.from('66e834cc91d805df11fa0ecb', 'hex')]), jsonTextPlacements('{"collection":"","oid":"66e834cc91d805df11fa0ecb","fields":{}}') ), rawCase( 'dbpointer:escaped-collection', 0x0c, Buffer.concat([bsonString('my_\\"coll\\"\\\\name/☃'), Buffer.from('66e834cc91d805df11fa0ecb', 'hex')]), jsonTextPlacements( '{"collection":"my_\\\\\\"coll\\\\\\"\\\\\\\\name/☃","oid":"66e834cc91d805df11fa0ecb","fields":{}}' ) ), rawCase( 'dbpointer:different-oid', 0x0c, Buffer.concat([bsonString('other'), Buffer.from('00112233445566778899aabb', 'hex')]), jsonTextPlacements('{"collection":"other","oid":"00112233445566778899aabb","fields":{}}') ), rawCase( 'dbpointer:with-db', 0x0c, Buffer.concat([bsonString('mydb.mycollection'), Buffer.from('66e834cc91d805df11fa0ecb', 'hex')]), jsonTextPlacements('{"collection":"mycollection","oid":"66e834cc91d805df11fa0ecb","db":"mydb","fields":{}}') ), serializableCase( 'codeScope', new Code('return x;', { x: 1 }), jsonTextPlacements('{"code":"return x;","scope":{"x":1}}') ), serializableCase('int32', new Int32(123), placements(123n, '[123]', '{"nested":123}')), serializableCase( 'timestamp', Timestamp.fromBits(123, 456), placements(1958505087099n, '[1958505087099]', '{"nested":1958505087099}') ), serializableCase( 'int64', Long.fromBigInt(9007199254740993n), placements(9007199254740993n, '[9007199254740993]', '{"nested":9007199254740993}') ), serializableCase('decimal128', Decimal128.fromString('1234.5678'), jsonStringPlacements('1234.5678')), serializableCase('minKey', new MinKey(), placements(null, '[null]', '{"nested":null}')), serializableCase('maxKey', new MaxKey(), placements(null, '[null]', '{"nested":null}')), serializableCase( 'binary:default', new Binary(Buffer.from([0, 1, 2, 255]), Binary.SUBTYPE_DEFAULT), placements(new Uint8Array([0, 1, 2, 255]), '[null]', '{}') ), serializableCase( 'binary:function', new Binary(Buffer.from([1, 2, 3]), Binary.SUBTYPE_FUNCTION), placements(new Uint8Array([1, 2, 3]), '[null]', '{}') ), serializableCase( 'binary:byteArray', new Binary(Buffer.from([4, 5, 6]), Binary.SUBTYPE_BYTE_ARRAY), placements(new Uint8Array([4, 5, 6]), '[null]', '{}') ), serializableCase( 'binary:uuidOld', new Binary(uuidBytes, Binary.SUBTYPE_UUID_OLD), placements(new Uint8Array(uuidBytes), '[null]', '{}') ), serializableCase( 'binary:uuid', new Binary(uuidBytes, Binary.SUBTYPE_UUID), jsonStringPlacements('00112233-4455-6677-8899-aabbccddeeff') ), serializableCase( 'binary:md5', new Binary(Buffer.from([7, 8, 9]), Binary.SUBTYPE_MD5), placements(new Uint8Array([7, 8, 9]), '[null]', '{}') ), serializableCase( 'binary:encrypted', new Binary(Buffer.from([10, 11, 12]), Binary.SUBTYPE_ENCRYPTED), placements(new Uint8Array([10, 11, 12]), '[null]', '{}') ), serializableCase( 'binary:column', new Binary(Buffer.from([13, 14, 15]), Binary.SUBTYPE_COLUMN), placements(new Uint8Array([13, 14, 15]), '[null]', '{}') ), serializableCase( 'binary:sensitive', new Binary(Buffer.from([16, 17, 18]), Binary.SUBTYPE_SENSITIVE), placements(new Uint8Array([16, 17, 18]), '[null]', '{}') ), serializableCase( 'binary:vector', new Binary(Buffer.from([19, 20, 21]), Binary.SUBTYPE_VECTOR), placements(new Uint8Array([19, 20, 21]), '[null]', '{}') ), serializableCase( 'binary:userDefined', new Binary(Buffer.from([22, 23, 24]), Binary.SUBTYPE_USER_DEFINED), placements(new Uint8Array([22, 23, 24]), '[null]', '{}') ), // Degenerate arrays: The string keys are not spec-compliant, but ignored by the parsers. { name: 'array:invalid-key:alpha', buildBuffer: (placement) => rawCaseDocument( `array:invalid-key:alpha:${placement}`, placement, 0x04, bsonDocument([bsonElement(0x10, 'alpha', int32(1))]) ), expected: placements('[1]', '[[1]]', '{"nested":[1]}') }, { name: 'array:invalid-key:leading-zero', buildBuffer: (placement) => rawCaseDocument( `array:invalid-key:leading-zero:${placement}`, placement, 0x04, bsonDocument([bsonElement(0x10, '01', int32(1))]) ), expected: placements('[1]', '[[1]]', '{"nested":[1]}') }, { name: 'array:invalid-key:gap', buildBuffer: (placement) => rawCaseDocument( `array:invalid-key:gap:${placement}`, placement, 0x04, bsonDocument([bsonElement(0x10, '1', int32(1))]) ), expected: placements('[1]', '[[1]]', '{"nested":[1]}') }, { name: 'array:invalid-key:negative', buildBuffer: (placement) => rawCaseDocument( `array:invalid-key:negative:${placement}`, placement, 0x04, bsonDocument([bsonElement(0x10, '-1', int32(1))]) ), expected: placements('[1]', '[[1]]', '{"nested":[1]}') }, { name: 'array:invalid-key:mixed', buildBuffer: (placement) => rawCaseDocument( `array:invalid-key:mixed:${placement}`, placement, 0x04, bsonDocument([bsonElement(0x10, '0', int32(1)), bsonElement(0x10, 'alpha', int32(2))]) ), expected: placements('[1,2]', '[[1,2]]', '{"nested":[1,2]}') }, { name: 'array:invalid-key:reversed', buildBuffer: (placement) => rawCaseDocument( `array:invalid-key:reversed:${placement}`, placement, 0x04, bsonDocument([bsonElement(0x10, '1', int32(1)), bsonElement(0x10, '0', int32(2))]) ), expected: placements('[1,2]', '[[1,2]]', '{"nested":[1,2]}') } ]; const INVALID_UUID_LENGTHS = [0, 1, 15, 17] as const; const DATE_COMPATIBILITY_CASES = [ { name: 'legacy-edition', context: CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY, normal: '2023-03-06 13:47:00.000Z', positiveExtended: '+010000-01-01 00:00:00.000Z', negativeExtended: '-000001-12-31 23:59:59.999Z' }, { name: 'sync-streams-edition', context: new CompatibilityContext({ edition: CompatibilityEdition.SYNC_STREAMS }), normal: '2023-03-06T13:47:00.000Z', positiveExtended: '+010000-01-01T00:00:00.000Z', negativeExtended: '-000001-12-31T23:59:59.999Z' }, { name: 'compiled-streams-edition', context: new CompatibilityContext({ edition: CompatibilityEdition.COMPILED_STREAMS }), normal: '2023-03-06T13:47:00.000Z', positiveExtended: '+010000-01-01T00:00:00.000Z', negativeExtended: '-000001-12-31T23:59:59.999Z' }, { name: 'legacy-with-iso-override-enabled', context: new CompatibilityContext({ edition: CompatibilityEdition.LEGACY, overrides: new Map([[CompatibilityOption.timestampsIso8601, true]]) }), normal: '2023-03-06T13:47:00.000Z', positiveExtended: '+010000-01-01T00:00:00.000Z', negativeExtended: '-000001-12-31T23:59:59.999Z' }, { name: 'sync-streams-with-iso-override-disabled', context: new CompatibilityContext({ edition: CompatibilityEdition.SYNC_STREAMS, overrides: new Map([[CompatibilityOption.timestampsIso8601, false]]) }), normal: '2023-03-06 13:47:00.000Z', positiveExtended: '+010000-01-01 00:00:00.000Z', negativeExtended: '-000001-12-31 23:59:59.999Z' }, { name: 'sync-streams-seconds-precision', context: new CompatibilityContext({ edition: CompatibilityEdition.SYNC_STREAMS, maxTimeValuePrecision: TimeValuePrecision.seconds }), normal: '2023-03-06T13:47:00Z', positiveExtended: '+010000-01-01T00:00:00Z', negativeExtended: '-000001-12-31T23:59:59Z' }, { name: 'sync-streams-milliseconds-precision', context: new CompatibilityContext({ edition: CompatibilityEdition.SYNC_STREAMS, maxTimeValuePrecision: TimeValuePrecision.milliseconds }), normal: '2023-03-06T13:47:00.000Z', positiveExtended: '+010000-01-01T00:00:00.000Z', negativeExtended: '-000001-12-31T23:59:59.999Z' }, { name: 'sync-streams-microseconds-clamped', context: new CompatibilityContext({ edition: CompatibilityEdition.SYNC_STREAMS, maxTimeValuePrecision: TimeValuePrecision.microseconds }), normal: '2023-03-06T13:47:00.000Z', positiveExtended: '+010000-01-01T00:00:00.000Z', negativeExtended: '-000001-12-31T23:59:59.999Z' }, { name: 'sync-streams-nanoseconds-clamped', context: new CompatibilityContext({ edition: CompatibilityEdition.SYNC_STREAMS, maxTimeValuePrecision: TimeValuePrecision.nanoseconds }), normal: '2023-03-06T13:47:00.000Z', positiveExtended: '+010000-01-01T00:00:00.000Z', negativeExtended: '-000001-12-31T23:59:59.999Z' }, { name: 'legacy-with-iso-override-and-seconds', context: new CompatibilityContext({ edition: CompatibilityEdition.LEGACY, overrides: new Map([[CompatibilityOption.timestampsIso8601, true]]), maxTimeValuePrecision: TimeValuePrecision.seconds }), normal: '2023-03-06T13:47:00Z', positiveExtended: '+010000-01-01T00:00:00Z', negativeExtended: '-000001-12-31T23:59:59Z' }, { name: 'sync-streams-iso-disabled-ignores-precision', context: new CompatibilityContext({ edition: CompatibilityEdition.SYNC_STREAMS, overrides: new Map([[CompatibilityOption.timestampsIso8601, false]]), maxTimeValuePrecision: TimeValuePrecision.seconds }), normal: '2023-03-06 13:47:00.000Z', positiveExtended: '+010000-01-01 00:00:00.000Z', negativeExtended: '-000001-12-31 23:59:59.999Z' } ] as const; for (const length of INVALID_UUID_LENGTHS) { testCases.push( serializableCase( `binary:uuid:invalid-length:${length}`, new Binary(Buffer.alloc(length, 0x11), Binary.SUBTYPE_UUID), placements(new Uint8Array(Buffer.alloc(length, 0x11).buffer), '[null]', '{}') ) ); } for (const dateCase of DATE_COMPATIBILITY_CASES) { testCases.push( { name: `date:compat:${dateCase.name}`, context: dateCase.context, buildBuffer: dateBufferForPlacement, expectedId: (placement) => `compatibility-date:${placement}`, expected: jsonStringPlacements(dateCase.normal) }, { name: `date:+010000:compat:${dateCase.name}`, context: dateCase.context, buildBuffer: (placement) => serializeCaseDocument(`compatibility-date:+010000:${placement}`, placement, positiveExtendedDate), expectedId: (placement) => `compatibility-date:+010000:${placement}`, expected: jsonStringPlacements(dateCase.positiveExtended) }, { name: `date:-000001:compat:${dateCase.name}`, context: dateCase.context, buildBuffer: (placement) => serializeCaseDocument(`compatibility-date:-000001:${placement}`, placement, negativeExtendedDate), expectedId: (placement) => `compatibility-date:-000001:${placement}`, expected: jsonStringPlacements(dateCase.negativeExtended) } ); } describe('SourceRowConverter.rawToSqliteRow expected output', () => { for (const parityCase of testCases) { const context = parityCase.context ?? CONTEXT; const legacyConverter = new LegacySourceRowConverter(context); const directConverter = new DirectSourceRowConverter(context); for (const placement of PLACEMENTS) { test(`default output for ${parityCase.name} as ${placementLabel(placement)}`, () => { expectNormalizedRow(legacyConverter, parityCase.buildBuffer(placement), { _id: parityCase.expectedId?.(placement) ?? `${parityCase.name}:${placement}`, value: parityCase.expected[placement] }); }); test(`custom output for ${parityCase.name} as ${placementLabel(placement)}`, () => { expectNormalizedRow(directConverter, parityCase.buildBuffer(placement), { _id: parityCase.expectedId?.(placement) ?? `${parityCase.name}:${placement}`, value: parityCase.expected[placement] }); }); } } test('default output for 21 nested object levels', () => { expectNormalizedRow( legacyConverter, BSON.serialize({ _id: 'depth-21', value: deepNestedObject(21) }) as Buffer, { _id: 'depth-21', value: depth21Expected } ); }); test('custom output for 21 nested object levels', () => { expectNormalizedRow( directConverter, BSON.serialize({ _id: 'depth-21', value: deepNestedObject(21) }) as Buffer, { _id: 'depth-21', value: depth21Expected } ); }); }); describe('SourceRowConverter.rawToSqliteRow regex option preservation', () => { // These cases intentionally diverge from the default implementation: // The default implementation parsed to a JS-compatible RegExp, converting // some options such as s -> g, and failing hard on some invalid cases such as "ii". // The custom implementation preserves options as-is, even when invalid according to the BSON spec. // Additionally, the default implementation performed some RegExp pattern normalization, which we don't do // in the custom parser. const regexDivergenceCases = [ { name: 'regex', buildBuffer: (placement: Placement) => serializeCaseDocument(`regex:${placement}`, placement, new BSONRegExp('a\\s+"b"', 'ims')), testDescription: 'preserves BSON regex options on custom converter', defaultExpected: jsonTextPlacements('{"pattern":"a\\\\s+\\"b\\"","options":"gim"}'), customExpected: jsonTextPlacements('{"pattern":"a\\\\s+\\"b\\"","options":"ims"}') }, { name: 'regex:flags:s', buildBuffer: (placement: Placement) => serializeCaseDocument(`regex:flags:s:${placement}`, placement, new BSONRegExp('a', 's')), testDescription: 'preserves BSON regex options on custom converter', defaultExpected: jsonTextPlacements('{"pattern":"a","options":"g"}'), customExpected: jsonTextPlacements('{"pattern":"a","options":"s"}') }, { name: 'regex:flags:x', buildBuffer: (placement: Placement) => serializeCaseDocument(`regex:flags:x:${placement}`, placement, new BSONRegExp('a', 'x')), testDescription: 'preserves BSON regex options on custom converter', defaultExpected: jsonTextPlacements('{"pattern":"a","options":""}'), customExpected: jsonTextPlacements('{"pattern":"a","options":"x"}') }, { name: 'regex:flags:u', buildBuffer: (placement: Placement) => serializeCaseDocument(`regex:flags:u:${placement}`, placement, new BSONRegExp('a', 'u')), testDescription: 'preserves BSON regex options on custom converter', defaultExpected: jsonTextPlacements('{"pattern":"a","options":""}'), customExpected: jsonTextPlacements('{"pattern":"a","options":"u"}') }, { name: 'regex:flags:imsxu', buildBuffer: (placement: Placement) => serializeCaseDocument(`regex:flags:imsxu:${placement}`, placement, new BSONRegExp('a', 'imsxu')), testDescription: 'preserves BSON regex options on custom converter', defaultExpected: jsonTextPlacements('{"pattern":"a","options":"gim"}'), customExpected: jsonTextPlacements('{"pattern":"a","options":"imsux"}') }, { name: 'regex:raw:quote-and-backslash-options', buildBuffer: (placement: Placement) => rawCaseDocument( `regex:raw:quote-and-backslash-options:${placement}`, placement, 0x0b, Buffer.concat([cstring('a"b\\c\n\t☃'), cstring('i"\\x')]) ), testDescription: 'escapes special characters correctly', defaultExpected: placements( '{"pattern":"a\\"b\\\\c\\\\n\\t☃","options":"i"}', '[{"pattern":"a\\"b\\\\c\\\\n\\t☃","options":"i"}]', '{"nested":{"pattern":"a\\"b\\\\c\\\\n\\t☃","options":"i"}}' ), customExpected: placements( '{"pattern":"a\\"b\\\\c\\n\\t☃","options":"i\\"\\\\x"}', '[{"pattern":"a\\"b\\\\c\\n\\t☃","options":"i\\"\\\\x"}]', '{"nested":{"pattern":"a\\"b\\\\c\\n\\t☃","options":"i\\"\\\\x"}}' ) }, { name: 'regex:raw:quoted-options-only', buildBuffer: (placement: Placement) => rawCaseDocument( `regex:raw:quoted-options-only:${placement}`, placement, 0x0b, Buffer.concat([cstring('line1\nline2\t"q"'), cstring('"\\')]) ), testDescription: 'escapes special characters correctly', defaultExpected: placements( '{"pattern":"line1\\\\nline2\\t\\"q\\"","options":""}', '[{"pattern":"line1\\\\nline2\\t\\"q\\"","options":""}]', '{"nested":{"pattern":"line1\\\\nline2\\t\\"q\\"","options":""}}' ), customExpected: placements( '{"pattern":"line1\\nline2\\t\\"q\\"","options":"\\"\\\\"}', '[{"pattern":"line1\\nline2\\t\\"q\\"","options":"\\"\\\\"}]', '{"nested":{"pattern":"line1\\nline2\\t\\"q\\"","options":"\\"\\\\"}}' ) }, { name: 'regex:raw:unsorted-valid-options', buildBuffer: (placement: Placement) => rawCaseDocument( `regex:raw:unsorted-valid-options:${placement}`, placement, 0x0b, Buffer.concat([cstring('a'), cstring('mi')]) ), testDescription: 'documents default-vs-raw normalization differences', defaultExpected: placements( '{"pattern":"a","options":"im"}', '[{"pattern":"a","options":"im"}]', '{"nested":{"pattern":"a","options":"im"}}' ), customExpected: placements( '{"pattern":"a","options":"mi"}', '[{"pattern":"a","options":"mi"}]', '{"nested":{"pattern":"a","options":"mi"}}' ) }, { name: 'regex:raw:unsorted-mixed-options', buildBuffer: (placement: Placement) => rawCaseDocument( `regex:raw:unsorted-mixed-options:${placement}`, placement, 0x0b, Buffer.concat([cstring('a'), cstring('xim')]) ), testDescription: 'documents default-vs-raw normalization differences', defaultExpected: placements( '{"pattern":"a","options":"im"}', '[{"pattern":"a","options":"im"}]', '{"nested":{"pattern":"a","options":"im"}}' ), customExpected: placements( '{"pattern":"a","options":"xim"}', '[{"pattern":"a","options":"xim"}]', '{"nested":{"pattern":"a","options":"xim"}}' ) }, { name: 'regex:raw:empty-pattern', buildBuffer: (placement: Placement) => rawCaseDocument( `regex:raw:empty-pattern:${placement}`, placement, 0x0b, Buffer.concat([cstring(''), cstring('x"\\')]) ), testDescription: 'documents default-vs-raw normalization differences', defaultExpected: placements( '{"pattern":"(?:)","options":""}', '[{"pattern":"(?:)","options":""}]', '{"nested":{"pattern":"(?:)","options":""}}' ), customExpected: placements( '{"pattern":"","options":"x\\"\\\\"}', '[{"pattern":"","options":"x\\"\\\\"}]', '{"nested":{"pattern":"","options":"x\\"\\\\"}}' ) } ] as const; for (const regexCase of regexDivergenceCases) { for (const placement of PLACEMENTS) { test(`${regexCase.name} ${regexCase.testDescription} as ${placementLabel(placement)}`, () => { const source = regexCase.buildBuffer(placement); // Parity is intentionally not expected here. The default path converts BSON regexes // through JS RegExp.flags or reconstructs JS RegExp semantics, while the raw // path preserves the BSON pattern/options bytes as-is. expectNormalizedRow(legacyConverter, source, { _id: `${regexCase.name}:${placement}`, value: regexCase.defaultExpected[placement] }); expectNormalizedRow(directConverter, source, { _id: `${regexCase.name}:${placement}`, value: regexCase.customExpected[placement] }); }); } } test('unsupported BSON regex flag is preserved only on the custom raw converter', () => { const source = rawCaseDocument('regex:invalid:z', 'top', 0x0b, Buffer.concat([cstring('a'), cstring('z')])); expectNormalizedRow(legacyConverter, source, { _id: 'regex:invalid:z', value: '{"pattern":"a","options":""}' }); expectNormalizedRow(directConverter, source, { _id: 'regex:invalid:z', value: '{"pattern":"a","options":"z"}' }); }); test('duplicate BSON regex flags are preserved only on the custom raw converter', () => { const source = rawCaseDocument('regex:invalid:ii', 'top', 0x0b, Buffer.concat([cstring('a'), cstring('ii')])); expectRowFailure(legacyConverter, source, "Invalid flags supplied to RegExp constructor 'ii'"); // The raw converter preserves the BSON option string even when it is not valid JS RegExp flags. expectNormalizedRow(directConverter, source, { _id: 'regex:invalid:ii', value: '{"pattern":"a","options":"ii"}' }); }); }); describe('SourceRowConverter.rawToSqliteRow invalid UTF-8', () => { // The upstream bson parser validates UTF-8 strings by default. // Our custom parser accepts invalid UTF-8 strings, using the replacement character in JSON. test('invalid UTF-8 in top-level string is accepted only on the custom raw converter', () => { const source = bsonDocument([ bsonElement(0x02, '_id', bsonString('invalid-utf8:top-string')), bsonElement(0x02, 'value', Buffer.concat([int32(2), Buffer.from([0xff, 0x00])])) ]); expectRowFailure(legacyConverter, source, 'Invalid UTF-8 string in BSON document'); expectNormalizedRow(directConverter, source, { _id: 'invalid-utf8:top-string', value: '�' }); }); test('invalid UTF-8 in nested string is accepted only on the custom raw converter', () => { const source = bsonDocument([ bsonElement(0x02, '_id', bsonString('invalid-utf8:nested-string')), bsonElement( 0x03, 'value', bsonDocument([bsonElement(0x02, 'nested', Buffer.concat([int32(2), Buffer.from([0xff, 0x00])]))]) ) ]); expectRowFailure(legacyConverter, source, 'Invalid UTF-8 string in BSON document'); expectNormalizedRow(directConverter, source, { _id: 'invalid-utf8:nested-string', value: '{"nested":"�"}' }); }); }); describe('SourceRowConverter.rawToSqliteRow malformed BSON lengths', () => { test('overlong top-level string length fails instead of hanging', () => { const source = bsonDocument([ bsonElement(0x02, '_id', bsonString('malformed-length:top-string')), // Declares far more string bytes than exist in the document. bsonElement(0x02, 'value', Buffer.concat([int32(1000), Buffer.from([0xff, 0x00])])) ]); expect(captureRow(legacyConverter, source).ok).toBe(false); expectRowFailure(directConverter, source, 'Invalid BSON string length'); }); }); describe('SourceRowConverter.rawToSqliteRow fuzz', () => { test('matches across randomized supported documents', () => { const rng = makeRng(0x5eedc0de); for (let i = 0; i < 150; i++) { const source = BSON.serialize( { _id: `fuzz:${i}:${randomString(rng, 0, 6)}`, [`root:${randomString(rng, 0, 5)}`]: randomSupportedValue(rng), value: randomSupportedValue(rng) }, { ignoreUndefined: false } ) as Buffer; expectRowParity(source); } }); test('matches on a large nested string that grows the JSON writer buffer', () => { const source = BSON.serialize({ _id: 'large-string', value: { nested: 'x'.repeat(1024 * 1024 + 4096) } }) as Buffer; expectRowParity(source); }); test('matches on escape-heavy keys and values', () => { const source = BSON.serialize({ _id: 'escapes', 'quote"slash\\newline\n': { '\tcontrol\u0001': ['line 1\nline 2', '"quoted"', '☃'] }, value: { 中: ['\\', '"', '\r', '\t', '\u0001'] } }) as Buffer; expectRowParity(source); }); test('matches on large escape-heavy nested values serialized from BSON', () => { // For this test to work, this must be larger than the default buffer size const escapeHeavy = '\u0001'.repeat(1024 * 400); const source = BSON.serialize({ _id: 'escapes-large', value: { nested: escapeHeavy } }) as Buffer; expectRowParity(source); }); test('matches 21 nested object levels', () => { expectRowParity( BSON.serialize({ _id: 'depth-21', value: deepNestedObject(21) }) as Buffer ); }); }); describe('SourceRowConverter.rawToSqliteRow full output parity', () => { test('matches replicaId when row parity succeeds', () => { const source = BSON.serialize({ _id: 'replica-id', value: new Int32(7) }) as Buffer; expect(captureOutput(directConverter, source)).toEqual(captureOutput(legacyConverter, source)); }); test('default full output matches expected replicaId and row', () => { const source = BSON.serialize({ _id: 'replica-id', value: new Int32(7) }) as Buffer; expect(captureOutput(legacyConverter, source)).toEqual({ ok: true, output: normalize({ replicaId: 'replica-id', row: { _id: 'replica-id', value: 7n } }) }); }); test('custom full output matches expected replicaId and row', () => { const source = BSON.serialize({ _id: 'replica-id', value: new Int32(7) }) as Buffer; expect(captureOutput(directConverter, source)).toEqual({ ok: true, output: normalize({ replicaId: 'replica-id', row: { _id: 'replica-id', value: 7n } }) }); }); }); describe('SourceRowConverter.rawToSqliteRow DBRef handling', () => { test('documents why DBRef differs from deprecated DBPointer handling', () => { // These are _not_ serialized as a DBPointer value. It is a normal document, // with $ref, $id, $db keys. const source = BSON.serialize({ _id: 'dbref', value: new DBRef('mycollection', new ObjectId('66e834cc91d805df11fa0ecb'), 'mydb', { foo: 'bar' }) }) as Buffer; // LegacySourceRowConverter serializes the DBRef instance fields. expectNormalizedRow(legacyConverter, source, { _id: 'dbref', value: '{"collection":"mycollection","oid":"66e834cc91d805df11fa0ecb","db":"mydb","fields":{"foo":"bar"}}' }); // DirectSourceRowConverter serializes the raw BSON document shape for DBRef. // We intentionally do not attempt to convert this type. expectNormalizedRow(directConverter, source, { _id: 'dbref', value: '{"$ref":"mycollection","$id":"66e834cc91d805df11fa0ecb","$db":"mydb","foo":"bar"}' }); }); }); function serializeCaseDocument(id: string, placement: Placement, value: unknown): Buffer { if (placement === 'top') { return BSON.serialize({ _id: id, value }, { ignoreUndefined: false }) as Buffer; } if (placement === 'array') { return BSON.serialize({ _id: id, value: [value] }, { ignoreUndefined: false }) as Buffer; } return BSON.serialize({ _id: id, value: { nested: value } }, { ignoreUndefined: false }) as Buffer; } function int32(value: number): Buffer { const bytes = Buffer.alloc(4); bytes.writeInt32LE(value); return bytes; } function cstring(value: string): Buffer { return Buffer.concat([Buffer.from(value, 'utf8'), Buffer.from([0])]); } function bsonString(value: string): Buffer { const bytes = Buffer.from(value, 'utf8'); return Buffer.concat([int32(bytes.length + 1), bytes, Buffer.from([0])]); } function bsonDocument(elements: Buffer[]): Buffer { const body = Buffer.concat([...elements, Buffer.from([0])]); return Buffer.concat([int32(body.length + 4), body]); } function bsonElement(type: number, key: string, payload: Buffer = Buffer.alloc(0)): Buffer { return Buffer.concat([Buffer.from([type]), cstring(key), payload]); } function rawCaseDocument(id: string, placement: Placement, type: number, payload: Buffer): Buffer { const valueElement = placement === 'top' ? bsonElement(type, 'value', payload) : placement === 'array' ? bsonElement(0x04, 'value', bsonDocument([bsonElement(type, '0', payload)])) : bsonElement(0x03, 'value', bsonDocument([bsonElement(type, 'nested', payload)])); return bsonDocument([bsonElement(0x02, '_id', bsonString(id)), valueElement]); } function placements(top: unknown, array: unknown, nested: unknown): ExpectedPlacements { return { top, array, nested }; } function jsonStringPlacements(top: string): ExpectedPlacements { return placements(top, `[${JSON.stringify(top)}]`, `{"nested":${JSON.stringify(top)}}`); } function jsonTextPlacements(top: string): ExpectedPlacements { return placements(top, `[${top}]`, `{"nested":${top}}`); } function serializableCase(name: string, value: unknown, expected: ExpectedPlacements): ConverterCase { return { name, buildBuffer: (placement) => serializeCaseDocument(`${name}:${placement}`, placement, value), expected }; } function rawCase(name: string, type: number, payload: Buffer, expected: ExpectedPlacements): ConverterCase { return { name, buildBuffer: (placement) => rawCaseDocument(`${name}:${placement}`, placement, type, payload), expected }; } function dateBufferForPlacement(placement: Placement): Buffer { return serializeCaseDocument(`compatibility-date:${placement}`, placement, normalDate); } /** * Normalize Uint8Array -> hex string to make test output nicer. */ function normalize(value: unknown): unknown { if (Buffer.isBuffer(value)) { throw new Error(`Unexpected buffer: Use standard Uint8Array instead.`); } if (value instanceof Uint8Array) { return { __bytes: Buffer.from(value).toString('hex') }; } if (Array.isArray(value)) { return value.map((entry) => normalize(entry)); } if (value != null && typeof value === 'object') { const record: Record = {}; for (const key of Object.keys(value).sort()) { record[key] = normalize((value as Record)[key]); } return record; } return value; } function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } function captureRow(converter: SourceRowConverter, source: Buffer): RowCapture { try { return { ok: true, row: normalize(converter.rawToSqliteRow(source).row) }; } catch (error) { return { ok: false, message: errorMessage(error) }; } } function expectRowFailure(converter: SourceRowConverter, source: Buffer, message: string) { expect(captureRow(converter, source)).toEqual({ ok: false, message }); } function captureOutput(converter: SourceRowConverter, source: Buffer): OutputCapture { try { return { ok: true, output: normalize(converter.rawToSqliteRow(source)) }; } catch (error) { return { ok: false, message: errorMessage(error) }; } } function expectRowParity(source: Buffer) { expect(captureRow(directConverter, source)).toEqual(captureRow(legacyConverter, source)); } function expectNormalizedRow(converter: SourceRowConverter, source: Buffer, expected: Record) { expect(captureRow(converter, source)).toEqual({ ok: true, row: normalize(expected) }); } function placementLabel(placement: Placement): string { switch (placement) { case 'top': return 'top-level field'; case 'array': return 'embedded in array'; case 'nested': return 'embedded in nested document'; } } function makeRng(seed: number) { let state = seed >>> 0; return () => { state = (state + 0x6d2b79f5) >>> 0; let next = Math.imul(state ^ (state >>> 15), 1 | state); next ^= next + Math.imul(next ^ (next >>> 7), 61 | next); return ((next ^ (next >>> 14)) >>> 0) / 4294967296; }; } function randomInt(rng: () => number, min: number, max: number) { return Math.floor(rng() * (max - min + 1)) + min; } function pick(rng: () => number, values: T[]): T { return values[randomInt(rng, 0, values.length - 1)]; } function randomString(rng: () => number, minLength: number, maxLength: number) { const alphabet = ['a', 'Z', '0', '9', ' ', '"', '\\', '\n', '\r', '\t', '/', '\u0001', 'é', '☃', '中']; const length = randomInt(rng, minLength, maxLength); let output = ''; for (let i = 0; i < length; i++) { output += pick(rng, alphabet); } return output; } function randomObjectId(rng: () => number) { const bytes = Buffer.alloc(12); for (let i = 0; i < bytes.length; i++) { bytes[i] = randomInt(rng, 0, 255); } return new ObjectId(bytes); } function randomSafeDate(rng: () => number) { const min = Date.UTC(2000, 0, 1); const max = Date.UTC(2035, 11, 31, 23, 59, 59, 999); return new Date(Math.floor(rng() * (max - min + 1)) + min); } function randomLeaf(rng: () => number) { switch (randomInt(rng, 0, 10)) { case 0: return new Double(Number((rng() * 1000 - 500).toFixed(6))); case 1: return randomString(rng, 0, 24); case 2: return randomInt(rng, 0, 1) === 0; case 3: return null; case 4: return randomObjectId(rng); case 5: return randomSafeDate(rng); case 6: return new Int32(randomInt(rng, -5000, 5000)); case 7: return Long.fromBigInt(BigInt(randomInt(rng, -5000, 5000)) * 1000000000000n + 17n); case 8: return Decimal128.fromString( `${randomInt(rng, -999, 999)}.${randomInt(rng, 0, 9999).toString().padStart(4, '0')}` ); case 9: return rng() < 0.5 ? new MinKey() : new MaxKey(); default: return new Double(Number((rng() * 1000 - 500).toFixed(6))); } } function randomSupportedValue(rng: () => number, depth = 0): unknown { if (depth >= 3) { return randomLeaf(rng); } switch (randomInt(rng, 0, 4)) { case 0: case 1: return randomLeaf(rng); case 2: { const length = randomInt(rng, 0, 4); return Array.from({ length }, () => randomSupportedValue(rng, depth + 1)); } default: { const entries = randomInt(rng, 0, 4); const record: Record = {}; for (let i = 0; i < entries; i++) { record[`key:${i}:${randomString(rng, 0, 8)}`] = randomSupportedValue(rng, depth + 1); } return record; } } } function deepNestedObject(depth: number): unknown { let value: unknown = 1; for (let i = 0; i < depth; i++) { value = { nested: value }; } return value; }