# Snapshot report for `src/tokenizer/__tests__/tokenizer-spec.js`

The actual snapshot is saved in `tokenizer-spec.js.snap`.

Generated by [AVA](https://ava.li).

## ignores comments

> Snapshot 1

    [
      {
        end: {
          col: 3,
          line: 3,
          sourceLine: '  2',
        },
        start: {
          col: 2,
          line: 3,
          sourceLine: '  2',
        },
        type: 'Constant',
        value: '2',
      },
    ]

## ignores multiline comments

> Snapshot 1

    [
      {
        end: {
          col: 5,
          line: 5,
          sourceLine: '    2 * 2',
        },
        start: {
          col: 4,
          line: 5,
          sourceLine: '    2 * 2',
        },
        type: 'Constant',
        value: '2',
      },
      {
        end: {
          col: 7,
          line: 5,
          sourceLine: '    2 * 2',
        },
        start: {
          col: 6,
          line: 5,
          sourceLine: '    2 * 2',
        },
        type: 'Punctuator',
        value: '*',
      },
      {
        end: {
          col: 9,
          line: 5,
          sourceLine: '    2 * 2',
        },
        start: {
          col: 8,
          line: 5,
          sourceLine: '    2 * 2',
        },
        type: 'Constant',
        value: '2',
      },
    ]

## ignores multiline comments and compiles correctly

> Snapshot 1

    [
      {
        end: {
          col: 10,
          line: 3,
          sourceLine: '    export function test() {',
        },
        start: {
          col: 4,
          line: 3,
          sourceLine: '    export function test() {',
        },
        type: 'Keyword',
        value: 'export',
      },
      {
        end: {
          col: 19,
          line: 3,
          sourceLine: '    export function test() {',
        },
        start: {
          col: 11,
          line: 3,
          sourceLine: '    export function test() {',
        },
        type: 'Keyword',
        value: 'function',
      },
      {
        end: {
          col: 24,
          line: 3,
          sourceLine: '    export function test() {',
        },
        start: {
          col: 20,
          line: 3,
          sourceLine: '    export function test() {',
        },
        type: 'Identifier',
        value: 'test',
      },
      {
        end: {
          col: 25,
          line: 3,
          sourceLine: '    export function test() {',
        },
        start: {
          col: 24,
          line: 3,
          sourceLine: '    export function test() {',
        },
        type: 'Punctuator',
        value: '(',
      },
      {
        end: {
          col: 26,
          line: 3,
          sourceLine: '    export function test() {',
        },
        start: {
          col: 25,
          line: 3,
          sourceLine: '    export function test() {',
        },
        type: 'Punctuator',
        value: ')',
      },
      {
        end: {
          col: 28,
          line: 3,
          sourceLine: '    export function test() {',
        },
        start: {
          col: 27,
          line: 3,
          sourceLine: '    export function test() {',
        },
        type: 'Punctuator',
        value: '{',
      },
      {
        end: {
          col: 12,
          line: 4,
          sourceLine: '      return 2/* inline comment */ * 2;',
        },
        start: {
          col: 6,
          line: 4,
          sourceLine: '      return 2/* inline comment */ * 2;',
        },
        type: 'Keyword',
        value: 'return',
      },
      {
        end: {
          col: 14,
          line: 4,
          sourceLine: '      return 2/* inline comment */ * 2;',
        },
        start: {
          col: 13,
          line: 4,
          sourceLine: '      return 2/* inline comment */ * 2;',
        },
        type: 'Constant',
        value: '2',
      },
      {
        end: {
          col: 36,
          line: 4,
          sourceLine: '      return 2/* inline comment */ * 2;',
        },
        start: {
          col: 35,
          line: 4,
          sourceLine: '      return 2/* inline comment */ * 2;',
        },
        type: 'Punctuator',
        value: '*',
      },
      {
        end: {
          col: 38,
          line: 4,
          sourceLine: '      return 2/* inline comment */ * 2;',
        },
        start: {
          col: 37,
          line: 4,
          sourceLine: '      return 2/* inline comment */ * 2;',
        },
        type: 'Constant',
        value: '2',
      },
      {
        end: {
          col: 39,
          line: 4,
          sourceLine: '      return 2/* inline comment */ * 2;',
        },
        start: {
          col: 38,
          line: 4,
          sourceLine: '      return 2/* inline comment */ * 2;',
        },
        type: 'Punctuator',
        value: ';',
      },
      {
        end: {
          col: 5,
          line: 5,
          sourceLine: '    }',
        },
        start: {
          col: 4,
          line: 5,
          sourceLine: '    }',
        },
        type: 'Punctuator',
        value: '}',
      },
    ]

## ignores one-liner multiline comments

> Snapshot 1

    [
      {
        end: {
          col: 5,
          line: 3,
          sourceLine: '    2',
        },
        start: {
          col: 4,
          line: 3,
          sourceLine: '    2',
        },
        type: 'Constant',
        value: '2',
      },
    ]

## ignores singleline tokens within multiline and vice-versa

> Snapshot 1

    []

## ignores whitespace at the end of file

> Snapshot 1

    [
      {
        end: {
          col: 10,
          line: 2,
          sourceLine: '  function test(): i32 {',
        },
        start: {
          col: 2,
          line: 2,
          sourceLine: '  function test(): i32 {',
        },
        type: 'Keyword',
        value: 'function',
      },
      {
        end: {
          col: 15,
          line: 2,
          sourceLine: '  function test(): i32 {',
        },
        start: {
          col: 11,
          line: 2,
          sourceLine: '  function test(): i32 {',
        },
        type: 'Identifier',
        value: 'test',
      },
      {
        end: {
          col: 16,
          line: 2,
          sourceLine: '  function test(): i32 {',
        },
        start: {
          col: 15,
          line: 2,
          sourceLine: '  function test(): i32 {',
        },
        type: 'Punctuator',
        value: '(',
      },
      {
        end: {
          col: 17,
          line: 2,
          sourceLine: '  function test(): i32 {',
        },
        start: {
          col: 16,
          line: 2,
          sourceLine: '  function test(): i32 {',
        },
        type: 'Punctuator',
        value: ')',
      },
      {
        end: {
          col: 18,
          line: 2,
          sourceLine: '  function test(): i32 {',
        },
        start: {
          col: 17,
          line: 2,
          sourceLine: '  function test(): i32 {',
        },
        type: 'Punctuator',
        value: ':',
      },
      {
        end: {
          col: 22,
          line: 2,
          sourceLine: '  function test(): i32 {',
        },
        start: {
          col: 19,
          line: 2,
          sourceLine: '  function test(): i32 {',
        },
        type: 'Type',
        value: 'i32',
      },
      {
        end: {
          col: 24,
          line: 2,
          sourceLine: '  function test(): i32 {',
        },
        start: {
          col: 23,
          line: 2,
          sourceLine: '  function test(): i32 {',
        },
        type: 'Punctuator',
        value: '{',
      },
      {
        end: {
          col: 10,
          line: 3,
          sourceLine: '    return 2 + 2;',
        },
        start: {
          col: 4,
          line: 3,
          sourceLine: '    return 2 + 2;',
        },
        type: 'Keyword',
        value: 'return',
      },
      {
        end: {
          col: 12,
          line: 3,
          sourceLine: '    return 2 + 2;',
        },
        start: {
          col: 11,
          line: 3,
          sourceLine: '    return 2 + 2;',
        },
        type: 'Constant',
        value: '2',
      },
      {
        end: {
          col: 14,
          line: 3,
          sourceLine: '    return 2 + 2;',
        },
        start: {
          col: 13,
          line: 3,
          sourceLine: '    return 2 + 2;',
        },
        type: 'Punctuator',
        value: '+',
      },
      {
        end: {
          col: 16,
          line: 3,
          sourceLine: '    return 2 + 2;',
        },
        start: {
          col: 15,
          line: 3,
          sourceLine: '    return 2 + 2;',
        },
        type: 'Constant',
        value: '2',
      },
      {
        end: {
          col: 17,
          line: 3,
          sourceLine: '    return 2 + 2;',
        },
        start: {
          col: 16,
          line: 3,
          sourceLine: '    return 2 + 2;',
        },
        type: 'Punctuator',
        value: ';',
      },
      {
        end: {
          col: 3,
          line: 4,
          sourceLine: '  }',
        },
        start: {
          col: 2,
          line: 4,
          sourceLine: '  }',
        },
        type: 'Punctuator',
        value: '}',
      },
    ]

## number literals

> Snapshot 1

    [
      '1e10',
      '0b101',
      '0xFF',
      '0B0101',
      '0o10',
      '0b10',
      '0xff',
      '0xE10',
    ]

## parses a stream into tokens

> Snapshot 1

    [
      {
        end: {
          col: 3,
          line: 1,
          sourceLine: 'let x: i32 = 2;',
        },
        start: {
          col: 0,
          line: 1,
          sourceLine: 'let x: i32 = 2;',
        },
        type: 'Keyword',
        value: 'let',
      },
      {
        end: {
          col: 5,
          line: 1,
          sourceLine: 'let x: i32 = 2;',
        },
        start: {
          col: 4,
          line: 1,
          sourceLine: 'let x: i32 = 2;',
        },
        type: 'Identifier',
        value: 'x',
      },
      {
        end: {
          col: 6,
          line: 1,
          sourceLine: 'let x: i32 = 2;',
        },
        start: {
          col: 5,
          line: 1,
          sourceLine: 'let x: i32 = 2;',
        },
        type: 'Punctuator',
        value: ':',
      },
      {
        end: {
          col: 10,
          line: 1,
          sourceLine: 'let x: i32 = 2;',
        },
        start: {
          col: 7,
          line: 1,
          sourceLine: 'let x: i32 = 2;',
        },
        type: 'Type',
        value: 'i32',
      },
      {
        end: {
          col: 12,
          line: 1,
          sourceLine: 'let x: i32 = 2;',
        },
        start: {
          col: 11,
          line: 1,
          sourceLine: 'let x: i32 = 2;',
        },
        type: 'Punctuator',
        value: '=',
      },
      {
        end: {
          col: 14,
          line: 1,
          sourceLine: 'let x: i32 = 2;',
        },
        start: {
          col: 13,
          line: 1,
          sourceLine: 'let x: i32 = 2;',
        },
        type: 'Constant',
        value: '2',
      },
      {
        end: {
          col: 15,
          line: 1,
          sourceLine: 'let x: i32 = 2;',
        },
        start: {
          col: 14,
          line: 1,
          sourceLine: 'let x: i32 = 2;',
        },
        type: 'Punctuator',
        value: ';',
      },
    ]

## parses basic strings

> Snapshot 1

    [
      {
        end: {
          col: 18,
          line: 1,
          sourceLine: '\'this is a string\' "and so is this"',
        },
        start: {
          col: 0,
          line: 1,
          sourceLine: '\'this is a string\' "and so is this"',
        },
        type: 'StringLiteral',
        value: '\'this is a string\'',
      },
      {
        end: {
          col: 35,
          line: 1,
          sourceLine: '\'this is a string\' "and so is this"',
        },
        start: {
          col: 19,
          line: 1,
          sourceLine: '\'this is a string\' "and so is this"',
        },
        type: 'StringLiteral',
        value: '"and so is this"',
      },
    ]

## parses identifiers with numbers

> Snapshot 1

    [
      {
        end: {
          col: 13,
          line: 1,
          sourceLine: 'test1foo42bar ',
        },
        start: {
          col: 0,
          line: 1,
          sourceLine: 'test1foo42bar ',
        },
        type: 'Identifier',
        value: 'test1foo42bar',
      },
    ]

## parses strings with escaped string

> Snapshot 1

    [
      {
        end: {
          col: 28,
          line: 1,
          sourceLine: '"string start \\" string end" \'start \\\' end \'',
        },
        start: {
          col: 0,
          line: 1,
          sourceLine: '"string start \\" string end" \'start \\\' end \'',
        },
        type: 'StringLiteral',
        value: '"string start \\" string end"',
      },
      {
        end: {
          col: 44,
          line: 1,
          sourceLine: '"string start \\" string end" \'start \\\' end \'',
        },
        start: {
          col: 29,
          line: 1,
          sourceLine: '"string start \\" string end" \'start \\\' end \'',
        },
        type: 'StringLiteral',
        value: '\'start \\\' end \'',
      },
    ]

## parses strings within strings

> Snapshot 1

    [
      {
        end: {
          col: 37,
          line: 1,
          sourceLine: '"here is a string with a \'substring\'"',
        },
        start: {
          col: 0,
          line: 1,
          sourceLine: '"here is a string with a \'substring\'"',
        },
        type: 'StringLiteral',
        value: '"here is a string with a \'substring\'"',
      },
    ]