all files / liquidjs/src/ tokenizer.js

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
1×
1×
1×
1×
1×
 
1×
340×
 
339×
339×
339×
339×
339×
 
339×
640×
264×
 
640×
 
 
 
332×
98×
 
332×
332×
 
1×
424×
424×
 
 
 
 
 
 
 
 
 
 
424×
7×
 
417×
417×
417×
 
 
1×
216×
 
 
 
 
 
 
 
 
 
 
 
1×
362×
362×
 
362×
 
 
 
 
 
 
 
1×
339×
339×
 
339×
 
640×
640×
640×
640×
 
 
 
 
1×
1×
 
const lexical = require('./lexical.js')
const TokenizationError = require('./util/error.js').TokenizationError
const _ = require('./util/underscore.js')
const whiteSpaceCtrl = require('./whitespace-ctrl.js')
const assert = require('./util/assert.js')
 
function parse (input, file, options) {
  assert(_.isString(input), 'illegal input')
 
  var rLiquid = /({%-?([\s\S]*?)-?%})|({{-?([\s\S]*?)-?}})/g
  var currIndent = 0
  var lineNumber = LineNumber(input)
  var lastMatchEnd = 0
  var tokens = []
 
  for (var match; (match = rLiquid.exec(input)); lastMatchEnd = rLiquid.lastIndex) {
    if (match.index > lastMatchEnd) {
      tokens.push(parseHTMLToken(lastMatchEnd, match.index))
    }
    tokens.push(match[1]
      ? parseTagToken(match[1], match[2].trim(), match.index)
      : parseValueToken(match[3], match[4].trim(), match.index))
  }
  if (input.length > lastMatchEnd) {
    tokens.push(parseHTMLToken(lastMatchEnd, input.length))
  }
  whiteSpaceCtrl(tokens, options)
  return tokens
 
  function parseTagToken (raw, value, pos) {
    var match = value.match(lexical.tagLine)
    var token = {
      type: 'tag',
      indent: currIndent,
      line: lineNumber.get(pos),
      trim_left: raw.slice(0, 3) === '{%-',
      trim_right: raw.slice(-3) === '-%}',
      raw,
      value,
      input,
      file
    }
    if (!match) {
      throw new TokenizationError(`illegal tag syntax`, token)
    }
    token.name = match[1]
    token.args = match[2]
    return token
  }
 
  function parseValueToken (raw, value, pos) {
    return {
      type: 'value',
      line: lineNumber.get(pos),
      trim_left: raw.slice(0, 3) === '{{-',
      trim_right: raw.slice(-3) === '-}}',
      raw,
      value,
      input,
      file
    }
  }
 
  function parseHTMLToken (begin, end) {
    var htmlFragment = input.slice(begin, end)
    currIndent = _.last((htmlFragment).split('\n')).length
 
    return {
      type: 'html',
      raw: htmlFragment,
      value: htmlFragment
    }
  }
}
 
function LineNumber (html) {
  var parsedLinesCount = 0
  var lastMatchBegin = -1
 
  return {
    get: function (pos) {
      var lines = html.slice(lastMatchBegin + 1, pos).split('\n')
      parsedLinesCount += lines.length - 1
      lastMatchBegin = pos
      return parsedLinesCount + 1
    }
  }
}
 
exports.parse = parse
exports.whiteSpaceCtrl = whiteSpaceCtrl