1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
|
/**
* The Lexer class handles tokenizing the input in various ways. Since our
* parser expects us to be able to backtrack, the lexer allows lexing from any
* given starting point.
*
* Its main exposed function is the `lex` function, which takes a position to
* lex from and a type of token to lex. It defers to the appropriate `_innerLex`
* function.
*
* The various `_innerLex` functions perform the actual lexing of different
* kinds.
*/
var matchAt = require("match-at");
var ParseError = require("./ParseError");
// The main lexer class
function Lexer(input) {
this._input = input;
}
// The resulting token returned from `lex`.
function Token(text, data, position) {
this.text = text;
this.data = data;
this.position = position;
}
/* The following tokenRegex
* - matches typical whitespace (but not NBSP etc.) using its first group
* - matches symbol combinations which result in a single output character
* - does not match any control character \x00-\x1f except whitespace
* - does not match a bare backslash
* - matches any ASCII character except those just mentioned
* - does not match the BMP private use area \uE000-\uF8FF
* - does not match bare surrogate code units
* - matches any BMP character except for those just described
* - matches any valid Unicode surrogate pair
* - matches a backslash followed by one or more letters
* - matches a backslash followed by any BMP character, including newline
* Just because the Lexer matches something doesn't mean it's valid input:
* If there is no matching function or symbol definition, the Parser will
* still reject the input.
*/
var tokenRegex = new RegExp(
"([ \r\n\t]+)|(" + // whitespace
"---?" + // special combinations
"|[!-\\[\\]-\u2027\u202A-\uD7FF\uF900-\uFFFF]" + // single codepoint
"|[\uD800-\uDBFF][\uDC00-\uDFFF]" + // surrogate pair
"|\\\\(?:[a-zA-Z]+|[^\uD800-\uDFFF])" + // function name
")"
);
var whitespaceRegex = /\s*/;
/**
* This function lexes a single normal token. It takes a position and
* whether it should completely ignore whitespace or not.
*/
Lexer.prototype._innerLex = function(pos, ignoreWhitespace) {
var input = this._input;
if (pos === input.length) {
return new Token("EOF", null, pos);
}
var match = matchAt(tokenRegex, input, pos);
if (match === null) {
throw new ParseError(
"Unexpected character: '" + input[pos] + "'",
this, pos);
} else if (match[2]) { // matched non-whitespace
return new Token(match[2], null, pos + match[2].length);
} else if (ignoreWhitespace) {
return this._innerLex(pos + match[1].length, true);
} else { // concatenate whitespace to a single space
return new Token(" ", null, pos + match[1].length);
}
};
// A regex to match a CSS color (like #ffffff or BlueViolet)
var cssColor = /#[a-z0-9]+|[a-z]+/i;
/**
* This function lexes a CSS color.
*/
Lexer.prototype._innerLexColor = function(pos) {
var input = this._input;
// Ignore whitespace
var whitespace = matchAt(whitespaceRegex, input, pos)[0];
pos += whitespace.length;
var match;
if ((match = matchAt(cssColor, input, pos))) {
// If we look like a color, return a color
return new Token(match[0], null, pos + match[0].length);
} else {
throw new ParseError("Invalid color", this, pos);
}
};
// A regex to match a dimension. Dimensions look like
// "1.2em" or ".4pt" or "1 ex"
var sizeRegex = /(-?)\s*(\d+(?:\.\d*)?|\.\d+)\s*([a-z]{2})/;
/**
* This function lexes a dimension.
*/
Lexer.prototype._innerLexSize = function(pos) {
var input = this._input;
// Ignore whitespace
var whitespace = matchAt(whitespaceRegex, input, pos)[0];
pos += whitespace.length;
var match;
if ((match = matchAt(sizeRegex, input, pos))) {
var unit = match[3];
// We only currently handle "em" and "ex" units
if (unit !== "em" && unit !== "ex") {
throw new ParseError("Invalid unit: '" + unit + "'", this, pos);
}
return new Token(match[0], {
number: +(match[1] + match[2]),
unit: unit,
}, pos + match[0].length);
}
throw new ParseError("Invalid size", this, pos);
};
/**
* This function lexes a string of whitespace.
*/
Lexer.prototype._innerLexWhitespace = function(pos) {
var input = this._input;
var whitespace = matchAt(whitespaceRegex, input, pos)[0];
pos += whitespace.length;
return new Token(whitespace[0], null, pos);
};
/**
* This function lexes a single token starting at `pos` and of the given mode.
* Based on the mode, we defer to one of the `_innerLex` functions.
*/
Lexer.prototype.lex = function(pos, mode) {
if (mode === "math") {
return this._innerLex(pos, true);
} else if (mode === "text") {
return this._innerLex(pos, false);
} else if (mode === "color") {
return this._innerLexColor(pos);
} else if (mode === "size") {
return this._innerLexSize(pos);
} else if (mode === "whitespace") {
return this._innerLexWhitespace(pos);
}
};
module.exports = Lexer;
|
