diff options
Diffstat (limited to 'infrastructure/ace/www/lexer_support.js')
| -rw-r--r-- | infrastructure/ace/www/lexer_support.js | 1068 |
1 files changed, 0 insertions, 1068 deletions
diff --git a/infrastructure/ace/www/lexer_support.js b/infrastructure/ace/www/lexer_support.js deleted file mode 100644 index 3d54f5c..0000000 --- a/infrastructure/ace/www/lexer_support.js +++ /dev/null @@ -1,1068 +0,0 @@ -/** - * Copyright 2009 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS-IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* This file is also a Helma module, referenced by its path! */ - -AceLexer = (function lexer_init() { - -// utility functions, make this file self-contained - -function forEach(array, func) { - for(var i=0;i<array.length;i++) { - var result = func(array[i], i); - if (result) break; - } -} - -function map(array, func) { - var result = []; - // must remain compatible with "arguments" pseudo-array - for(var i=0;i<array.length;i++) { - if (func) result.push(func(array[i], i)); - else result.push(array[i]); - } - return result; -} - -function filter(array, func) { - var result = []; - // must remain compatible with "arguments" pseudo-array - for(var i=0;i<array.length;i++) { - if (func(array[i], i)) result.push(array[i]); - } - return result; -} - -function isArray(testObject) { - return testObject && typeof testObject === 'object' && - !(testObject.propertyIsEnumerable('length')) && - typeof testObject.length === 'number'; -} - -// these three lines inspired by Steven Levithan's XRegExp -var singleLineRegex = /(?:[^[\\.]+|\\(?:[\S\s]|$)|\[\^?]?(?:[^\\\]]+|\\(?:[\S\s]|$))*]?)+|\./g; -var backReferenceRegex = /(?:[^\\[]+|\\(?:[^0-9]|$)|\[\^?]?(?:[^\\\]]+|\\(?:[\S\s]|$))*]?)+|\\([0-9]+)/g; -var parenFindingRegex = /(?:[^[(\\]+|\\(?:[\S\s]|$)|\[\^?]?(?:[^\\\]]+|\\(?:[\S\s]|$))*]?|\((?=\?))+|(\()/g; - -// Creates a function that, when called with (string, startIndex), finds the first of the patterns that -// matches the string starting at startIndex (and anchored there). Expects startIndex < string.length. -// The function returns a structure containing "whichCase", a number 0..(patterns.length-1) giving the -// index of the pattern that matched, or -1 if no pattern did, and "result", an array of the kind -// returned by RegExp.exec called on that pattern, or the array that would be returned by matching /[\S\s]/ -// (any single character) if no other pattern matched. Supports the flags 'i', 'm', and 's', where the -// effect of 's' is to make dots match all characters, including newlines. -// Patterns in general are not allowed to match zero-width strings, but a pattern that is specified -// as a regular expression literal with the 'm' flag is considered special, and may be zero-width, -// though as a consequence the match cannot include the final newline of the document. (Other flags -// on regular expression literals are ignored; use the "flags" argument instead.) -function makeRegexSwitch(patterns, flags) { - var numPatterns = patterns.length; - var patternStrings = map(patterns, function (p) { - if ((typeof p) == "string") - return p; // a string - else return p.source; // assume it's a regex - }); - var patternZeros = map(patterns, function (p) { - // using "multiline" is a special way to indicate the reg-ex is zero-width - return ((typeof p) != "string") && p.multiline; - }); - patternStrings.push("[\\S\\s]"); // default case - patternZeros.push(false); - // how many capturing groups each pattern has - var numGroups = map(patternStrings, function (p) { - var count = 0; - p.replace(parenFindingRegex, function (full,paren,offset) { if (paren) count++; }); - return count; - }); - // the group number for each case of the switch - var caseGroupNums = []; - var idx = 1; - forEach(numGroups, function (n) { caseGroupNums.push(idx); idx += n+1; }); - // make a big alternation of capturing groups - var alternation = map(patternStrings, function(p, pi) { - // correct the back-reference numbers - p = p.replace(backReferenceRegex, function (full, num) { - if (num) return "\\"+((+num)+caseGroupNums[pi]); - else return full; - }); - var extra = (patternZeros[pi] ? "[\\S\\s]": ""); // tack on another char for zero-widths - return '('+p+extra+')'; - }).join('|'); - // process regex flags - flags = (flags || ""); - var realFlags = "g"; - for(var i=0;i<flags.length;i++) { - var f = flags.charAt(i); - if (f == "i" || f == "m") realFlags += f; - else if (f == "s") { - alternation = alternation.replace(singleLineRegex, - function (x) { return x==='.' ? "[\\S\\s]" : x; }); - } - } - //console.log(alternation); - var bigRegex = new RegExp(alternation, realFlags); - return function (string, matchIndex) { - bigRegex.lastIndex = matchIndex; - var execResult = bigRegex.exec(string); - var whichCase; - var resultArray = []; - // search linearly for which case matched in the alternation - for(var i=0;i<=numPatterns;i++) { - var groupNum = caseGroupNums[i]; - if (execResult[groupNum]) { - whichCase = i; - for(var j=0;j<=numGroups[i];j++) { - var r = execResult[groupNum+j]; - if (patternZeros[i] && j==0) { - r = r.substring(0, r.length-1); - } - resultArray[j] = r; - } - break; - } - } - if (whichCase == numPatterns) - whichCase = -1; // default case - return {whichCase: whichCase, result: resultArray}; - } -} - - -var tokenClasses = { - 'Token': '', - - 'Text': '', - 'TEST': 'test', - 'Whitespace': 'w', - 'Error': 'err', - 'Other': 'x', - 'Dirty': 'dirty', - - 'Keyword': 'k', - 'Keyword.Constant': 'kc', - 'Keyword.Declaration': 'kd', - 'Keyword.Pseudo': 'kp', - 'Keyword.Reserved': 'kr', - 'Keyword.Type': 'kt', - - 'Name': 'n', - 'Name.Attribute': 'na', - 'Name.Builtin': 'nb', - 'Name.Builtin.Pseudo': 'bp', - 'Name.Class': 'nc', - 'Name.Constant': 'no', - 'Name.Decorator': 'nd', - 'Name.Entity': 'ni', - 'Name.Exception': 'ne', - 'Name.Function': 'nf', - 'Name.Property': 'py', - 'Name.Label': 'nl', - 'Name.Namespace': 'nn', - 'Name.Other': 'nx', - 'Name.Tag': 'nt', - 'Name.Variable': 'nv', - 'Name.Variable.Class': 'vc', - 'Name.Variable.Global': 'vg', - 'Name.Variable.Instance': 'vi', - - 'Literal': 'l', - 'Literal.Date': 'ld', - - 'String': 's', - 'String.Backtick': 'sb', - 'String.Char': 'sc', - 'String.Doc': 'sd', - 'String.Double': 's2', - 'String.Escape': 'se', - 'String.Heredoc': 'sh', - 'String.Interpol': 'si', - 'String.Other': 'sx', - 'String.Regex': 'sr', - 'String.Single': 's1', - 'String.Symbol': 'ss', - - 'Number': 'm', - 'Number.Float': 'mf', - 'Number.Hex': 'mh', - 'Number.Integer': 'mi', - 'Number.Integer.Long': 'il', - 'Number.Oct': 'mo', - - 'Operator': 'o', - 'Operator.Word': 'ow', - - 'Punctuation': 'p', - - 'Comment': 'c', - 'Comment.Multiline': 'cm', - 'Comment.Preproc': 'cp', - 'Comment.Single': 'c1', - 'Comment.Special': 'cs', - - 'Generic': 'g', - 'Generic.Deleted': 'gd', - 'Generic.Emph': 'ge', - 'Generic.Error': 'gr', - 'Generic.Heading': 'gh', - 'Generic.Inserted': 'gi', - 'Generic.Output': 'go', - 'Generic.Prompt': 'gp', - 'Generic.Strong': 'gs', - 'Generic.Subheading': 'gu', - 'Generic.Traceback': 'gt' -} - - -function makeTokenProducer(regexData, flags) { - var data = {}; - var procCasesMap = {}; - - // topological sort of state dependencies - var statesToProcess = []; - var sortedStates = []; - var sortedStatesMap = {}; - for(var state in regexData) statesToProcess.push(state); - while (statesToProcess.length > 0) { - var state = statesToProcess.shift(); - var stateReady = true; - forEach(regexData[state], function (c) { - if ((typeof c) == "object" && c.include) { - var otherState = c.include; - if (/!$/.exec(otherState)) { - otherState = otherState.substring(0, otherState.length-1); - } - if (! sortedStatesMap[otherState]) { - stateReady = false; - return true; - } - } - }); - if (stateReady) { - sortedStates.push(state); - sortedStatesMap[state] = true; - } - else { - // move to end of queue - statesToProcess.push(state); - } - } - - forEach(sortedStates, function(state) { - var cases = regexData[state]; - var procCases = []; - forEach(cases, function (c) { - if ((typeof c) == "object" && c.include) { - var otherState = c.include; - var isBang = false; - if (/!$/.exec(otherState)) { - // "bang" include, returns to other state - otherState = otherState.substring(0, otherState.length-1); - isBang = true; - } - forEach(procCasesMap[otherState], function (d) { - var dd = [d[0], d[1], d[2]]; - if (isBang) { - if (! (d[2] && d[2][0] && d[2][0].indexOf('#pop') != 0)) { - dd[2] = ['#pop', otherState].concat(d[2] || []); - } - } - procCases.push(dd); - }); - } - else procCases.push(c); - }); - procCasesMap[state] = procCases; - data[state] = { - switcher: makeRegexSwitch(map(procCases, function(x) { return x[0]; }), flags), - tokenTypes: map(procCases, function(x) { return x[1]; }), - stateEffects: map(procCases, function(y) { - var x = y[2]; - if (!x) return []; - if (isArray(x)) return x; - return [x]; - }) - } - }); - - // mutates stateStack, calls tokenFunc on each new token in order, returns new index - return function(string, startIndex, stateStack, tokenFunc) { - var stateBefore = stateStack.join('/'); - - while (true) { // loop until non-zero-length token - var stateData = data[stateStack[stateStack.length-1]]; - var switcherResult = stateData.switcher(string, startIndex); - var whichCase = switcherResult.whichCase; - var regexResult = switcherResult.result; - var tokenTypes, stateEffects; - if (whichCase < 0) { - tokenTypes = 'Error'; - stateEffects = null; - } - else { - tokenTypes = stateData.tokenTypes[whichCase]; - stateEffects = stateData.stateEffects[whichCase]; - } - - if (stateEffects) { - forEach(stateEffects, function (se) { - if (se === '#pop') stateStack.pop(); - else if (se === '#popall') { - while (stateStack.length > 0) stateStack.pop(); - } - else stateStack.push(se); - }); - } - var stateAfter = stateStack.join('/'); - - if (regexResult[0].length > 0) { - if ((typeof tokenTypes) === "object" && tokenTypes.bygroups) { - var types = tokenTypes.bygroups; - forEach(types, function (t,i) { - var tkn = { width:regexResult[i+1].length, type:t }; - if (i == 0) tkn.stateBefore = stateBefore; - if (i == (types.length-1)) tkn.stateAfter = stateAfter; - tokenFunc(tkn); - }); - } - else { - tokenFunc({ width:regexResult[0].length, type:tokenTypes, - stateBefore:stateBefore, stateAfter:stateAfter }); - } - return startIndex + regexResult[0].length; - } - } - } -} - -function makeSimpleLexer(tokenProducer) { - function lexString(str, tokenFunc) { - var state = ['root']; - var idx = 0; - while (idx < str.length) { - var i = idx; - idx = tokenProducer(str, idx, state, function (tkn) { - tokenFunc(str.substr(i, tkn.width), tkn.type); - i += tkn.width; - }); - } - } - function lexAsLines(str, tokenFunc, newLineFunc) { - str += "\n"; - var nextNewline = str.indexOf('\n'); - var curIndex = 0; - - lexString(str, function(txt, typ) { - var wid = txt.length; - var widthLeft = wid; - while (widthLeft > 0 && curIndex + wid > nextNewline) { - var w = nextNewline - curIndex; - if (w > 0) { - tokenFunc(str.substr(curIndex, w), typ); - } - curIndex += (w+1); - widthLeft -= (w+1); - if (curIndex < str.length) { - newLineFunc(); - nextNewline = str.indexOf("\n", curIndex); - } - } - if (widthLeft > 0) { - tokenFunc(str.substr(curIndex, widthLeft), typ); - curIndex += widthLeft; - } - }); - } - return {lexString:lexString, lexAsLines:lexAsLines}; -} - -var txtTokenProducer = makeTokenProducer( - { - 'root': [ - [/.*?\n/, 'Text'], - [/.+/, 'Text'] - ] - }, 's'); - -var jsTokenProducer = makeTokenProducer( - { - 'root': [ - [/\/\*[^\w\n]+appjet:version[^\w\n]+[0-9.]+[^\w\n]+\*\/[^\w\n]*(?=\n)/, - 'Comment.Special', 'main'], - [/(?:)/m, 'Text', ['main', 'regex-ready', 'linestart']] - ], - 'whitespace' : [ - [/\n/, 'Text', 'linestart'], - [/[^\S\n]+/, 'Text'], - [/\/\*/, 'Comment', 'longcomment'] - ], - 'common' : [ - {include:'whitespace'}, - [/\"/, 'String.Double', 'dstr'], - [/\'/, 'String.Single', 'sstr'] - ], - 'regex-ready' : [ - {include:'whitespace'}, - [/\/(?:[^[\\\n\/]|\\.|\[\^?]?(?:[^\\\]\n]|\\.)+\]?)+\/[gim]*/, 'String.Regex'], - [/(?:)/m, 'Text', '#pop'] - ], - 'main': [ - [/\"\"\"/, 'String.Doc', 'mstr'], - {include:"common"}, - [/<!--/, 'Comment'], - [/\/\/.*?(?=\n)/, 'Comment'], - [/[\{\}\[\]\(;]/, 'Punctuation', 'regex-ready'], - [/[\).]/, 'Punctuation'], - [/[~\^\*!%&<>\|=:,\/?\\]/, 'Operator', 'regex-ready'], - [/[+-]/, 'Operator'], - ['(import|break|case|catch|const|continue|default|delete|do|else|'+ - 'export|for|function|if|in|instanceof|label|new|return|switch|this|'+ - 'throw|try|typeof|var|void|while|with|abstract|boolean|byte|catch|char|'+ - 'class|const|debugger|double|enum|extends|final|finally|float|goto|implements|'+ - 'int|interface|long|native|package|private|protected|public|short|static|super|'+ - 'synchronized|throws|transient|volatile|let|yield)\\b', 'Keyword'], - [/(true|false|null|NaN|Infinity|undefined)\b/, 'Keyword.Constant'], - [/[$a-zA-Z_][a-zA-Z0-9_]*/, 'Name.Other'], - [/[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?/, 'Number.Float'], - [/0x[0-9a-f]+/, 'Number.Hex'], - [/[0-9]+/, 'Number.Integer'] - ], - 'csscommon': [ // common outside of style rule brackets - {include:'common'}, - [/\{/, 'Punctuation', 'csscontent'], - [/\:[a-zA-Z0-9_-]+/, 'Name.Decorator'], - [/\.[a-zA-Z0-9_-]+/, 'Name.Class'], - [/\#[a-zA-Z0-9_-]+/, 'Name.Function'], - [/[a-zA-Z0-9_-]+/, 'Name.Tag'], - [/[~\^\*!%&\[\]\(\)<>\|+=@:;,.\/?-]/, 'Operator'] - ], - 'cssmain': [ - [/(@media)([^\S\n]+)(\w+)([^\S\n]*)(\{)/, {bygroups:['Keyword', 'Text', 'String', - 'Text', 'Punctuation']}, 'cssmedia'], - {include:'csscommon'} - ], - 'cssmedia': [ - {include:'csscommon'}, - [/\}/, 'Punctuation', '#pop'] - ], - 'csscontent': [ - {include:'common'}, - [/\}/, 'Punctuation', '#pop'], - [/url\(.*?\)/, 'String.Other'], - ['(azimuth|background-attachment|background-color|'+ - 'background-image|background-position|background-repeat|'+ - 'background|border-bottom-color|border-bottom-style|'+ - 'border-bottom-width|border-left-color|border-left-style|'+ - 'border-left-width|border-right|border-right-color|'+ - 'border-right-style|border-right-width|border-top-color|'+ - 'border-top-style|border-top-width|border-bottom|'+ - 'border-collapse|border-left|border-width|border-color|'+ - 'border-spacing|border-style|border-top|border|caption-side|'+ - 'clear|clip|color|content|counter-increment|counter-reset|'+ - 'cue-after|cue-before|cue|cursor|direction|display|'+ - 'elevation|empty-cells|float|font-family|font-size|'+ - 'font-size-adjust|font-stretch|font-style|font-variant|'+ - 'font-weight|font|height|letter-spacing|line-height|'+ - 'list-style-type|list-style-image|list-style-position|'+ - 'list-style|margin-bottom|margin-left|margin-right|'+ - 'margin-top|margin|marker-offset|marks|max-height|max-width|'+ - 'min-height|min-width|opacity|orphans|outline|outline-color|'+ - 'outline-style|outline-width|overflow|padding-bottom|'+ - 'padding-left|padding-right|padding-top|padding|page|'+ - 'page-break-after|page-break-before|page-break-inside|'+ - 'pause-after|pause-before|pause|pitch|pitch-range|'+ - 'play-during|position|quotes|richness|right|size|'+ - 'speak-header|speak-numeral|speak-punctuation|speak|'+ - 'speech-rate|stress|table-layout|text-align|text-decoration|'+ - 'text-indent|text-shadow|text-transform|top|unicode-bidi|'+ - 'vertical-align|visibility|voice-family|volume|white-space|'+ - 'widows|width|word-spacing|z-index|bottom|left|'+ - 'above|absolute|always|armenian|aural|auto|avoid|baseline|'+ - 'behind|below|bidi-override|blink|block|bold|bolder|both|'+ - 'capitalize|center-left|center-right|center|circle|'+ - 'cjk-ideographic|close-quote|collapse|condensed|continuous|'+ - 'crop|crosshair|cross|cursive|dashed|decimal-leading-zero|'+ - 'decimal|default|digits|disc|dotted|double|e-resize|embed|'+ - 'extra-condensed|extra-expanded|expanded|fantasy|far-left|'+ - 'far-right|faster|fast|fixed|georgian|groove|hebrew|help|'+ - 'hidden|hide|higher|high|hiragana-iroha|hiragana|icon|'+ - 'inherit|inline-table|inline|inset|inside|invert|italic|'+ - 'justify|katakana-iroha|katakana|landscape|larger|large|'+ - 'left-side|leftwards|level|lighter|line-through|list-item|'+ - 'loud|lower-alpha|lower-greek|lower-roman|lowercase|ltr|'+ - 'lower|low|medium|message-box|middle|mix|monospace|'+ - 'n-resize|narrower|ne-resize|no-close-quote|no-open-quote|'+ - 'no-repeat|none|normal|nowrap|nw-resize|oblique|once|'+ - 'open-quote|outset|outside|overline|pointer|portrait|px|'+ - 'relative|repeat-x|repeat-y|repeat|rgb|ridge|right-side|'+ - 'rightwards|s-resize|sans-serif|scroll|se-resize|'+ - 'semi-condensed|semi-expanded|separate|serif|show|silent|'+ - 'slow|slower|small-caps|small-caption|smaller|soft|solid|'+ - 'spell-out|square|static|status-bar|super|sw-resize|'+ - 'table-caption|table-cell|table-column|table-column-group|'+ - 'table-footer-group|table-header-group|table-row|'+ - 'table-row-group|text|text-bottom|text-top|thick|thin|'+ - 'transparent|ultra-condensed|ultra-expanded|underline|'+ - 'upper-alpha|upper-latin|upper-roman|uppercase|url|'+ - 'visible|w-resize|wait|wider|x-fast|x-high|x-large|x-loud|'+ - 'x-low|x-small|x-soft|xx-large|xx-small|yes)\\b', 'Keyword'], - ['(indigo|gold|firebrick|indianred|yellow|darkolivegreen|'+ - 'darkseagreen|mediumvioletred|mediumorchid|chartreuse|'+ - 'mediumslateblue|black|springgreen|crimson|lightsalmon|brown|'+ - 'turquoise|olivedrab|cyan|silver|skyblue|gray|darkturquoise|'+ - 'goldenrod|darkgreen|darkviolet|darkgray|lightpink|teal|'+ - 'darkmagenta|lightgoldenrodyellow|lavender|yellowgreen|thistle|'+ - 'violet|navy|orchid|blue|ghostwhite|honeydew|cornflowerblue|'+ - 'darkblue|darkkhaki|mediumpurple|cornsilk|red|bisque|slategray|'+ - 'darkcyan|khaki|wheat|deepskyblue|darkred|steelblue|aliceblue|'+ - 'gainsboro|mediumturquoise|floralwhite|coral|purple|lightgrey|'+ - 'lightcyan|darksalmon|beige|azure|lightsteelblue|oldlace|'+ - 'greenyellow|royalblue|lightseagreen|mistyrose|sienna|'+ - 'lightcoral|orangered|navajowhite|lime|palegreen|burlywood|'+ - 'seashell|mediumspringgreen|fuchsia|papayawhip|blanchedalmond|'+ - 'peru|aquamarine|white|darkslategray|ivory|dodgerblue|'+ - 'lemonchiffon|chocolate|orange|forestgreen|slateblue|olive|'+ - 'mintcream|antiquewhite|darkorange|cadetblue|moccasin|'+ - 'limegreen|saddlebrown|darkslateblue|lightskyblue|deeppink|'+ - 'plum|aqua|darkgoldenrod|maroon|sandybrown|magenta|tan|'+ - 'rosybrown|pink|lightblue|palevioletred|mediumseagreen|'+ - 'dimgray|powderblue|seagreen|snow|mediumblue|midnightblue|'+ - 'paleturquoise|palegoldenrod|whitesmoke|darkorchid|salmon|'+ - 'lightslategray|lawngreen|lightgreen|tomato|hotpink|'+ - 'lightyellow|lavenderblush|linen|mediumaquamarine|green|'+ - 'blueviolet|peachpuff)\\b', 'Name.Builtin'], - [/\!important/, 'Comment.Preproc'], - [/\#[a-zA-Z0-9]{1,6}/, 'Number'], - [/[\.-]?[0-9]*[\.]?[0-9]+(em|px|\%|pt|pc|in|mm|cm|ex)/, 'Number'], - [/-?[0-9]+/, 'Number'], - [/[~\^\*!%&<>\|+=@:,.\/?-]+/, 'Operator'], - [/[\[\]();]+/, 'Punctuation'], - [/[a-zA-Z][a-zA-Z0-9]+/, 'Name'] - ], - 'linestart': [ - [/\/\*[^\w\n]+appjet:css[^\w\n]+\*\/[^\w\n]*(?=\n)/, 'Comment.Special', - ['#popall', 'root', 'cssmain']], - [/\/\*[^\w\n]+appjet:(\w+)[^\w\n]+\*\/[^\w\n]*(?=\n)/, 'Comment.Special', - ['#popall', 'root', 'main', 'regex-ready']], - [/(?:)/m, 'Text', '#pop'] - ], - 'dstr': [ - [/\"/, 'String.Double', '#pop'], - [/(?=\n)/m, 'String.Double', '#pop'], - [/(\\\\|\\\"|[^\"\n])+/, 'String.Double'] - ], - 'sstr': [ - [/\'/, 'String.Single', '#pop'], - [/(?=\n)/m, 'String.Single', '#pop'], - [/(\\\\|\\\'|[^\'\n])+/, 'String.Single'] - ], - 'longcomment': [ - [/\*\//, 'Comment', '#pop'], - [/\n/, 'Comment'], - [/.+?(?:\n|(?=\*\/))/, 'Comment'] - ], - 'mstr': [ - [/(\\\"\"\"|\n)/, 'String.Doc'], - [/\"\"\"/, 'String.Doc', '#pop'], - [/.+?(?=\\"""|"""|\n)/, 'String.Doc'] - ] - } -); - -function escapeHTML(s) { - var re = /[&<>\'\" ]/g; - if (! re.MAP) { - // persisted across function calls! - re.MAP = { - '&': '&', - '<': '<', - '>': '>', - '"': '"', - "'": ''', - ' ': ' ' - }; - } - return s.replace(re, function(c) { return re.MAP[c]; }); -} - -var simpleLexer = makeSimpleLexer(jsTokenProducer); - -function codeStringToHTML(codeString) { - var atLineStart = true; - var html = []; - function tokenFunc(txt, type) { - var cls = tokenClasses[type]; - if (cls) html.push('<tt class="',tokenClasses[type],'">'); - else html.push('<tt>'); - html.push(escapeHTML(txt),'</tt>'); - atLineStart = false; - } - function newLineFunc() { - html.push('<br/>\n'); - atLineStart = true; - } - simpleLexer.lexAsLines(codeString, tokenFunc, newLineFunc); - if (atLineStart) html.push('<br/>\n'); - return html.join(''); -} - -/* ========== Incremental Lexer for ACE ========== */ - -function makeIncrementalLexer(tokenProducer) { - - var tokens = newSkipList(); - var buffer = ""; - var nextId = 1; - var dirtyTokenKeys = []; - var uncoloredRanges = []; - - //top.dbg_uncoloredRanges = function() { return uncoloredRanges; } - //top.dbg_dirtyTokenKeys = function() { return dirtyTokenKeys; } - - function mergeRangesIfTouching(a, b) { - // if a = [a0,a1] and b = [b0,b1] are overlapping or touching, return a single merged range - // else return null - var a0 = a[0], a1 = a[1], b0 = b[0], b1 = b[1]; - if (a1 < b0) return null; - if (b1 < a0) return null; - var c0 = ((a0 < b0) ? a0 : b0); - var c1 = ((a1 > b1) ? a1 : b1); - return [c0,c1]; - } - - function addUncoloredRange(rng) { - // shouldn't this merge existing ranges if the new range overlaps multiple ones? - var done = false; - forEach(uncoloredRanges, function (x, i) { - var merged = mergeRangesIfTouching(x, rng); - if (merged) { - uncoloredRanges[i] = merged; - done = true; - return true; - } - }); - if (! done) { - uncoloredRanges.push(rng); - } - } - - function removeUncoloredRange(rng) { - var i = uncoloredRanges.length-1; - while (i >= 0) { - removeUncoloredRangeFrom(rng, i); - i--; - } - } - - function removeUncoloredRangeFrom(rangeToRemove, containingRangeIndex) { - var idx = containingRangeIndex; - var cont = uncoloredRanges[idx]; - var rem0 = rangeToRemove[0], rem1 = rangeToRemove[1]; - // limit to containing range - if (rem0 < cont[0]) rem0 = cont[0]; - if (rem1 > cont[1]) rem1 = cont[1]; - if (rem1 <= rem0) return; - // splice out uncoloredRanes[containingRangeIndex] for 0, 1, or 2 ranges - uncoloredRanges.splice(idx, 1); - if (cont[0] < rem0) - uncoloredRanges.splice(idx, 0, [cont[0], rem0]); - if (rem1 < cont[1]) - uncoloredRanges.splice(idx, 0, [rem1, cont[1]]); - } - - function prepareTokens(tokenArray) { - forEach(tokenArray, function (t) { - t.key = "$"+(nextId++); - }); - return tokenArray; - } - - function roundBackToTokenBoundary(charOffset) { - return tokens.indexOfOffset(charOffset); - } - function roundForwardToTokenBoundary(charOffset) { - var tokenEnd; - if (charOffset == tokens.totalWidth()) - tokenEnd = tokens.length(); - else { - var endToken = tokens.keyAtOffset(charOffset); - tokenEnd = tokens.indexOfKey(endToken); - // round up to nearest token boundary - if (charOffset > tokens.offsetOfKey(endToken)) { - tokenEnd++; - } - } - return tokenEnd; - } - - // findLexingStartPoint and findLexingEndPoint take a character boundary - // (0 .. buffer.length) and return a token boundary (0 .. tokens.length()) - // that, if not at the document edge, is such that the next token outside - // the boundary has a pre/post lexing state associated with it (i.e is not - // a dirty-region token or in the middle of a multi-token lexing rule). - - function findLexingStartPoint(startChar) { - if (tokens.length() == 0) return 0; - var tokenStart = roundBackToTokenBoundary(startChar); - // expand to not break up a series of tokens from the same - // lexing rule, and to include dirty regions - if (tokenStart > 0) { - var tokenBefore = tokens.atIndex(tokenStart - 1); - while (tokenBefore && (! tokenBefore.stateAfter)) { - tokenStart--; - tokenBefore = tokens.prev(tokenBefore); - } - } - return tokenStart; - } - - function findLexingEndPoint(endChar) { - if (tokens.length() == 0) return 0; - var tokenEnd = roundForwardToTokenBoundary(endChar); - // expand to not break up a series of tokens from the same - // lexing rule, and to include dirty regions - if (tokenEnd < tokens.length()) { - var tokenAfter = tokens.atIndex(tokenEnd); - while (tokenAfter && (! tokenAfter.stateBefore)) { - tokenEnd++; - tokenAfter = tokens.next(tokenAfter); - } - } - return tokenEnd; - } - - function updateBuffer(newBuffer, spliceStart, charsRemoved, charsAdded) { - buffer = newBuffer; - - // back up to new line - if (spliceStart > 0) { - var newStart = buffer.lastIndexOf('\n', spliceStart-1) + 1; - var charsBack = spliceStart - newStart; - spliceStart -= charsBack; - charsRemoved += charsBack; - charsAdded += charsBack; - } - // expand to lexing points - var tokenRangeStart = findLexingStartPoint(spliceStart); - var tokenRangeEnd = findLexingEndPoint(spliceStart + charsRemoved); - - var dirtyWidth = 0; - // make sure to mark at least one token dirty so that deletions correctly cause - // rehighlighting; in practice doesn't come up often except when an entire line - // is cleanly deleted, like deleting a blank line (which doesn't usually affect highlighting) - while (dirtyWidth == 0) { - var curStart = tokens.offsetOfIndex(tokenRangeStart); - var curEnd = tokens.offsetOfIndex(tokenRangeEnd); - dirtyWidth = (curEnd - curStart) + (charsAdded - charsRemoved); - if (dirtyWidth == 0) { - if (curEnd >= tokens.totalWidth()) break; - tokenRangeEnd = findLexingEndPoint(curEnd+1); - } - } - - var dirtyTokens = []; // 0 or 1 of them - if (dirtyWidth > 0) { - dirtyTokens.push({ width: dirtyWidth, type: 'Dirty' }); - } - - //console.log("%d, %d, %d, %d", charsRemoved, charsAdded, - //(curEnd - curStart), dirtyWidth); - - tokens.splice(tokenRangeStart, tokenRangeEnd - tokenRangeStart, - prepareTokens(dirtyTokens)); - - if (tokens.totalWidth() != buffer.length) { - console.error("updateBuffer: Bad total token width: "+ - tokens.totalWidth()+" not "+buffer.length); - } - - forEach(dirtyTokens, function (t) { dirtyTokenKeys.push(t.key); }); - dirtyTokenKeys = filter(dirtyTokenKeys, function (k) { return tokens.containsKey(k); }); - //console.log("after update: %s", toSource(tokens.slice())); - - function applySpliceToIndex(i) { - if (i <= spliceStart) return i; - if (i >= (spliceStart + charsRemoved)) return i + charsAdded - charsRemoved; - return spliceStart; - } - for(var i=uncoloredRanges.length-1; i>=0; i--) { - var r = uncoloredRanges[i]; - r[0] = applySpliceToIndex(r[0]); - r[1] = applySpliceToIndex(r[1]); - if (r[1] <= r[0]) uncoloredRanges.splice(i, 1); - } - } - - function processDirtyToken(dirtyToken, isTimeUp, stopAtChar) { - - //console.time("lexing"); - //var p = PROFILER("lex", false); - var stateStack; - if (! tokens.prev(dirtyToken)) stateStack = ['root']; - else stateStack = tokens.prev(dirtyToken).stateAfter.split('/'); - var newTokens = []; - var dirtyTokenIndex = tokens.indexOfEntry(dirtyToken); - var tokenCount = 0; - var startTime = (new Date()).getTime(); - var stopBasedOnChar = (typeof(stopAtChar) == "number"); - //p.mark("tokenize"); - - var curOffset = tokens.offsetOfEntry(dirtyToken); - var startedOffset = curOffset; - var oldToken = dirtyToken; - var oldTokenOffset = curOffset; - var done = false; - - while ((! done) && (! isTimeUp()) && (! (stopBasedOnChar && curOffset >= stopAtChar))) { - curOffset = tokenProducer(buffer, curOffset, stateStack, - function (t) { newTokens.push(t); }); - while (oldToken && (oldTokenOffset + oldToken.width <= curOffset)) { - oldTokenOffset += oldToken.width; - oldToken = tokens.next(oldToken); - } - if (curOffset == tokens.totalWidth()) { - // hit the end - done = true; - } - else if (oldTokenOffset == curOffset) { - // at a token boundary, the beginning of oldTokenOffset - if (stateStack.join('/') === oldToken.stateBefore) { - // state matches up, we can stop - done = true; - } - } - } - - var endedOffset = curOffset; - var dist = endedOffset - startedOffset; - var tokensToRemove; - var newDirtyToken; - if (dist < dirtyToken.width) { - tokens.setEntryWidth(dirtyToken, dirtyToken.width - dist); - tokensToRemove = 0; - newDirtyToken = dirtyToken; - } - else { - var nextLexingPoint = findLexingEndPoint(endedOffset); - var lexingPointChar = tokens.offsetOfIndex(nextLexingPoint); - if (lexingPointChar == endedOffset && (! done) && endedOffset < tokens.totalWidth()) { - // happened to stop at token boundary before end, but not done lexing, - // so make next token dirty - nextLexingPoint = findLexingEndPoint(endedOffset+1); - lexingPointChar = tokens.offsetOfIndex(nextLexingPoint); - } - var dirtyCharsLeft = lexingPointChar - endedOffset; - if (dirtyCharsLeft > 0) { - newDirtyToken = { width: dirtyCharsLeft, type: 'Dirty' }; - newTokens.push(newDirtyToken); - } - tokensToRemove = nextLexingPoint - dirtyTokenIndex; - } - - //p.mark("prepare"); - prepareTokens(newTokens); - //p.mark("remove"); - tokens.splice(dirtyTokenIndex, tokensToRemove, []); - //p.mark("insert"); - tokens.splice(dirtyTokenIndex, 0, newTokens); - if (tokens.totalWidth() != buffer.length) - console.error("processDirtyToken: Bad total token width: "+ - tokens.totalWidth()+" not "+buffer.length); - //p.end(); - - addUncoloredRange([startedOffset, endedOffset]); - - //console.log("processed chars %d to %d", startedOffset, endedOffset); - //console.timeEnd("lexing"); - - return (newDirtyToken && newDirtyToken.key); - } - - function lexSomeDirty(filter, isTimeUp) { - var newDirtyTokenKeys = []; - - forEach(dirtyTokenKeys, function (dirtyKey) { - if (! tokens.containsKey(dirtyKey)) return; - var dirtyToken = tokens.atKey(dirtyKey); - var filterResult; - if ((! isTimeUp()) && ((filterResult = filter(dirtyToken)))) { - var stopAtChar; - if ((typeof filterResult) == "object" && (typeof filterResult.stopAtChar) == "number") { - stopAtChar = filterResult.stopAtChar; - } - var tkn = processDirtyToken(dirtyToken, isTimeUp, filterResult.stopAtChar); - if (tkn) newDirtyTokenKeys.push(tkn); - } - else { - // leave the token behind - newDirtyTokenKeys.push(dirtyKey); - } - - if (tokens.totalWidth() != buffer.length) - console.error("Bad total token width: "+tokens.totalWidth()+" not "+buffer.length); - - }); - - dirtyTokenKeys = newDirtyTokenKeys; - } - - function lexCharRange(charRange, isTimeUp) { - //var startTime = (new Date()).getTime(); - //function isTimeUp() { return ((new Date()).getTime() - startTime) > timeLimit; } - - if (isTimeUp()) return; - - lexSomeDirty(function (dirtyToken) { - var start = tokens.offsetOfEntry(dirtyToken); - var end = start + dirtyToken.width; - if (end <= charRange[0]) return false; - if (start >= charRange[1]) return false; - //console.log("tokenStart: %d, tokenEnd: %d, visStart: %d, visEnd: %d", - //start, end, charRange[0], charRange[1]); - var result = {}; - if (charRange[1] < end) { - result.stopAtChar = charRange[1]; - } - return result; - }, isTimeUp); - - //if (isTimeUp()) return; - - /* - // highlight the visible area - var i = uncoloredRanges.length-1; - // iterate backwards because we change the array - while (i >= 0) { - var rng = uncoloredRanges[i]; - var start = rng[0], end = rng[1]; - if (start < viewRange[0]) start = viewRange[0]; - if (end > viewRange[1]) end = viewRange[1]; - if (end > start) { - var charsRecolored = recolorFunc(start, end-start, - isTimeUp, getSpansForRange); - removeUncoloredSubrange([start, start+charsRecolored], i); - } - if (isTimeUp()) break; - i--; - }*/ - } - - function tokenToString(tkn) { - return toSource({width:tkn.width, type:tkn.type, stateBefore:tkn.stateBefore, stateAfter:tkn.stateAfter}); - } - - // Calls func(startChar, endChar) on each range of characters that needs to be colored - // in the DOM, based on calls to getSpansForRange (which removes chars from consideration) - // and lexCharRange (which calculates new colors and adds chars for consideration). - // There are usually relatively few uncolored ranges, each of which may be many lines, - // even the whole document. - // func must return true iff any tokens are accessed through getSpansForRange during - // the call. func should not do new lexing. - function forEachUncoloredRange(func, isTimeUp) { - var i = 0; - // uncoloredRanges will change during this function! - // Terminates is time runs out, whole document is colored, - // or the func "passes" on all ranges by returning false. - while (i < uncoloredRanges.length && ! isTimeUp()) { - var rng = uncoloredRanges[i]; - var returnVal = func(rng[0], rng[1], isTimeUp); - if (returnVal) { - // func did something, uncolored ranges may have changed around - i = 0; - } - else { - i++; - } - } - } - - // Like forEachUncoloredRange, but "cropped" to the char range given. For example, - // if no "uncolored ranges" extend by a non-zero amount into the char range, - // func will never be called. - function forEachUncoloredSubrange(startChar, endChar, func, isTimeUp) { - forEachUncoloredRange(function (s, e, isTimeUp2) { - if (s < startChar) s = startChar; - if (e > endChar) e = endChar; - if (e > s) { - return func(s, e, isTimeUp2); - } - return false; - }, isTimeUp); - } - - // This function takes note of what it's passed, and assumes that part of the - // DOM has been taken care of (unless justPeek). - // The "func" takes arguments tokenWidth and tokenClass, and is called on each - // token in the range, with the widths adding up to the range size. - function getSpansForRange(startChar, endChar, func, justPeek) { - - if (startChar == endChar) return; - - var startToken = tokens.atOffset(startChar); - var startTokenStart = tokens.offsetOfEntry(startToken); - var curOffset = startChar; - var curToken = startToken; - while (curOffset < endChar) { - var spanEnd; - if (curToken === startToken) { - spanEnd = startTokenStart + startToken.width; - } - else { - spanEnd = curOffset + curToken.width; - } - if (spanEnd > endChar) spanEnd = endChar; - if (spanEnd > curOffset) { - func(spanEnd - curOffset, tokenClasses[curToken.type]); - } - curOffset = spanEnd; - curToken = tokens.next(curToken); - } - - if (! justPeek) removeUncoloredRange([startChar, endChar]); - } - - function markRangeUncolored(start, end) { - addUncoloredRange([start, end]); - } - - return { - updateBuffer: updateBuffer, - lexCharRange: lexCharRange, - getSpansForRange: getSpansForRange, - forEachUncoloredSubrange: forEachUncoloredSubrange, - markRangeUncolored: markRangeUncolored - }; -} - -/* ========== End Incremental Lexer ========== */ - -tokenProds = {js: jsTokenProducer, txt: txtTokenProducer}; - -function getTokenProducer(type) { - return tokenProds[type || 'txt'] || tokenProds['txt']; -} - -function getIncrementalLexer(type) { - return makeIncrementalLexer(getTokenProducer(type)); -} -function getSimpleLexer(type) { - return makeSimpleLexer(getTokenProducer(type)); -} - -return {getIncrementalLexer:getIncrementalLexer, getSimpleLexer:getSimpleLexer, - codeStringToHTML:codeStringToHTML}; - -})(); |