diff options
Diffstat (limited to 'trunk/infrastructure/yuicompressor')
19 files changed, 11957 insertions, 0 deletions
diff --git a/trunk/infrastructure/yuicompressor/lib/jargs-1.0.jar b/trunk/infrastructure/yuicompressor/lib/jargs-1.0.jar Binary files differnew file mode 100644 index 0000000..cdbc80b --- /dev/null +++ b/trunk/infrastructure/yuicompressor/lib/jargs-1.0.jar diff --git a/trunk/infrastructure/yuicompressor/lib/rhino-yuicompressor.jar b/trunk/infrastructure/yuicompressor/lib/rhino-yuicompressor.jar Binary files differnew file mode 100644 index 0000000..b99560e --- /dev/null +++ b/trunk/infrastructure/yuicompressor/lib/rhino-yuicompressor.jar diff --git a/trunk/infrastructure/yuicompressor/make.sh b/trunk/infrastructure/yuicompressor/make.sh new file mode 100755 index 0000000..947aafa --- /dev/null +++ b/trunk/infrastructure/yuicompressor/make.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Copyright 2009 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS-IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +mkdir -p build +( + cd build + find ../lib -name '*.jar' | xargs -n1 jar xf + rm -rf META-INF + cd .. + javac -d build -classpath lib/jargs-1.0.jar:lib/rhino-yuicompressor.jar `find src -name '*.java'` + cd build + jar cf ../../lib/yuicompressor-2.4-appjet.jar * +) +rm -rf build diff --git a/trunk/infrastructure/yuicompressor/src/com/yahoo/platform/yui/compressor/Bootstrap.java b/trunk/infrastructure/yuicompressor/src/com/yahoo/platform/yui/compressor/Bootstrap.java new file mode 100644 index 0000000..1b95aca --- /dev/null +++ b/trunk/infrastructure/yuicompressor/src/com/yahoo/platform/yui/compressor/Bootstrap.java @@ -0,0 +1,22 @@ +/*
+ * YUI Compressor
+ * Author: Julien Lecomte <jlecomte@yahoo-inc.com>
+ * Copyright (c) 2007, Yahoo! Inc. All rights reserved.
+ * Code licensed under the BSD License:
+ * http://developer.yahoo.net/yui/license.txt
+ */
+
+package com.yahoo.platform.yui.compressor;
+
+import java.lang.reflect.Method;
+
+public class Bootstrap {
+
+ public static void main(String args[]) throws Exception {
+ ClassLoader loader = new JarClassLoader();
+ Thread.currentThread().setContextClassLoader(loader);
+ Class c = loader.loadClass(YUICompressor.class.getName());
+ Method main = c.getMethod("main", new Class[]{String[].class});
+ main.invoke(null, new Object[]{args});
+ }
+}
\ No newline at end of file diff --git a/trunk/infrastructure/yuicompressor/src/com/yahoo/platform/yui/compressor/CssCompressor.java b/trunk/infrastructure/yuicompressor/src/com/yahoo/platform/yui/compressor/CssCompressor.java new file mode 100644 index 0000000..68b4de9 --- /dev/null +++ b/trunk/infrastructure/yuicompressor/src/com/yahoo/platform/yui/compressor/CssCompressor.java @@ -0,0 +1,188 @@ +/* + * YUI Compressor + * Author: Julien Lecomte <jlecomte@yahoo-inc.com> + * Copyright (c) 2007, Yahoo! Inc. All rights reserved. + * Code licensed under the BSD License: + * http://developer.yahoo.net/yui/license.txt + * + * This code is a port of Isaac Schlueter's cssmin utility. + */ + +package com.yahoo.platform.yui.compressor; + +import java.io.IOException; +import java.io.Reader; +import java.io.Writer; +import java.util.regex.Pattern; +import java.util.regex.Matcher; + +public class CssCompressor { + + private StringBuffer srcsb = new StringBuffer(); + + public CssCompressor(Reader in) throws IOException { + // Read the stream... + int c; + while ((c = in.read()) != -1) { + srcsb.append((char) c); + } + } + + public void compress(Writer out, int linebreakpos) + throws IOException { + + Pattern p; + Matcher m; + String css; + StringBuffer sb; + int startIndex, endIndex; + + // Remove all comment blocks... + startIndex = 0; + boolean iemac = false; + boolean preserve = false; + sb = new StringBuffer(srcsb.toString()); + while ((startIndex = sb.indexOf("/*", startIndex)) >= 0) { + preserve = sb.length() > startIndex + 2 && sb.charAt(startIndex + 2) == '!'; + endIndex = sb.indexOf("*/", startIndex + 2); + if (endIndex < 0) { + if (!preserve) { + sb.delete(startIndex, sb.length()); + } + } else if (endIndex >= startIndex + 2) { + if (sb.charAt(endIndex-1) == '\\') { + // Looks like a comment to hide rules from IE Mac. + // Leave this comment, and the following one, alone... + startIndex = endIndex + 2; + iemac = true; + } else if (iemac) { + startIndex = endIndex + 2; + iemac = false; + } else if (!preserve) { + sb.delete(startIndex, endIndex + 2); + } else { + startIndex = endIndex + 2; + } + } + } + + css = sb.toString(); + + // Normalize all whitespace strings to single spaces. Easier to work with that way. + css = css.replaceAll("\\s+", " "); + + // Make a pseudo class for the Box Model Hack + css = css.replaceAll("\"\\\\\"}\\\\\"\"", "___PSEUDOCLASSBMH___"); + + // Remove the spaces before the things that should not have spaces before them. + // But, be careful not to turn "p :link {...}" into "p:link{...}" + // Swap out any pseudo-class colons with the token, and then swap back. + sb = new StringBuffer(); + p = Pattern.compile("(^|\\})(([^\\{:])+:)+([^\\{]*\\{)"); + m = p.matcher(css); + while (m.find()) { + String s = m.group(); + s = s.replaceAll(":", "___PSEUDOCLASSCOLON___"); + m.appendReplacement(sb, s); + } + m.appendTail(sb); + css = sb.toString(); + css = css.replaceAll("\\s+([!{};:>+\\(\\)\\],])", "$1"); + css = css.replaceAll("___PSEUDOCLASSCOLON___", ":"); + + // Remove the spaces after the things that should not have spaces after them. + css = css.replaceAll("([!{}:;>+\\(\\[,])\\s+", "$1"); + + // Add the semicolon where it's missing. + css = css.replaceAll("([^;\\}])}", "$1;}"); + + // Replace 0(px,em,%) with 0. + css = css.replaceAll("([\\s:])(0)(px|em|%|in|cm|mm|pc|pt|ex)", "$1$2"); + + // Replace 0 0 0 0; with 0. + css = css.replaceAll(":0 0 0 0;", ":0;"); + css = css.replaceAll(":0 0 0;", ":0;"); + css = css.replaceAll(":0 0;", ":0;"); + // Replace background-position:0; with background-position:0 0; + css = css.replaceAll("background-position:0;", "background-position:0 0;"); + + // Replace 0.6 to .6, but only when preceded by : or a white-space + css = css.replaceAll("(:|\\s)0+\\.(\\d+)", "$1.$2"); + + // Shorten colors from rgb(51,102,153) to #336699 + // This makes it more likely that it'll get further compressed in the next step. + p = Pattern.compile("rgb\\s*\\(\\s*([0-9,\\s]+)\\s*\\)"); + m = p.matcher(css); + sb = new StringBuffer(); + while (m.find()) { + String[] rgbcolors = m.group(1).split(","); + StringBuffer hexcolor = new StringBuffer("#"); + for (int i = 0; i < rgbcolors.length; i++) { + int val = Integer.parseInt(rgbcolors[i]); + if (val < 16) { + hexcolor.append("0"); + } + hexcolor.append(Integer.toHexString(val)); + } + m.appendReplacement(sb, hexcolor.toString()); + } + m.appendTail(sb); + css = sb.toString(); + + // Shorten colors from #AABBCC to #ABC. Note that we want to make sure + // the color is not preceded by either ", " or =. Indeed, the property + // filter: chroma(color="#FFFFFF"); + // would become + // filter: chroma(color="#FFF"); + // which makes the filter break in IE. + p = Pattern.compile("([^\"'=\\s])(\\s*)#([0-9a-fA-F])([0-9a-fA-F])([0-9a-fA-F])([0-9a-fA-F])([0-9a-fA-F])([0-9a-fA-F])"); + m = p.matcher(css); + sb = new StringBuffer(); + while (m.find()) { + // Test for AABBCC pattern + if (m.group(3).equalsIgnoreCase(m.group(4)) && + m.group(5).equalsIgnoreCase(m.group(6)) && + m.group(7).equalsIgnoreCase(m.group(8))) { + m.appendReplacement(sb, m.group(1) + m.group(2) + "#" + m.group(3) + m.group(5) + m.group(7)); + } else { + m.appendReplacement(sb, m.group()); + } + } + m.appendTail(sb); + css = sb.toString(); + + // Remove empty rules. + css = css.replaceAll("[^\\}]+\\{;\\}", ""); + + if (linebreakpos >= 0) { + // Some source control tools don't like it when files containing lines longer + // than, say 8000 characters, are checked in. The linebreak option is used in + // that case to split long lines after a specific column. + int i = 0; + int linestartpos = 0; + sb = new StringBuffer(css); + while (i < sb.length()) { + char c = sb.charAt(i++); + if (c == '}' && i - linestartpos > linebreakpos) { + sb.insert(i, '\n'); + linestartpos = i; + } + } + + css = sb.toString(); + } + + // Replace the pseudo class for the Box Model Hack + css = css.replaceAll("___PSEUDOCLASSBMH___", "\"\\\\\"}\\\\\"\""); + + // Replace multiple semi-colons in a row by a single one + // See SF bug #1980989 + css = css.replaceAll(";;+", ";"); + + // Trim the final string (for any leading or trailing white spaces) + css = css.trim(); + + // Write the output... + out.write(css); + } +} diff --git a/trunk/infrastructure/yuicompressor/src/com/yahoo/platform/yui/compressor/JarClassLoader.java b/trunk/infrastructure/yuicompressor/src/com/yahoo/platform/yui/compressor/JarClassLoader.java new file mode 100644 index 0000000..a6d3e13 --- /dev/null +++ b/trunk/infrastructure/yuicompressor/src/com/yahoo/platform/yui/compressor/JarClassLoader.java @@ -0,0 +1,158 @@ +/* + * YUI Compressor + * Author: Julien Lecomte <jlecomte@yahoo-inc.com> + * Copyright (c) 2007, Yahoo! Inc. All rights reserved. + * Code licensed under the BSD License: + * http://developer.yahoo.net/yui/license.txt + */ + +package com.yahoo.platform.yui.compressor; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Enumeration; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; + +public class JarClassLoader extends ClassLoader { + + private static String jarPath; + + public Class loadClass(String name) throws ClassNotFoundException { + + // First check if the class is already loaded + Class c = findLoadedClass(name); + if (c == null) { + c = findClass(name); + } + + if (c == null) { + c = ClassLoader.getSystemClassLoader().loadClass(name); + } + + return c; + } + + private static String getJarPath() { + + if (jarPath != null) { + return jarPath; + } + + String classname = JarClassLoader.class.getName().replace('.', '/') + ".class"; + String classpath = System.getProperty("java.class.path"); + String classpaths[] = classpath.split(System.getProperty("path.separator")); + + for (int i = 0; i < classpaths.length; i++) { + + String path = classpaths[i]; + JarFile jarFile = null; + JarEntry jarEntry = null; + + try { + jarFile = new JarFile(path); + jarEntry = findJarEntry(jarFile, classname); + } catch (IOException ioe) { + /* ignore */ + } finally { + if (jarFile != null) { + try { + jarFile.close(); + } catch (IOException ioe) { + /* ignore */ + } + } + } + + if (jarEntry != null) { + jarPath = path; + break; + } + } + + return jarPath; + } + + private static JarEntry findJarEntry(JarFile jarFile, String entryName) { + + Enumeration entries = jarFile.entries(); + + while (entries.hasMoreElements()) { + JarEntry entry = (JarEntry) entries.nextElement(); + if (entry.getName().equals(entryName)) { + return entry; + } + } + + return null; + } + + protected Class findClass(String name) { + + Class c = null; + String jarPath = getJarPath(); + + if (jarPath != null) { + JarFile jarFile = null; + try { + jarFile = new JarFile(jarPath); + c = loadClassData(jarFile, name); + } catch (IOException ioe) { + /* ignore */ + } finally { + if (jarFile != null) { + try { + jarFile.close(); + } catch (IOException ioe) { + /* ignore */ + } + } + } + } + + return c; + } + + private Class loadClassData(JarFile jarFile, String className) { + + String entryName = className.replace('.', '/') + ".class"; + JarEntry jarEntry = findJarEntry(jarFile, entryName); + if (jarEntry == null) { + return null; + } + + // Create the necessary package if needed... + int index = className.lastIndexOf('.'); + if (index >= 0) { + String packageName = className.substring(0, index); + if (getPackage(packageName) == null) { + definePackage(packageName, "", "", "", "", "", "", null); + } + } + + // Read the Jar File entry and define the class... + Class c = null; + try { + InputStream is = jarFile.getInputStream(jarEntry); + ByteArrayOutputStream os = new ByteArrayOutputStream(); + copy(is, os); + byte[] bytes = os.toByteArray(); + c = defineClass(className, bytes, 0, bytes.length); + } catch (IOException ioe) { + /* ignore */ + } + + return c; + } + + private void copy(InputStream in, OutputStream out) throws IOException { + byte[] buf = new byte[1024]; + while (true) { + int len = in.read(buf); + if (len < 0) break; + out.write(buf, 0, len); + } + } +}
\ No newline at end of file diff --git a/trunk/infrastructure/yuicompressor/src/com/yahoo/platform/yui/compressor/JavaScriptCompressor.java b/trunk/infrastructure/yuicompressor/src/com/yahoo/platform/yui/compressor/JavaScriptCompressor.java new file mode 100644 index 0000000..e69ae1a --- /dev/null +++ b/trunk/infrastructure/yuicompressor/src/com/yahoo/platform/yui/compressor/JavaScriptCompressor.java @@ -0,0 +1,1307 @@ +/* + * YUI Compressor + * Author: Julien Lecomte <jlecomte@yahoo-inc.com> + * Copyright (c) 2007, Yahoo! Inc. All rights reserved. + * Code licensed under the BSD License: + * http://developer.yahoo.net/yui/license.txt + */ + +package com.yahoo.platform.yui.compressor; + +import yuicompressor.org.mozilla.javascript.*; + +import java.io.IOException; +import java.io.Reader; +import java.io.Writer; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class JavaScriptCompressor { + + static final ArrayList ones; + static final ArrayList twos; + static final ArrayList threes; + + static final Set builtin = new HashSet(); + static final Map literals = new Hashtable(); + static final Set reserved = new HashSet(); + + static { + + // This list contains all the 3 characters or less built-in global + // symbols available in a browser. Please add to this list if you + // see anything missing. + builtin.add("NaN"); + builtin.add("top"); + + ones = new ArrayList(); + for (char c = 'A'; c <= 'Z'; c++) + ones.add(Character.toString(c)); + for (char c = 'a'; c <= 'z'; c++) + ones.add(Character.toString(c)); + + twos = new ArrayList(); + for (int i = 0; i < ones.size(); i++) { + String one = (String) ones.get(i); + for (char c = 'A'; c <= 'Z'; c++) + twos.add(one + Character.toString(c)); + for (char c = 'a'; c <= 'z'; c++) + twos.add(one + Character.toString(c)); + for (char c = '0'; c <= '9'; c++) + twos.add(one + Character.toString(c)); + } + + // Remove two-letter JavaScript reserved words and built-in globals... + twos.remove("as"); + twos.remove("is"); + twos.remove("do"); + twos.remove("if"); + twos.remove("in"); + twos.removeAll(builtin); + + threes = new ArrayList(); + for (int i = 0; i < twos.size(); i++) { + String two = (String) twos.get(i); + for (char c = 'A'; c <= 'Z'; c++) + threes.add(two + Character.toString(c)); + for (char c = 'a'; c <= 'z'; c++) + threes.add(two + Character.toString(c)); + for (char c = '0'; c <= '9'; c++) + threes.add(two + Character.toString(c)); + } + + // Remove three-letter JavaScript reserved words and built-in globals... + threes.remove("for"); + threes.remove("int"); + threes.remove("new"); + threes.remove("try"); + threes.remove("use"); + threes.remove("var"); + threes.removeAll(builtin); + + // That's up to ((26+26)*(1+(26+26+10)))*(1+(26+26+10))-8 + // (206,380 symbols per scope) + + // The following list comes from org/mozilla/javascript/Decompiler.java... + literals.put(new Integer(Token.GET), "get "); + literals.put(new Integer(Token.SET), "set "); + literals.put(new Integer(Token.TRUE), "true"); + literals.put(new Integer(Token.FALSE), "false"); + literals.put(new Integer(Token.NULL), "null"); + literals.put(new Integer(Token.THIS), "this"); + literals.put(new Integer(Token.FUNCTION), "function"); + literals.put(new Integer(Token.COMMA), ","); + literals.put(new Integer(Token.LC), "{"); + literals.put(new Integer(Token.RC), "}"); + literals.put(new Integer(Token.LP), "("); + literals.put(new Integer(Token.RP), ")"); + literals.put(new Integer(Token.LB), "["); + literals.put(new Integer(Token.RB), "]"); + literals.put(new Integer(Token.DOT), "."); + literals.put(new Integer(Token.NEW), "new "); + literals.put(new Integer(Token.DELPROP), "delete "); + literals.put(new Integer(Token.IF), "if"); + literals.put(new Integer(Token.ELSE), "else"); + literals.put(new Integer(Token.FOR), "for"); + literals.put(new Integer(Token.IN), " in "); + literals.put(new Integer(Token.WITH), "with"); + literals.put(new Integer(Token.WHILE), "while"); + literals.put(new Integer(Token.DO), "do"); + literals.put(new Integer(Token.TRY), "try"); + literals.put(new Integer(Token.CATCH), "catch"); + literals.put(new Integer(Token.FINALLY), "finally"); + literals.put(new Integer(Token.THROW), "throw"); + literals.put(new Integer(Token.SWITCH), "switch"); + literals.put(new Integer(Token.BREAK), "break"); + literals.put(new Integer(Token.CONTINUE), "continue"); + literals.put(new Integer(Token.CASE), "case"); + literals.put(new Integer(Token.DEFAULT), "default"); + literals.put(new Integer(Token.RETURN), "return"); + literals.put(new Integer(Token.VAR), "var "); + literals.put(new Integer(Token.SEMI), ";"); + literals.put(new Integer(Token.ASSIGN), "="); + literals.put(new Integer(Token.ASSIGN_ADD), "+="); + literals.put(new Integer(Token.ASSIGN_SUB), "-="); + literals.put(new Integer(Token.ASSIGN_MUL), "*="); + literals.put(new Integer(Token.ASSIGN_DIV), "/="); + literals.put(new Integer(Token.ASSIGN_MOD), "%="); + literals.put(new Integer(Token.ASSIGN_BITOR), "|="); + literals.put(new Integer(Token.ASSIGN_BITXOR), "^="); + literals.put(new Integer(Token.ASSIGN_BITAND), "&="); + literals.put(new Integer(Token.ASSIGN_LSH), "<<="); + literals.put(new Integer(Token.ASSIGN_RSH), ">>="); + literals.put(new Integer(Token.ASSIGN_URSH), ">>>="); + literals.put(new Integer(Token.HOOK), "?"); + literals.put(new Integer(Token.OBJECTLIT), ":"); + literals.put(new Integer(Token.COLON), ":"); + literals.put(new Integer(Token.OR), "||"); + literals.put(new Integer(Token.AND), "&&"); + literals.put(new Integer(Token.BITOR), "|"); + literals.put(new Integer(Token.BITXOR), "^"); + literals.put(new Integer(Token.BITAND), "&"); + literals.put(new Integer(Token.SHEQ), "==="); + literals.put(new Integer(Token.SHNE), "!=="); + literals.put(new Integer(Token.EQ), "=="); + literals.put(new Integer(Token.NE), "!="); + literals.put(new Integer(Token.LE), "<="); + literals.put(new Integer(Token.LT), "<"); + literals.put(new Integer(Token.GE), ">="); + literals.put(new Integer(Token.GT), ">"); + literals.put(new Integer(Token.INSTANCEOF), " instanceof "); + literals.put(new Integer(Token.LSH), "<<"); + literals.put(new Integer(Token.RSH), ">>"); + literals.put(new Integer(Token.URSH), ">>>"); + literals.put(new Integer(Token.TYPEOF), "typeof"); + literals.put(new Integer(Token.VOID), "void "); + literals.put(new Integer(Token.CONST), "const "); + literals.put(new Integer(Token.NOT), "!"); + literals.put(new Integer(Token.BITNOT), "~"); + literals.put(new Integer(Token.POS), "+"); + literals.put(new Integer(Token.NEG), "-"); + literals.put(new Integer(Token.INC), "++"); + literals.put(new Integer(Token.DEC), "--"); + literals.put(new Integer(Token.ADD), "+"); + literals.put(new Integer(Token.SUB), "-"); + literals.put(new Integer(Token.MUL), "*"); + literals.put(new Integer(Token.DIV), "/"); + literals.put(new Integer(Token.MOD), "%"); + literals.put(new Integer(Token.COLONCOLON), "::"); + literals.put(new Integer(Token.DOTDOT), ".."); + literals.put(new Integer(Token.DOTQUERY), ".("); + literals.put(new Integer(Token.XMLATTR), "@"); + + // See http://developer.mozilla.org/en/docs/Core_JavaScript_1.5_Reference:Reserved_Words + + // JavaScript 1.5 reserved words + reserved.add("break"); + reserved.add("case"); + reserved.add("catch"); + reserved.add("continue"); + reserved.add("default"); + reserved.add("delete"); + reserved.add("do"); + reserved.add("else"); + reserved.add("finally"); + reserved.add("for"); + reserved.add("function"); + reserved.add("if"); + reserved.add("in"); + reserved.add("instanceof"); + reserved.add("new"); + reserved.add("return"); + reserved.add("switch"); + reserved.add("this"); + reserved.add("throw"); + reserved.add("try"); + reserved.add("typeof"); + reserved.add("var"); + reserved.add("void"); + reserved.add("while"); + reserved.add("with"); + // Words reserved for future use + reserved.add("abstract"); + reserved.add("boolean"); + reserved.add("byte"); + reserved.add("char"); + reserved.add("class"); + reserved.add("const"); + reserved.add("debugger"); + reserved.add("double"); + reserved.add("enum"); + reserved.add("export"); + reserved.add("extends"); + reserved.add("final"); + reserved.add("float"); + reserved.add("goto"); + reserved.add("implements"); + reserved.add("import"); + reserved.add("int"); + reserved.add("interface"); + reserved.add("long"); + reserved.add("native"); + reserved.add("package"); + reserved.add("private"); + reserved.add("protected"); + reserved.add("public"); + reserved.add("short"); + reserved.add("static"); + reserved.add("super"); + reserved.add("synchronized"); + reserved.add("throws"); + reserved.add("transient"); + reserved.add("volatile"); + // These are not reserved, but should be taken into account + // in isValidIdentifier (See jslint source code) + reserved.add("arguments"); + reserved.add("eval"); + reserved.add("true"); + reserved.add("false"); + reserved.add("Infinity"); + reserved.add("NaN"); + reserved.add("null"); + reserved.add("undefined"); + } + + private static int countChar(String haystack, char needle) { + int idx = 0; + int count = 0; + int length = haystack.length(); + while (idx < length) { + char c = haystack.charAt(idx++); + if (c == needle) { + count++; + } + } + return count; + } + + private static int printSourceString(String source, int offset, StringBuffer sb) { + int length = source.charAt(offset); + ++offset; + if ((0x8000 & length) != 0) { + length = ((0x7FFF & length) << 16) | source.charAt(offset); + ++offset; + } + if (sb != null) { + String str = source.substring(offset, offset + length); + sb.append(str); + } + return offset + length; + } + + private static int printSourceNumber(String source, + int offset, StringBuffer sb) { + double number = 0.0; + char type = source.charAt(offset); + ++offset; + if (type == 'S') { + if (sb != null) { + number = source.charAt(offset); + } + ++offset; + } else if (type == 'J' || type == 'D') { + if (sb != null) { + long lbits; + lbits = (long) source.charAt(offset) << 48; + lbits |= (long) source.charAt(offset + 1) << 32; + lbits |= (long) source.charAt(offset + 2) << 16; + lbits |= (long) source.charAt(offset + 3); + if (type == 'J') { + number = lbits; + } else { + number = Double.longBitsToDouble(lbits); + } + } + offset += 4; + } else { + // Bad source + throw new RuntimeException(); + } + if (sb != null) { + sb.append(ScriptRuntime.numberToString(number, 10)); + } + return offset; + } + + private static ArrayList parse(Reader in, ErrorReporter reporter) + throws IOException, EvaluatorException { + + CompilerEnvirons env = new CompilerEnvirons(); + Parser parser = new Parser(env, reporter); + parser.parse(in, null, 1); + String source = parser.getEncodedSource(); + + int offset = 0; + int length = source.length(); + ArrayList tokens = new ArrayList(); + StringBuffer sb = new StringBuffer(); + + while (offset < length) { + int tt = source.charAt(offset++); + switch (tt) { + + case Token.SPECIALCOMMENT: + case Token.NAME: + case Token.REGEXP: + case Token.STRING: + sb.setLength(0); + offset = printSourceString(source, offset, sb); + tokens.add(new JavaScriptToken(tt, sb.toString())); + break; + + case Token.NUMBER: + sb.setLength(0); + offset = printSourceNumber(source, offset, sb); + tokens.add(new JavaScriptToken(tt, sb.toString())); + break; + + default: + String literal = (String) literals.get(new Integer(tt)); + if (literal != null) { + tokens.add(new JavaScriptToken(tt, literal)); + } + break; + } + } + + return tokens; + } + + private static void processStringLiterals(ArrayList tokens, boolean merge) { + + String tv; + int i, length = tokens.size(); + JavaScriptToken token, prevToken, nextToken; + + if (merge) { + + // Concatenate string literals that are being appended wherever + // it is safe to do so. Note that we take care of the case: + // "a" + "b".toUpperCase() + + for (i = 0; i < length; i++) { + token = (JavaScriptToken) tokens.get(i); + switch (token.getType()) { + + case Token.ADD: + if (i > 0 && i < length) { + prevToken = (JavaScriptToken) tokens.get(i - 1); + nextToken = (JavaScriptToken) tokens.get(i + 1); + if (prevToken.getType() == Token.STRING && nextToken.getType() == Token.STRING && + (i == length - 1 || ((JavaScriptToken) tokens.get(i + 2)).getType() != Token.DOT)) { + tokens.set(i - 1, new JavaScriptToken(Token.STRING, + prevToken.getValue() + nextToken.getValue())); + tokens.remove(i + 1); + tokens.remove(i); + i = i - 1; + length = length - 2; + break; + } + } + } + } + + } + + // Second pass... + + for (i = 0; i < length; i++) { + // APPJET modifications in this loop + token = (JavaScriptToken) tokens.get(i); + if (token.getType() == Token.STRING || token.getType() == Token.REGEXP) { + tv = token.getValue(); + if (token.getType() == Token.STRING) { + + // Finally, add the quoting characters and escape the string. We use + // the quoting character that minimizes the amount of escaping to save + // a few additional bytes. + + char quotechar; + int singleQuoteCount = countChar(tv, '\''); + int doubleQuoteCount = countChar(tv, '"'); + if (doubleQuoteCount <= singleQuoteCount) { + quotechar = '"'; + } else { + quotechar = '\''; + } + + tv = quotechar + escapeString(tv, quotechar) + quotechar; + } + + // String concatenation transforms the old script scheme: + // '<scr'+'ipt ...><'+'/script>' + // into the following: + // '<script ...></script>' + // which breaks if this code is embedded inside an HTML document. + // Since this is not the right way to do this, let's fix the code by + // transforming all "</script" into "<\/script" + + // (Anti-malware software can be sensitive to open tags) + tv = tv.replace("<script", "\\x3cscript"); + tv = tv.replace("</script", "\\x3c/script"); + + tokens.set(i, new JavaScriptToken(token.getType(), tv)); + } + } + } + + // Add necessary escaping that was removed in Rhino's tokenizer. + private static String escapeString(String s, char quotechar) { + + assert quotechar == '"' || quotechar == '\''; + + if (s == null) { + return null; + } + + StringBuffer sb = new StringBuffer(); + for (int i = 0, L = s.length(); i < L; i++) { + int c = s.charAt(i); + if (c == quotechar) { + sb.append("\\"); + } + sb.append((char) c); + } + + return sb.toString(); + } + + /* + * Simple check to see whether a string is a valid identifier name. + * If a string matches this pattern, it means it IS a valid + * identifier name. If a string doesn't match it, it does not + * necessarily mean it is not a valid identifier name. + */ + private static final Pattern SIMPLE_IDENTIFIER_NAME_PATTERN = Pattern.compile("^[a-zA-Z_][a-zA-Z0-9_]*$"); + + private static boolean isValidIdentifier(String s) { + Matcher m = SIMPLE_IDENTIFIER_NAME_PATTERN.matcher(s); + return (m.matches() && !reserved.contains(s)); + } + + /* + * Transforms obj["foo"] into obj.foo whenever possible, saving 3 bytes. + */ + private static void optimizeObjectMemberAccess(ArrayList tokens) { + + String tv; + int i, length; + JavaScriptToken token; + + for (i = 0, length = tokens.size(); i < length; i++) { + + if (((JavaScriptToken) tokens.get(i)).getType() == Token.LB && + i > 0 && i < length - 2 && + ((JavaScriptToken) tokens.get(i - 1)).getType() == Token.NAME && + ((JavaScriptToken) tokens.get(i + 1)).getType() == Token.STRING && + ((JavaScriptToken) tokens.get(i + 2)).getType() == Token.RB) { + token = (JavaScriptToken) tokens.get(i + 1); + tv = token.getValue(); + tv = tv.substring(1, tv.length() - 1); + if (isValidIdentifier(tv)) { + tokens.set(i, new JavaScriptToken(Token.DOT, ".")); + tokens.set(i + 1, new JavaScriptToken(Token.NAME, tv)); + tokens.remove(i + 2); + i = i + 2; + length = length - 1; + } + } + } + } + + /* + * Transforms 'foo': ... into foo: ... whenever possible, saving 2 bytes. + */ + private static void optimizeObjLitMemberDecl(ArrayList tokens) { + + String tv; + int i, length; + JavaScriptToken token; + + for (i = 0, length = tokens.size(); i < length; i++) { + if (((JavaScriptToken) tokens.get(i)).getType() == Token.OBJECTLIT && + i > 0 && ((JavaScriptToken) tokens.get(i - 1)).getType() == Token.STRING) { + token = (JavaScriptToken) tokens.get(i - 1); + tv = token.getValue(); + tv = tv.substring(1, tv.length() - 1); + if (isValidIdentifier(tv)) { + tokens.set(i - 1, new JavaScriptToken(Token.NAME, tv)); + } + } + } + } + + private ErrorReporter logger; + + private boolean munge; + private boolean verbose; + + private static final int BUILDING_SYMBOL_TREE = 1; + private static final int CHECKING_SYMBOL_TREE = 2; + + private int mode; + private int offset; + private int braceNesting; + private ArrayList tokens; + private Stack scopes = new Stack(); + private ScriptOrFnScope globalScope = new ScriptOrFnScope(-1, null); + private Hashtable indexedScopes = new Hashtable(); + + public JavaScriptCompressor(Reader in, ErrorReporter reporter) + throws IOException, EvaluatorException { + + this.logger = reporter; + this.tokens = parse(in, reporter); + } + + public void compress(Writer out, int linebreak, boolean munge, boolean verbose, + boolean preserveAllSemiColons, boolean disableOptimizations) + throws IOException { + + this.munge = munge; + this.verbose = verbose; + + processStringLiterals(this.tokens, !disableOptimizations); + + if (!disableOptimizations) { + optimizeObjectMemberAccess(this.tokens); + optimizeObjLitMemberDecl(this.tokens); + } + + buildSymbolTree(); + // DO NOT TOUCH this.tokens BETWEEN THESE TWO PHASES (BECAUSE OF this.indexedScopes) + mungeSymboltree(); + StringBuffer sb = printSymbolTree(linebreak, preserveAllSemiColons); + + out.write(sb.toString()); + } + + private ScriptOrFnScope getCurrentScope() { + return (ScriptOrFnScope) scopes.peek(); + } + + private void enterScope(ScriptOrFnScope scope) { + scopes.push(scope); + } + + private void leaveCurrentScope() { + scopes.pop(); + } + + private JavaScriptToken consumeToken() { + return (JavaScriptToken) tokens.get(offset++); + } + + private JavaScriptToken getToken(int delta) { + return (JavaScriptToken) tokens.get(offset + delta); + } + + /* + * Returns the identifier for the specified symbol defined in + * the specified scope or in any scope above it. Returns null + * if this symbol does not have a corresponding identifier. + */ + private JavaScriptIdentifier getIdentifier(String symbol, ScriptOrFnScope scope) { + JavaScriptIdentifier identifier; + while (scope != null) { + identifier = scope.getIdentifier(symbol); + if (identifier != null) { + return identifier; + } + scope = scope.getParentScope(); + } + return null; + } + + /* + * If either 'eval' or 'with' is used in a local scope, we must make + * sure that all containing local scopes don't get munged. Otherwise, + * the obfuscation would potentially introduce bugs. + */ + private void protectScopeFromObfuscation(ScriptOrFnScope scope) { + assert scope != null; + + if (scope == globalScope) { + // The global scope does not get obfuscated, + // so we don't need to worry about it... + return; + } + + // Find the highest local scope containing the specified scope. + while (scope.getParentScope() != globalScope) { + scope = scope.getParentScope(); + } + + assert scope.getParentScope() == globalScope; + scope.preventMunging(); + } + + private String getDebugString(int max) { + assert max > 0; + StringBuffer result = new StringBuffer(); + int start = Math.max(offset - max, 0); + int end = Math.min(offset + max, tokens.size()); + for (int i = start; i < end; i++) { + JavaScriptToken token = (JavaScriptToken) tokens.get(i); + if (i == offset - 1) { + result.append(" ---> "); + } + result.append(token.getValue()); + if (i == offset - 1) { + result.append(" <--- "); + } + } + return result.toString(); + } + + private void warn(String message, boolean showDebugString) { + if (verbose) { + if (showDebugString) { + message = message + "\n" + getDebugString(10); + } + logger.warning(message, null, -1, null, -1); + } + } + + private void parseFunctionDeclaration() { + + String symbol; + JavaScriptToken token; + ScriptOrFnScope currentScope, fnScope; + JavaScriptIdentifier identifier; + + currentScope = getCurrentScope(); + + token = consumeToken(); + if (token.getType() == Token.NAME) { + if (mode == BUILDING_SYMBOL_TREE) { + // Get the name of the function and declare it in the current scope. + symbol = token.getValue(); + if (currentScope.getIdentifier(symbol) != null) { + warn("The function " + symbol + " has already been declared in the same scope...", true); + } + currentScope.declareIdentifier(symbol); + } + token = consumeToken(); + } + + assert token.getType() == Token.LP; + if (mode == BUILDING_SYMBOL_TREE) { + fnScope = new ScriptOrFnScope(braceNesting, currentScope); + indexedScopes.put(new Integer(offset), fnScope); + } else { + fnScope = (ScriptOrFnScope) indexedScopes.get(new Integer(offset)); + } + + // Parse function arguments. + int argpos = 0; + while ((token = consumeToken()).getType() != Token.RP) { + assert token.getType() == Token.NAME || + token.getType() == Token.COMMA; + if (token.getType() == Token.NAME && mode == BUILDING_SYMBOL_TREE) { + symbol = token.getValue(); + identifier = fnScope.declareIdentifier(symbol); + if (symbol.equals("$super") && argpos == 0) { + // Exception for Prototype 1.6... + identifier.preventMunging(); + } + argpos++; + } + } + + token = consumeToken(); + assert token.getType() == Token.LC; + braceNesting++; + + token = getToken(0); + if (token.getType() == Token.STRING && + getToken(1).getType() == Token.SEMI) { + // This is a hint. Hints are empty statements that look like + // "localvar1:nomunge, localvar2:nomunge"; They allow developers + // to prevent specific symbols from getting obfuscated (some heretic + // implementations, such as Prototype 1.6, require specific variable + // names, such as $super for example, in order to work appropriately. + // Note: right now, only "nomunge" is supported in the right hand side + // of a hint. However, in the future, the right hand side may contain + // other values. + consumeToken(); + String hints = token.getValue(); + // Remove the leading and trailing quotes... + hints = hints.substring(1, hints.length() - 1).trim(); + StringTokenizer st1 = new StringTokenizer(hints, ","); + while (st1.hasMoreTokens()) { + String hint = st1.nextToken(); + int idx = hint.indexOf(':'); + if (idx <= 0 || idx >= hint.length() - 1) { + if (mode == BUILDING_SYMBOL_TREE) { + // No need to report the error twice, hence the test... + warn("Invalid hint syntax: " + hint, true); + } + break; + } + String variableName = hint.substring(0, idx).trim(); + String variableType = hint.substring(idx + 1).trim(); + if (mode == BUILDING_SYMBOL_TREE) { + fnScope.addHint(variableName, variableType); + } else if (mode == CHECKING_SYMBOL_TREE) { + identifier = fnScope.getIdentifier(variableName); + if (identifier != null) { + if (variableType.equals("nomunge")) { + identifier.preventMunging(); + } else { + warn("Unsupported hint value: " + hint, true); + } + } else { + warn("Hint refers to an unknown identifier: " + hint, true); + } + } + } + } + + parseScope(fnScope); + } + + private void parseCatch() { + + String symbol; + JavaScriptToken token; + ScriptOrFnScope currentScope; + JavaScriptIdentifier identifier; + + token = getToken(-1); + assert token.getType() == Token.CATCH; + token = consumeToken(); + assert token.getType() == Token.LP; + token = consumeToken(); + assert token.getType() == Token.NAME; + + symbol = token.getValue(); + currentScope = getCurrentScope(); + + if (mode == BUILDING_SYMBOL_TREE) { + // We must declare the exception identifier in the containing function + // scope to avoid errors related to the obfuscation process. No need to + // display a warning if the symbol was already declared here... + currentScope.declareIdentifier(symbol); + } else { + identifier = getIdentifier(symbol, currentScope); + identifier.incrementRefcount(); + } + + token = consumeToken(); + assert token.getType() == Token.RP; + } + + private void parseExpression() { + + // Parse the expression until we encounter a comma or a semi-colon + // in the same brace nesting, bracket nesting and paren nesting. + // Parse functions if any... + + String symbol; + JavaScriptToken token; + ScriptOrFnScope currentScope; + JavaScriptIdentifier identifier; + + int expressionBraceNesting = braceNesting; + int bracketNesting = 0; + int parensNesting = 0; + + int length = tokens.size(); + + while (offset < length) { + + token = consumeToken(); + currentScope = getCurrentScope(); + + switch (token.getType()) { + + case Token.SEMI: + case Token.COMMA: + if (braceNesting == expressionBraceNesting && + bracketNesting == 0 && + parensNesting == 0) { + return; + } + break; + + case Token.FUNCTION: + parseFunctionDeclaration(); + break; + + case Token.LC: + braceNesting++; + break; + + case Token.RC: + braceNesting--; + assert braceNesting >= expressionBraceNesting; + break; + + case Token.LB: + bracketNesting++; + break; + + case Token.RB: + bracketNesting--; + break; + + case Token.LP: + parensNesting++; + break; + + case Token.RP: + parensNesting--; + break; + + case Token.SPECIALCOMMENT: + if (mode == BUILDING_SYMBOL_TREE) { + protectScopeFromObfuscation(currentScope); + warn("Using JScript conditional comments is not recommended." + (munge ? " Moreover, using JScript conditional comments reduces the level of compression!" : ""), true); + } + break; + + case Token.NAME: + symbol = token.getValue(); + + if (mode == BUILDING_SYMBOL_TREE) { + + if (symbol.equals("eval")) { + + protectScopeFromObfuscation(currentScope); + warn("Using 'eval' is not recommended." + (munge ? " Moreover, using 'eval' reduces the level of compression!" : ""), true); + + } + + } else if (mode == CHECKING_SYMBOL_TREE) { + + if ((offset < 2 || + (getToken(-2).getType() != Token.DOT && + getToken(-2).getType() != Token.GET && + getToken(-2).getType() != Token.SET)) && + getToken(0).getType() != Token.OBJECTLIT) { + + identifier = getIdentifier(symbol, currentScope); + + if (identifier == null) { + + if (symbol.length() <= 3 && !builtin.contains(symbol)) { + // Here, we found an undeclared and un-namespaced symbol that is + // 3 characters or less in length. Declare it in the global scope. + // We don't need to declare longer symbols since they won't cause + // any conflict with other munged symbols. + globalScope.declareIdentifier(symbol); + warn("Found an undeclared symbol: " + symbol, true); + } + + } else { + + identifier.incrementRefcount(); + } + } + } + break; + } + } + } + + private void parseScope(ScriptOrFnScope scope) { + + String symbol; + JavaScriptToken token; + JavaScriptIdentifier identifier; + + int length = tokens.size(); + + enterScope(scope); + + while (offset < length) { + + token = consumeToken(); + + switch (token.getType()) { + + case Token.VAR: + + if (mode == BUILDING_SYMBOL_TREE && scope.incrementVarCount() > 1) { + warn("Try to use a single 'var' statement per scope.", true); + } + + /* FALLSTHROUGH */ + + case Token.CONST: + + // The var keyword is followed by at least one symbol name. + // If several symbols follow, they are comma separated. + for (; ;) { + token = consumeToken(); + + assert token.getType() == Token.NAME; + + if (mode == BUILDING_SYMBOL_TREE) { + symbol = token.getValue(); + if (scope.getIdentifier(symbol) == null) { + scope.declareIdentifier(symbol); + } else { + warn("The variable " + symbol + " has already been declared in the same scope...", true); + } + } + + token = getToken(0); + + assert token.getType() == Token.SEMI || + token.getType() == Token.ASSIGN || + token.getType() == Token.COMMA || + token.getType() == Token.IN; + + if (token.getType() == Token.IN) { + break; + } else { + parseExpression(); + token = getToken(-1); + if (token.getType() == Token.SEMI) { + break; + } + } + } + break; + + case Token.FUNCTION: + parseFunctionDeclaration(); + break; + + case Token.LC: + braceNesting++; + break; + + case Token.RC: + braceNesting--; + assert braceNesting >= scope.getBraceNesting(); + if (braceNesting == scope.getBraceNesting()) { + leaveCurrentScope(); + return; + } + break; + + case Token.WITH: + if (mode == BUILDING_SYMBOL_TREE) { + // Inside a 'with' block, it is impossible to figure out + // statically whether a symbol is a local variable or an + // object member. As a consequence, the only thing we can + // do is turn the obfuscation off for the highest scope + // containing the 'with' block. + protectScopeFromObfuscation(scope); + warn("Using 'with' is not recommended." + (munge ? " Moreover, using 'with' reduces the level of compression!" : ""), true); + } + break; + + case Token.CATCH: + parseCatch(); + break; + + case Token.SPECIALCOMMENT: + if (mode == BUILDING_SYMBOL_TREE) { + protectScopeFromObfuscation(scope); + warn("Using JScript conditional comments is not recommended." + (munge ? " Moreover, using JScript conditional comments reduces the level of compression." : ""), true); + } + break; + + case Token.NAME: + symbol = token.getValue(); + + if (mode == BUILDING_SYMBOL_TREE) { + + if (symbol.equals("eval")) { + + protectScopeFromObfuscation(scope); + warn("Using 'eval' is not recommended." + (munge ? " Moreover, using 'eval' reduces the level of compression!" : ""), true); + + } + + } else if (mode == CHECKING_SYMBOL_TREE) { + + if ((offset < 2 || getToken(-2).getType() != Token.DOT) && + getToken(0).getType() != Token.OBJECTLIT) { + + identifier = getIdentifier(symbol, scope); + + if (identifier == null) { + + if (symbol.length() <= 3 && !builtin.contains(symbol)) { + // Here, we found an undeclared and un-namespaced symbol that is + // 3 characters or less in length. Declare it in the global scope. + // We don't need to declare longer symbols since they won't cause + // any conflict with other munged symbols. + globalScope.declareIdentifier(symbol); + warn("Found an undeclared symbol: " + symbol, true); + } + + } else { + + identifier.incrementRefcount(); + } + } + } + break; + } + } + } + + private void buildSymbolTree() { + offset = 0; + braceNesting = 0; + scopes.clear(); + indexedScopes.clear(); + indexedScopes.put(new Integer(0), globalScope); + mode = BUILDING_SYMBOL_TREE; + parseScope(globalScope); + } + + private void mungeSymboltree() { + + if (!munge) { + return; + } + + // One problem with obfuscation resides in the use of undeclared + // and un-namespaced global symbols that are 3 characters or less + // in length. Here is an example: + // + // var declaredGlobalVar; + // + // function declaredGlobalFn() { + // var localvar; + // localvar = abc; // abc is an undeclared global symbol + // } + // + // In the example above, there is a slim chance that localvar may be + // munged to 'abc', conflicting with the undeclared global symbol + // abc, creating a potential bug. The following code detects such + // global symbols. This must be done AFTER the entire file has been + // parsed, and BEFORE munging the symbol tree. Note that declaring + // extra symbols in the global scope won't hurt. + // + // Note: Since we go through all the tokens to do this, we also use + // the opportunity to count how many times each identifier is used. + + offset = 0; + braceNesting = 0; + scopes.clear(); + mode = CHECKING_SYMBOL_TREE; + parseScope(globalScope); + globalScope.munge(); + } + + private StringBuffer printSymbolTree(int linebreakpos, boolean preserveAllSemiColons) + throws IOException { + + offset = 0; + braceNesting = 0; + scopes.clear(); + + String symbol; + JavaScriptToken token; + ScriptOrFnScope currentScope; + JavaScriptIdentifier identifier; + + int length = tokens.size(); + StringBuffer result = new StringBuffer(); + + int linestartpos = 0; + + enterScope(globalScope); + + while (offset < length) { + + token = consumeToken(); + symbol = token.getValue(); + currentScope = getCurrentScope(); + + switch (token.getType()) { + + case Token.NAME: + + if (offset >= 2 && getToken(-2).getType() == Token.DOT || + getToken(0).getType() == Token.OBJECTLIT) { + + result.append(symbol); + + } else { + + identifier = getIdentifier(symbol, currentScope); + if (identifier != null) { + if (identifier.getMungedValue() != null) { + result.append(identifier.getMungedValue()); + } else { + result.append(symbol); + } + if (currentScope != globalScope && identifier.getRefcount() == 0) { + warn("The symbol " + symbol + " is declared but is apparently never used.\nThis code can probably be written in a more compact way.", true); + } + } else { + result.append(symbol); + } + } + break; + + case Token.REGEXP: + case Token.NUMBER: + case Token.STRING: + result.append(symbol); + break; + + case Token.ADD: + case Token.SUB: + result.append((String) literals.get(new Integer(token.getType()))); + if (offset < length) { + token = getToken(0); + if (token.getType() == Token.INC || + token.getType() == Token.DEC || + token.getType() == Token.ADD || + token.getType() == Token.DEC) { + // Handle the case x +/- ++/-- y + // We must keep a white space here. Otherwise, x +++ y would be + // interpreted as x ++ + y by the compiler, which is a bug (due + // to the implicit assignment being done on the wrong variable) + result.append(' '); + } else if (token.getType() == Token.POS && getToken(-1).getType() == Token.ADD || + token.getType() == Token.NEG && getToken(-1).getType() == Token.SUB) { + // Handle the case x + + y and x - - y + result.append(' '); + } + } + break; + + case Token.FUNCTION: + result.append("function"); + token = consumeToken(); + if (token.getType() == Token.NAME) { + result.append(' '); + symbol = token.getValue(); + identifier = getIdentifier(symbol, currentScope); + assert identifier != null; + if (identifier.getMungedValue() != null) { + result.append(identifier.getMungedValue()); + } else { + result.append(symbol); + } + if (currentScope != globalScope && identifier.getRefcount() == 0) { + warn("The symbol " + symbol + " is declared but is apparently never used.\nThis code can probably be written in a more compact way.", true); + } + token = consumeToken(); + } + assert token.getType() == Token.LP; + result.append('('); + currentScope = (ScriptOrFnScope) indexedScopes.get(new Integer(offset)); + enterScope(currentScope); + while ((token = consumeToken()).getType() != Token.RP) { + assert token.getType() == Token.NAME || token.getType() == Token.COMMA; + if (token.getType() == Token.NAME) { + symbol = token.getValue(); + identifier = getIdentifier(symbol, currentScope); + assert identifier != null; + if (identifier.getMungedValue() != null) { + result.append(identifier.getMungedValue()); + } else { + result.append(symbol); + } + } else if (token.getType() == Token.COMMA) { + result.append(','); + } + } + result.append(')'); + token = consumeToken(); + assert token.getType() == Token.LC; + result.append('{'); + braceNesting++; + token = getToken(0); + if (token.getType() == Token.STRING && + getToken(1).getType() == Token.SEMI) { + // This is a hint. Skip it! + consumeToken(); + consumeToken(); + } + break; + + case Token.RETURN: + case Token.TYPEOF: + result.append(literals.get(new Integer(token.getType()))); + // No space needed after 'return' and 'typeof' when followed + // by '(', '[', '{', a string or a regexp. + if (offset < length) { + token = getToken(0); + if (token.getType() != Token.LP && + token.getType() != Token.LB && + token.getType() != Token.LC && + token.getType() != Token.STRING && + token.getType() != Token.REGEXP && + token.getType() != Token.SEMI) { + result.append(' '); + } + } + break; + + case Token.CASE: + case Token.THROW: + result.append(literals.get(new Integer(token.getType()))); + // White-space needed after 'case' and 'throw' when not followed by a string. + if (offset < length && getToken(0).getType() != Token.STRING) { + result.append(' '); + } + break; + + case Token.BREAK: + case Token.CONTINUE: + result.append(literals.get(new Integer(token.getType()))); + if (offset < length && getToken(0).getType() != Token.SEMI) { + // If 'break' or 'continue' is not followed by a semi-colon, it must + // be followed by a label, hence the need for a white space. + result.append(' '); + } + break; + + case Token.LC: + result.append('{'); + braceNesting++; + break; + + case Token.RC: + result.append('}'); + braceNesting--; + assert braceNesting >= currentScope.getBraceNesting(); + if (braceNesting == currentScope.getBraceNesting()) { + leaveCurrentScope(); + } + break; + + case Token.SEMI: + // No need to output a semi-colon if the next character is a right-curly... + if (preserveAllSemiColons || offset < length && getToken(0).getType() != Token.RC) { + result.append(';'); + } + + if (linebreakpos >= 0 && result.length() - linestartpos > linebreakpos) { + // Some source control tools don't like it when files containing lines longer + // than, say 8000 characters, are checked in. The linebreak option is used in + // that case to split long lines after a specific column. + result.append('\n'); + linestartpos = result.length(); + } + break; + + case Token.SPECIALCOMMENT: + if (result.length() > 0 && result.charAt(result.length() - 1) != '\n') { + result.append("\n"); + } + result.append("/*"); + result.append(symbol); + result.append("*/\n"); + break; + + default: + String literal = (String) literals.get(new Integer(token.getType())); + if (literal != null) { + result.append(literal); + } else { + warn("This symbol cannot be printed: " + symbol, true); + } + break; + } + } + + // Append a semi-colon at the end, even if unnecessary semi-colons are + // supposed to be removed. This is especially useful when concatenating + // several minified files (the absence of an ending semi-colon at the + // end of one file may very likely cause a syntax error) + if (!preserveAllSemiColons && result.length() > 0) { + if (result.charAt(result.length() - 1) == '\n') { + result.setCharAt(result.length() - 1, ';'); + } else { + result.append(';'); + } + } + + return result; + } +} diff --git a/trunk/infrastructure/yuicompressor/src/com/yahoo/platform/yui/compressor/JavaScriptIdentifier.java b/trunk/infrastructure/yuicompressor/src/com/yahoo/platform/yui/compressor/JavaScriptIdentifier.java new file mode 100644 index 0000000..8668f49 --- /dev/null +++ b/trunk/infrastructure/yuicompressor/src/com/yahoo/platform/yui/compressor/JavaScriptIdentifier.java @@ -0,0 +1,55 @@ +/* + * YUI Compressor + * Author: Julien Lecomte <jlecomte@yahoo-inc.com> + * Copyright (c) 2007, Yahoo! Inc. All rights reserved. + * Code licensed under the BSD License: + * http://developer.yahoo.net/yui/license.txt + */ + +package com.yahoo.platform.yui.compressor; + +import yuicompressor.org.mozilla.javascript.Token; + +/** + * JavaScriptIdentifier represents a variable/function identifier. + */ +class JavaScriptIdentifier extends JavaScriptToken { + + private int refcount = 0; + private String mungedValue; + private ScriptOrFnScope declaredScope; + private boolean markedForMunging = true; + + JavaScriptIdentifier(String value, ScriptOrFnScope declaredScope) { + super(Token.NAME, value); + this.declaredScope = declaredScope; + } + + ScriptOrFnScope getDeclaredScope() { + return declaredScope; + } + + void setMungedValue(String value) { + mungedValue = value; + } + + String getMungedValue() { + return mungedValue; + } + + void preventMunging() { + markedForMunging = false; + } + + boolean isMarkedForMunging() { + return markedForMunging; + } + + void incrementRefcount() { + refcount++; + } + + int getRefcount() { + return refcount; + } +} diff --git a/trunk/infrastructure/yuicompressor/src/com/yahoo/platform/yui/compressor/JavaScriptToken.java b/trunk/infrastructure/yuicompressor/src/com/yahoo/platform/yui/compressor/JavaScriptToken.java new file mode 100644 index 0000000..fee21d9 --- /dev/null +++ b/trunk/infrastructure/yuicompressor/src/com/yahoo/platform/yui/compressor/JavaScriptToken.java @@ -0,0 +1,28 @@ +/* + * YUI Compressor + * Author: Julien Lecomte <jlecomte@yahoo-inc.com> + * Copyright (c) 2007, Yahoo! Inc. All rights reserved. + * Code licensed under the BSD License: + * http://developer.yahoo.net/yui/license.txt + */ + +package com.yahoo.platform.yui.compressor; + +public class JavaScriptToken { + + private int type; + private String value; + + JavaScriptToken(int type, String value) { + this.type = type; + this.value = value; + } + + int getType() { + return type; + } + + String getValue() { + return value; + } +} diff --git a/trunk/infrastructure/yuicompressor/src/com/yahoo/platform/yui/compressor/ScriptOrFnScope.java b/trunk/infrastructure/yuicompressor/src/com/yahoo/platform/yui/compressor/ScriptOrFnScope.java new file mode 100644 index 0000000..c1a2e47 --- /dev/null +++ b/trunk/infrastructure/yuicompressor/src/com/yahoo/platform/yui/compressor/ScriptOrFnScope.java @@ -0,0 +1,169 @@ +/* + * YUI Compressor + * Author: Julien Lecomte <jlecomte@yahoo-inc.com> + * Copyright (c) 2007, Yahoo! Inc. All rights reserved. + * Code licensed under the BSD License: + * http://developer.yahoo.net/yui/license.txt + */ + +package com.yahoo.platform.yui.compressor; + +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.Hashtable; + +class ScriptOrFnScope { + + private int braceNesting; + private ScriptOrFnScope parentScope; + private ArrayList subScopes; + private Hashtable identifiers = new Hashtable(); + private Hashtable hints = new Hashtable(); + private boolean markedForMunging = true; + private int varcount = 0; + + ScriptOrFnScope(int braceNesting, ScriptOrFnScope parentScope) { + this.braceNesting = braceNesting; + this.parentScope = parentScope; + this.subScopes = new ArrayList(); + if (parentScope != null) { + parentScope.subScopes.add(this); + } + } + + int getBraceNesting() { + return braceNesting; + } + + ScriptOrFnScope getParentScope() { + return parentScope; + } + + JavaScriptIdentifier declareIdentifier(String symbol) { + JavaScriptIdentifier identifier = (JavaScriptIdentifier) identifiers.get(symbol); + if (identifier == null) { + identifier = new JavaScriptIdentifier(symbol, this); + identifiers.put(symbol, identifier); + } + return identifier; + } + + JavaScriptIdentifier getIdentifier(String symbol) { + return (JavaScriptIdentifier) identifiers.get(symbol); + } + + void addHint(String variableName, String variableType) { + hints.put(variableName, variableType); + } + + void preventMunging() { + if (parentScope != null) { + // The symbols in the global scope don't get munged, + // but the sub-scopes it contains do get munged. + markedForMunging = false; + } + } + + private ArrayList getUsedSymbols() { + ArrayList result = new ArrayList(); + Enumeration elements = identifiers.elements(); + while (elements.hasMoreElements()) { + JavaScriptIdentifier identifier = (JavaScriptIdentifier) elements.nextElement(); + String mungedValue = identifier.getMungedValue(); + if (mungedValue == null) { + mungedValue = identifier.getValue(); + } + result.add(mungedValue); + } + return result; + } + + private ArrayList getAllUsedSymbols() { + ArrayList result = new ArrayList(); + ScriptOrFnScope scope = this; + while (scope != null) { + result.addAll(scope.getUsedSymbols()); + scope = scope.parentScope; + } + return result; + } + + int incrementVarCount() { + varcount++; + return varcount; + } + + void munge() { + + if (!markedForMunging) { + // Stop right here if this scope was flagged as unsafe for munging. + return; + } + + int pickFromSet = 1; + + // Do not munge symbols in the global scope! + if (parentScope != null) { + + ArrayList freeSymbols = new ArrayList(); + + freeSymbols.addAll(JavaScriptCompressor.ones); + freeSymbols.removeAll(getAllUsedSymbols()); + if (freeSymbols.size() == 0) { + pickFromSet = 2; + freeSymbols.addAll(JavaScriptCompressor.twos); + freeSymbols.removeAll(getAllUsedSymbols()); + } + if (freeSymbols.size() == 0) { + pickFromSet = 3; + freeSymbols.addAll(JavaScriptCompressor.threes); + freeSymbols.removeAll(getAllUsedSymbols()); + } + if (freeSymbols.size() == 0) { + throw new IllegalStateException("The YUI Compressor ran out of symbols. Aborting..."); + } + + // APPJET: sort identifiers by popularity + JavaScriptIdentifier idArray[] = ((Hashtable<String,JavaScriptIdentifier>)identifiers).values().toArray(new JavaScriptIdentifier[0]); + java.util.Arrays.sort(idArray, new java.util.Comparator<JavaScriptIdentifier>() { + public int compare(JavaScriptIdentifier i1, JavaScriptIdentifier i2) { + return i2.getRefcount() - i1.getRefcount(); // positive if i2 is more popular, indicating i2 should come first + } + }); + java.util.Iterator<JavaScriptIdentifier> elements = java.util.Arrays.asList(idArray).iterator(); + + //Enumeration elements = identifiers.elements(); + while (elements.hasNext()) { + if (freeSymbols.size() == 0) { + pickFromSet++; + if (pickFromSet == 2) { + freeSymbols.addAll(JavaScriptCompressor.twos); + } else if (pickFromSet == 3) { + freeSymbols.addAll(JavaScriptCompressor.threes); + } else { + throw new IllegalStateException("The YUI Compressor ran out of symbols. Aborting..."); + } + // It is essential to remove the symbols already used in + // the containing scopes, or some of the variables declared + // in the containing scopes will be redeclared, which can + // lead to errors. + freeSymbols.removeAll(getAllUsedSymbols()); + } + + String mungedValue; + JavaScriptIdentifier identifier = (JavaScriptIdentifier) elements.next(); + if (identifier.isMarkedForMunging()) { + mungedValue = (String) freeSymbols.remove(0); + } else { + mungedValue = identifier.getValue(); + } + identifier.setMungedValue(mungedValue); + } + } + + for (int i = 0; i < subScopes.size(); i++) { + ScriptOrFnScope scope = (ScriptOrFnScope) subScopes.get(i); + scope.munge(); + } + } +} diff --git a/trunk/infrastructure/yuicompressor/src/com/yahoo/platform/yui/compressor/YUICompressor.java b/trunk/infrastructure/yuicompressor/src/com/yahoo/platform/yui/compressor/YUICompressor.java new file mode 100644 index 0000000..dcbaff4 --- /dev/null +++ b/trunk/infrastructure/yuicompressor/src/com/yahoo/platform/yui/compressor/YUICompressor.java @@ -0,0 +1,232 @@ +/* + * YUI Compressor + * Author: Julien Lecomte <jlecomte@yahoo-inc.com> + * Copyright (c) 2007, Yahoo! Inc. All rights reserved. + * Code licensed under the BSD License: + * http://developer.yahoo.net/yui/license.txt + */ + +package com.yahoo.platform.yui.compressor; + +import jargs.gnu.CmdLineParser; +import yuicompressor.org.mozilla.javascript.ErrorReporter; +import yuicompressor.org.mozilla.javascript.EvaluatorException; + +import java.io.*; +import java.nio.charset.Charset; + +public class YUICompressor { + + public static void main(String args[]) { + + CmdLineParser parser = new CmdLineParser(); + CmdLineParser.Option typeOpt = parser.addStringOption("type"); + CmdLineParser.Option verboseOpt = parser.addBooleanOption('v', "verbose"); + CmdLineParser.Option nomungeOpt = parser.addBooleanOption("nomunge"); + CmdLineParser.Option linebreakOpt = parser.addStringOption("line-break"); + CmdLineParser.Option preserveSemiOpt = parser.addBooleanOption("preserve-semi"); + CmdLineParser.Option disableOptimizationsOpt = parser.addBooleanOption("disable-optimizations"); + CmdLineParser.Option helpOpt = parser.addBooleanOption('h', "help"); + CmdLineParser.Option charsetOpt = parser.addStringOption("charset"); + CmdLineParser.Option outputFilenameOpt = parser.addStringOption('o', "output"); + + Reader in = null; + Writer out = null; + + try { + + parser.parse(args); + + Boolean help = (Boolean) parser.getOptionValue(helpOpt); + if (help != null && help.booleanValue()) { + usage(); + System.exit(0); + } + + boolean verbose = parser.getOptionValue(verboseOpt) != null; + + String charset = (String) parser.getOptionValue(charsetOpt); + if (charset == null || !Charset.isSupported(charset)) { + charset = System.getProperty("file.encoding"); + if (charset == null) { + charset = "UTF-8"; + } + if (verbose) { + System.err.println("\n[INFO] Using charset " + charset); + } + } + + String[] fileArgs = parser.getRemainingArgs(); + String type = (String) parser.getOptionValue(typeOpt); + + if (fileArgs.length == 0) { + + if (type == null || !type.equalsIgnoreCase("js") && !type.equalsIgnoreCase("css")) { + usage(); + System.exit(1); + } + + in = new InputStreamReader(System.in, charset); + + } else { + + if (type != null && !type.equalsIgnoreCase("js") && !type.equalsIgnoreCase("css")) { + usage(); + System.exit(1); + } + + String inputFilename = fileArgs[0]; + + if (type == null) { + int idx = inputFilename.lastIndexOf('.'); + if (idx >= 0 && idx < inputFilename.length() - 1) { + type = inputFilename.substring(idx + 1); + } + } + + if (type == null || !type.equalsIgnoreCase("js") && !type.equalsIgnoreCase("css")) { + usage(); + System.exit(1); + } + + in = new InputStreamReader(new FileInputStream(inputFilename), charset); + } + + int linebreakpos = -1; + String linebreakstr = (String) parser.getOptionValue(linebreakOpt); + if (linebreakstr != null) { + try { + linebreakpos = Integer.parseInt(linebreakstr, 10); + } catch (NumberFormatException e) { + usage(); + System.exit(1); + } + } + + String outputFilename = (String) parser.getOptionValue(outputFilenameOpt); + + if (type.equalsIgnoreCase("js")) { + + try { + + JavaScriptCompressor compressor = new JavaScriptCompressor(in, new ErrorReporter() { + + public void warning(String message, String sourceName, + int line, String lineSource, int lineOffset) { + if (line < 0) { + System.err.println("\n[WARNING] " + message); + } else { + System.err.println("\n[WARNING] " + line + ':' + lineOffset + ':' + message); + } + } + + public void error(String message, String sourceName, + int line, String lineSource, int lineOffset) { + if (line < 0) { + System.err.println("\n[ERROR] " + message); + } else { + System.err.println("\n[ERROR] " + line + ':' + lineOffset + ':' + message); + } + } + + public EvaluatorException runtimeError(String message, String sourceName, + int line, String lineSource, int lineOffset) { + error(message, sourceName, line, lineSource, lineOffset); + return new EvaluatorException(message); + } + }); + + // Close the input stream first, and then open the output stream, + // in case the output file should override the input file. + in.close(); in = null; + + if (outputFilename == null) { + out = new OutputStreamWriter(System.out, charset); + } else { + out = new OutputStreamWriter(new FileOutputStream(outputFilename), charset); + } + + boolean munge = parser.getOptionValue(nomungeOpt) == null; + boolean preserveAllSemiColons = parser.getOptionValue(preserveSemiOpt) != null; + boolean disableOptimizations = parser.getOptionValue(disableOptimizationsOpt) != null; + + compressor.compress(out, linebreakpos, munge, verbose, + preserveAllSemiColons, disableOptimizations); + + } catch (EvaluatorException e) { + + e.printStackTrace(); + // Return a special error code used specifically by the web front-end. + System.exit(2); + + } + + } else if (type.equalsIgnoreCase("css")) { + + CssCompressor compressor = new CssCompressor(in); + + // Close the input stream first, and then open the output stream, + // in case the output file should override the input file. + in.close(); in = null; + + if (outputFilename == null) { + out = new OutputStreamWriter(System.out, charset); + } else { + out = new OutputStreamWriter(new FileOutputStream(outputFilename), charset); + } + + compressor.compress(out, linebreakpos); + } + + } catch (CmdLineParser.OptionException e) { + + usage(); + System.exit(1); + + } catch (IOException e) { + + e.printStackTrace(); + System.exit(1); + + } finally { + + if (in != null) { + try { + in.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + if (out != null) { + try { + out.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + } + + private static void usage() { + System.out.println( + "\nUsage: java -jar yuicompressor-x.y.z.jar [options] [input file]\n\n" + + + "Global Options\n" + + " -h, --help Displays this information\n" + + " --type <js|css> Specifies the type of the input file\n" + + " --charset <charset> Read the input file using <charset>\n" + + " --line-break <column> Insert a line break after the specified column number\n" + + " -v, --verbose Display informational messages and warnings\n" + + " -o <file> Place the output into <file>. Defaults to stdout.\n\n" + + + "JavaScript Options\n" + + " --nomunge Minify only, do not obfuscate\n" + + " --preserve-semi Preserve all semicolons\n" + + " --disable-optimizations Disable all micro optimizations\n\n" + + + "If no input file is specified, it defaults to stdin. In this case, the 'type'\n" + + "option is required. Otherwise, the 'type' option is required only if the input\n" + + "file extension is neither 'js' nor 'css'."); + } +} diff --git a/trunk/infrastructure/yuicompressor/src/yuicompressor/org/mozilla/javascript/Decompiler.java b/trunk/infrastructure/yuicompressor/src/yuicompressor/org/mozilla/javascript/Decompiler.java new file mode 100644 index 0000000..348c568 --- /dev/null +++ b/trunk/infrastructure/yuicompressor/src/yuicompressor/org/mozilla/javascript/Decompiler.java @@ -0,0 +1,916 @@ +/* -*- Mode: java; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is Rhino code, released + * May 6, 1999. + * + * The Initial Developer of the Original Code is + * Netscape Communications Corporation. + * Portions created by the Initial Developer are Copyright (C) 1997-1999 + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * Mike Ang + * Igor Bukanov + * Bob Jervis + * Mike McCabe + * + * Alternatively, the contents of this file may be used under the terms of + * the GNU General Public License Version 2 or later (the "GPL"), in which + * case the provisions of the GPL are applicable instead of those above. If + * you wish to allow use of your version of this file only under the terms of + * the GPL and not to allow others to use your version of this file under the + * MPL, indicate your decision by deleting the provisions above and replacing + * them with the notice and other provisions required by the GPL. If you do + * not delete the provisions above, a recipient may use your version of this + * file under either the MPL or the GPL. + * + * ***** END LICENSE BLOCK ***** */ + +package yuicompressor.org.mozilla.javascript; + +/** + * The following class save decompilation information about the source. + * Source information is returned from the parser as a String + * associated with function nodes and with the toplevel script. When + * saved in the constant pool of a class, this string will be UTF-8 + * encoded, and token values will occupy a single byte. + + * Source is saved (mostly) as token numbers. The tokens saved pretty + * much correspond to the token stream of a 'canonical' representation + * of the input program, as directed by the parser. (There were a few + * cases where tokens could have been left out where decompiler could + * easily reconstruct them, but I left them in for clarity). (I also + * looked adding source collection to TokenStream instead, where I + * could have limited the changes to a few lines in getToken... but + * this wouldn't have saved any space in the resulting source + * representation, and would have meant that I'd have to duplicate + * parser logic in the decompiler to disambiguate situations where + * newlines are important.) The function decompile expands the + * tokens back into their string representations, using simple + * lookahead to correct spacing and indentation. + * + * Assignments are saved as two-token pairs (Token.ASSIGN, op). Number tokens + * are stored inline, as a NUMBER token, a character representing the type, and + * either 1 or 4 characters representing the bit-encoding of the number. String + * types NAME, STRING and OBJECT are currently stored as a token type, + * followed by a character giving the length of the string (assumed to + * be less than 2^16), followed by the characters of the string + * inlined into the source string. Changing this to some reference to + * to the string in the compiled class' constant pool would probably + * save a lot of space... but would require some method of deriving + * the final constant pool entry from information available at parse + * time. + */ +public class Decompiler +{ + /** + * Flag to indicate that the decompilation should omit the + * function header and trailing brace. + */ + public static final int ONLY_BODY_FLAG = 1 << 0; + + /** + * Flag to indicate that the decompilation generates toSource result. + */ + public static final int TO_SOURCE_FLAG = 1 << 1; + + /** + * Decompilation property to specify initial ident value. + */ + public static final int INITIAL_INDENT_PROP = 1; + + /** + * Decompilation property to specify default identation offset. + */ + public static final int INDENT_GAP_PROP = 2; + + /** + * Decompilation property to specify identation offset for case labels. + */ + public static final int CASE_GAP_PROP = 3; + + // Marker to denote the last RC of function so it can be distinguished from + // the last RC of object literals in case of function expressions + private static final int FUNCTION_END = Token.LAST_TOKEN + 1; + + String getEncodedSource() + { + return sourceToString(0); + } + + int getCurrentOffset() + { + return sourceTop; + } + + int markFunctionStart(int functionType) + { + int savedOffset = getCurrentOffset(); + addToken(Token.FUNCTION); + append((char)functionType); + return savedOffset; + } + + int markFunctionEnd(int functionStart) + { + int offset = getCurrentOffset(); + append((char)FUNCTION_END); + return offset; + } + + void addToken(int token) + { + if (!(0 <= token && token <= Token.LAST_TOKEN)) + throw new IllegalArgumentException(); + + append((char)token); + } + + void addEOL(int token) + { + if (!(0 <= token && token <= Token.LAST_TOKEN)) + throw new IllegalArgumentException(); + + append((char)token); + append((char)Token.EOL); + } + + void addName(String str) + { + addToken(Token.NAME); + appendString(str); + } + + void addString(String str) + { + addToken(Token.STRING); + appendString(str); + } + + void addRegexp(String regexp, String flags) + { + addToken(Token.REGEXP); + appendString('/' + regexp + '/' + flags); + } + + void addJScriptConditionalComment(String str) + { + addToken(Token.SPECIALCOMMENT); + appendString(str); + } + + void addNumber(double n) + { + addToken(Token.NUMBER); + + /* encode the number in the source stream. + * Save as NUMBER type (char | char char char char) + * where type is + * 'D' - double, 'S' - short, 'J' - long. + + * We need to retain float vs. integer type info to keep the + * behavior of liveconnect type-guessing the same after + * decompilation. (Liveconnect tries to present 1.0 to Java + * as a float/double) + * OPT: This is no longer true. We could compress the format. + + * This may not be the most space-efficient encoding; + * the chars created below may take up to 3 bytes in + * constant pool UTF-8 encoding, so a Double could take + * up to 12 bytes. + */ + + long lbits = (long)n; + if (lbits != n) { + // if it's floating point, save as a Double bit pattern. + // (12/15/97 our scanner only returns Double for f.p.) + lbits = Double.doubleToLongBits(n); + append('D'); + append((char)(lbits >> 48)); + append((char)(lbits >> 32)); + append((char)(lbits >> 16)); + append((char)lbits); + } + else { + // we can ignore negative values, bc they're already prefixed + // by NEG + if (lbits < 0) Kit.codeBug(); + + // will it fit in a char? + // this gives a short encoding for integer values up to 2^16. + if (lbits <= Character.MAX_VALUE) { + append('S'); + append((char)lbits); + } + else { // Integral, but won't fit in a char. Store as a long. + append('J'); + append((char)(lbits >> 48)); + append((char)(lbits >> 32)); + append((char)(lbits >> 16)); + append((char)lbits); + } + } + } + + private void appendString(String str) + { + int L = str.length(); + int lengthEncodingSize = 1; + if (L >= 0x8000) { + lengthEncodingSize = 2; + } + int nextTop = sourceTop + lengthEncodingSize + L; + if (nextTop > sourceBuffer.length) { + increaseSourceCapacity(nextTop); + } + if (L >= 0x8000) { + // Use 2 chars to encode strings exceeding 32K, were the highest + // bit in the first char indicates presence of the next byte + sourceBuffer[sourceTop] = (char)(0x8000 | (L >>> 16)); + ++sourceTop; + } + sourceBuffer[sourceTop] = (char)L; + ++sourceTop; + str.getChars(0, L, sourceBuffer, sourceTop); + sourceTop = nextTop; + } + + private void append(char c) + { + if (sourceTop == sourceBuffer.length) { + increaseSourceCapacity(sourceTop + 1); + } + sourceBuffer[sourceTop] = c; + ++sourceTop; + } + + private void increaseSourceCapacity(int minimalCapacity) + { + // Call this only when capacity increase is must + if (minimalCapacity <= sourceBuffer.length) Kit.codeBug(); + int newCapacity = sourceBuffer.length * 2; + if (newCapacity < minimalCapacity) { + newCapacity = minimalCapacity; + } + char[] tmp = new char[newCapacity]; + System.arraycopy(sourceBuffer, 0, tmp, 0, sourceTop); + sourceBuffer = tmp; + } + + private String sourceToString(int offset) + { + if (offset < 0 || sourceTop < offset) Kit.codeBug(); + return new String(sourceBuffer, offset, sourceTop - offset); + } + + /** + * Decompile the source information associated with this js + * function/script back into a string. For the most part, this + * just means translating tokens back to their string + * representations; there's a little bit of lookahead logic to + * decide the proper spacing/indentation. Most of the work in + * mapping the original source to the prettyprinted decompiled + * version is done by the parser. + * + * @param source encoded source tree presentation + * + * @param flags flags to select output format + * + * @param properties indentation properties + * + */ + public static String decompile(String source, int flags, + UintMap properties) + { + int length = source.length(); + if (length == 0) { return ""; } + + int indent = properties.getInt(INITIAL_INDENT_PROP, 0); + if (indent < 0) throw new IllegalArgumentException(); + int indentGap = properties.getInt(INDENT_GAP_PROP, 4); + if (indentGap < 0) throw new IllegalArgumentException(); + int caseGap = properties.getInt(CASE_GAP_PROP, 2); + if (caseGap < 0) throw new IllegalArgumentException(); + + StringBuffer result = new StringBuffer(); + boolean justFunctionBody = (0 != (flags & Decompiler.ONLY_BODY_FLAG)); + boolean toSource = (0 != (flags & Decompiler.TO_SOURCE_FLAG)); + + // Spew tokens in source, for debugging. + // as TYPE number char + if (printSource) { + System.err.println("length:" + length); + for (int i = 0; i < length; ++i) { + // Note that tokenToName will fail unless Context.printTrees + // is true. + String tokenname = null; + if (Token.printNames) { + tokenname = Token.name(source.charAt(i)); + } + if (tokenname == null) { + tokenname = "---"; + } + String pad = tokenname.length() > 7 + ? "\t" + : "\t\t"; + System.err.println + (tokenname + + pad + (int)source.charAt(i) + + "\t'" + ScriptRuntime.escapeString + (source.substring(i, i+1)) + + "'"); + } + System.err.println(); + } + + int braceNesting = 0; + boolean afterFirstEOL = false; + int i = 0; + int topFunctionType; + if (source.charAt(i) == Token.SCRIPT) { + ++i; + topFunctionType = -1; + } else { + topFunctionType = source.charAt(i + 1); + } + + if (!toSource) { + // add an initial newline to exactly match js. + result.append('\n'); + for (int j = 0; j < indent; j++) + result.append(' '); + } else { + if (topFunctionType == FunctionNode.FUNCTION_EXPRESSION) { + result.append('('); + } + } + + while (i < length) { + switch(source.charAt(i)) { + case Token.GET: + case Token.SET: + result.append(source.charAt(i) == Token.GET ? "get " : "set "); + ++i; + i = printSourceString(source, i + 1, false, result); + // Now increment one more to get past the FUNCTION token + ++i; + break; + + case Token.NAME: + case Token.REGEXP: // re-wrapped in '/'s in parser... + i = printSourceString(source, i + 1, false, result); + continue; + + case Token.STRING: + i = printSourceString(source, i + 1, true, result); + continue; + + case Token.NUMBER: + i = printSourceNumber(source, i + 1, result); + continue; + + case Token.TRUE: + result.append("true"); + break; + + case Token.FALSE: + result.append("false"); + break; + + case Token.NULL: + result.append("null"); + break; + + case Token.THIS: + result.append("this"); + break; + + case Token.FUNCTION: + ++i; // skip function type + result.append("function "); + break; + + case FUNCTION_END: + // Do nothing + break; + + case Token.COMMA: + result.append(", "); + break; + + case Token.LC: + ++braceNesting; + if (Token.EOL == getNext(source, length, i)) + indent += indentGap; + result.append('{'); + break; + + case Token.RC: { + --braceNesting; + /* don't print the closing RC if it closes the + * toplevel function and we're called from + * decompileFunctionBody. + */ + if (justFunctionBody && braceNesting == 0) + break; + + result.append('}'); + switch (getNext(source, length, i)) { + case Token.EOL: + case FUNCTION_END: + indent -= indentGap; + break; + case Token.WHILE: + case Token.ELSE: + indent -= indentGap; + result.append(' '); + break; + } + break; + } + case Token.LP: + result.append('('); + break; + + case Token.RP: + result.append(')'); + if (Token.LC == getNext(source, length, i)) + result.append(' '); + break; + + case Token.LB: + result.append('['); + break; + + case Token.RB: + result.append(']'); + break; + + case Token.EOL: { + if (toSource) break; + boolean newLine = true; + if (!afterFirstEOL) { + afterFirstEOL = true; + if (justFunctionBody) { + /* throw away just added 'function name(...) {' + * and restore the original indent + */ + result.setLength(0); + indent -= indentGap; + newLine = false; + } + } + if (newLine) { + result.append('\n'); + } + + /* add indent if any tokens remain, + * less setback if next token is + * a label, case or default. + */ + if (i + 1 < length) { + int less = 0; + int nextToken = source.charAt(i + 1); + if (nextToken == Token.CASE + || nextToken == Token.DEFAULT) + { + less = indentGap - caseGap; + } else if (nextToken == Token.RC) { + less = indentGap; + } + + /* elaborate check against label... skip past a + * following inlined NAME and look for a COLON. + */ + else if (nextToken == Token.NAME) { + int afterName = getSourceStringEnd(source, i + 2); + if (source.charAt(afterName) == Token.COLON) + less = indentGap; + } + + for (; less < indent; less++) + result.append(' '); + } + break; + } + case Token.DOT: + result.append('.'); + break; + + case Token.NEW: + result.append("new "); + break; + + case Token.DELPROP: + result.append("delete "); + break; + + case Token.IF: + result.append("if "); + break; + + case Token.ELSE: + result.append("else "); + break; + + case Token.FOR: + result.append("for "); + break; + + case Token.IN: + result.append(" in "); + break; + + case Token.WITH: + result.append("with "); + break; + + case Token.WHILE: + result.append("while "); + break; + + case Token.DO: + result.append("do "); + break; + + case Token.TRY: + result.append("try "); + break; + + case Token.CATCH: + result.append("catch "); + break; + + case Token.FINALLY: + result.append("finally "); + break; + + case Token.THROW: + result.append("throw "); + break; + + case Token.SWITCH: + result.append("switch "); + break; + + case Token.BREAK: + result.append("break"); + if (Token.NAME == getNext(source, length, i)) + result.append(' '); + break; + + case Token.CONTINUE: + result.append("continue"); + if (Token.NAME == getNext(source, length, i)) + result.append(' '); + break; + + case Token.CASE: + result.append("case "); + break; + + case Token.DEFAULT: + result.append("default"); + break; + + case Token.RETURN: + result.append("return"); + if (Token.SEMI != getNext(source, length, i)) + result.append(' '); + break; + + case Token.VAR: + result.append("var "); + break; + + case Token.SEMI: + result.append(';'); + if (Token.EOL != getNext(source, length, i)) { + // separators in FOR + result.append(' '); + } + break; + + case Token.ASSIGN: + result.append(" = "); + break; + + case Token.ASSIGN_ADD: + result.append(" += "); + break; + + case Token.ASSIGN_SUB: + result.append(" -= "); + break; + + case Token.ASSIGN_MUL: + result.append(" *= "); + break; + + case Token.ASSIGN_DIV: + result.append(" /= "); + break; + + case Token.ASSIGN_MOD: + result.append(" %= "); + break; + + case Token.ASSIGN_BITOR: + result.append(" |= "); + break; + + case Token.ASSIGN_BITXOR: + result.append(" ^= "); + break; + + case Token.ASSIGN_BITAND: + result.append(" &= "); + break; + + case Token.ASSIGN_LSH: + result.append(" <<= "); + break; + + case Token.ASSIGN_RSH: + result.append(" >>= "); + break; + + case Token.ASSIGN_URSH: + result.append(" >>>= "); + break; + + case Token.HOOK: + result.append(" ? "); + break; + + case Token.OBJECTLIT: + // pun OBJECTLIT to mean colon in objlit property + // initialization. + // This needs to be distinct from COLON in the general case + // to distinguish from the colon in a ternary... which needs + // different spacing. + result.append(':'); + break; + + case Token.COLON: + if (Token.EOL == getNext(source, length, i)) + // it's the end of a label + result.append(':'); + else + // it's the middle part of a ternary + result.append(" : "); + break; + + case Token.OR: + result.append(" || "); + break; + + case Token.AND: + result.append(" && "); + break; + + case Token.BITOR: + result.append(" | "); + break; + + case Token.BITXOR: + result.append(" ^ "); + break; + + case Token.BITAND: + result.append(" & "); + break; + + case Token.SHEQ: + result.append(" === "); + break; + + case Token.SHNE: + result.append(" !== "); + break; + + case Token.EQ: + result.append(" == "); + break; + + case Token.NE: + result.append(" != "); + break; + + case Token.LE: + result.append(" <= "); + break; + + case Token.LT: + result.append(" < "); + break; + + case Token.GE: + result.append(" >= "); + break; + + case Token.GT: + result.append(" > "); + break; + + case Token.INSTANCEOF: + result.append(" instanceof "); + break; + + case Token.LSH: + result.append(" << "); + break; + + case Token.RSH: + result.append(" >> "); + break; + + case Token.URSH: + result.append(" >>> "); + break; + + case Token.TYPEOF: + result.append("typeof "); + break; + + case Token.VOID: + result.append("void "); + break; + + case Token.CONST: + result.append("const "); + break; + + case Token.NOT: + result.append('!'); + break; + + case Token.BITNOT: + result.append('~'); + break; + + case Token.POS: + result.append('+'); + break; + + case Token.NEG: + result.append('-'); + break; + + case Token.INC: + result.append("++"); + break; + + case Token.DEC: + result.append("--"); + break; + + case Token.ADD: + result.append(" + "); + break; + + case Token.SUB: + result.append(" - "); + break; + + case Token.MUL: + result.append(" * "); + break; + + case Token.DIV: + result.append(" / "); + break; + + case Token.MOD: + result.append(" % "); + break; + + case Token.COLONCOLON: + result.append("::"); + break; + + case Token.DOTDOT: + result.append(".."); + break; + + case Token.DOTQUERY: + result.append(".("); + break; + + case Token.XMLATTR: + result.append('@'); + break; + + default: + // If we don't know how to decompile it, raise an exception. + throw new RuntimeException("Token: " + + Token.name(source.charAt(i))); + } + ++i; + } + + if (!toSource) { + // add that trailing newline if it's an outermost function. + if (!justFunctionBody) + result.append('\n'); + } else { + if (topFunctionType == FunctionNode.FUNCTION_EXPRESSION) { + result.append(')'); + } + } + + return result.toString(); + } + + private static int getNext(String source, int length, int i) + { + return (i + 1 < length) ? source.charAt(i + 1) : Token.EOF; + } + + private static int getSourceStringEnd(String source, int offset) + { + return printSourceString(source, offset, false, null); + } + + private static int printSourceString(String source, int offset, + boolean asQuotedString, + StringBuffer sb) + { + int length = source.charAt(offset); + ++offset; + if ((0x8000 & length) != 0) { + length = ((0x7FFF & length) << 16) | source.charAt(offset); + ++offset; + } + if (sb != null) { + String str = source.substring(offset, offset + length); + if (!asQuotedString) { + sb.append(str); + } else { + sb.append('"'); + sb.append(ScriptRuntime.escapeString(str)); + sb.append('"'); + } + } + return offset + length; + } + + private static int printSourceNumber(String source, int offset, + StringBuffer sb) + { + double number = 0.0; + char type = source.charAt(offset); + ++offset; + if (type == 'S') { + if (sb != null) { + int ival = source.charAt(offset); + number = ival; + } + ++offset; + } else if (type == 'J' || type == 'D') { + if (sb != null) { + long lbits; + lbits = (long)source.charAt(offset) << 48; + lbits |= (long)source.charAt(offset + 1) << 32; + lbits |= (long)source.charAt(offset + 2) << 16; + lbits |= source.charAt(offset + 3); + if (type == 'J') { + number = lbits; + } else { + number = Double.longBitsToDouble(lbits); + } + } + offset += 4; + } else { + // Bad source + throw new RuntimeException(); + } + if (sb != null) { + sb.append(ScriptRuntime.numberToString(number, 10)); + } + return offset; + } + + private char[] sourceBuffer = new char[128]; + +// Per script/function source buffer top: parent source does not include a +// nested functions source and uses function index as a reference instead. + private int sourceTop; + +// whether to do a debug print of the source information, when decompiling. + private static final boolean printSource = false; + +} diff --git a/trunk/infrastructure/yuicompressor/src/yuicompressor/org/mozilla/javascript/Decompiler.java.orig b/trunk/infrastructure/yuicompressor/src/yuicompressor/org/mozilla/javascript/Decompiler.java.orig new file mode 100644 index 0000000..cdb00b7 --- /dev/null +++ b/trunk/infrastructure/yuicompressor/src/yuicompressor/org/mozilla/javascript/Decompiler.java.orig @@ -0,0 +1,910 @@ +/* -*- Mode: java; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is Rhino code, released + * May 6, 1999. + * + * The Initial Developer of the Original Code is + * Netscape Communications Corporation. + * Portions created by the Initial Developer are Copyright (C) 1997-1999 + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * Mike Ang + * Igor Bukanov + * Bob Jervis + * Mike McCabe + * + * Alternatively, the contents of this file may be used under the terms of + * the GNU General Public License Version 2 or later (the "GPL"), in which + * case the provisions of the GPL are applicable instead of those above. If + * you wish to allow use of your version of this file only under the terms of + * the GPL and not to allow others to use your version of this file under the + * MPL, indicate your decision by deleting the provisions above and replacing + * them with the notice and other provisions required by the GPL. If you do + * not delete the provisions above, a recipient may use your version of this + * file under either the MPL or the GPL. + * + * ***** END LICENSE BLOCK ***** */ + +package org.mozilla.javascript; + +/** + * The following class save decompilation information about the source. + * Source information is returned from the parser as a String + * associated with function nodes and with the toplevel script. When + * saved in the constant pool of a class, this string will be UTF-8 + * encoded, and token values will occupy a single byte. + + * Source is saved (mostly) as token numbers. The tokens saved pretty + * much correspond to the token stream of a 'canonical' representation + * of the input program, as directed by the parser. (There were a few + * cases where tokens could have been left out where decompiler could + * easily reconstruct them, but I left them in for clarity). (I also + * looked adding source collection to TokenStream instead, where I + * could have limited the changes to a few lines in getToken... but + * this wouldn't have saved any space in the resulting source + * representation, and would have meant that I'd have to duplicate + * parser logic in the decompiler to disambiguate situations where + * newlines are important.) The function decompile expands the + * tokens back into their string representations, using simple + * lookahead to correct spacing and indentation. + * + * Assignments are saved as two-token pairs (Token.ASSIGN, op). Number tokens + * are stored inline, as a NUMBER token, a character representing the type, and + * either 1 or 4 characters representing the bit-encoding of the number. String + * types NAME, STRING and OBJECT are currently stored as a token type, + * followed by a character giving the length of the string (assumed to + * be less than 2^16), followed by the characters of the string + * inlined into the source string. Changing this to some reference to + * to the string in the compiled class' constant pool would probably + * save a lot of space... but would require some method of deriving + * the final constant pool entry from information available at parse + * time. + */ +public class Decompiler +{ + /** + * Flag to indicate that the decompilation should omit the + * function header and trailing brace. + */ + public static final int ONLY_BODY_FLAG = 1 << 0; + + /** + * Flag to indicate that the decompilation generates toSource result. + */ + public static final int TO_SOURCE_FLAG = 1 << 1; + + /** + * Decompilation property to specify initial ident value. + */ + public static final int INITIAL_INDENT_PROP = 1; + + /** + * Decompilation property to specify default identation offset. + */ + public static final int INDENT_GAP_PROP = 2; + + /** + * Decompilation property to specify identation offset for case labels. + */ + public static final int CASE_GAP_PROP = 3; + + // Marker to denote the last RC of function so it can be distinguished from + // the last RC of object literals in case of function expressions + private static final int FUNCTION_END = Token.LAST_TOKEN + 1; + + String getEncodedSource() + { + return sourceToString(0); + } + + int getCurrentOffset() + { + return sourceTop; + } + + int markFunctionStart(int functionType) + { + int savedOffset = getCurrentOffset(); + addToken(Token.FUNCTION); + append((char)functionType); + return savedOffset; + } + + int markFunctionEnd(int functionStart) + { + int offset = getCurrentOffset(); + append((char)FUNCTION_END); + return offset; + } + + void addToken(int token) + { + if (!(0 <= token && token <= Token.LAST_TOKEN)) + throw new IllegalArgumentException(); + + append((char)token); + } + + void addEOL(int token) + { + if (!(0 <= token && token <= Token.LAST_TOKEN)) + throw new IllegalArgumentException(); + + append((char)token); + append((char)Token.EOL); + } + + void addName(String str) + { + addToken(Token.NAME); + appendString(str); + } + + void addString(String str) + { + addToken(Token.STRING); + appendString(str); + } + + void addRegexp(String regexp, String flags) + { + addToken(Token.REGEXP); + appendString('/' + regexp + '/' + flags); + } + + void addNumber(double n) + { + addToken(Token.NUMBER); + + /* encode the number in the source stream. + * Save as NUMBER type (char | char char char char) + * where type is + * 'D' - double, 'S' - short, 'J' - long. + + * We need to retain float vs. integer type info to keep the + * behavior of liveconnect type-guessing the same after + * decompilation. (Liveconnect tries to present 1.0 to Java + * as a float/double) + * OPT: This is no longer true. We could compress the format. + + * This may not be the most space-efficient encoding; + * the chars created below may take up to 3 bytes in + * constant pool UTF-8 encoding, so a Double could take + * up to 12 bytes. + */ + + long lbits = (long)n; + if (lbits != n) { + // if it's floating point, save as a Double bit pattern. + // (12/15/97 our scanner only returns Double for f.p.) + lbits = Double.doubleToLongBits(n); + append('D'); + append((char)(lbits >> 48)); + append((char)(lbits >> 32)); + append((char)(lbits >> 16)); + append((char)lbits); + } + else { + // we can ignore negative values, bc they're already prefixed + // by NEG + if (lbits < 0) Kit.codeBug(); + + // will it fit in a char? + // this gives a short encoding for integer values up to 2^16. + if (lbits <= Character.MAX_VALUE) { + append('S'); + append((char)lbits); + } + else { // Integral, but won't fit in a char. Store as a long. + append('J'); + append((char)(lbits >> 48)); + append((char)(lbits >> 32)); + append((char)(lbits >> 16)); + append((char)lbits); + } + } + } + + private void appendString(String str) + { + int L = str.length(); + int lengthEncodingSize = 1; + if (L >= 0x8000) { + lengthEncodingSize = 2; + } + int nextTop = sourceTop + lengthEncodingSize + L; + if (nextTop > sourceBuffer.length) { + increaseSourceCapacity(nextTop); + } + if (L >= 0x8000) { + // Use 2 chars to encode strings exceeding 32K, were the highest + // bit in the first char indicates presence of the next byte + sourceBuffer[sourceTop] = (char)(0x8000 | (L >>> 16)); + ++sourceTop; + } + sourceBuffer[sourceTop] = (char)L; + ++sourceTop; + str.getChars(0, L, sourceBuffer, sourceTop); + sourceTop = nextTop; + } + + private void append(char c) + { + if (sourceTop == sourceBuffer.length) { + increaseSourceCapacity(sourceTop + 1); + } + sourceBuffer[sourceTop] = c; + ++sourceTop; + } + + private void increaseSourceCapacity(int minimalCapacity) + { + // Call this only when capacity increase is must + if (minimalCapacity <= sourceBuffer.length) Kit.codeBug(); + int newCapacity = sourceBuffer.length * 2; + if (newCapacity < minimalCapacity) { + newCapacity = minimalCapacity; + } + char[] tmp = new char[newCapacity]; + System.arraycopy(sourceBuffer, 0, tmp, 0, sourceTop); + sourceBuffer = tmp; + } + + private String sourceToString(int offset) + { + if (offset < 0 || sourceTop < offset) Kit.codeBug(); + return new String(sourceBuffer, offset, sourceTop - offset); + } + + /** + * Decompile the source information associated with this js + * function/script back into a string. For the most part, this + * just means translating tokens back to their string + * representations; there's a little bit of lookahead logic to + * decide the proper spacing/indentation. Most of the work in + * mapping the original source to the prettyprinted decompiled + * version is done by the parser. + * + * @param source encoded source tree presentation + * + * @param flags flags to select output format + * + * @param properties indentation properties + * + */ + public static String decompile(String source, int flags, + UintMap properties) + { + int length = source.length(); + if (length == 0) { return ""; } + + int indent = properties.getInt(INITIAL_INDENT_PROP, 0); + if (indent < 0) throw new IllegalArgumentException(); + int indentGap = properties.getInt(INDENT_GAP_PROP, 4); + if (indentGap < 0) throw new IllegalArgumentException(); + int caseGap = properties.getInt(CASE_GAP_PROP, 2); + if (caseGap < 0) throw new IllegalArgumentException(); + + StringBuffer result = new StringBuffer(); + boolean justFunctionBody = (0 != (flags & Decompiler.ONLY_BODY_FLAG)); + boolean toSource = (0 != (flags & Decompiler.TO_SOURCE_FLAG)); + + // Spew tokens in source, for debugging. + // as TYPE number char + if (printSource) { + System.err.println("length:" + length); + for (int i = 0; i < length; ++i) { + // Note that tokenToName will fail unless Context.printTrees + // is true. + String tokenname = null; + if (Token.printNames) { + tokenname = Token.name(source.charAt(i)); + } + if (tokenname == null) { + tokenname = "---"; + } + String pad = tokenname.length() > 7 + ? "\t" + : "\t\t"; + System.err.println + (tokenname + + pad + (int)source.charAt(i) + + "\t'" + ScriptRuntime.escapeString + (source.substring(i, i+1)) + + "'"); + } + System.err.println(); + } + + int braceNesting = 0; + boolean afterFirstEOL = false; + int i = 0; + int topFunctionType; + if (source.charAt(i) == Token.SCRIPT) { + ++i; + topFunctionType = -1; + } else { + topFunctionType = source.charAt(i + 1); + } + + if (!toSource) { + // add an initial newline to exactly match js. + result.append('\n'); + for (int j = 0; j < indent; j++) + result.append(' '); + } else { + if (topFunctionType == FunctionNode.FUNCTION_EXPRESSION) { + result.append('('); + } + } + + while (i < length) { + switch(source.charAt(i)) { + case Token.GET: + case Token.SET: + result.append(source.charAt(i) == Token.GET ? "get " : "set "); + ++i; + i = printSourceString(source, i + 1, false, result); + // Now increment one more to get past the FUNCTION token + ++i; + break; + + case Token.NAME: + case Token.REGEXP: // re-wrapped in '/'s in parser... + i = printSourceString(source, i + 1, false, result); + continue; + + case Token.STRING: + i = printSourceString(source, i + 1, true, result); + continue; + + case Token.NUMBER: + i = printSourceNumber(source, i + 1, result); + continue; + + case Token.TRUE: + result.append("true"); + break; + + case Token.FALSE: + result.append("false"); + break; + + case Token.NULL: + result.append("null"); + break; + + case Token.THIS: + result.append("this"); + break; + + case Token.FUNCTION: + ++i; // skip function type + result.append("function "); + break; + + case FUNCTION_END: + // Do nothing + break; + + case Token.COMMA: + result.append(", "); + break; + + case Token.LC: + ++braceNesting; + if (Token.EOL == getNext(source, length, i)) + indent += indentGap; + result.append('{'); + break; + + case Token.RC: { + --braceNesting; + /* don't print the closing RC if it closes the + * toplevel function and we're called from + * decompileFunctionBody. + */ + if (justFunctionBody && braceNesting == 0) + break; + + result.append('}'); + switch (getNext(source, length, i)) { + case Token.EOL: + case FUNCTION_END: + indent -= indentGap; + break; + case Token.WHILE: + case Token.ELSE: + indent -= indentGap; + result.append(' '); + break; + } + break; + } + case Token.LP: + result.append('('); + break; + + case Token.RP: + result.append(')'); + if (Token.LC == getNext(source, length, i)) + result.append(' '); + break; + + case Token.LB: + result.append('['); + break; + + case Token.RB: + result.append(']'); + break; + + case Token.EOL: { + if (toSource) break; + boolean newLine = true; + if (!afterFirstEOL) { + afterFirstEOL = true; + if (justFunctionBody) { + /* throw away just added 'function name(...) {' + * and restore the original indent + */ + result.setLength(0); + indent -= indentGap; + newLine = false; + } + } + if (newLine) { + result.append('\n'); + } + + /* add indent if any tokens remain, + * less setback if next token is + * a label, case or default. + */ + if (i + 1 < length) { + int less = 0; + int nextToken = source.charAt(i + 1); + if (nextToken == Token.CASE + || nextToken == Token.DEFAULT) + { + less = indentGap - caseGap; + } else if (nextToken == Token.RC) { + less = indentGap; + } + + /* elaborate check against label... skip past a + * following inlined NAME and look for a COLON. + */ + else if (nextToken == Token.NAME) { + int afterName = getSourceStringEnd(source, i + 2); + if (source.charAt(afterName) == Token.COLON) + less = indentGap; + } + + for (; less < indent; less++) + result.append(' '); + } + break; + } + case Token.DOT: + result.append('.'); + break; + + case Token.NEW: + result.append("new "); + break; + + case Token.DELPROP: + result.append("delete "); + break; + + case Token.IF: + result.append("if "); + break; + + case Token.ELSE: + result.append("else "); + break; + + case Token.FOR: + result.append("for "); + break; + + case Token.IN: + result.append(" in "); + break; + + case Token.WITH: + result.append("with "); + break; + + case Token.WHILE: + result.append("while "); + break; + + case Token.DO: + result.append("do "); + break; + + case Token.TRY: + result.append("try "); + break; + + case Token.CATCH: + result.append("catch "); + break; + + case Token.FINALLY: + result.append("finally "); + break; + + case Token.THROW: + result.append("throw "); + break; + + case Token.SWITCH: + result.append("switch "); + break; + + case Token.BREAK: + result.append("break"); + if (Token.NAME == getNext(source, length, i)) + result.append(' '); + break; + + case Token.CONTINUE: + result.append("continue"); + if (Token.NAME == getNext(source, length, i)) + result.append(' '); + break; + + case Token.CASE: + result.append("case "); + break; + + case Token.DEFAULT: + result.append("default"); + break; + + case Token.RETURN: + result.append("return"); + if (Token.SEMI != getNext(source, length, i)) + result.append(' '); + break; + + case Token.VAR: + result.append("var "); + break; + + case Token.SEMI: + result.append(';'); + if (Token.EOL != getNext(source, length, i)) { + // separators in FOR + result.append(' '); + } + break; + + case Token.ASSIGN: + result.append(" = "); + break; + + case Token.ASSIGN_ADD: + result.append(" += "); + break; + + case Token.ASSIGN_SUB: + result.append(" -= "); + break; + + case Token.ASSIGN_MUL: + result.append(" *= "); + break; + + case Token.ASSIGN_DIV: + result.append(" /= "); + break; + + case Token.ASSIGN_MOD: + result.append(" %= "); + break; + + case Token.ASSIGN_BITOR: + result.append(" |= "); + break; + + case Token.ASSIGN_BITXOR: + result.append(" ^= "); + break; + + case Token.ASSIGN_BITAND: + result.append(" &= "); + break; + + case Token.ASSIGN_LSH: + result.append(" <<= "); + break; + + case Token.ASSIGN_RSH: + result.append(" >>= "); + break; + + case Token.ASSIGN_URSH: + result.append(" >>>= "); + break; + + case Token.HOOK: + result.append(" ? "); + break; + + case Token.OBJECTLIT: + // pun OBJECTLIT to mean colon in objlit property + // initialization. + // This needs to be distinct from COLON in the general case + // to distinguish from the colon in a ternary... which needs + // different spacing. + result.append(':'); + break; + + case Token.COLON: + if (Token.EOL == getNext(source, length, i)) + // it's the end of a label + result.append(':'); + else + // it's the middle part of a ternary + result.append(" : "); + break; + + case Token.OR: + result.append(" || "); + break; + + case Token.AND: + result.append(" && "); + break; + + case Token.BITOR: + result.append(" | "); + break; + + case Token.BITXOR: + result.append(" ^ "); + break; + + case Token.BITAND: + result.append(" & "); + break; + + case Token.SHEQ: + result.append(" === "); + break; + + case Token.SHNE: + result.append(" !== "); + break; + + case Token.EQ: + result.append(" == "); + break; + + case Token.NE: + result.append(" != "); + break; + + case Token.LE: + result.append(" <= "); + break; + + case Token.LT: + result.append(" < "); + break; + + case Token.GE: + result.append(" >= "); + break; + + case Token.GT: + result.append(" > "); + break; + + case Token.INSTANCEOF: + result.append(" instanceof "); + break; + + case Token.LSH: + result.append(" << "); + break; + + case Token.RSH: + result.append(" >> "); + break; + + case Token.URSH: + result.append(" >>> "); + break; + + case Token.TYPEOF: + result.append("typeof "); + break; + + case Token.VOID: + result.append("void "); + break; + + case Token.CONST: + result.append("const "); + break; + + case Token.NOT: + result.append('!'); + break; + + case Token.BITNOT: + result.append('~'); + break; + + case Token.POS: + result.append('+'); + break; + + case Token.NEG: + result.append('-'); + break; + + case Token.INC: + result.append("++"); + break; + + case Token.DEC: + result.append("--"); + break; + + case Token.ADD: + result.append(" + "); + break; + + case Token.SUB: + result.append(" - "); + break; + + case Token.MUL: + result.append(" * "); + break; + + case Token.DIV: + result.append(" / "); + break; + + case Token.MOD: + result.append(" % "); + break; + + case Token.COLONCOLON: + result.append("::"); + break; + + case Token.DOTDOT: + result.append(".."); + break; + + case Token.DOTQUERY: + result.append(".("); + break; + + case Token.XMLATTR: + result.append('@'); + break; + + default: + // If we don't know how to decompile it, raise an exception. + throw new RuntimeException("Token: " + + Token.name(source.charAt(i))); + } + ++i; + } + + if (!toSource) { + // add that trailing newline if it's an outermost function. + if (!justFunctionBody) + result.append('\n'); + } else { + if (topFunctionType == FunctionNode.FUNCTION_EXPRESSION) { + result.append(')'); + } + } + + return result.toString(); + } + + private static int getNext(String source, int length, int i) + { + return (i + 1 < length) ? source.charAt(i + 1) : Token.EOF; + } + + private static int getSourceStringEnd(String source, int offset) + { + return printSourceString(source, offset, false, null); + } + + private static int printSourceString(String source, int offset, + boolean asQuotedString, + StringBuffer sb) + { + int length = source.charAt(offset); + ++offset; + if ((0x8000 & length) != 0) { + length = ((0x7FFF & length) << 16) | source.charAt(offset); + ++offset; + } + if (sb != null) { + String str = source.substring(offset, offset + length); + if (!asQuotedString) { + sb.append(str); + } else { + sb.append('"'); + sb.append(ScriptRuntime.escapeString(str)); + sb.append('"'); + } + } + return offset + length; + } + + private static int printSourceNumber(String source, int offset, + StringBuffer sb) + { + double number = 0.0; + char type = source.charAt(offset); + ++offset; + if (type == 'S') { + if (sb != null) { + int ival = source.charAt(offset); + number = ival; + } + ++offset; + } else if (type == 'J' || type == 'D') { + if (sb != null) { + long lbits; + lbits = (long)source.charAt(offset) << 48; + lbits |= (long)source.charAt(offset + 1) << 32; + lbits |= (long)source.charAt(offset + 2) << 16; + lbits |= source.charAt(offset + 3); + if (type == 'J') { + number = lbits; + } else { + number = Double.longBitsToDouble(lbits); + } + } + offset += 4; + } else { + // Bad source + throw new RuntimeException(); + } + if (sb != null) { + sb.append(ScriptRuntime.numberToString(number, 10)); + } + return offset; + } + + private char[] sourceBuffer = new char[128]; + +// Per script/function source buffer top: parent source does not include a +// nested functions source and uses function index as a reference instead. + private int sourceTop; + +// whether to do a debug print of the source information, when decompiling. + private static final boolean printSource = false; + +} diff --git a/trunk/infrastructure/yuicompressor/src/yuicompressor/org/mozilla/javascript/Parser.java b/trunk/infrastructure/yuicompressor/src/yuicompressor/org/mozilla/javascript/Parser.java new file mode 100644 index 0000000..bf830a8 --- /dev/null +++ b/trunk/infrastructure/yuicompressor/src/yuicompressor/org/mozilla/javascript/Parser.java @@ -0,0 +1,2170 @@ +/* -*- Mode: java; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is Rhino code, released + * May 6, 1999. + * + * The Initial Developer of the Original Code is + * Netscape Communications Corporation. + * Portions created by the Initial Developer are Copyright (C) 1997-1999 + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * Mike Ang + * Igor Bukanov + * Yuh-Ruey Chen + * Ethan Hugg + * Bob Jervis + * Terry Lucas + * Mike McCabe + * Milen Nankov + * + * Alternatively, the contents of this file may be used under the terms of + * the GNU General Public License Version 2 or later (the "GPL"), in which + * case the provisions of the GPL are applicable instead of those above. If + * you wish to allow use of your version of this file only under the terms of + * the GPL and not to allow others to use your version of this file under the + * MPL, indicate your decision by deleting the provisions above and replacing + * them with the notice and other provisions required by the GPL. If you do + * not delete the provisions above, a recipient may use your version of this + * file under either the MPL or the GPL. + * + * ***** END LICENSE BLOCK ***** */ + +package yuicompressor.org.mozilla.javascript; + +import java.io.Reader; +import java.io.IOException; +import java.util.Hashtable; + +/** + * This class implements the JavaScript parser. + * + * It is based on the C source files jsparse.c and jsparse.h + * in the jsref package. + * + * @see TokenStream + * + * @author Mike McCabe + * @author Brendan Eich + */ + +public class Parser +{ + // TokenInformation flags : currentFlaggedToken stores them together + // with token type + final static int + CLEAR_TI_MASK = 0xFFFF, // mask to clear token information bits + TI_AFTER_EOL = 1 << 16, // first token of the source line + TI_CHECK_LABEL = 1 << 17; // indicates to check for label + + CompilerEnvirons compilerEnv; + private ErrorReporter errorReporter; + private String sourceURI; + boolean calledByCompileFunction; + + private TokenStream ts; + private int currentFlaggedToken; + private int syntaxErrorCount; + + private IRFactory nf; + + private int nestingOfFunction; + + private Decompiler decompiler; + private String encodedSource; + +// The following are per function variables and should be saved/restored +// during function parsing. +// XXX Move to separated class? + ScriptOrFnNode currentScriptOrFn; + private int nestingOfWith; + private Hashtable labelSet; // map of label names into nodes + private ObjArray loopSet; + private ObjArray loopAndSwitchSet; + private boolean hasReturnValue; + private int functionEndFlags; +// end of per function variables + + // Exception to unwind + private static class ParserException extends RuntimeException + { + static final long serialVersionUID = 5882582646773765630L; + } + + public Parser(CompilerEnvirons compilerEnv, ErrorReporter errorReporter) + { + this.compilerEnv = compilerEnv; + this.errorReporter = errorReporter; + } + + protected Decompiler createDecompiler(CompilerEnvirons compilerEnv) + { + return new Decompiler(); + } + + void addStrictWarning(String messageId, String messageArg) + { + if (compilerEnv.isStrictMode()) + addWarning(messageId, messageArg); + } + + void addWarning(String messageId, String messageArg) + { + String message = ScriptRuntime.getMessage1(messageId, messageArg); + if (compilerEnv.reportWarningAsError()) { + ++syntaxErrorCount; + errorReporter.error(message, sourceURI, ts.getLineno(), + ts.getLine(), ts.getOffset()); + } else + errorReporter.warning(message, sourceURI, ts.getLineno(), + ts.getLine(), ts.getOffset()); + } + + void addError(String messageId) + { + ++syntaxErrorCount; + String message = ScriptRuntime.getMessage0(messageId); + errorReporter.error(message, sourceURI, ts.getLineno(), + ts.getLine(), ts.getOffset()); + } + + void addError(String messageId, String messageArg) + { + ++syntaxErrorCount; + String message = ScriptRuntime.getMessage1(messageId, messageArg); + errorReporter.error(message, sourceURI, ts.getLineno(), + ts.getLine(), ts.getOffset()); + } + + RuntimeException reportError(String messageId) + { + addError(messageId); + + // Throw a ParserException exception to unwind the recursive descent + // parse. + throw new ParserException(); + } + + private int peekToken() + throws IOException + { + int tt = currentFlaggedToken; + if (tt == Token.EOF) { + + while ((tt = ts.getToken()) == Token.SPECIALCOMMENT) { + /* Support for JScript conditional comments */ + decompiler.addJScriptConditionalComment(ts.getString()); + } + + if (tt == Token.EOL) { + do { + tt = ts.getToken(); + + if (tt == Token.SPECIALCOMMENT) { + /* Support for JScript conditional comments */ + decompiler.addJScriptConditionalComment(ts.getString()); + } + + } while (tt == Token.EOL || tt == Token.SPECIALCOMMENT); + tt |= TI_AFTER_EOL; + } + currentFlaggedToken = tt; + } + return tt & CLEAR_TI_MASK; + } + + private int peekFlaggedToken() + throws IOException + { + peekToken(); + return currentFlaggedToken; + } + + private void consumeToken() + { + currentFlaggedToken = Token.EOF; + } + + private int nextToken() + throws IOException + { + int tt = peekToken(); + consumeToken(); + return tt; + } + + private int nextFlaggedToken() + throws IOException + { + peekToken(); + int ttFlagged = currentFlaggedToken; + consumeToken(); + return ttFlagged; + } + + private boolean matchToken(int toMatch) + throws IOException + { + int tt = peekToken(); + if (tt != toMatch) { + return false; + } + consumeToken(); + return true; + } + + private int peekTokenOrEOL() + throws IOException + { + int tt = peekToken(); + // Check for last peeked token flags + if ((currentFlaggedToken & TI_AFTER_EOL) != 0) { + tt = Token.EOL; + } + return tt; + } + + private void setCheckForLabel() + { + if ((currentFlaggedToken & CLEAR_TI_MASK) != Token.NAME) + throw Kit.codeBug(); + currentFlaggedToken |= TI_CHECK_LABEL; + } + + private void mustMatchToken(int toMatch, String messageId) + throws IOException, ParserException + { + if (!matchToken(toMatch)) { + reportError(messageId); + } + } + + private void mustHaveXML() + { + if (!compilerEnv.isXmlAvailable()) { + reportError("msg.XML.not.available"); + } + } + + public String getEncodedSource() + { + return encodedSource; + } + + public boolean eof() + { + return ts.eof(); + } + + boolean insideFunction() + { + return nestingOfFunction != 0; + } + + private Node enterLoop(Node loopLabel) + { + Node loop = nf.createLoopNode(loopLabel, ts.getLineno()); + if (loopSet == null) { + loopSet = new ObjArray(); + if (loopAndSwitchSet == null) { + loopAndSwitchSet = new ObjArray(); + } + } + loopSet.push(loop); + loopAndSwitchSet.push(loop); + return loop; + } + + private void exitLoop() + { + loopSet.pop(); + loopAndSwitchSet.pop(); + } + + private Node enterSwitch(Node switchSelector, int lineno) + { + Node switchNode = nf.createSwitch(switchSelector, lineno); + if (loopAndSwitchSet == null) { + loopAndSwitchSet = new ObjArray(); + } + loopAndSwitchSet.push(switchNode); + return switchNode; + } + + private void exitSwitch() + { + loopAndSwitchSet.pop(); + } + + /* + * Build a parse tree from the given sourceString. + * + * @return an Object representing the parsed + * program. If the parse fails, null will be returned. (The + * parse failure will result in a call to the ErrorReporter from + * CompilerEnvirons.) + */ + public ScriptOrFnNode parse(String sourceString, + String sourceURI, int lineno) + { + this.sourceURI = sourceURI; + this.ts = new TokenStream(this, null, sourceString, lineno); + try { + return parse(); + } catch (IOException ex) { + // Should never happen + throw new IllegalStateException(); + } + } + + /* + * Build a parse tree from the given sourceString. + * + * @return an Object representing the parsed + * program. If the parse fails, null will be returned. (The + * parse failure will result in a call to the ErrorReporter from + * CompilerEnvirons.) + */ + public ScriptOrFnNode parse(Reader sourceReader, + String sourceURI, int lineno) + throws IOException + { + this.sourceURI = sourceURI; + this.ts = new TokenStream(this, sourceReader, null, lineno); + return parse(); + } + + private ScriptOrFnNode parse() + throws IOException + { + this.decompiler = createDecompiler(compilerEnv); + this.nf = new IRFactory(this); + currentScriptOrFn = nf.createScript(); + int sourceStartOffset = decompiler.getCurrentOffset(); + this.encodedSource = null; + decompiler.addToken(Token.SCRIPT); + + this.currentFlaggedToken = Token.EOF; + this.syntaxErrorCount = 0; + + int baseLineno = ts.getLineno(); // line number where source starts + + /* so we have something to add nodes to until + * we've collected all the source */ + Node pn = nf.createLeaf(Token.BLOCK); + + try { + for (;;) { + int tt = peekToken(); + + if (tt <= Token.EOF) { + break; + } + + Node n; + if (tt == Token.FUNCTION) { + consumeToken(); + try { + n = function(calledByCompileFunction + ? FunctionNode.FUNCTION_EXPRESSION + : FunctionNode.FUNCTION_STATEMENT); + } catch (ParserException e) { + break; + } + } else { + n = statement(); + } + nf.addChildToBack(pn, n); + } + } catch (StackOverflowError ex) { + String msg = ScriptRuntime.getMessage0( + "msg.too.deep.parser.recursion"); + throw Context.reportRuntimeError(msg, sourceURI, + ts.getLineno(), null, 0); + } + + if (this.syntaxErrorCount != 0) { + String msg = String.valueOf(this.syntaxErrorCount); + msg = ScriptRuntime.getMessage1("msg.got.syntax.errors", msg); + throw errorReporter.runtimeError(msg, sourceURI, baseLineno, + null, 0); + } + + currentScriptOrFn.setSourceName(sourceURI); + currentScriptOrFn.setBaseLineno(baseLineno); + currentScriptOrFn.setEndLineno(ts.getLineno()); + + int sourceEndOffset = decompiler.getCurrentOffset(); + currentScriptOrFn.setEncodedSourceBounds(sourceStartOffset, + sourceEndOffset); + + nf.initScript(currentScriptOrFn, pn); + + if (compilerEnv.isGeneratingSource()) { + encodedSource = decompiler.getEncodedSource(); + } + this.decompiler = null; // It helps GC + + return currentScriptOrFn; + } + + /* + * The C version of this function takes an argument list, + * which doesn't seem to be needed for tree generation... + * it'd only be useful for checking argument hiding, which + * I'm not doing anyway... + */ + private Node parseFunctionBody() + throws IOException + { + ++nestingOfFunction; + Node pn = nf.createBlock(ts.getLineno()); + try { + bodyLoop: for (;;) { + Node n; + int tt = peekToken(); + switch (tt) { + case Token.ERROR: + case Token.EOF: + case Token.RC: + break bodyLoop; + + case Token.FUNCTION: + consumeToken(); + n = function(FunctionNode.FUNCTION_STATEMENT); + break; + default: + n = statement(); + break; + } + nf.addChildToBack(pn, n); + } + } catch (ParserException e) { + // Ignore it + } finally { + --nestingOfFunction; + } + + return pn; + } + + private Node function(int functionType) + throws IOException, ParserException + { + int syntheticType = functionType; + int baseLineno = ts.getLineno(); // line number where source starts + + int functionSourceStart = decompiler.markFunctionStart(functionType); + String name; + Node memberExprNode = null; + if (matchToken(Token.NAME)) { + name = ts.getString(); + decompiler.addName(name); + if (!matchToken(Token.LP)) { + if (compilerEnv.isAllowMemberExprAsFunctionName()) { + // Extension to ECMA: if 'function <name>' does not follow + // by '(', assume <name> starts memberExpr + Node memberExprHead = nf.createName(name); + name = ""; + memberExprNode = memberExprTail(false, memberExprHead); + } + mustMatchToken(Token.LP, "msg.no.paren.parms"); + } + } else if (matchToken(Token.LP)) { + // Anonymous function + name = ""; + } else { + name = ""; + if (compilerEnv.isAllowMemberExprAsFunctionName()) { + // Note that memberExpr can not start with '(' like + // in function (1+2).toString(), because 'function (' already + // processed as anonymous function + memberExprNode = memberExpr(false); + } + mustMatchToken(Token.LP, "msg.no.paren.parms"); + } + + if (memberExprNode != null) { + syntheticType = FunctionNode.FUNCTION_EXPRESSION; + } + + boolean nested = insideFunction(); + + FunctionNode fnNode = nf.createFunction(name); + if (nested || nestingOfWith > 0) { + // 1. Nested functions are not affected by the dynamic scope flag + // as dynamic scope is already a parent of their scope. + // 2. Functions defined under the with statement also immune to + // this setup, in which case dynamic scope is ignored in favor + // of with object. + fnNode.itsIgnoreDynamicScope = true; + } + + int functionIndex = currentScriptOrFn.addFunction(fnNode); + + int functionSourceEnd; + + ScriptOrFnNode savedScriptOrFn = currentScriptOrFn; + currentScriptOrFn = fnNode; + int savedNestingOfWith = nestingOfWith; + nestingOfWith = 0; + Hashtable savedLabelSet = labelSet; + labelSet = null; + ObjArray savedLoopSet = loopSet; + loopSet = null; + ObjArray savedLoopAndSwitchSet = loopAndSwitchSet; + loopAndSwitchSet = null; + boolean savedHasReturnValue = hasReturnValue; + int savedFunctionEndFlags = functionEndFlags; + + Node body; + try { + decompiler.addToken(Token.LP); + if (!matchToken(Token.RP)) { + boolean first = true; + do { + if (!first) + decompiler.addToken(Token.COMMA); + first = false; + mustMatchToken(Token.NAME, "msg.no.parm"); + String s = ts.getString(); + if (fnNode.hasParamOrVar(s)) { + addWarning("msg.dup.parms", s); + } + fnNode.addParam(s); + decompiler.addName(s); + } while (matchToken(Token.COMMA)); + + mustMatchToken(Token.RP, "msg.no.paren.after.parms"); + } + decompiler.addToken(Token.RP); + + mustMatchToken(Token.LC, "msg.no.brace.body"); + decompiler.addEOL(Token.LC); + body = parseFunctionBody(); + mustMatchToken(Token.RC, "msg.no.brace.after.body"); + + if (compilerEnv.isStrictMode() && !body.hasConsistentReturnUsage()) + { + String msg = name.length() > 0 ? "msg.no.return.value" + : "msg.anon.no.return.value"; + addStrictWarning(msg, name); + } + + decompiler.addToken(Token.RC); + functionSourceEnd = decompiler.markFunctionEnd(functionSourceStart); + if (functionType != FunctionNode.FUNCTION_EXPRESSION) { + // Add EOL only if function is not part of expression + // since it gets SEMI + EOL from Statement in that case + decompiler.addToken(Token.EOL); + } + } + finally { + hasReturnValue = savedHasReturnValue; + functionEndFlags = savedFunctionEndFlags; + loopAndSwitchSet = savedLoopAndSwitchSet; + loopSet = savedLoopSet; + labelSet = savedLabelSet; + nestingOfWith = savedNestingOfWith; + currentScriptOrFn = savedScriptOrFn; + } + + fnNode.setEncodedSourceBounds(functionSourceStart, functionSourceEnd); + fnNode.setSourceName(sourceURI); + fnNode.setBaseLineno(baseLineno); + fnNode.setEndLineno(ts.getLineno()); + + if (name != null) { + int index = currentScriptOrFn.getParamOrVarIndex(name); + if (index >= 0 && index < currentScriptOrFn.getParamCount()) + addStrictWarning("msg.var.hides.arg", name); + } + + Node pn = nf.initFunction(fnNode, functionIndex, body, syntheticType); + if (memberExprNode != null) { + pn = nf.createAssignment(Token.ASSIGN, memberExprNode, pn); + if (functionType != FunctionNode.FUNCTION_EXPRESSION) { + // XXX check JScript behavior: should it be createExprStatement? + pn = nf.createExprStatementNoReturn(pn, baseLineno); + } + } + return pn; + } + + private Node statements() + throws IOException + { + Node pn = nf.createBlock(ts.getLineno()); + + int tt; + while((tt = peekToken()) > Token.EOF && tt != Token.RC) { + nf.addChildToBack(pn, statement()); + } + + return pn; + } + + private Node condition() + throws IOException, ParserException + { + mustMatchToken(Token.LP, "msg.no.paren.cond"); + decompiler.addToken(Token.LP); + Node pn = expr(false); + mustMatchToken(Token.RP, "msg.no.paren.after.cond"); + decompiler.addToken(Token.RP); + + // Report strict warning on code like "if (a = 7) ...". Suppress the + // warning if the condition is parenthesized, like "if ((a = 7)) ...". + if (pn.getProp(Node.PARENTHESIZED_PROP) == null && + (pn.getType() == Token.SETNAME || pn.getType() == Token.SETPROP || + pn.getType() == Token.SETELEM)) + { + addStrictWarning("msg.equal.as.assign", ""); + } + return pn; + } + + // match a NAME; return null if no match. + private Node matchJumpLabelName() + throws IOException, ParserException + { + Node label = null; + + int tt = peekTokenOrEOL(); + if (tt == Token.NAME) { + consumeToken(); + String name = ts.getString(); + decompiler.addName(name); + if (labelSet != null) { + label = (Node)labelSet.get(name); + } + if (label == null) { + reportError("msg.undef.label"); + } + } + + return label; + } + + private Node statement() + throws IOException + { + try { + Node pn = statementHelper(null); + if (pn != null) { + if (compilerEnv.isStrictMode() && !pn.hasSideEffects()) + addStrictWarning("msg.no.side.effects", ""); + return pn; + } + } catch (ParserException e) { } + + // skip to end of statement + int lineno = ts.getLineno(); + guessingStatementEnd: for (;;) { + int tt = peekTokenOrEOL(); + consumeToken(); + switch (tt) { + case Token.ERROR: + case Token.EOF: + case Token.EOL: + case Token.SEMI: + break guessingStatementEnd; + } + } + return nf.createExprStatement(nf.createName("error"), lineno); + } + + /** + * Whether the "catch (e: e instanceof Exception) { ... }" syntax + * is implemented. + */ + + private Node statementHelper(Node statementLabel) + throws IOException, ParserException + { + Node pn = null; + + int tt; + + tt = peekToken(); + + switch(tt) { + case Token.IF: { + consumeToken(); + + decompiler.addToken(Token.IF); + int lineno = ts.getLineno(); + Node cond = condition(); + decompiler.addEOL(Token.LC); + Node ifTrue = statement(); + Node ifFalse = null; + if (matchToken(Token.ELSE)) { + decompiler.addToken(Token.RC); + decompiler.addToken(Token.ELSE); + decompiler.addEOL(Token.LC); + ifFalse = statement(); + } + decompiler.addEOL(Token.RC); + pn = nf.createIf(cond, ifTrue, ifFalse, lineno); + return pn; + } + + case Token.SWITCH: { + consumeToken(); + + decompiler.addToken(Token.SWITCH); + int lineno = ts.getLineno(); + mustMatchToken(Token.LP, "msg.no.paren.switch"); + decompiler.addToken(Token.LP); + pn = enterSwitch(expr(false), lineno); + try { + mustMatchToken(Token.RP, "msg.no.paren.after.switch"); + decompiler.addToken(Token.RP); + mustMatchToken(Token.LC, "msg.no.brace.switch"); + decompiler.addEOL(Token.LC); + + boolean hasDefault = false; + switchLoop: for (;;) { + tt = nextToken(); + Node caseExpression; + switch (tt) { + case Token.RC: + break switchLoop; + + case Token.CASE: + decompiler.addToken(Token.CASE); + caseExpression = expr(false); + mustMatchToken(Token.COLON, "msg.no.colon.case"); + decompiler.addEOL(Token.COLON); + break; + + case Token.DEFAULT: + if (hasDefault) { + reportError("msg.double.switch.default"); + } + decompiler.addToken(Token.DEFAULT); + hasDefault = true; + caseExpression = null; + mustMatchToken(Token.COLON, "msg.no.colon.case"); + decompiler.addEOL(Token.COLON); + break; + + default: + reportError("msg.bad.switch"); + break switchLoop; + } + + Node block = nf.createLeaf(Token.BLOCK); + while ((tt = peekToken()) != Token.RC + && tt != Token.CASE + && tt != Token.DEFAULT + && tt != Token.EOF) + { + nf.addChildToBack(block, statement()); + } + + // caseExpression == null => add default lable + nf.addSwitchCase(pn, caseExpression, block); + } + decompiler.addEOL(Token.RC); + nf.closeSwitch(pn); + } finally { + exitSwitch(); + } + return pn; + } + + case Token.WHILE: { + consumeToken(); + decompiler.addToken(Token.WHILE); + + Node loop = enterLoop(statementLabel); + try { + Node cond = condition(); + decompiler.addEOL(Token.LC); + Node body = statement(); + decompiler.addEOL(Token.RC); + pn = nf.createWhile(loop, cond, body); + } finally { + exitLoop(); + } + return pn; + } + + case Token.DO: { + consumeToken(); + decompiler.addToken(Token.DO); + decompiler.addEOL(Token.LC); + + Node loop = enterLoop(statementLabel); + try { + Node body = statement(); + decompiler.addToken(Token.RC); + mustMatchToken(Token.WHILE, "msg.no.while.do"); + decompiler.addToken(Token.WHILE); + Node cond = condition(); + pn = nf.createDoWhile(loop, body, cond); + } finally { + exitLoop(); + } + // Always auto-insert semicon to follow SpiderMonkey: + // It is required by EMAScript but is ignored by the rest of + // world, see bug 238945 + matchToken(Token.SEMI); + decompiler.addEOL(Token.SEMI); + return pn; + } + + case Token.FOR: { + consumeToken(); + boolean isForEach = false; + decompiler.addToken(Token.FOR); + + Node loop = enterLoop(statementLabel); + try { + + Node init; // Node init is also foo in 'foo in Object' + Node cond; // Node cond is also object in 'foo in Object' + Node incr = null; // to kill warning + Node body; + + // See if this is a for each () instead of just a for () + if (matchToken(Token.NAME)) { + decompiler.addName(ts.getString()); + if (ts.getString().equals("each")) { + isForEach = true; + } else { + reportError("msg.no.paren.for"); + } + } + + mustMatchToken(Token.LP, "msg.no.paren.for"); + decompiler.addToken(Token.LP); + tt = peekToken(); + if (tt == Token.SEMI) { + init = nf.createLeaf(Token.EMPTY); + } else { + if (tt == Token.VAR) { + // set init to a var list or initial + consumeToken(); // consume the 'var' token + init = variables(Token.FOR); + } + else { + init = expr(true); + } + } + + if (matchToken(Token.IN)) { + decompiler.addToken(Token.IN); + // 'cond' is the object over which we're iterating + cond = expr(false); + } else { // ordinary for loop + mustMatchToken(Token.SEMI, "msg.no.semi.for"); + decompiler.addToken(Token.SEMI); + if (peekToken() == Token.SEMI) { + // no loop condition + cond = nf.createLeaf(Token.EMPTY); + } else { + cond = expr(false); + } + + mustMatchToken(Token.SEMI, "msg.no.semi.for.cond"); + decompiler.addToken(Token.SEMI); + if (peekToken() == Token.RP) { + incr = nf.createLeaf(Token.EMPTY); + } else { + incr = expr(false); + } + } + + mustMatchToken(Token.RP, "msg.no.paren.for.ctrl"); + decompiler.addToken(Token.RP); + decompiler.addEOL(Token.LC); + body = statement(); + decompiler.addEOL(Token.RC); + + if (incr == null) { + // cond could be null if 'in obj' got eaten + // by the init node. + pn = nf.createForIn(loop, init, cond, body, isForEach); + } else { + pn = nf.createFor(loop, init, cond, incr, body); + } + } finally { + exitLoop(); + } + return pn; + } + + case Token.TRY: { + consumeToken(); + int lineno = ts.getLineno(); + + Node tryblock; + Node catchblocks = null; + Node finallyblock = null; + + decompiler.addToken(Token.TRY); + decompiler.addEOL(Token.LC); + tryblock = statement(); + decompiler.addEOL(Token.RC); + + catchblocks = nf.createLeaf(Token.BLOCK); + + boolean sawDefaultCatch = false; + int peek = peekToken(); + if (peek == Token.CATCH) { + while (matchToken(Token.CATCH)) { + if (sawDefaultCatch) { + reportError("msg.catch.unreachable"); + } + decompiler.addToken(Token.CATCH); + mustMatchToken(Token.LP, "msg.no.paren.catch"); + decompiler.addToken(Token.LP); + + mustMatchToken(Token.NAME, "msg.bad.catchcond"); + String varName = ts.getString(); + decompiler.addName(varName); + + Node catchCond = null; + if (matchToken(Token.IF)) { + decompiler.addToken(Token.IF); + catchCond = expr(false); + } else { + sawDefaultCatch = true; + } + + mustMatchToken(Token.RP, "msg.bad.catchcond"); + decompiler.addToken(Token.RP); + mustMatchToken(Token.LC, "msg.no.brace.catchblock"); + decompiler.addEOL(Token.LC); + + nf.addChildToBack(catchblocks, + nf.createCatch(varName, catchCond, + statements(), + ts.getLineno())); + + mustMatchToken(Token.RC, "msg.no.brace.after.body"); + decompiler.addEOL(Token.RC); + } + } else if (peek != Token.FINALLY) { + mustMatchToken(Token.FINALLY, "msg.try.no.catchfinally"); + } + + if (matchToken(Token.FINALLY)) { + decompiler.addToken(Token.FINALLY); + decompiler.addEOL(Token.LC); + finallyblock = statement(); + decompiler.addEOL(Token.RC); + } + + pn = nf.createTryCatchFinally(tryblock, catchblocks, + finallyblock, lineno); + + return pn; + } + + case Token.THROW: { + consumeToken(); + if (peekTokenOrEOL() == Token.EOL) { + // ECMAScript does not allow new lines before throw expression, + // see bug 256617 + reportError("msg.bad.throw.eol"); + } + + int lineno = ts.getLineno(); + decompiler.addToken(Token.THROW); + pn = nf.createThrow(expr(false), lineno); + break; + } + + case Token.BREAK: { + consumeToken(); + int lineno = ts.getLineno(); + + decompiler.addToken(Token.BREAK); + + // matchJumpLabelName only matches if there is one + Node breakStatement = matchJumpLabelName(); + if (breakStatement == null) { + if (loopAndSwitchSet == null || loopAndSwitchSet.size() == 0) { + reportError("msg.bad.break"); + return null; + } + breakStatement = (Node)loopAndSwitchSet.peek(); + } + pn = nf.createBreak(breakStatement, lineno); + break; + } + + case Token.CONTINUE: { + consumeToken(); + int lineno = ts.getLineno(); + + decompiler.addToken(Token.CONTINUE); + + Node loop; + // matchJumpLabelName only matches if there is one + Node label = matchJumpLabelName(); + if (label == null) { + if (loopSet == null || loopSet.size() == 0) { + reportError("msg.continue.outside"); + return null; + } + loop = (Node)loopSet.peek(); + } else { + loop = nf.getLabelLoop(label); + if (loop == null) { + reportError("msg.continue.nonloop"); + return null; + } + } + pn = nf.createContinue(loop, lineno); + break; + } + + case Token.WITH: { + consumeToken(); + + decompiler.addToken(Token.WITH); + int lineno = ts.getLineno(); + mustMatchToken(Token.LP, "msg.no.paren.with"); + decompiler.addToken(Token.LP); + Node obj = expr(false); + mustMatchToken(Token.RP, "msg.no.paren.after.with"); + decompiler.addToken(Token.RP); + decompiler.addEOL(Token.LC); + + ++nestingOfWith; + Node body; + try { + body = statement(); + } finally { + --nestingOfWith; + } + + decompiler.addEOL(Token.RC); + + pn = nf.createWith(obj, body, lineno); + return pn; + } + + case Token.CONST: + case Token.VAR: { + consumeToken(); + pn = variables(tt); + break; + } + + case Token.RETURN: { + if (!insideFunction()) { + reportError("msg.bad.return"); + } + consumeToken(); + decompiler.addToken(Token.RETURN); + int lineno = ts.getLineno(); + + Node retExpr; + /* This is ugly, but we don't want to require a semicolon. */ + tt = peekTokenOrEOL(); + switch (tt) { + case Token.SEMI: + case Token.RC: + case Token.EOF: + case Token.EOL: + case Token.ERROR: + retExpr = null; + break; + default: + retExpr = expr(false); + hasReturnValue = true; + } + pn = nf.createReturn(retExpr, lineno); + + // see if we need a strict mode warning + if (retExpr == null) { + if (functionEndFlags == Node.END_RETURNS_VALUE) + addStrictWarning("msg.return.inconsistent", ""); + + functionEndFlags |= Node.END_RETURNS; + } else { + if (functionEndFlags == Node.END_RETURNS) + addStrictWarning("msg.return.inconsistent", ""); + + functionEndFlags |= Node.END_RETURNS_VALUE; + } + + break; + } + + case Token.LC: + consumeToken(); + if (statementLabel != null) { + decompiler.addToken(Token.LC); + } + pn = statements(); + mustMatchToken(Token.RC, "msg.no.brace.block"); + if (statementLabel != null) { + decompiler.addEOL(Token.RC); + } + return pn; + + case Token.ERROR: + // Fall thru, to have a node for error recovery to work on + case Token.SEMI: + consumeToken(); + pn = nf.createLeaf(Token.EMPTY); + return pn; + + case Token.FUNCTION: { + consumeToken(); + pn = function(FunctionNode.FUNCTION_EXPRESSION_STATEMENT); + return pn; + } + + case Token.DEFAULT : + consumeToken(); + mustHaveXML(); + + decompiler.addToken(Token.DEFAULT); + int nsLine = ts.getLineno(); + + if (!(matchToken(Token.NAME) + && ts.getString().equals("xml"))) + { + reportError("msg.bad.namespace"); + } + decompiler.addName(" xml"); + + if (!(matchToken(Token.NAME) + && ts.getString().equals("namespace"))) + { + reportError("msg.bad.namespace"); + } + decompiler.addName(" namespace"); + + if (!matchToken(Token.ASSIGN)) { + reportError("msg.bad.namespace"); + } + decompiler.addToken(Token.ASSIGN); + + Node expr = expr(false); + pn = nf.createDefaultNamespace(expr, nsLine); + break; + + case Token.NAME: { + int lineno = ts.getLineno(); + String name = ts.getString(); + setCheckForLabel(); + pn = expr(false); + if (pn.getType() != Token.LABEL) { + pn = nf.createExprStatement(pn, lineno); + } else { + // Parsed the label: push back token should be + // colon that primaryExpr left untouched. + if (peekToken() != Token.COLON) Kit.codeBug(); + consumeToken(); + // depend on decompiling lookahead to guess that that + // last name was a label. + decompiler.addName(name); + decompiler.addEOL(Token.COLON); + + if (labelSet == null) { + labelSet = new Hashtable(); + } else if (labelSet.containsKey(name)) { + reportError("msg.dup.label"); + } + + boolean firstLabel; + if (statementLabel == null) { + firstLabel = true; + statementLabel = pn; + } else { + // Discard multiple label nodes and use only + // the first: it allows to simplify IRFactory + firstLabel = false; + } + labelSet.put(name, statementLabel); + try { + pn = statementHelper(statementLabel); + } finally { + labelSet.remove(name); + } + if (firstLabel) { + pn = nf.createLabeledStatement(statementLabel, pn); + } + return pn; + } + break; + } + + default: { + int lineno = ts.getLineno(); + pn = expr(false); + pn = nf.createExprStatement(pn, lineno); + break; + } + } + + int ttFlagged = peekFlaggedToken(); + switch (ttFlagged & CLEAR_TI_MASK) { + case Token.SEMI: + // Consume ';' as a part of expression + consumeToken(); + break; + case Token.ERROR: + case Token.EOF: + case Token.RC: + // Autoinsert ; + break; + default: + if ((ttFlagged & TI_AFTER_EOL) == 0) { + // Report error if no EOL or autoinsert ; otherwise + reportError("msg.no.semi.stmt"); + } + break; + } + decompiler.addEOL(Token.SEMI); + + return pn; + } + + /** + * Parse a 'var' or 'const' statement, or a 'var' init list in a for + * statement. + * @param context A token value: either VAR, CONST or FOR depending on + * context. + * @return The parsed statement + * @throws IOException + * @throws ParserException + */ + private Node variables(int context) + throws IOException, ParserException + { + Node pn; + boolean first = true; + + if (context == Token.CONST){ + pn = nf.createVariables(Token.CONST, ts.getLineno()); + decompiler.addToken(Token.CONST); + } else { + pn = nf.createVariables(Token.VAR, ts.getLineno()); + decompiler.addToken(Token.VAR); + } + + for (;;) { + Node name; + Node init; + mustMatchToken(Token.NAME, "msg.bad.var"); + String s = ts.getString(); + + if (!first) + decompiler.addToken(Token.COMMA); + first = false; + + decompiler.addName(s); + + if (context == Token.CONST) { + if (!currentScriptOrFn.addConst(s)) { + // We know it's already defined, since addConst passes if + // it's not defined at all. The addVar call just confirms + // what it is. + if (currentScriptOrFn.addVar(s) != ScriptOrFnNode.DUPLICATE_CONST) + addError("msg.var.redecl", s); + else + addError("msg.const.redecl", s); + } + } else { + int dupState = currentScriptOrFn.addVar(s); + if (dupState == ScriptOrFnNode.DUPLICATE_CONST) + addError("msg.const.redecl", s); + else if (dupState == ScriptOrFnNode.DUPLICATE_PARAMETER) + addStrictWarning("msg.var.hides.arg", s); + else if (dupState == ScriptOrFnNode.DUPLICATE_VAR) + addStrictWarning("msg.var.redecl", s); + } + name = nf.createName(s); + + // omitted check for argument hiding + + if (matchToken(Token.ASSIGN)) { + decompiler.addToken(Token.ASSIGN); + + init = assignExpr(context == Token.FOR); + nf.addChildToBack(name, init); + } + nf.addChildToBack(pn, name); + if (!matchToken(Token.COMMA)) + break; + } + return pn; + } + + private Node expr(boolean inForInit) + throws IOException, ParserException + { + Node pn = assignExpr(inForInit); + while (matchToken(Token.COMMA)) { + decompiler.addToken(Token.COMMA); + if (compilerEnv.isStrictMode() && !pn.hasSideEffects()) + addStrictWarning("msg.no.side.effects", ""); + pn = nf.createBinary(Token.COMMA, pn, assignExpr(inForInit)); + } + return pn; + } + + private Node assignExpr(boolean inForInit) + throws IOException, ParserException + { + Node pn = condExpr(inForInit); + + int tt = peekToken(); + if (Token.FIRST_ASSIGN <= tt && tt <= Token.LAST_ASSIGN) { + consumeToken(); + decompiler.addToken(tt); + pn = nf.createAssignment(tt, pn, assignExpr(inForInit)); + } + + return pn; + } + + private Node condExpr(boolean inForInit) + throws IOException, ParserException + { + Node pn = orExpr(inForInit); + + if (matchToken(Token.HOOK)) { + decompiler.addToken(Token.HOOK); + Node ifTrue = assignExpr(false); + mustMatchToken(Token.COLON, "msg.no.colon.cond"); + decompiler.addToken(Token.COLON); + Node ifFalse = assignExpr(inForInit); + return nf.createCondExpr(pn, ifTrue, ifFalse); + } + + return pn; + } + + private Node orExpr(boolean inForInit) + throws IOException, ParserException + { + Node pn = andExpr(inForInit); + if (matchToken(Token.OR)) { + decompiler.addToken(Token.OR); + pn = nf.createBinary(Token.OR, pn, orExpr(inForInit)); + } + + return pn; + } + + private Node andExpr(boolean inForInit) + throws IOException, ParserException + { + Node pn = bitOrExpr(inForInit); + if (matchToken(Token.AND)) { + decompiler.addToken(Token.AND); + pn = nf.createBinary(Token.AND, pn, andExpr(inForInit)); + } + + return pn; + } + + private Node bitOrExpr(boolean inForInit) + throws IOException, ParserException + { + Node pn = bitXorExpr(inForInit); + while (matchToken(Token.BITOR)) { + decompiler.addToken(Token.BITOR); + pn = nf.createBinary(Token.BITOR, pn, bitXorExpr(inForInit)); + } + return pn; + } + + private Node bitXorExpr(boolean inForInit) + throws IOException, ParserException + { + Node pn = bitAndExpr(inForInit); + while (matchToken(Token.BITXOR)) { + decompiler.addToken(Token.BITXOR); + pn = nf.createBinary(Token.BITXOR, pn, bitAndExpr(inForInit)); + } + return pn; + } + + private Node bitAndExpr(boolean inForInit) + throws IOException, ParserException + { + Node pn = eqExpr(inForInit); + while (matchToken(Token.BITAND)) { + decompiler.addToken(Token.BITAND); + pn = nf.createBinary(Token.BITAND, pn, eqExpr(inForInit)); + } + return pn; + } + + private Node eqExpr(boolean inForInit) + throws IOException, ParserException + { + Node pn = relExpr(inForInit); + for (;;) { + int tt = peekToken(); + switch (tt) { + case Token.EQ: + case Token.NE: + case Token.SHEQ: + case Token.SHNE: + consumeToken(); + int decompilerToken = tt; + int parseToken = tt; + if (compilerEnv.getLanguageVersion() == Context.VERSION_1_2) { + // JavaScript 1.2 uses shallow equality for == and != . + // In addition, convert === and !== for decompiler into + // == and != since the decompiler is supposed to show + // canonical source and in 1.2 ===, !== are allowed + // only as an alias to ==, !=. + switch (tt) { + case Token.EQ: + parseToken = Token.SHEQ; + break; + case Token.NE: + parseToken = Token.SHNE; + break; + case Token.SHEQ: + decompilerToken = Token.EQ; + break; + case Token.SHNE: + decompilerToken = Token.NE; + break; + } + } + decompiler.addToken(decompilerToken); + pn = nf.createBinary(parseToken, pn, relExpr(inForInit)); + continue; + } + break; + } + return pn; + } + + private Node relExpr(boolean inForInit) + throws IOException, ParserException + { + Node pn = shiftExpr(); + for (;;) { + int tt = peekToken(); + switch (tt) { + case Token.IN: + if (inForInit) + break; + // fall through + case Token.INSTANCEOF: + case Token.LE: + case Token.LT: + case Token.GE: + case Token.GT: + consumeToken(); + decompiler.addToken(tt); + pn = nf.createBinary(tt, pn, shiftExpr()); + continue; + } + break; + } + return pn; + } + + private Node shiftExpr() + throws IOException, ParserException + { + Node pn = addExpr(); + for (;;) { + int tt = peekToken(); + switch (tt) { + case Token.LSH: + case Token.URSH: + case Token.RSH: + consumeToken(); + decompiler.addToken(tt); + pn = nf.createBinary(tt, pn, addExpr()); + continue; + } + break; + } + return pn; + } + + private Node addExpr() + throws IOException, ParserException + { + Node pn = mulExpr(); + for (;;) { + int tt = peekToken(); + if (tt == Token.ADD || tt == Token.SUB) { + consumeToken(); + decompiler.addToken(tt); + // flushNewLines + pn = nf.createBinary(tt, pn, mulExpr()); + continue; + } + break; + } + + return pn; + } + + private Node mulExpr() + throws IOException, ParserException + { + Node pn = unaryExpr(); + for (;;) { + int tt = peekToken(); + switch (tt) { + case Token.MUL: + case Token.DIV: + case Token.MOD: + consumeToken(); + decompiler.addToken(tt); + pn = nf.createBinary(tt, pn, unaryExpr()); + continue; + } + break; + } + + return pn; + } + + private Node unaryExpr() + throws IOException, ParserException + { + int tt; + + tt = peekToken(); + + switch(tt) { + case Token.VOID: + case Token.NOT: + case Token.BITNOT: + case Token.TYPEOF: + consumeToken(); + decompiler.addToken(tt); + return nf.createUnary(tt, unaryExpr()); + + case Token.ADD: + consumeToken(); + // Convert to special POS token in decompiler and parse tree + decompiler.addToken(Token.POS); + return nf.createUnary(Token.POS, unaryExpr()); + + case Token.SUB: + consumeToken(); + // Convert to special NEG token in decompiler and parse tree + decompiler.addToken(Token.NEG); + return nf.createUnary(Token.NEG, unaryExpr()); + + case Token.INC: + case Token.DEC: + consumeToken(); + decompiler.addToken(tt); + return nf.createIncDec(tt, false, memberExpr(true)); + + case Token.DELPROP: + consumeToken(); + decompiler.addToken(Token.DELPROP); + return nf.createUnary(Token.DELPROP, unaryExpr()); + + case Token.ERROR: + consumeToken(); + break; + + // XML stream encountered in expression. + case Token.LT: + if (compilerEnv.isXmlAvailable()) { + consumeToken(); + Node pn = xmlInitializer(); + return memberExprTail(true, pn); + } + // Fall thru to the default handling of RELOP + + default: + Node pn = memberExpr(true); + + // Don't look across a newline boundary for a postfix incop. + tt = peekTokenOrEOL(); + if (tt == Token.INC || tt == Token.DEC) { + consumeToken(); + decompiler.addToken(tt); + return nf.createIncDec(tt, true, pn); + } + return pn; + } + return nf.createName("err"); // Only reached on error. Try to continue. + + } + + private Node xmlInitializer() throws IOException + { + int tt = ts.getFirstXMLToken(); + if (tt != Token.XML && tt != Token.XMLEND) { + reportError("msg.syntax"); + return null; + } + + /* Make a NEW node to append to. */ + Node pnXML = nf.createLeaf(Token.NEW); + + String xml = ts.getString(); + boolean fAnonymous = xml.trim().startsWith("<>"); + + Node pn = nf.createName(fAnonymous ? "XMLList" : "XML"); + nf.addChildToBack(pnXML, pn); + + pn = null; + Node expr; + for (;;tt = ts.getNextXMLToken()) { + switch (tt) { + case Token.XML: + xml = ts.getString(); + decompiler.addName(xml); + mustMatchToken(Token.LC, "msg.syntax"); + decompiler.addToken(Token.LC); + expr = (peekToken() == Token.RC) + ? nf.createString("") + : expr(false); + mustMatchToken(Token.RC, "msg.syntax"); + decompiler.addToken(Token.RC); + if (pn == null) { + pn = nf.createString(xml); + } else { + pn = nf.createBinary(Token.ADD, pn, nf.createString(xml)); + } + if (ts.isXMLAttribute()) { + /* Need to put the result in double quotes */ + expr = nf.createUnary(Token.ESCXMLATTR, expr); + Node prepend = nf.createBinary(Token.ADD, + nf.createString("\""), + expr); + expr = nf.createBinary(Token.ADD, + prepend, + nf.createString("\"")); + } else { + expr = nf.createUnary(Token.ESCXMLTEXT, expr); + } + pn = nf.createBinary(Token.ADD, pn, expr); + break; + case Token.XMLEND: + xml = ts.getString(); + decompiler.addName(xml); + if (pn == null) { + pn = nf.createString(xml); + } else { + pn = nf.createBinary(Token.ADD, pn, nf.createString(xml)); + } + + nf.addChildToBack(pnXML, pn); + return pnXML; + default: + reportError("msg.syntax"); + return null; + } + } + } + + private void argumentList(Node listNode) + throws IOException, ParserException + { + boolean matched; + matched = matchToken(Token.RP); + if (!matched) { + boolean first = true; + do { + if (!first) + decompiler.addToken(Token.COMMA); + first = false; + nf.addChildToBack(listNode, assignExpr(false)); + } while (matchToken(Token.COMMA)); + + mustMatchToken(Token.RP, "msg.no.paren.arg"); + } + decompiler.addToken(Token.RP); + } + + private Node memberExpr(boolean allowCallSyntax) + throws IOException, ParserException + { + int tt; + + Node pn; + + /* Check for new expressions. */ + tt = peekToken(); + if (tt == Token.NEW) { + /* Eat the NEW token. */ + consumeToken(); + decompiler.addToken(Token.NEW); + + /* Make a NEW node to append to. */ + pn = nf.createCallOrNew(Token.NEW, memberExpr(false)); + + if (matchToken(Token.LP)) { + decompiler.addToken(Token.LP); + /* Add the arguments to pn, if any are supplied. */ + argumentList(pn); + } + + /* XXX there's a check in the C source against + * "too many constructor arguments" - how many + * do we claim to support? + */ + + /* Experimental syntax: allow an object literal to follow a new expression, + * which will mean a kind of anonymous class built with the JavaAdapter. + * the object literal will be passed as an additional argument to the constructor. + */ + tt = peekToken(); + if (tt == Token.LC) { + nf.addChildToBack(pn, primaryExpr()); + } + } else { + pn = primaryExpr(); + } + + return memberExprTail(allowCallSyntax, pn); + } + + private Node memberExprTail(boolean allowCallSyntax, Node pn) + throws IOException, ParserException + { + tailLoop: + for (;;) { + int tt = peekToken(); + switch (tt) { + + case Token.DOT: + case Token.DOTDOT: + { + int memberTypeFlags; + String s; + + consumeToken(); + decompiler.addToken(tt); + memberTypeFlags = 0; + if (tt == Token.DOTDOT) { + mustHaveXML(); + memberTypeFlags = Node.DESCENDANTS_FLAG; + } + if (!compilerEnv.isXmlAvailable()) { + mustMatchToken(Token.NAME, "msg.no.name.after.dot"); + s = ts.getString(); + decompiler.addName(s); + pn = nf.createPropertyGet(pn, null, s, memberTypeFlags); + break; + } + + tt = nextToken(); + switch (tt) { + // handles: name, ns::name, ns::*, ns::[expr] + case Token.NAME: + s = ts.getString(); + decompiler.addName(s); + pn = propertyName(pn, s, memberTypeFlags); + break; + + // handles: *, *::name, *::*, *::[expr] + case Token.MUL: + decompiler.addName("*"); + pn = propertyName(pn, "*", memberTypeFlags); + break; + + // handles: '@attr', '@ns::attr', '@ns::*', '@ns::*', + // '@::attr', '@::*', '@*', '@*::attr', '@*::*' + case Token.XMLATTR: + decompiler.addToken(Token.XMLATTR); + pn = attributeAccess(pn, memberTypeFlags); + break; + + default: + reportError("msg.no.name.after.dot"); + } + } + break; + + case Token.DOTQUERY: + consumeToken(); + mustHaveXML(); + decompiler.addToken(Token.DOTQUERY); + pn = nf.createDotQuery(pn, expr(false), ts.getLineno()); + mustMatchToken(Token.RP, "msg.no.paren"); + decompiler.addToken(Token.RP); + break; + + case Token.LB: + consumeToken(); + decompiler.addToken(Token.LB); + pn = nf.createElementGet(pn, null, expr(false), 0); + mustMatchToken(Token.RB, "msg.no.bracket.index"); + decompiler.addToken(Token.RB); + break; + + case Token.LP: + if (!allowCallSyntax) { + break tailLoop; + } + consumeToken(); + decompiler.addToken(Token.LP); + pn = nf.createCallOrNew(Token.CALL, pn); + /* Add the arguments to pn, if any are supplied. */ + argumentList(pn); + break; + + default: + break tailLoop; + } + } + return pn; + } + + /* + * Xml attribute expression: + * '@attr', '@ns::attr', '@ns::*', '@ns::*', '@*', '@*::attr', '@*::*' + */ + private Node attributeAccess(Node pn, int memberTypeFlags) + throws IOException + { + memberTypeFlags |= Node.ATTRIBUTE_FLAG; + int tt = nextToken(); + + switch (tt) { + // handles: @name, @ns::name, @ns::*, @ns::[expr] + case Token.NAME: + { + String s = ts.getString(); + decompiler.addName(s); + pn = propertyName(pn, s, memberTypeFlags); + } + break; + + // handles: @*, @*::name, @*::*, @*::[expr] + case Token.MUL: + decompiler.addName("*"); + pn = propertyName(pn, "*", memberTypeFlags); + break; + + // handles @[expr] + case Token.LB: + decompiler.addToken(Token.LB); + pn = nf.createElementGet(pn, null, expr(false), memberTypeFlags); + mustMatchToken(Token.RB, "msg.no.bracket.index"); + decompiler.addToken(Token.RB); + break; + + default: + reportError("msg.no.name.after.xmlAttr"); + pn = nf.createPropertyGet(pn, null, "?", memberTypeFlags); + break; + } + + return pn; + } + + /** + * Check if :: follows name in which case it becomes qualified name + */ + private Node propertyName(Node pn, String name, int memberTypeFlags) + throws IOException, ParserException + { + String namespace = null; + if (matchToken(Token.COLONCOLON)) { + decompiler.addToken(Token.COLONCOLON); + namespace = name; + + int tt = nextToken(); + switch (tt) { + // handles name::name + case Token.NAME: + name = ts.getString(); + decompiler.addName(name); + break; + + // handles name::* + case Token.MUL: + decompiler.addName("*"); + name = "*"; + break; + + // handles name::[expr] + case Token.LB: + decompiler.addToken(Token.LB); + pn = nf.createElementGet(pn, namespace, expr(false), + memberTypeFlags); + mustMatchToken(Token.RB, "msg.no.bracket.index"); + decompiler.addToken(Token.RB); + return pn; + + default: + reportError("msg.no.name.after.coloncolon"); + name = "?"; + } + } + + pn = nf.createPropertyGet(pn, namespace, name, memberTypeFlags); + return pn; + } + + private Node primaryExpr() + throws IOException, ParserException + { + Node pn; + + int ttFlagged = nextFlaggedToken(); + int tt = ttFlagged & CLEAR_TI_MASK; + + switch(tt) { + + case Token.FUNCTION: + return function(FunctionNode.FUNCTION_EXPRESSION); + + case Token.LB: { + ObjArray elems = new ObjArray(); + int skipCount = 0; + decompiler.addToken(Token.LB); + boolean after_lb_or_comma = true; + for (;;) { + tt = peekToken(); + + if (tt == Token.COMMA) { + consumeToken(); + decompiler.addToken(Token.COMMA); + if (!after_lb_or_comma) { + after_lb_or_comma = true; + } else { + elems.add(null); + ++skipCount; + } + } else if (tt == Token.RB) { + consumeToken(); + decompiler.addToken(Token.RB); + break; + } else { + if (!after_lb_or_comma) { + reportError("msg.no.bracket.arg"); + } + elems.add(assignExpr(false)); + after_lb_or_comma = false; + } + } + return nf.createArrayLiteral(elems, skipCount); + } + + case Token.LC: { + ObjArray elems = new ObjArray(); + decompiler.addToken(Token.LC); + if (!matchToken(Token.RC)) { + + boolean first = true; + commaloop: + do { + Object property; + + if (!first) + decompiler.addToken(Token.COMMA); + else + first = false; + + tt = peekToken(); + switch(tt) { + case Token.NAME: + case Token.STRING: + consumeToken(); + // map NAMEs to STRINGs in object literal context + // but tell the decompiler the proper type + String s = ts.getString(); + if (tt == Token.NAME) { + if (s.equals("get") && + peekToken() == Token.NAME) { + decompiler.addToken(Token.GET); + consumeToken(); + s = ts.getString(); + decompiler.addName(s); + property = ScriptRuntime.getIndexObject(s); + if (!getterSetterProperty(elems, property, + true)) + break commaloop; + break; + } else if (s.equals("set") && + peekToken() == Token.NAME) { + decompiler.addToken(Token.SET); + consumeToken(); + s = ts.getString(); + decompiler.addName(s); + property = ScriptRuntime.getIndexObject(s); + if (!getterSetterProperty(elems, property, + false)) + break commaloop; + break; + } + decompiler.addName(s); + } else { + decompiler.addString(s); + } + property = ScriptRuntime.getIndexObject(s); + plainProperty(elems, property); + break; + + case Token.NUMBER: + consumeToken(); + double n = ts.getNumber(); + decompiler.addNumber(n); + property = ScriptRuntime.getIndexObject(n); + plainProperty(elems, property); + break; + + case Token.RC: + // trailing comma is OK. + break commaloop; + default: + reportError("msg.bad.prop"); + break commaloop; + } + } while (matchToken(Token.COMMA)); + + mustMatchToken(Token.RC, "msg.no.brace.prop"); + } + decompiler.addToken(Token.RC); + return nf.createObjectLiteral(elems); + } + + case Token.LP: + + /* Brendan's IR-jsparse.c makes a new node tagged with + * TOK_LP here... I'm not sure I understand why. Isn't + * the grouping already implicit in the structure of the + * parse tree? also TOK_LP is already overloaded (I + * think) in the C IR as 'function call.' */ + decompiler.addToken(Token.LP); + pn = expr(false); + pn.putProp(Node.PARENTHESIZED_PROP, Boolean.TRUE); + decompiler.addToken(Token.RP); + mustMatchToken(Token.RP, "msg.no.paren"); + return pn; + + case Token.XMLATTR: + mustHaveXML(); + decompiler.addToken(Token.XMLATTR); + pn = attributeAccess(null, 0); + return pn; + + case Token.NAME: { + String name = ts.getString(); + if ((ttFlagged & TI_CHECK_LABEL) != 0) { + if (peekToken() == Token.COLON) { + // Do not consume colon, it is used as unwind indicator + // to return to statementHelper. + // XXX Better way? + return nf.createLabel(ts.getLineno()); + } + } + + decompiler.addName(name); + if (compilerEnv.isXmlAvailable()) { + pn = propertyName(null, name, 0); + } else { + pn = nf.createName(name); + } + return pn; + } + + case Token.NUMBER: { + double n = ts.getNumber(); + decompiler.addNumber(n); + return nf.createNumber(n); + } + + case Token.STRING: { + String s = ts.getString(); + decompiler.addString(s); + return nf.createString(s); + } + + case Token.DIV: + case Token.ASSIGN_DIV: { + // Got / or /= which should be treated as regexp in fact + ts.readRegExp(tt); + String flags = ts.regExpFlags; + ts.regExpFlags = null; + String re = ts.getString(); + decompiler.addRegexp(re, flags); + int index = currentScriptOrFn.addRegexp(re, flags); + return nf.createRegExp(index); + } + + case Token.NULL: + case Token.THIS: + case Token.FALSE: + case Token.TRUE: + decompiler.addToken(tt); + return nf.createLeaf(tt); + + case Token.RESERVED: + reportError("msg.reserved.id"); + break; + + case Token.ERROR: + /* the scanner or one of its subroutines reported the error. */ + break; + + case Token.EOF: + reportError("msg.unexpected.eof"); + break; + + default: + reportError("msg.syntax"); + break; + } + return null; // should never reach here + } + + private void plainProperty(ObjArray elems, Object property) + throws IOException { + mustMatchToken(Token.COLON, "msg.no.colon.prop"); + + // OBJLIT is used as ':' in object literal for + // decompilation to solve spacing ambiguity. + decompiler.addToken(Token.OBJECTLIT); + elems.add(property); + elems.add(assignExpr(false)); + } + + private boolean getterSetterProperty(ObjArray elems, Object property, + boolean isGetter) throws IOException { + Node f = function(FunctionNode.FUNCTION_EXPRESSION); + if (f.getType() != Token.FUNCTION) { + reportError("msg.bad.prop"); + return false; + } + int fnIndex = f.getExistingIntProp(Node.FUNCTION_PROP); + FunctionNode fn = currentScriptOrFn.getFunctionNode(fnIndex); + if (fn.getFunctionName().length() != 0) { + reportError("msg.bad.prop"); + return false; + } + elems.add(property); + if (isGetter) { + elems.add(nf.createUnary(Token.GET, f)); + } else { + elems.add(nf.createUnary(Token.SET, f)); + } + return true; + } +} diff --git a/trunk/infrastructure/yuicompressor/src/yuicompressor/org/mozilla/javascript/Parser.java.orig b/trunk/infrastructure/yuicompressor/src/yuicompressor/org/mozilla/javascript/Parser.java.orig new file mode 100644 index 0000000..628bb42 --- /dev/null +++ b/trunk/infrastructure/yuicompressor/src/yuicompressor/org/mozilla/javascript/Parser.java.orig @@ -0,0 +1,2159 @@ +/* -*- Mode: java; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is Rhino code, released + * May 6, 1999. + * + * The Initial Developer of the Original Code is + * Netscape Communications Corporation. + * Portions created by the Initial Developer are Copyright (C) 1997-1999 + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * Mike Ang + * Igor Bukanov + * Yuh-Ruey Chen + * Ethan Hugg + * Bob Jervis + * Terry Lucas + * Mike McCabe + * Milen Nankov + * + * Alternatively, the contents of this file may be used under the terms of + * the GNU General Public License Version 2 or later (the "GPL"), in which + * case the provisions of the GPL are applicable instead of those above. If + * you wish to allow use of your version of this file only under the terms of + * the GPL and not to allow others to use your version of this file under the + * MPL, indicate your decision by deleting the provisions above and replacing + * them with the notice and other provisions required by the GPL. If you do + * not delete the provisions above, a recipient may use your version of this + * file under either the MPL or the GPL. + * + * ***** END LICENSE BLOCK ***** */ + +package org.mozilla.javascript; + +import java.io.Reader; +import java.io.IOException; +import java.util.Hashtable; + +/** + * This class implements the JavaScript parser. + * + * It is based on the C source files jsparse.c and jsparse.h + * in the jsref package. + * + * @see TokenStream + * + * @author Mike McCabe + * @author Brendan Eich + */ + +public class Parser +{ + // TokenInformation flags : currentFlaggedToken stores them together + // with token type + final static int + CLEAR_TI_MASK = 0xFFFF, // mask to clear token information bits + TI_AFTER_EOL = 1 << 16, // first token of the source line + TI_CHECK_LABEL = 1 << 17; // indicates to check for label + + CompilerEnvirons compilerEnv; + private ErrorReporter errorReporter; + private String sourceURI; + boolean calledByCompileFunction; + + private TokenStream ts; + private int currentFlaggedToken; + private int syntaxErrorCount; + + private IRFactory nf; + + private int nestingOfFunction; + + private Decompiler decompiler; + private String encodedSource; + +// The following are per function variables and should be saved/restored +// during function parsing. +// XXX Move to separated class? + ScriptOrFnNode currentScriptOrFn; + private int nestingOfWith; + private Hashtable labelSet; // map of label names into nodes + private ObjArray loopSet; + private ObjArray loopAndSwitchSet; + private boolean hasReturnValue; + private int functionEndFlags; +// end of per function variables + + // Exception to unwind + private static class ParserException extends RuntimeException + { + static final long serialVersionUID = 5882582646773765630L; + } + + public Parser(CompilerEnvirons compilerEnv, ErrorReporter errorReporter) + { + this.compilerEnv = compilerEnv; + this.errorReporter = errorReporter; + } + + protected Decompiler createDecompiler(CompilerEnvirons compilerEnv) + { + return new Decompiler(); + } + + void addStrictWarning(String messageId, String messageArg) + { + if (compilerEnv.isStrictMode()) + addWarning(messageId, messageArg); + } + + void addWarning(String messageId, String messageArg) + { + String message = ScriptRuntime.getMessage1(messageId, messageArg); + if (compilerEnv.reportWarningAsError()) { + ++syntaxErrorCount; + errorReporter.error(message, sourceURI, ts.getLineno(), + ts.getLine(), ts.getOffset()); + } else + errorReporter.warning(message, sourceURI, ts.getLineno(), + ts.getLine(), ts.getOffset()); + } + + void addError(String messageId) + { + ++syntaxErrorCount; + String message = ScriptRuntime.getMessage0(messageId); + errorReporter.error(message, sourceURI, ts.getLineno(), + ts.getLine(), ts.getOffset()); + } + + void addError(String messageId, String messageArg) + { + ++syntaxErrorCount; + String message = ScriptRuntime.getMessage1(messageId, messageArg); + errorReporter.error(message, sourceURI, ts.getLineno(), + ts.getLine(), ts.getOffset()); + } + + RuntimeException reportError(String messageId) + { + addError(messageId); + + // Throw a ParserException exception to unwind the recursive descent + // parse. + throw new ParserException(); + } + + private int peekToken() + throws IOException + { + int tt = currentFlaggedToken; + if (tt == Token.EOF) { + tt = ts.getToken(); + if (tt == Token.EOL) { + do { + tt = ts.getToken(); + } while (tt == Token.EOL); + tt |= TI_AFTER_EOL; + } + currentFlaggedToken = tt; + } + return tt & CLEAR_TI_MASK; + } + + private int peekFlaggedToken() + throws IOException + { + peekToken(); + return currentFlaggedToken; + } + + private void consumeToken() + { + currentFlaggedToken = Token.EOF; + } + + private int nextToken() + throws IOException + { + int tt = peekToken(); + consumeToken(); + return tt; + } + + private int nextFlaggedToken() + throws IOException + { + peekToken(); + int ttFlagged = currentFlaggedToken; + consumeToken(); + return ttFlagged; + } + + private boolean matchToken(int toMatch) + throws IOException + { + int tt = peekToken(); + if (tt != toMatch) { + return false; + } + consumeToken(); + return true; + } + + private int peekTokenOrEOL() + throws IOException + { + int tt = peekToken(); + // Check for last peeked token flags + if ((currentFlaggedToken & TI_AFTER_EOL) != 0) { + tt = Token.EOL; + } + return tt; + } + + private void setCheckForLabel() + { + if ((currentFlaggedToken & CLEAR_TI_MASK) != Token.NAME) + throw Kit.codeBug(); + currentFlaggedToken |= TI_CHECK_LABEL; + } + + private void mustMatchToken(int toMatch, String messageId) + throws IOException, ParserException + { + if (!matchToken(toMatch)) { + reportError(messageId); + } + } + + private void mustHaveXML() + { + if (!compilerEnv.isXmlAvailable()) { + reportError("msg.XML.not.available"); + } + } + + public String getEncodedSource() + { + return encodedSource; + } + + public boolean eof() + { + return ts.eof(); + } + + boolean insideFunction() + { + return nestingOfFunction != 0; + } + + private Node enterLoop(Node loopLabel) + { + Node loop = nf.createLoopNode(loopLabel, ts.getLineno()); + if (loopSet == null) { + loopSet = new ObjArray(); + if (loopAndSwitchSet == null) { + loopAndSwitchSet = new ObjArray(); + } + } + loopSet.push(loop); + loopAndSwitchSet.push(loop); + return loop; + } + + private void exitLoop() + { + loopSet.pop(); + loopAndSwitchSet.pop(); + } + + private Node enterSwitch(Node switchSelector, int lineno) + { + Node switchNode = nf.createSwitch(switchSelector, lineno); + if (loopAndSwitchSet == null) { + loopAndSwitchSet = new ObjArray(); + } + loopAndSwitchSet.push(switchNode); + return switchNode; + } + + private void exitSwitch() + { + loopAndSwitchSet.pop(); + } + + /* + * Build a parse tree from the given sourceString. + * + * @return an Object representing the parsed + * program. If the parse fails, null will be returned. (The + * parse failure will result in a call to the ErrorReporter from + * CompilerEnvirons.) + */ + public ScriptOrFnNode parse(String sourceString, + String sourceURI, int lineno) + { + this.sourceURI = sourceURI; + this.ts = new TokenStream(this, null, sourceString, lineno); + try { + return parse(); + } catch (IOException ex) { + // Should never happen + throw new IllegalStateException(); + } + } + + /* + * Build a parse tree from the given sourceString. + * + * @return an Object representing the parsed + * program. If the parse fails, null will be returned. (The + * parse failure will result in a call to the ErrorReporter from + * CompilerEnvirons.) + */ + public ScriptOrFnNode parse(Reader sourceReader, + String sourceURI, int lineno) + throws IOException + { + this.sourceURI = sourceURI; + this.ts = new TokenStream(this, sourceReader, null, lineno); + return parse(); + } + + private ScriptOrFnNode parse() + throws IOException + { + this.decompiler = createDecompiler(compilerEnv); + this.nf = new IRFactory(this); + currentScriptOrFn = nf.createScript(); + int sourceStartOffset = decompiler.getCurrentOffset(); + this.encodedSource = null; + decompiler.addToken(Token.SCRIPT); + + this.currentFlaggedToken = Token.EOF; + this.syntaxErrorCount = 0; + + int baseLineno = ts.getLineno(); // line number where source starts + + /* so we have something to add nodes to until + * we've collected all the source */ + Node pn = nf.createLeaf(Token.BLOCK); + + try { + for (;;) { + int tt = peekToken(); + + if (tt <= Token.EOF) { + break; + } + + Node n; + if (tt == Token.FUNCTION) { + consumeToken(); + try { + n = function(calledByCompileFunction + ? FunctionNode.FUNCTION_EXPRESSION + : FunctionNode.FUNCTION_STATEMENT); + } catch (ParserException e) { + break; + } + } else { + n = statement(); + } + nf.addChildToBack(pn, n); + } + } catch (StackOverflowError ex) { + String msg = ScriptRuntime.getMessage0( + "msg.too.deep.parser.recursion"); + throw Context.reportRuntimeError(msg, sourceURI, + ts.getLineno(), null, 0); + } + + if (this.syntaxErrorCount != 0) { + String msg = String.valueOf(this.syntaxErrorCount); + msg = ScriptRuntime.getMessage1("msg.got.syntax.errors", msg); + throw errorReporter.runtimeError(msg, sourceURI, baseLineno, + null, 0); + } + + currentScriptOrFn.setSourceName(sourceURI); + currentScriptOrFn.setBaseLineno(baseLineno); + currentScriptOrFn.setEndLineno(ts.getLineno()); + + int sourceEndOffset = decompiler.getCurrentOffset(); + currentScriptOrFn.setEncodedSourceBounds(sourceStartOffset, + sourceEndOffset); + + nf.initScript(currentScriptOrFn, pn); + + if (compilerEnv.isGeneratingSource()) { + encodedSource = decompiler.getEncodedSource(); + } + this.decompiler = null; // It helps GC + + return currentScriptOrFn; + } + + /* + * The C version of this function takes an argument list, + * which doesn't seem to be needed for tree generation... + * it'd only be useful for checking argument hiding, which + * I'm not doing anyway... + */ + private Node parseFunctionBody() + throws IOException + { + ++nestingOfFunction; + Node pn = nf.createBlock(ts.getLineno()); + try { + bodyLoop: for (;;) { + Node n; + int tt = peekToken(); + switch (tt) { + case Token.ERROR: + case Token.EOF: + case Token.RC: + break bodyLoop; + + case Token.FUNCTION: + consumeToken(); + n = function(FunctionNode.FUNCTION_STATEMENT); + break; + default: + n = statement(); + break; + } + nf.addChildToBack(pn, n); + } + } catch (ParserException e) { + // Ignore it + } finally { + --nestingOfFunction; + } + + return pn; + } + + private Node function(int functionType) + throws IOException, ParserException + { + int syntheticType = functionType; + int baseLineno = ts.getLineno(); // line number where source starts + + int functionSourceStart = decompiler.markFunctionStart(functionType); + String name; + Node memberExprNode = null; + if (matchToken(Token.NAME)) { + name = ts.getString(); + decompiler.addName(name); + if (!matchToken(Token.LP)) { + if (compilerEnv.isAllowMemberExprAsFunctionName()) { + // Extension to ECMA: if 'function <name>' does not follow + // by '(', assume <name> starts memberExpr + Node memberExprHead = nf.createName(name); + name = ""; + memberExprNode = memberExprTail(false, memberExprHead); + } + mustMatchToken(Token.LP, "msg.no.paren.parms"); + } + } else if (matchToken(Token.LP)) { + // Anonymous function + name = ""; + } else { + name = ""; + if (compilerEnv.isAllowMemberExprAsFunctionName()) { + // Note that memberExpr can not start with '(' like + // in function (1+2).toString(), because 'function (' already + // processed as anonymous function + memberExprNode = memberExpr(false); + } + mustMatchToken(Token.LP, "msg.no.paren.parms"); + } + + if (memberExprNode != null) { + syntheticType = FunctionNode.FUNCTION_EXPRESSION; + } + + boolean nested = insideFunction(); + + FunctionNode fnNode = nf.createFunction(name); + if (nested || nestingOfWith > 0) { + // 1. Nested functions are not affected by the dynamic scope flag + // as dynamic scope is already a parent of their scope. + // 2. Functions defined under the with statement also immune to + // this setup, in which case dynamic scope is ignored in favor + // of with object. + fnNode.itsIgnoreDynamicScope = true; + } + + int functionIndex = currentScriptOrFn.addFunction(fnNode); + + int functionSourceEnd; + + ScriptOrFnNode savedScriptOrFn = currentScriptOrFn; + currentScriptOrFn = fnNode; + int savedNestingOfWith = nestingOfWith; + nestingOfWith = 0; + Hashtable savedLabelSet = labelSet; + labelSet = null; + ObjArray savedLoopSet = loopSet; + loopSet = null; + ObjArray savedLoopAndSwitchSet = loopAndSwitchSet; + loopAndSwitchSet = null; + boolean savedHasReturnValue = hasReturnValue; + int savedFunctionEndFlags = functionEndFlags; + + Node body; + try { + decompiler.addToken(Token.LP); + if (!matchToken(Token.RP)) { + boolean first = true; + do { + if (!first) + decompiler.addToken(Token.COMMA); + first = false; + mustMatchToken(Token.NAME, "msg.no.parm"); + String s = ts.getString(); + if (fnNode.hasParamOrVar(s)) { + addWarning("msg.dup.parms", s); + } + fnNode.addParam(s); + decompiler.addName(s); + } while (matchToken(Token.COMMA)); + + mustMatchToken(Token.RP, "msg.no.paren.after.parms"); + } + decompiler.addToken(Token.RP); + + mustMatchToken(Token.LC, "msg.no.brace.body"); + decompiler.addEOL(Token.LC); + body = parseFunctionBody(); + mustMatchToken(Token.RC, "msg.no.brace.after.body"); + + if (compilerEnv.isStrictMode() && !body.hasConsistentReturnUsage()) + { + String msg = name.length() > 0 ? "msg.no.return.value" + : "msg.anon.no.return.value"; + addStrictWarning(msg, name); + } + + decompiler.addToken(Token.RC); + functionSourceEnd = decompiler.markFunctionEnd(functionSourceStart); + if (functionType != FunctionNode.FUNCTION_EXPRESSION) { + // Add EOL only if function is not part of expression + // since it gets SEMI + EOL from Statement in that case + decompiler.addToken(Token.EOL); + } + } + finally { + hasReturnValue = savedHasReturnValue; + functionEndFlags = savedFunctionEndFlags; + loopAndSwitchSet = savedLoopAndSwitchSet; + loopSet = savedLoopSet; + labelSet = savedLabelSet; + nestingOfWith = savedNestingOfWith; + currentScriptOrFn = savedScriptOrFn; + } + + fnNode.setEncodedSourceBounds(functionSourceStart, functionSourceEnd); + fnNode.setSourceName(sourceURI); + fnNode.setBaseLineno(baseLineno); + fnNode.setEndLineno(ts.getLineno()); + + if (name != null) { + int index = currentScriptOrFn.getParamOrVarIndex(name); + if (index >= 0 && index < currentScriptOrFn.getParamCount()) + addStrictWarning("msg.var.hides.arg", name); + } + + Node pn = nf.initFunction(fnNode, functionIndex, body, syntheticType); + if (memberExprNode != null) { + pn = nf.createAssignment(Token.ASSIGN, memberExprNode, pn); + if (functionType != FunctionNode.FUNCTION_EXPRESSION) { + // XXX check JScript behavior: should it be createExprStatement? + pn = nf.createExprStatementNoReturn(pn, baseLineno); + } + } + return pn; + } + + private Node statements() + throws IOException + { + Node pn = nf.createBlock(ts.getLineno()); + + int tt; + while((tt = peekToken()) > Token.EOF && tt != Token.RC) { + nf.addChildToBack(pn, statement()); + } + + return pn; + } + + private Node condition() + throws IOException, ParserException + { + mustMatchToken(Token.LP, "msg.no.paren.cond"); + decompiler.addToken(Token.LP); + Node pn = expr(false); + mustMatchToken(Token.RP, "msg.no.paren.after.cond"); + decompiler.addToken(Token.RP); + + // Report strict warning on code like "if (a = 7) ...". Suppress the + // warning if the condition is parenthesized, like "if ((a = 7)) ...". + if (pn.getProp(Node.PARENTHESIZED_PROP) == null && + (pn.getType() == Token.SETNAME || pn.getType() == Token.SETPROP || + pn.getType() == Token.SETELEM)) + { + addStrictWarning("msg.equal.as.assign", ""); + } + return pn; + } + + // match a NAME; return null if no match. + private Node matchJumpLabelName() + throws IOException, ParserException + { + Node label = null; + + int tt = peekTokenOrEOL(); + if (tt == Token.NAME) { + consumeToken(); + String name = ts.getString(); + decompiler.addName(name); + if (labelSet != null) { + label = (Node)labelSet.get(name); + } + if (label == null) { + reportError("msg.undef.label"); + } + } + + return label; + } + + private Node statement() + throws IOException + { + try { + Node pn = statementHelper(null); + if (pn != null) { + if (compilerEnv.isStrictMode() && !pn.hasSideEffects()) + addStrictWarning("msg.no.side.effects", ""); + return pn; + } + } catch (ParserException e) { } + + // skip to end of statement + int lineno = ts.getLineno(); + guessingStatementEnd: for (;;) { + int tt = peekTokenOrEOL(); + consumeToken(); + switch (tt) { + case Token.ERROR: + case Token.EOF: + case Token.EOL: + case Token.SEMI: + break guessingStatementEnd; + } + } + return nf.createExprStatement(nf.createName("error"), lineno); + } + + /** + * Whether the "catch (e: e instanceof Exception) { ... }" syntax + * is implemented. + */ + + private Node statementHelper(Node statementLabel) + throws IOException, ParserException + { + Node pn = null; + + int tt; + + tt = peekToken(); + + switch(tt) { + case Token.IF: { + consumeToken(); + + decompiler.addToken(Token.IF); + int lineno = ts.getLineno(); + Node cond = condition(); + decompiler.addEOL(Token.LC); + Node ifTrue = statement(); + Node ifFalse = null; + if (matchToken(Token.ELSE)) { + decompiler.addToken(Token.RC); + decompiler.addToken(Token.ELSE); + decompiler.addEOL(Token.LC); + ifFalse = statement(); + } + decompiler.addEOL(Token.RC); + pn = nf.createIf(cond, ifTrue, ifFalse, lineno); + return pn; + } + + case Token.SWITCH: { + consumeToken(); + + decompiler.addToken(Token.SWITCH); + int lineno = ts.getLineno(); + mustMatchToken(Token.LP, "msg.no.paren.switch"); + decompiler.addToken(Token.LP); + pn = enterSwitch(expr(false), lineno); + try { + mustMatchToken(Token.RP, "msg.no.paren.after.switch"); + decompiler.addToken(Token.RP); + mustMatchToken(Token.LC, "msg.no.brace.switch"); + decompiler.addEOL(Token.LC); + + boolean hasDefault = false; + switchLoop: for (;;) { + tt = nextToken(); + Node caseExpression; + switch (tt) { + case Token.RC: + break switchLoop; + + case Token.CASE: + decompiler.addToken(Token.CASE); + caseExpression = expr(false); + mustMatchToken(Token.COLON, "msg.no.colon.case"); + decompiler.addEOL(Token.COLON); + break; + + case Token.DEFAULT: + if (hasDefault) { + reportError("msg.double.switch.default"); + } + decompiler.addToken(Token.DEFAULT); + hasDefault = true; + caseExpression = null; + mustMatchToken(Token.COLON, "msg.no.colon.case"); + decompiler.addEOL(Token.COLON); + break; + + default: + reportError("msg.bad.switch"); + break switchLoop; + } + + Node block = nf.createLeaf(Token.BLOCK); + while ((tt = peekToken()) != Token.RC + && tt != Token.CASE + && tt != Token.DEFAULT + && tt != Token.EOF) + { + nf.addChildToBack(block, statement()); + } + + // caseExpression == null => add default lable + nf.addSwitchCase(pn, caseExpression, block); + } + decompiler.addEOL(Token.RC); + nf.closeSwitch(pn); + } finally { + exitSwitch(); + } + return pn; + } + + case Token.WHILE: { + consumeToken(); + decompiler.addToken(Token.WHILE); + + Node loop = enterLoop(statementLabel); + try { + Node cond = condition(); + decompiler.addEOL(Token.LC); + Node body = statement(); + decompiler.addEOL(Token.RC); + pn = nf.createWhile(loop, cond, body); + } finally { + exitLoop(); + } + return pn; + } + + case Token.DO: { + consumeToken(); + decompiler.addToken(Token.DO); + decompiler.addEOL(Token.LC); + + Node loop = enterLoop(statementLabel); + try { + Node body = statement(); + decompiler.addToken(Token.RC); + mustMatchToken(Token.WHILE, "msg.no.while.do"); + decompiler.addToken(Token.WHILE); + Node cond = condition(); + pn = nf.createDoWhile(loop, body, cond); + } finally { + exitLoop(); + } + // Always auto-insert semicon to follow SpiderMonkey: + // It is required by EMAScript but is ignored by the rest of + // world, see bug 238945 + matchToken(Token.SEMI); + decompiler.addEOL(Token.SEMI); + return pn; + } + + case Token.FOR: { + consumeToken(); + boolean isForEach = false; + decompiler.addToken(Token.FOR); + + Node loop = enterLoop(statementLabel); + try { + + Node init; // Node init is also foo in 'foo in Object' + Node cond; // Node cond is also object in 'foo in Object' + Node incr = null; // to kill warning + Node body; + + // See if this is a for each () instead of just a for () + if (matchToken(Token.NAME)) { + decompiler.addName(ts.getString()); + if (ts.getString().equals("each")) { + isForEach = true; + } else { + reportError("msg.no.paren.for"); + } + } + + mustMatchToken(Token.LP, "msg.no.paren.for"); + decompiler.addToken(Token.LP); + tt = peekToken(); + if (tt == Token.SEMI) { + init = nf.createLeaf(Token.EMPTY); + } else { + if (tt == Token.VAR) { + // set init to a var list or initial + consumeToken(); // consume the 'var' token + init = variables(Token.FOR); + } + else { + init = expr(true); + } + } + + if (matchToken(Token.IN)) { + decompiler.addToken(Token.IN); + // 'cond' is the object over which we're iterating + cond = expr(false); + } else { // ordinary for loop + mustMatchToken(Token.SEMI, "msg.no.semi.for"); + decompiler.addToken(Token.SEMI); + if (peekToken() == Token.SEMI) { + // no loop condition + cond = nf.createLeaf(Token.EMPTY); + } else { + cond = expr(false); + } + + mustMatchToken(Token.SEMI, "msg.no.semi.for.cond"); + decompiler.addToken(Token.SEMI); + if (peekToken() == Token.RP) { + incr = nf.createLeaf(Token.EMPTY); + } else { + incr = expr(false); + } + } + + mustMatchToken(Token.RP, "msg.no.paren.for.ctrl"); + decompiler.addToken(Token.RP); + decompiler.addEOL(Token.LC); + body = statement(); + decompiler.addEOL(Token.RC); + + if (incr == null) { + // cond could be null if 'in obj' got eaten + // by the init node. + pn = nf.createForIn(loop, init, cond, body, isForEach); + } else { + pn = nf.createFor(loop, init, cond, incr, body); + } + } finally { + exitLoop(); + } + return pn; + } + + case Token.TRY: { + consumeToken(); + int lineno = ts.getLineno(); + + Node tryblock; + Node catchblocks = null; + Node finallyblock = null; + + decompiler.addToken(Token.TRY); + decompiler.addEOL(Token.LC); + tryblock = statement(); + decompiler.addEOL(Token.RC); + + catchblocks = nf.createLeaf(Token.BLOCK); + + boolean sawDefaultCatch = false; + int peek = peekToken(); + if (peek == Token.CATCH) { + while (matchToken(Token.CATCH)) { + if (sawDefaultCatch) { + reportError("msg.catch.unreachable"); + } + decompiler.addToken(Token.CATCH); + mustMatchToken(Token.LP, "msg.no.paren.catch"); + decompiler.addToken(Token.LP); + + mustMatchToken(Token.NAME, "msg.bad.catchcond"); + String varName = ts.getString(); + decompiler.addName(varName); + + Node catchCond = null; + if (matchToken(Token.IF)) { + decompiler.addToken(Token.IF); + catchCond = expr(false); + } else { + sawDefaultCatch = true; + } + + mustMatchToken(Token.RP, "msg.bad.catchcond"); + decompiler.addToken(Token.RP); + mustMatchToken(Token.LC, "msg.no.brace.catchblock"); + decompiler.addEOL(Token.LC); + + nf.addChildToBack(catchblocks, + nf.createCatch(varName, catchCond, + statements(), + ts.getLineno())); + + mustMatchToken(Token.RC, "msg.no.brace.after.body"); + decompiler.addEOL(Token.RC); + } + } else if (peek != Token.FINALLY) { + mustMatchToken(Token.FINALLY, "msg.try.no.catchfinally"); + } + + if (matchToken(Token.FINALLY)) { + decompiler.addToken(Token.FINALLY); + decompiler.addEOL(Token.LC); + finallyblock = statement(); + decompiler.addEOL(Token.RC); + } + + pn = nf.createTryCatchFinally(tryblock, catchblocks, + finallyblock, lineno); + + return pn; + } + + case Token.THROW: { + consumeToken(); + if (peekTokenOrEOL() == Token.EOL) { + // ECMAScript does not allow new lines before throw expression, + // see bug 256617 + reportError("msg.bad.throw.eol"); + } + + int lineno = ts.getLineno(); + decompiler.addToken(Token.THROW); + pn = nf.createThrow(expr(false), lineno); + break; + } + + case Token.BREAK: { + consumeToken(); + int lineno = ts.getLineno(); + + decompiler.addToken(Token.BREAK); + + // matchJumpLabelName only matches if there is one + Node breakStatement = matchJumpLabelName(); + if (breakStatement == null) { + if (loopAndSwitchSet == null || loopAndSwitchSet.size() == 0) { + reportError("msg.bad.break"); + return null; + } + breakStatement = (Node)loopAndSwitchSet.peek(); + } + pn = nf.createBreak(breakStatement, lineno); + break; + } + + case Token.CONTINUE: { + consumeToken(); + int lineno = ts.getLineno(); + + decompiler.addToken(Token.CONTINUE); + + Node loop; + // matchJumpLabelName only matches if there is one + Node label = matchJumpLabelName(); + if (label == null) { + if (loopSet == null || loopSet.size() == 0) { + reportError("msg.continue.outside"); + return null; + } + loop = (Node)loopSet.peek(); + } else { + loop = nf.getLabelLoop(label); + if (loop == null) { + reportError("msg.continue.nonloop"); + return null; + } + } + pn = nf.createContinue(loop, lineno); + break; + } + + case Token.WITH: { + consumeToken(); + + decompiler.addToken(Token.WITH); + int lineno = ts.getLineno(); + mustMatchToken(Token.LP, "msg.no.paren.with"); + decompiler.addToken(Token.LP); + Node obj = expr(false); + mustMatchToken(Token.RP, "msg.no.paren.after.with"); + decompiler.addToken(Token.RP); + decompiler.addEOL(Token.LC); + + ++nestingOfWith; + Node body; + try { + body = statement(); + } finally { + --nestingOfWith; + } + + decompiler.addEOL(Token.RC); + + pn = nf.createWith(obj, body, lineno); + return pn; + } + + case Token.CONST: + case Token.VAR: { + consumeToken(); + pn = variables(tt); + break; + } + + case Token.RETURN: { + if (!insideFunction()) { + reportError("msg.bad.return"); + } + consumeToken(); + decompiler.addToken(Token.RETURN); + int lineno = ts.getLineno(); + + Node retExpr; + /* This is ugly, but we don't want to require a semicolon. */ + tt = peekTokenOrEOL(); + switch (tt) { + case Token.SEMI: + case Token.RC: + case Token.EOF: + case Token.EOL: + case Token.ERROR: + retExpr = null; + break; + default: + retExpr = expr(false); + hasReturnValue = true; + } + pn = nf.createReturn(retExpr, lineno); + + // see if we need a strict mode warning + if (retExpr == null) { + if (functionEndFlags == Node.END_RETURNS_VALUE) + addStrictWarning("msg.return.inconsistent", ""); + + functionEndFlags |= Node.END_RETURNS; + } else { + if (functionEndFlags == Node.END_RETURNS) + addStrictWarning("msg.return.inconsistent", ""); + + functionEndFlags |= Node.END_RETURNS_VALUE; + } + + break; + } + + case Token.LC: + consumeToken(); + if (statementLabel != null) { + decompiler.addToken(Token.LC); + } + pn = statements(); + mustMatchToken(Token.RC, "msg.no.brace.block"); + if (statementLabel != null) { + decompiler.addEOL(Token.RC); + } + return pn; + + case Token.ERROR: + // Fall thru, to have a node for error recovery to work on + case Token.SEMI: + consumeToken(); + pn = nf.createLeaf(Token.EMPTY); + return pn; + + case Token.FUNCTION: { + consumeToken(); + pn = function(FunctionNode.FUNCTION_EXPRESSION_STATEMENT); + return pn; + } + + case Token.DEFAULT : + consumeToken(); + mustHaveXML(); + + decompiler.addToken(Token.DEFAULT); + int nsLine = ts.getLineno(); + + if (!(matchToken(Token.NAME) + && ts.getString().equals("xml"))) + { + reportError("msg.bad.namespace"); + } + decompiler.addName(" xml"); + + if (!(matchToken(Token.NAME) + && ts.getString().equals("namespace"))) + { + reportError("msg.bad.namespace"); + } + decompiler.addName(" namespace"); + + if (!matchToken(Token.ASSIGN)) { + reportError("msg.bad.namespace"); + } + decompiler.addToken(Token.ASSIGN); + + Node expr = expr(false); + pn = nf.createDefaultNamespace(expr, nsLine); + break; + + case Token.NAME: { + int lineno = ts.getLineno(); + String name = ts.getString(); + setCheckForLabel(); + pn = expr(false); + if (pn.getType() != Token.LABEL) { + pn = nf.createExprStatement(pn, lineno); + } else { + // Parsed the label: push back token should be + // colon that primaryExpr left untouched. + if (peekToken() != Token.COLON) Kit.codeBug(); + consumeToken(); + // depend on decompiling lookahead to guess that that + // last name was a label. + decompiler.addName(name); + decompiler.addEOL(Token.COLON); + + if (labelSet == null) { + labelSet = new Hashtable(); + } else if (labelSet.containsKey(name)) { + reportError("msg.dup.label"); + } + + boolean firstLabel; + if (statementLabel == null) { + firstLabel = true; + statementLabel = pn; + } else { + // Discard multiple label nodes and use only + // the first: it allows to simplify IRFactory + firstLabel = false; + } + labelSet.put(name, statementLabel); + try { + pn = statementHelper(statementLabel); + } finally { + labelSet.remove(name); + } + if (firstLabel) { + pn = nf.createLabeledStatement(statementLabel, pn); + } + return pn; + } + break; + } + + default: { + int lineno = ts.getLineno(); + pn = expr(false); + pn = nf.createExprStatement(pn, lineno); + break; + } + } + + int ttFlagged = peekFlaggedToken(); + switch (ttFlagged & CLEAR_TI_MASK) { + case Token.SEMI: + // Consume ';' as a part of expression + consumeToken(); + break; + case Token.ERROR: + case Token.EOF: + case Token.RC: + // Autoinsert ; + break; + default: + if ((ttFlagged & TI_AFTER_EOL) == 0) { + // Report error if no EOL or autoinsert ; otherwise + reportError("msg.no.semi.stmt"); + } + break; + } + decompiler.addEOL(Token.SEMI); + + return pn; + } + + /** + * Parse a 'var' or 'const' statement, or a 'var' init list in a for + * statement. + * @param context A token value: either VAR, CONST or FOR depending on + * context. + * @return The parsed statement + * @throws IOException + * @throws ParserException + */ + private Node variables(int context) + throws IOException, ParserException + { + Node pn; + boolean first = true; + + if (context == Token.CONST){ + pn = nf.createVariables(Token.CONST, ts.getLineno()); + decompiler.addToken(Token.CONST); + } else { + pn = nf.createVariables(Token.VAR, ts.getLineno()); + decompiler.addToken(Token.VAR); + } + + for (;;) { + Node name; + Node init; + mustMatchToken(Token.NAME, "msg.bad.var"); + String s = ts.getString(); + + if (!first) + decompiler.addToken(Token.COMMA); + first = false; + + decompiler.addName(s); + + if (context == Token.CONST) { + if (!currentScriptOrFn.addConst(s)) { + // We know it's already defined, since addConst passes if + // it's not defined at all. The addVar call just confirms + // what it is. + if (currentScriptOrFn.addVar(s) != ScriptOrFnNode.DUPLICATE_CONST) + addError("msg.var.redecl", s); + else + addError("msg.const.redecl", s); + } + } else { + int dupState = currentScriptOrFn.addVar(s); + if (dupState == ScriptOrFnNode.DUPLICATE_CONST) + addError("msg.const.redecl", s); + else if (dupState == ScriptOrFnNode.DUPLICATE_PARAMETER) + addStrictWarning("msg.var.hides.arg", s); + else if (dupState == ScriptOrFnNode.DUPLICATE_VAR) + addStrictWarning("msg.var.redecl", s); + } + name = nf.createName(s); + + // omitted check for argument hiding + + if (matchToken(Token.ASSIGN)) { + decompiler.addToken(Token.ASSIGN); + + init = assignExpr(context == Token.FOR); + nf.addChildToBack(name, init); + } + nf.addChildToBack(pn, name); + if (!matchToken(Token.COMMA)) + break; + } + return pn; + } + + private Node expr(boolean inForInit) + throws IOException, ParserException + { + Node pn = assignExpr(inForInit); + while (matchToken(Token.COMMA)) { + decompiler.addToken(Token.COMMA); + if (compilerEnv.isStrictMode() && !pn.hasSideEffects()) + addStrictWarning("msg.no.side.effects", ""); + pn = nf.createBinary(Token.COMMA, pn, assignExpr(inForInit)); + } + return pn; + } + + private Node assignExpr(boolean inForInit) + throws IOException, ParserException + { + Node pn = condExpr(inForInit); + + int tt = peekToken(); + if (Token.FIRST_ASSIGN <= tt && tt <= Token.LAST_ASSIGN) { + consumeToken(); + decompiler.addToken(tt); + pn = nf.createAssignment(tt, pn, assignExpr(inForInit)); + } + + return pn; + } + + private Node condExpr(boolean inForInit) + throws IOException, ParserException + { + Node pn = orExpr(inForInit); + + if (matchToken(Token.HOOK)) { + decompiler.addToken(Token.HOOK); + Node ifTrue = assignExpr(false); + mustMatchToken(Token.COLON, "msg.no.colon.cond"); + decompiler.addToken(Token.COLON); + Node ifFalse = assignExpr(inForInit); + return nf.createCondExpr(pn, ifTrue, ifFalse); + } + + return pn; + } + + private Node orExpr(boolean inForInit) + throws IOException, ParserException + { + Node pn = andExpr(inForInit); + if (matchToken(Token.OR)) { + decompiler.addToken(Token.OR); + pn = nf.createBinary(Token.OR, pn, orExpr(inForInit)); + } + + return pn; + } + + private Node andExpr(boolean inForInit) + throws IOException, ParserException + { + Node pn = bitOrExpr(inForInit); + if (matchToken(Token.AND)) { + decompiler.addToken(Token.AND); + pn = nf.createBinary(Token.AND, pn, andExpr(inForInit)); + } + + return pn; + } + + private Node bitOrExpr(boolean inForInit) + throws IOException, ParserException + { + Node pn = bitXorExpr(inForInit); + while (matchToken(Token.BITOR)) { + decompiler.addToken(Token.BITOR); + pn = nf.createBinary(Token.BITOR, pn, bitXorExpr(inForInit)); + } + return pn; + } + + private Node bitXorExpr(boolean inForInit) + throws IOException, ParserException + { + Node pn = bitAndExpr(inForInit); + while (matchToken(Token.BITXOR)) { + decompiler.addToken(Token.BITXOR); + pn = nf.createBinary(Token.BITXOR, pn, bitAndExpr(inForInit)); + } + return pn; + } + + private Node bitAndExpr(boolean inForInit) + throws IOException, ParserException + { + Node pn = eqExpr(inForInit); + while (matchToken(Token.BITAND)) { + decompiler.addToken(Token.BITAND); + pn = nf.createBinary(Token.BITAND, pn, eqExpr(inForInit)); + } + return pn; + } + + private Node eqExpr(boolean inForInit) + throws IOException, ParserException + { + Node pn = relExpr(inForInit); + for (;;) { + int tt = peekToken(); + switch (tt) { + case Token.EQ: + case Token.NE: + case Token.SHEQ: + case Token.SHNE: + consumeToken(); + int decompilerToken = tt; + int parseToken = tt; + if (compilerEnv.getLanguageVersion() == Context.VERSION_1_2) { + // JavaScript 1.2 uses shallow equality for == and != . + // In addition, convert === and !== for decompiler into + // == and != since the decompiler is supposed to show + // canonical source and in 1.2 ===, !== are allowed + // only as an alias to ==, !=. + switch (tt) { + case Token.EQ: + parseToken = Token.SHEQ; + break; + case Token.NE: + parseToken = Token.SHNE; + break; + case Token.SHEQ: + decompilerToken = Token.EQ; + break; + case Token.SHNE: + decompilerToken = Token.NE; + break; + } + } + decompiler.addToken(decompilerToken); + pn = nf.createBinary(parseToken, pn, relExpr(inForInit)); + continue; + } + break; + } + return pn; + } + + private Node relExpr(boolean inForInit) + throws IOException, ParserException + { + Node pn = shiftExpr(); + for (;;) { + int tt = peekToken(); + switch (tt) { + case Token.IN: + if (inForInit) + break; + // fall through + case Token.INSTANCEOF: + case Token.LE: + case Token.LT: + case Token.GE: + case Token.GT: + consumeToken(); + decompiler.addToken(tt); + pn = nf.createBinary(tt, pn, shiftExpr()); + continue; + } + break; + } + return pn; + } + + private Node shiftExpr() + throws IOException, ParserException + { + Node pn = addExpr(); + for (;;) { + int tt = peekToken(); + switch (tt) { + case Token.LSH: + case Token.URSH: + case Token.RSH: + consumeToken(); + decompiler.addToken(tt); + pn = nf.createBinary(tt, pn, addExpr()); + continue; + } + break; + } + return pn; + } + + private Node addExpr() + throws IOException, ParserException + { + Node pn = mulExpr(); + for (;;) { + int tt = peekToken(); + if (tt == Token.ADD || tt == Token.SUB) { + consumeToken(); + decompiler.addToken(tt); + // flushNewLines + pn = nf.createBinary(tt, pn, mulExpr()); + continue; + } + break; + } + + return pn; + } + + private Node mulExpr() + throws IOException, ParserException + { + Node pn = unaryExpr(); + for (;;) { + int tt = peekToken(); + switch (tt) { + case Token.MUL: + case Token.DIV: + case Token.MOD: + consumeToken(); + decompiler.addToken(tt); + pn = nf.createBinary(tt, pn, unaryExpr()); + continue; + } + break; + } + + return pn; + } + + private Node unaryExpr() + throws IOException, ParserException + { + int tt; + + tt = peekToken(); + + switch(tt) { + case Token.VOID: + case Token.NOT: + case Token.BITNOT: + case Token.TYPEOF: + consumeToken(); + decompiler.addToken(tt); + return nf.createUnary(tt, unaryExpr()); + + case Token.ADD: + consumeToken(); + // Convert to special POS token in decompiler and parse tree + decompiler.addToken(Token.POS); + return nf.createUnary(Token.POS, unaryExpr()); + + case Token.SUB: + consumeToken(); + // Convert to special NEG token in decompiler and parse tree + decompiler.addToken(Token.NEG); + return nf.createUnary(Token.NEG, unaryExpr()); + + case Token.INC: + case Token.DEC: + consumeToken(); + decompiler.addToken(tt); + return nf.createIncDec(tt, false, memberExpr(true)); + + case Token.DELPROP: + consumeToken(); + decompiler.addToken(Token.DELPROP); + return nf.createUnary(Token.DELPROP, unaryExpr()); + + case Token.ERROR: + consumeToken(); + break; + + // XML stream encountered in expression. + case Token.LT: + if (compilerEnv.isXmlAvailable()) { + consumeToken(); + Node pn = xmlInitializer(); + return memberExprTail(true, pn); + } + // Fall thru to the default handling of RELOP + + default: + Node pn = memberExpr(true); + + // Don't look across a newline boundary for a postfix incop. + tt = peekTokenOrEOL(); + if (tt == Token.INC || tt == Token.DEC) { + consumeToken(); + decompiler.addToken(tt); + return nf.createIncDec(tt, true, pn); + } + return pn; + } + return nf.createName("err"); // Only reached on error. Try to continue. + + } + + private Node xmlInitializer() throws IOException + { + int tt = ts.getFirstXMLToken(); + if (tt != Token.XML && tt != Token.XMLEND) { + reportError("msg.syntax"); + return null; + } + + /* Make a NEW node to append to. */ + Node pnXML = nf.createLeaf(Token.NEW); + + String xml = ts.getString(); + boolean fAnonymous = xml.trim().startsWith("<>"); + + Node pn = nf.createName(fAnonymous ? "XMLList" : "XML"); + nf.addChildToBack(pnXML, pn); + + pn = null; + Node expr; + for (;;tt = ts.getNextXMLToken()) { + switch (tt) { + case Token.XML: + xml = ts.getString(); + decompiler.addName(xml); + mustMatchToken(Token.LC, "msg.syntax"); + decompiler.addToken(Token.LC); + expr = (peekToken() == Token.RC) + ? nf.createString("") + : expr(false); + mustMatchToken(Token.RC, "msg.syntax"); + decompiler.addToken(Token.RC); + if (pn == null) { + pn = nf.createString(xml); + } else { + pn = nf.createBinary(Token.ADD, pn, nf.createString(xml)); + } + if (ts.isXMLAttribute()) { + /* Need to put the result in double quotes */ + expr = nf.createUnary(Token.ESCXMLATTR, expr); + Node prepend = nf.createBinary(Token.ADD, + nf.createString("\""), + expr); + expr = nf.createBinary(Token.ADD, + prepend, + nf.createString("\"")); + } else { + expr = nf.createUnary(Token.ESCXMLTEXT, expr); + } + pn = nf.createBinary(Token.ADD, pn, expr); + break; + case Token.XMLEND: + xml = ts.getString(); + decompiler.addName(xml); + if (pn == null) { + pn = nf.createString(xml); + } else { + pn = nf.createBinary(Token.ADD, pn, nf.createString(xml)); + } + + nf.addChildToBack(pnXML, pn); + return pnXML; + default: + reportError("msg.syntax"); + return null; + } + } + } + + private void argumentList(Node listNode) + throws IOException, ParserException + { + boolean matched; + matched = matchToken(Token.RP); + if (!matched) { + boolean first = true; + do { + if (!first) + decompiler.addToken(Token.COMMA); + first = false; + nf.addChildToBack(listNode, assignExpr(false)); + } while (matchToken(Token.COMMA)); + + mustMatchToken(Token.RP, "msg.no.paren.arg"); + } + decompiler.addToken(Token.RP); + } + + private Node memberExpr(boolean allowCallSyntax) + throws IOException, ParserException + { + int tt; + + Node pn; + + /* Check for new expressions. */ + tt = peekToken(); + if (tt == Token.NEW) { + /* Eat the NEW token. */ + consumeToken(); + decompiler.addToken(Token.NEW); + + /* Make a NEW node to append to. */ + pn = nf.createCallOrNew(Token.NEW, memberExpr(false)); + + if (matchToken(Token.LP)) { + decompiler.addToken(Token.LP); + /* Add the arguments to pn, if any are supplied. */ + argumentList(pn); + } + + /* XXX there's a check in the C source against + * "too many constructor arguments" - how many + * do we claim to support? + */ + + /* Experimental syntax: allow an object literal to follow a new expression, + * which will mean a kind of anonymous class built with the JavaAdapter. + * the object literal will be passed as an additional argument to the constructor. + */ + tt = peekToken(); + if (tt == Token.LC) { + nf.addChildToBack(pn, primaryExpr()); + } + } else { + pn = primaryExpr(); + } + + return memberExprTail(allowCallSyntax, pn); + } + + private Node memberExprTail(boolean allowCallSyntax, Node pn) + throws IOException, ParserException + { + tailLoop: + for (;;) { + int tt = peekToken(); + switch (tt) { + + case Token.DOT: + case Token.DOTDOT: + { + int memberTypeFlags; + String s; + + consumeToken(); + decompiler.addToken(tt); + memberTypeFlags = 0; + if (tt == Token.DOTDOT) { + mustHaveXML(); + memberTypeFlags = Node.DESCENDANTS_FLAG; + } + if (!compilerEnv.isXmlAvailable()) { + mustMatchToken(Token.NAME, "msg.no.name.after.dot"); + s = ts.getString(); + decompiler.addName(s); + pn = nf.createPropertyGet(pn, null, s, memberTypeFlags); + break; + } + + tt = nextToken(); + switch (tt) { + // handles: name, ns::name, ns::*, ns::[expr] + case Token.NAME: + s = ts.getString(); + decompiler.addName(s); + pn = propertyName(pn, s, memberTypeFlags); + break; + + // handles: *, *::name, *::*, *::[expr] + case Token.MUL: + decompiler.addName("*"); + pn = propertyName(pn, "*", memberTypeFlags); + break; + + // handles: '@attr', '@ns::attr', '@ns::*', '@ns::*', + // '@::attr', '@::*', '@*', '@*::attr', '@*::*' + case Token.XMLATTR: + decompiler.addToken(Token.XMLATTR); + pn = attributeAccess(pn, memberTypeFlags); + break; + + default: + reportError("msg.no.name.after.dot"); + } + } + break; + + case Token.DOTQUERY: + consumeToken(); + mustHaveXML(); + decompiler.addToken(Token.DOTQUERY); + pn = nf.createDotQuery(pn, expr(false), ts.getLineno()); + mustMatchToken(Token.RP, "msg.no.paren"); + decompiler.addToken(Token.RP); + break; + + case Token.LB: + consumeToken(); + decompiler.addToken(Token.LB); + pn = nf.createElementGet(pn, null, expr(false), 0); + mustMatchToken(Token.RB, "msg.no.bracket.index"); + decompiler.addToken(Token.RB); + break; + + case Token.LP: + if (!allowCallSyntax) { + break tailLoop; + } + consumeToken(); + decompiler.addToken(Token.LP); + pn = nf.createCallOrNew(Token.CALL, pn); + /* Add the arguments to pn, if any are supplied. */ + argumentList(pn); + break; + + default: + break tailLoop; + } + } + return pn; + } + + /* + * Xml attribute expression: + * '@attr', '@ns::attr', '@ns::*', '@ns::*', '@*', '@*::attr', '@*::*' + */ + private Node attributeAccess(Node pn, int memberTypeFlags) + throws IOException + { + memberTypeFlags |= Node.ATTRIBUTE_FLAG; + int tt = nextToken(); + + switch (tt) { + // handles: @name, @ns::name, @ns::*, @ns::[expr] + case Token.NAME: + { + String s = ts.getString(); + decompiler.addName(s); + pn = propertyName(pn, s, memberTypeFlags); + } + break; + + // handles: @*, @*::name, @*::*, @*::[expr] + case Token.MUL: + decompiler.addName("*"); + pn = propertyName(pn, "*", memberTypeFlags); + break; + + // handles @[expr] + case Token.LB: + decompiler.addToken(Token.LB); + pn = nf.createElementGet(pn, null, expr(false), memberTypeFlags); + mustMatchToken(Token.RB, "msg.no.bracket.index"); + decompiler.addToken(Token.RB); + break; + + default: + reportError("msg.no.name.after.xmlAttr"); + pn = nf.createPropertyGet(pn, null, "?", memberTypeFlags); + break; + } + + return pn; + } + + /** + * Check if :: follows name in which case it becomes qualified name + */ + private Node propertyName(Node pn, String name, int memberTypeFlags) + throws IOException, ParserException + { + String namespace = null; + if (matchToken(Token.COLONCOLON)) { + decompiler.addToken(Token.COLONCOLON); + namespace = name; + + int tt = nextToken(); + switch (tt) { + // handles name::name + case Token.NAME: + name = ts.getString(); + decompiler.addName(name); + break; + + // handles name::* + case Token.MUL: + decompiler.addName("*"); + name = "*"; + break; + + // handles name::[expr] + case Token.LB: + decompiler.addToken(Token.LB); + pn = nf.createElementGet(pn, namespace, expr(false), + memberTypeFlags); + mustMatchToken(Token.RB, "msg.no.bracket.index"); + decompiler.addToken(Token.RB); + return pn; + + default: + reportError("msg.no.name.after.coloncolon"); + name = "?"; + } + } + + pn = nf.createPropertyGet(pn, namespace, name, memberTypeFlags); + return pn; + } + + private Node primaryExpr() + throws IOException, ParserException + { + Node pn; + + int ttFlagged = nextFlaggedToken(); + int tt = ttFlagged & CLEAR_TI_MASK; + + switch(tt) { + + case Token.FUNCTION: + return function(FunctionNode.FUNCTION_EXPRESSION); + + case Token.LB: { + ObjArray elems = new ObjArray(); + int skipCount = 0; + decompiler.addToken(Token.LB); + boolean after_lb_or_comma = true; + for (;;) { + tt = peekToken(); + + if (tt == Token.COMMA) { + consumeToken(); + decompiler.addToken(Token.COMMA); + if (!after_lb_or_comma) { + after_lb_or_comma = true; + } else { + elems.add(null); + ++skipCount; + } + } else if (tt == Token.RB) { + consumeToken(); + decompiler.addToken(Token.RB); + break; + } else { + if (!after_lb_or_comma) { + reportError("msg.no.bracket.arg"); + } + elems.add(assignExpr(false)); + after_lb_or_comma = false; + } + } + return nf.createArrayLiteral(elems, skipCount); + } + + case Token.LC: { + ObjArray elems = new ObjArray(); + decompiler.addToken(Token.LC); + if (!matchToken(Token.RC)) { + + boolean first = true; + commaloop: + do { + Object property; + + if (!first) + decompiler.addToken(Token.COMMA); + else + first = false; + + tt = peekToken(); + switch(tt) { + case Token.NAME: + case Token.STRING: + consumeToken(); + // map NAMEs to STRINGs in object literal context + // but tell the decompiler the proper type + String s = ts.getString(); + if (tt == Token.NAME) { + if (s.equals("get") && + peekToken() == Token.NAME) { + decompiler.addToken(Token.GET); + consumeToken(); + s = ts.getString(); + decompiler.addName(s); + property = ScriptRuntime.getIndexObject(s); + if (!getterSetterProperty(elems, property, + true)) + break commaloop; + break; + } else if (s.equals("set") && + peekToken() == Token.NAME) { + decompiler.addToken(Token.SET); + consumeToken(); + s = ts.getString(); + decompiler.addName(s); + property = ScriptRuntime.getIndexObject(s); + if (!getterSetterProperty(elems, property, + false)) + break commaloop; + break; + } + decompiler.addName(s); + } else { + decompiler.addString(s); + } + property = ScriptRuntime.getIndexObject(s); + plainProperty(elems, property); + break; + + case Token.NUMBER: + consumeToken(); + double n = ts.getNumber(); + decompiler.addNumber(n); + property = ScriptRuntime.getIndexObject(n); + plainProperty(elems, property); + break; + + case Token.RC: + // trailing comma is OK. + break commaloop; + default: + reportError("msg.bad.prop"); + break commaloop; + } + } while (matchToken(Token.COMMA)); + + mustMatchToken(Token.RC, "msg.no.brace.prop"); + } + decompiler.addToken(Token.RC); + return nf.createObjectLiteral(elems); + } + + case Token.LP: + + /* Brendan's IR-jsparse.c makes a new node tagged with + * TOK_LP here... I'm not sure I understand why. Isn't + * the grouping already implicit in the structure of the + * parse tree? also TOK_LP is already overloaded (I + * think) in the C IR as 'function call.' */ + decompiler.addToken(Token.LP); + pn = expr(false); + pn.putProp(Node.PARENTHESIZED_PROP, Boolean.TRUE); + decompiler.addToken(Token.RP); + mustMatchToken(Token.RP, "msg.no.paren"); + return pn; + + case Token.XMLATTR: + mustHaveXML(); + decompiler.addToken(Token.XMLATTR); + pn = attributeAccess(null, 0); + return pn; + + case Token.NAME: { + String name = ts.getString(); + if ((ttFlagged & TI_CHECK_LABEL) != 0) { + if (peekToken() == Token.COLON) { + // Do not consume colon, it is used as unwind indicator + // to return to statementHelper. + // XXX Better way? + return nf.createLabel(ts.getLineno()); + } + } + + decompiler.addName(name); + if (compilerEnv.isXmlAvailable()) { + pn = propertyName(null, name, 0); + } else { + pn = nf.createName(name); + } + return pn; + } + + case Token.NUMBER: { + double n = ts.getNumber(); + decompiler.addNumber(n); + return nf.createNumber(n); + } + + case Token.STRING: { + String s = ts.getString(); + decompiler.addString(s); + return nf.createString(s); + } + + case Token.DIV: + case Token.ASSIGN_DIV: { + // Got / or /= which should be treated as regexp in fact + ts.readRegExp(tt); + String flags = ts.regExpFlags; + ts.regExpFlags = null; + String re = ts.getString(); + decompiler.addRegexp(re, flags); + int index = currentScriptOrFn.addRegexp(re, flags); + return nf.createRegExp(index); + } + + case Token.NULL: + case Token.THIS: + case Token.FALSE: + case Token.TRUE: + decompiler.addToken(tt); + return nf.createLeaf(tt); + + case Token.RESERVED: + reportError("msg.reserved.id"); + break; + + case Token.ERROR: + /* the scanner or one of its subroutines reported the error. */ + break; + + case Token.EOF: + reportError("msg.unexpected.eof"); + break; + + default: + reportError("msg.syntax"); + break; + } + return null; // should never reach here + } + + private void plainProperty(ObjArray elems, Object property) + throws IOException { + mustMatchToken(Token.COLON, "msg.no.colon.prop"); + + // OBJLIT is used as ':' in object literal for + // decompilation to solve spacing ambiguity. + decompiler.addToken(Token.OBJECTLIT); + elems.add(property); + elems.add(assignExpr(false)); + } + + private boolean getterSetterProperty(ObjArray elems, Object property, + boolean isGetter) throws IOException { + Node f = function(FunctionNode.FUNCTION_EXPRESSION); + if (f.getType() != Token.FUNCTION) { + reportError("msg.bad.prop"); + return false; + } + int fnIndex = f.getExistingIntProp(Node.FUNCTION_PROP); + FunctionNode fn = currentScriptOrFn.getFunctionNode(fnIndex); + if (fn.getFunctionName().length() != 0) { + reportError("msg.bad.prop"); + return false; + } + elems.add(property); + if (isGetter) { + elems.add(nf.createUnary(Token.GET, f)); + } else { + elems.add(nf.createUnary(Token.SET, f)); + } + return true; + } +} diff --git a/trunk/infrastructure/yuicompressor/src/yuicompressor/org/mozilla/javascript/Token.java b/trunk/infrastructure/yuicompressor/src/yuicompressor/org/mozilla/javascript/Token.java new file mode 100644 index 0000000..f082d68 --- /dev/null +++ b/trunk/infrastructure/yuicompressor/src/yuicompressor/org/mozilla/javascript/Token.java @@ -0,0 +1,420 @@ +/* -*- Mode: java; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is Rhino code, released + * May 6, 1999. + * + * The Initial Developer of the Original Code is + * Netscape Communications Corporation. + * Portions created by the Initial Developer are Copyright (C) 1997-1999 + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * Roger Lawrence + * Mike McCabe + * Igor Bukanov + * Bob Jervis + * Milen Nankov + * + * Alternatively, the contents of this file may be used under the terms of + * the GNU General Public License Version 2 or later (the "GPL"), in which + * case the provisions of the GPL are applicable instead of those above. If + * you wish to allow use of your version of this file only under the terms of + * the GPL and not to allow others to use your version of this file under the + * MPL, indicate your decision by deleting the provisions above and replacing + * them with the notice and other provisions required by the GPL. If you do + * not delete the provisions above, a recipient may use your version of this + * file under either the MPL or the GPL. + * + * ***** END LICENSE BLOCK ***** */ + +package yuicompressor.org.mozilla.javascript; + +/** + * This class implements the JavaScript scanner. + * + * It is based on the C source files jsscan.c and jsscan.h + * in the jsref package. + * + * @see yuicompressor.org.mozilla.javascript.Parser + * + * @author Mike McCabe + * @author Brendan Eich + */ + +public class Token +{ + + // debug flags + public static final boolean printTrees = false; + static final boolean printICode = false; + static final boolean printNames = printTrees || printICode; + + /** + * Token types. These values correspond to JSTokenType values in + * jsscan.c. + */ + + public final static int + // start enum + ERROR = -1, // well-known as the only code < EOF + EOF = 0, // end of file token - (not EOF_CHAR) + EOL = 1, // end of line + + // Interpreter reuses the following as bytecodes + FIRST_BYTECODE_TOKEN = 2, + + ENTERWITH = 2, + LEAVEWITH = 3, + RETURN = 4, + GOTO = 5, + IFEQ = 6, + IFNE = 7, + SETNAME = 8, + BITOR = 9, + BITXOR = 10, + BITAND = 11, + EQ = 12, + NE = 13, + LT = 14, + LE = 15, + GT = 16, + GE = 17, + LSH = 18, + RSH = 19, + URSH = 20, + ADD = 21, + SUB = 22, + MUL = 23, + DIV = 24, + MOD = 25, + NOT = 26, + BITNOT = 27, + POS = 28, + NEG = 29, + NEW = 30, + DELPROP = 31, + TYPEOF = 32, + GETPROP = 33, + SETPROP = 34, + GETELEM = 35, + SETELEM = 36, + CALL = 37, + NAME = 38, + NUMBER = 39, + STRING = 40, + NULL = 41, + THIS = 42, + FALSE = 43, + TRUE = 44, + SHEQ = 45, // shallow equality (===) + SHNE = 46, // shallow inequality (!==) + REGEXP = 47, + BINDNAME = 48, + THROW = 49, + RETHROW = 50, // rethrow caught execetion: catch (e if ) use it + IN = 51, + INSTANCEOF = 52, + LOCAL_LOAD = 53, + GETVAR = 54, + SETVAR = 55, + CATCH_SCOPE = 56, + ENUM_INIT_KEYS = 57, + ENUM_INIT_VALUES = 58, + ENUM_NEXT = 59, + ENUM_ID = 60, + THISFN = 61, + RETURN_RESULT = 62, // to return prevoisly stored return result + ARRAYLIT = 63, // array literal + OBJECTLIT = 64, // object literal + GET_REF = 65, // *reference + SET_REF = 66, // *reference = something + DEL_REF = 67, // delete reference + REF_CALL = 68, // f(args) = something or f(args)++ + REF_SPECIAL = 69, // reference for special properties like __proto + + // For XML support: + DEFAULTNAMESPACE = 70, // default xml namespace = + ESCXMLATTR = 71, + ESCXMLTEXT = 72, + REF_MEMBER = 73, // Reference for x.@y, x..y etc. + REF_NS_MEMBER = 74, // Reference for x.ns::y, x..ns::y etc. + REF_NAME = 75, // Reference for @y, @[y] etc. + REF_NS_NAME = 76; // Reference for ns::y, @ns::y@[y] etc. + + // End of interpreter bytecodes + public final static int + LAST_BYTECODE_TOKEN = REF_NS_NAME, + + TRY = 77, + SEMI = 78, // semicolon + LB = 79, // left and right brackets + RB = 80, + LC = 81, // left and right curlies (braces) + RC = 82, + LP = 83, // left and right parentheses + RP = 84, + COMMA = 85, // comma operator + + ASSIGN = 86, // simple assignment (=) + ASSIGN_BITOR = 87, // |= + ASSIGN_BITXOR = 88, // ^= + ASSIGN_BITAND = 89, // |= + ASSIGN_LSH = 90, // <<= + ASSIGN_RSH = 91, // >>= + ASSIGN_URSH = 92, // >>>= + ASSIGN_ADD = 93, // += + ASSIGN_SUB = 94, // -= + ASSIGN_MUL = 95, // *= + ASSIGN_DIV = 96, // /= + ASSIGN_MOD = 97; // %= + + public final static int + FIRST_ASSIGN = ASSIGN, + LAST_ASSIGN = ASSIGN_MOD, + + HOOK = 98, // conditional (?:) + COLON = 99, + OR = 100, // logical or (||) + AND = 101, // logical and (&&) + INC = 102, // increment/decrement (++ --) + DEC = 103, + DOT = 104, // member operator (.) + FUNCTION = 105, // function keyword + EXPORT = 106, // export keyword + IMPORT = 107, // import keyword + IF = 108, // if keyword + ELSE = 109, // else keyword + SWITCH = 110, // switch keyword + CASE = 111, // case keyword + DEFAULT = 112, // default keyword + WHILE = 113, // while keyword + DO = 114, // do keyword + FOR = 115, // for keyword + BREAK = 116, // break keyword + CONTINUE = 117, // continue keyword + VAR = 118, // var keyword + WITH = 119, // with keyword + CATCH = 120, // catch keyword + FINALLY = 121, // finally keyword + VOID = 122, // void keyword + RESERVED = 123, // reserved keywords + + EMPTY = 124, + + /* types used for the parse tree - these never get returned + * by the scanner. + */ + + BLOCK = 125, // statement block + LABEL = 126, // label + TARGET = 127, + LOOP = 128, + EXPR_VOID = 129, // expression statement in functions + EXPR_RESULT = 130, // expression statement in scripts + JSR = 131, + SCRIPT = 132, // top-level node for entire script + TYPEOFNAME = 133, // for typeof(simple-name) + USE_STACK = 134, + SETPROP_OP = 135, // x.y op= something + SETELEM_OP = 136, // x[y] op= something + LOCAL_BLOCK = 137, + SET_REF_OP = 138, // *reference op= something + + // For XML support: + DOTDOT = 139, // member operator (..) + COLONCOLON = 140, // namespace::name + XML = 141, // XML type + DOTQUERY = 142, // .() -- e.g., x.emps.emp.(name == "terry") + XMLATTR = 143, // @ + XMLEND = 144, + + // Optimizer-only-tokens + TO_OBJECT = 145, + TO_DOUBLE = 146, + + GET = 147, // JS 1.5 get pseudo keyword + SET = 148, // JS 1.5 set pseudo keyword + CONST = 149, + SETCONST = 150, + SETCONSTVAR = 151, + + SPECIALCOMMENT = 152, // Internet Explorer conditional comment + + LAST_TOKEN = 153; + + public static String name(int token) + { + if (!printNames) { + return String.valueOf(token); + } + switch (token) { + case ERROR: return "ERROR"; + case EOF: return "EOF"; + case EOL: return "EOL"; + case ENTERWITH: return "ENTERWITH"; + case LEAVEWITH: return "LEAVEWITH"; + case RETURN: return "RETURN"; + case GOTO: return "GOTO"; + case IFEQ: return "IFEQ"; + case IFNE: return "IFNE"; + case SETNAME: return "SETNAME"; + case BITOR: return "BITOR"; + case BITXOR: return "BITXOR"; + case BITAND: return "BITAND"; + case EQ: return "EQ"; + case NE: return "NE"; + case LT: return "LT"; + case LE: return "LE"; + case GT: return "GT"; + case GE: return "GE"; + case LSH: return "LSH"; + case RSH: return "RSH"; + case URSH: return "URSH"; + case ADD: return "ADD"; + case SUB: return "SUB"; + case MUL: return "MUL"; + case DIV: return "DIV"; + case MOD: return "MOD"; + case NOT: return "NOT"; + case BITNOT: return "BITNOT"; + case POS: return "POS"; + case NEG: return "NEG"; + case NEW: return "NEW"; + case DELPROP: return "DELPROP"; + case TYPEOF: return "TYPEOF"; + case GETPROP: return "GETPROP"; + case SETPROP: return "SETPROP"; + case GETELEM: return "GETELEM"; + case SETELEM: return "SETELEM"; + case CALL: return "CALL"; + case NAME: return "NAME"; + case NUMBER: return "NUMBER"; + case STRING: return "STRING"; + case NULL: return "NULL"; + case THIS: return "THIS"; + case FALSE: return "FALSE"; + case TRUE: return "TRUE"; + case SHEQ: return "SHEQ"; + case SHNE: return "SHNE"; + case REGEXP: return "OBJECT"; + case BINDNAME: return "BINDNAME"; + case THROW: return "THROW"; + case RETHROW: return "RETHROW"; + case IN: return "IN"; + case INSTANCEOF: return "INSTANCEOF"; + case LOCAL_LOAD: return "LOCAL_LOAD"; + case GETVAR: return "GETVAR"; + case SETVAR: return "SETVAR"; + case CATCH_SCOPE: return "CATCH_SCOPE"; + case ENUM_INIT_KEYS: return "ENUM_INIT_KEYS"; + case ENUM_INIT_VALUES: return "ENUM_INIT_VALUES"; + case ENUM_NEXT: return "ENUM_NEXT"; + case ENUM_ID: return "ENUM_ID"; + case THISFN: return "THISFN"; + case RETURN_RESULT: return "RETURN_RESULT"; + case ARRAYLIT: return "ARRAYLIT"; + case OBJECTLIT: return "OBJECTLIT"; + case GET_REF: return "GET_REF"; + case SET_REF: return "SET_REF"; + case DEL_REF: return "DEL_REF"; + case REF_CALL: return "REF_CALL"; + case REF_SPECIAL: return "REF_SPECIAL"; + case DEFAULTNAMESPACE:return "DEFAULTNAMESPACE"; + case ESCXMLTEXT: return "ESCXMLTEXT"; + case ESCXMLATTR: return "ESCXMLATTR"; + case REF_MEMBER: return "REF_MEMBER"; + case REF_NS_MEMBER: return "REF_NS_MEMBER"; + case REF_NAME: return "REF_NAME"; + case REF_NS_NAME: return "REF_NS_NAME"; + case TRY: return "TRY"; + case SEMI: return "SEMI"; + case LB: return "LB"; + case RB: return "RB"; + case LC: return "LC"; + case RC: return "RC"; + case LP: return "LP"; + case RP: return "RP"; + case COMMA: return "COMMA"; + case ASSIGN: return "ASSIGN"; + case ASSIGN_BITOR: return "ASSIGN_BITOR"; + case ASSIGN_BITXOR: return "ASSIGN_BITXOR"; + case ASSIGN_BITAND: return "ASSIGN_BITAND"; + case ASSIGN_LSH: return "ASSIGN_LSH"; + case ASSIGN_RSH: return "ASSIGN_RSH"; + case ASSIGN_URSH: return "ASSIGN_URSH"; + case ASSIGN_ADD: return "ASSIGN_ADD"; + case ASSIGN_SUB: return "ASSIGN_SUB"; + case ASSIGN_MUL: return "ASSIGN_MUL"; + case ASSIGN_DIV: return "ASSIGN_DIV"; + case ASSIGN_MOD: return "ASSIGN_MOD"; + case HOOK: return "HOOK"; + case COLON: return "COLON"; + case OR: return "OR"; + case AND: return "AND"; + case INC: return "INC"; + case DEC: return "DEC"; + case DOT: return "DOT"; + case FUNCTION: return "FUNCTION"; + case EXPORT: return "EXPORT"; + case IMPORT: return "IMPORT"; + case IF: return "IF"; + case ELSE: return "ELSE"; + case SWITCH: return "SWITCH"; + case CASE: return "CASE"; + case DEFAULT: return "DEFAULT"; + case WHILE: return "WHILE"; + case DO: return "DO"; + case FOR: return "FOR"; + case BREAK: return "BREAK"; + case CONTINUE: return "CONTINUE"; + case VAR: return "VAR"; + case WITH: return "WITH"; + case CATCH: return "CATCH"; + case FINALLY: return "FINALLY"; + case RESERVED: return "RESERVED"; + case EMPTY: return "EMPTY"; + case BLOCK: return "BLOCK"; + case LABEL: return "LABEL"; + case TARGET: return "TARGET"; + case LOOP: return "LOOP"; + case EXPR_VOID: return "EXPR_VOID"; + case EXPR_RESULT: return "EXPR_RESULT"; + case JSR: return "JSR"; + case SCRIPT: return "SCRIPT"; + case TYPEOFNAME: return "TYPEOFNAME"; + case USE_STACK: return "USE_STACK"; + case SETPROP_OP: return "SETPROP_OP"; + case SETELEM_OP: return "SETELEM_OP"; + case LOCAL_BLOCK: return "LOCAL_BLOCK"; + case SET_REF_OP: return "SET_REF_OP"; + case DOTDOT: return "DOTDOT"; + case COLONCOLON: return "COLONCOLON"; + case XML: return "XML"; + case DOTQUERY: return "DOTQUERY"; + case XMLATTR: return "XMLATTR"; + case XMLEND: return "XMLEND"; + case TO_OBJECT: return "TO_OBJECT"; + case TO_DOUBLE: return "TO_DOUBLE"; + case GET: return "GET"; + case SET: return "SET"; + case CONST: return "CONST"; + case SETCONST: return "SETCONST"; + } + + // Token without name + throw new IllegalStateException(String.valueOf(token)); + } +} diff --git a/trunk/infrastructure/yuicompressor/src/yuicompressor/org/mozilla/javascript/Token.java.orig b/trunk/infrastructure/yuicompressor/src/yuicompressor/org/mozilla/javascript/Token.java.orig new file mode 100644 index 0000000..7f7cdc2 --- /dev/null +++ b/trunk/infrastructure/yuicompressor/src/yuicompressor/org/mozilla/javascript/Token.java.orig @@ -0,0 +1,417 @@ +/* -*- Mode: java; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is Rhino code, released + * May 6, 1999. + * + * The Initial Developer of the Original Code is + * Netscape Communications Corporation. + * Portions created by the Initial Developer are Copyright (C) 1997-1999 + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * Roger Lawrence + * Mike McCabe + * Igor Bukanov + * Bob Jervis + * Milen Nankov + * + * Alternatively, the contents of this file may be used under the terms of + * the GNU General Public License Version 2 or later (the "GPL"), in which + * case the provisions of the GPL are applicable instead of those above. If + * you wish to allow use of your version of this file only under the terms of + * the GPL and not to allow others to use your version of this file under the + * MPL, indicate your decision by deleting the provisions above and replacing + * them with the notice and other provisions required by the GPL. If you do + * not delete the provisions above, a recipient may use your version of this + * file under either the MPL or the GPL. + * + * ***** END LICENSE BLOCK ***** */ + +package org.mozilla.javascript; + +/** + * This class implements the JavaScript scanner. + * + * It is based on the C source files jsscan.c and jsscan.h + * in the jsref package. + * + * @see org.mozilla.javascript.Parser + * + * @author Mike McCabe + * @author Brendan Eich + */ + +public class Token +{ + + // debug flags + public static final boolean printTrees = false; + static final boolean printICode = false; + static final boolean printNames = printTrees || printICode; + + /** + * Token types. These values correspond to JSTokenType values in + * jsscan.c. + */ + + public final static int + // start enum + ERROR = -1, // well-known as the only code < EOF + EOF = 0, // end of file token - (not EOF_CHAR) + EOL = 1, // end of line + + // Interpreter reuses the following as bytecodes + FIRST_BYTECODE_TOKEN = 2, + + ENTERWITH = 2, + LEAVEWITH = 3, + RETURN = 4, + GOTO = 5, + IFEQ = 6, + IFNE = 7, + SETNAME = 8, + BITOR = 9, + BITXOR = 10, + BITAND = 11, + EQ = 12, + NE = 13, + LT = 14, + LE = 15, + GT = 16, + GE = 17, + LSH = 18, + RSH = 19, + URSH = 20, + ADD = 21, + SUB = 22, + MUL = 23, + DIV = 24, + MOD = 25, + NOT = 26, + BITNOT = 27, + POS = 28, + NEG = 29, + NEW = 30, + DELPROP = 31, + TYPEOF = 32, + GETPROP = 33, + SETPROP = 34, + GETELEM = 35, + SETELEM = 36, + CALL = 37, + NAME = 38, + NUMBER = 39, + STRING = 40, + NULL = 41, + THIS = 42, + FALSE = 43, + TRUE = 44, + SHEQ = 45, // shallow equality (===) + SHNE = 46, // shallow inequality (!==) + REGEXP = 47, + BINDNAME = 48, + THROW = 49, + RETHROW = 50, // rethrow caught execetion: catch (e if ) use it + IN = 51, + INSTANCEOF = 52, + LOCAL_LOAD = 53, + GETVAR = 54, + SETVAR = 55, + CATCH_SCOPE = 56, + ENUM_INIT_KEYS = 57, + ENUM_INIT_VALUES = 58, + ENUM_NEXT = 59, + ENUM_ID = 60, + THISFN = 61, + RETURN_RESULT = 62, // to return prevoisly stored return result + ARRAYLIT = 63, // array literal + OBJECTLIT = 64, // object literal + GET_REF = 65, // *reference + SET_REF = 66, // *reference = something + DEL_REF = 67, // delete reference + REF_CALL = 68, // f(args) = something or f(args)++ + REF_SPECIAL = 69, // reference for special properties like __proto + + // For XML support: + DEFAULTNAMESPACE = 70, // default xml namespace = + ESCXMLATTR = 71, + ESCXMLTEXT = 72, + REF_MEMBER = 73, // Reference for x.@y, x..y etc. + REF_NS_MEMBER = 74, // Reference for x.ns::y, x..ns::y etc. + REF_NAME = 75, // Reference for @y, @[y] etc. + REF_NS_NAME = 76; // Reference for ns::y, @ns::y@[y] etc. + + // End of interpreter bytecodes + public final static int + LAST_BYTECODE_TOKEN = REF_NS_NAME, + + TRY = 77, + SEMI = 78, // semicolon + LB = 79, // left and right brackets + RB = 80, + LC = 81, // left and right curlies (braces) + RC = 82, + LP = 83, // left and right parentheses + RP = 84, + COMMA = 85, // comma operator + + ASSIGN = 86, // simple assignment (=) + ASSIGN_BITOR = 87, // |= + ASSIGN_BITXOR = 88, // ^= + ASSIGN_BITAND = 89, // |= + ASSIGN_LSH = 90, // <<= + ASSIGN_RSH = 91, // >>= + ASSIGN_URSH = 92, // >>>= + ASSIGN_ADD = 93, // += + ASSIGN_SUB = 94, // -= + ASSIGN_MUL = 95, // *= + ASSIGN_DIV = 96, // /= + ASSIGN_MOD = 97; // %= + + public final static int + FIRST_ASSIGN = ASSIGN, + LAST_ASSIGN = ASSIGN_MOD, + + HOOK = 98, // conditional (?:) + COLON = 99, + OR = 100, // logical or (||) + AND = 101, // logical and (&&) + INC = 102, // increment/decrement (++ --) + DEC = 103, + DOT = 104, // member operator (.) + FUNCTION = 105, // function keyword + EXPORT = 106, // export keyword + IMPORT = 107, // import keyword + IF = 108, // if keyword + ELSE = 109, // else keyword + SWITCH = 110, // switch keyword + CASE = 111, // case keyword + DEFAULT = 112, // default keyword + WHILE = 113, // while keyword + DO = 114, // do keyword + FOR = 115, // for keyword + BREAK = 116, // break keyword + CONTINUE = 117, // continue keyword + VAR = 118, // var keyword + WITH = 119, // with keyword + CATCH = 120, // catch keyword + FINALLY = 121, // finally keyword + VOID = 122, // void keyword + RESERVED = 123, // reserved keywords + + EMPTY = 124, + + /* types used for the parse tree - these never get returned + * by the scanner. + */ + + BLOCK = 125, // statement block + LABEL = 126, // label + TARGET = 127, + LOOP = 128, + EXPR_VOID = 129, // expression statement in functions + EXPR_RESULT = 130, // expression statement in scripts + JSR = 131, + SCRIPT = 132, // top-level node for entire script + TYPEOFNAME = 133, // for typeof(simple-name) + USE_STACK = 134, + SETPROP_OP = 135, // x.y op= something + SETELEM_OP = 136, // x[y] op= something + LOCAL_BLOCK = 137, + SET_REF_OP = 138, // *reference op= something + + // For XML support: + DOTDOT = 139, // member operator (..) + COLONCOLON = 140, // namespace::name + XML = 141, // XML type + DOTQUERY = 142, // .() -- e.g., x.emps.emp.(name == "terry") + XMLATTR = 143, // @ + XMLEND = 144, + + // Optimizer-only-tokens + TO_OBJECT = 145, + TO_DOUBLE = 146, + + GET = 147, // JS 1.5 get pseudo keyword + SET = 148, // JS 1.5 set pseudo keyword + CONST = 149, + SETCONST = 150, + SETCONSTVAR = 151, + LAST_TOKEN = 152; + + public static String name(int token) + { + if (!printNames) { + return String.valueOf(token); + } + switch (token) { + case ERROR: return "ERROR"; + case EOF: return "EOF"; + case EOL: return "EOL"; + case ENTERWITH: return "ENTERWITH"; + case LEAVEWITH: return "LEAVEWITH"; + case RETURN: return "RETURN"; + case GOTO: return "GOTO"; + case IFEQ: return "IFEQ"; + case IFNE: return "IFNE"; + case SETNAME: return "SETNAME"; + case BITOR: return "BITOR"; + case BITXOR: return "BITXOR"; + case BITAND: return "BITAND"; + case EQ: return "EQ"; + case NE: return "NE"; + case LT: return "LT"; + case LE: return "LE"; + case GT: return "GT"; + case GE: return "GE"; + case LSH: return "LSH"; + case RSH: return "RSH"; + case URSH: return "URSH"; + case ADD: return "ADD"; + case SUB: return "SUB"; + case MUL: return "MUL"; + case DIV: return "DIV"; + case MOD: return "MOD"; + case NOT: return "NOT"; + case BITNOT: return "BITNOT"; + case POS: return "POS"; + case NEG: return "NEG"; + case NEW: return "NEW"; + case DELPROP: return "DELPROP"; + case TYPEOF: return "TYPEOF"; + case GETPROP: return "GETPROP"; + case SETPROP: return "SETPROP"; + case GETELEM: return "GETELEM"; + case SETELEM: return "SETELEM"; + case CALL: return "CALL"; + case NAME: return "NAME"; + case NUMBER: return "NUMBER"; + case STRING: return "STRING"; + case NULL: return "NULL"; + case THIS: return "THIS"; + case FALSE: return "FALSE"; + case TRUE: return "TRUE"; + case SHEQ: return "SHEQ"; + case SHNE: return "SHNE"; + case REGEXP: return "OBJECT"; + case BINDNAME: return "BINDNAME"; + case THROW: return "THROW"; + case RETHROW: return "RETHROW"; + case IN: return "IN"; + case INSTANCEOF: return "INSTANCEOF"; + case LOCAL_LOAD: return "LOCAL_LOAD"; + case GETVAR: return "GETVAR"; + case SETVAR: return "SETVAR"; + case CATCH_SCOPE: return "CATCH_SCOPE"; + case ENUM_INIT_KEYS: return "ENUM_INIT_KEYS"; + case ENUM_INIT_VALUES: return "ENUM_INIT_VALUES"; + case ENUM_NEXT: return "ENUM_NEXT"; + case ENUM_ID: return "ENUM_ID"; + case THISFN: return "THISFN"; + case RETURN_RESULT: return "RETURN_RESULT"; + case ARRAYLIT: return "ARRAYLIT"; + case OBJECTLIT: return "OBJECTLIT"; + case GET_REF: return "GET_REF"; + case SET_REF: return "SET_REF"; + case DEL_REF: return "DEL_REF"; + case REF_CALL: return "REF_CALL"; + case REF_SPECIAL: return "REF_SPECIAL"; + case DEFAULTNAMESPACE:return "DEFAULTNAMESPACE"; + case ESCXMLTEXT: return "ESCXMLTEXT"; + case ESCXMLATTR: return "ESCXMLATTR"; + case REF_MEMBER: return "REF_MEMBER"; + case REF_NS_MEMBER: return "REF_NS_MEMBER"; + case REF_NAME: return "REF_NAME"; + case REF_NS_NAME: return "REF_NS_NAME"; + case TRY: return "TRY"; + case SEMI: return "SEMI"; + case LB: return "LB"; + case RB: return "RB"; + case LC: return "LC"; + case RC: return "RC"; + case LP: return "LP"; + case RP: return "RP"; + case COMMA: return "COMMA"; + case ASSIGN: return "ASSIGN"; + case ASSIGN_BITOR: return "ASSIGN_BITOR"; + case ASSIGN_BITXOR: return "ASSIGN_BITXOR"; + case ASSIGN_BITAND: return "ASSIGN_BITAND"; + case ASSIGN_LSH: return "ASSIGN_LSH"; + case ASSIGN_RSH: return "ASSIGN_RSH"; + case ASSIGN_URSH: return "ASSIGN_URSH"; + case ASSIGN_ADD: return "ASSIGN_ADD"; + case ASSIGN_SUB: return "ASSIGN_SUB"; + case ASSIGN_MUL: return "ASSIGN_MUL"; + case ASSIGN_DIV: return "ASSIGN_DIV"; + case ASSIGN_MOD: return "ASSIGN_MOD"; + case HOOK: return "HOOK"; + case COLON: return "COLON"; + case OR: return "OR"; + case AND: return "AND"; + case INC: return "INC"; + case DEC: return "DEC"; + case DOT: return "DOT"; + case FUNCTION: return "FUNCTION"; + case EXPORT: return "EXPORT"; + case IMPORT: return "IMPORT"; + case IF: return "IF"; + case ELSE: return "ELSE"; + case SWITCH: return "SWITCH"; + case CASE: return "CASE"; + case DEFAULT: return "DEFAULT"; + case WHILE: return "WHILE"; + case DO: return "DO"; + case FOR: return "FOR"; + case BREAK: return "BREAK"; + case CONTINUE: return "CONTINUE"; + case VAR: return "VAR"; + case WITH: return "WITH"; + case CATCH: return "CATCH"; + case FINALLY: return "FINALLY"; + case RESERVED: return "RESERVED"; + case EMPTY: return "EMPTY"; + case BLOCK: return "BLOCK"; + case LABEL: return "LABEL"; + case TARGET: return "TARGET"; + case LOOP: return "LOOP"; + case EXPR_VOID: return "EXPR_VOID"; + case EXPR_RESULT: return "EXPR_RESULT"; + case JSR: return "JSR"; + case SCRIPT: return "SCRIPT"; + case TYPEOFNAME: return "TYPEOFNAME"; + case USE_STACK: return "USE_STACK"; + case SETPROP_OP: return "SETPROP_OP"; + case SETELEM_OP: return "SETELEM_OP"; + case LOCAL_BLOCK: return "LOCAL_BLOCK"; + case SET_REF_OP: return "SET_REF_OP"; + case DOTDOT: return "DOTDOT"; + case COLONCOLON: return "COLONCOLON"; + case XML: return "XML"; + case DOTQUERY: return "DOTQUERY"; + case XMLATTR: return "XMLATTR"; + case XMLEND: return "XMLEND"; + case TO_OBJECT: return "TO_OBJECT"; + case TO_DOUBLE: return "TO_DOUBLE"; + case GET: return "GET"; + case SET: return "SET"; + case CONST: return "CONST"; + case SETCONST: return "SETCONST"; + } + + // Token without name + throw new IllegalStateException(String.valueOf(token)); + } +} diff --git a/trunk/infrastructure/yuicompressor/src/yuicompressor/org/mozilla/javascript/TokenStream.java b/trunk/infrastructure/yuicompressor/src/yuicompressor/org/mozilla/javascript/TokenStream.java new file mode 100644 index 0000000..5bee827 --- /dev/null +++ b/trunk/infrastructure/yuicompressor/src/yuicompressor/org/mozilla/javascript/TokenStream.java @@ -0,0 +1,1381 @@ +/* -*- Mode: java; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is Rhino code, released + * May 6, 1999. + * + * The Initial Developer of the Original Code is + * Netscape Communications Corporation. + * Portions created by the Initial Developer are Copyright (C) 1997-1999 + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * Roger Lawrence + * Mike McCabe + * Igor Bukanov + * Ethan Hugg + * Bob Jervis + * Terry Lucas + * Milen Nankov + * + * Alternatively, the contents of this file may be used under the terms of + * the GNU General Public License Version 2 or later (the "GPL"), in which + * case the provisions of the GPL are applicable instead of those above. If + * you wish to allow use of your version of this file only under the terms of + * the GPL and not to allow others to use your version of this file under the + * MPL, indicate your decision by deleting the provisions above and replacing + * them with the notice and other provisions required by the GPL. If you do + * not delete the provisions above, a recipient may use your version of this + * file under either the MPL or the GPL. + * + * ***** END LICENSE BLOCK ***** */ + +package yuicompressor.org.mozilla.javascript; + +import java.io.*; + +/** + * This class implements the JavaScript scanner. + * + * It is based on the C source files jsscan.c and jsscan.h + * in the jsref package. + * + * @see yuicompressor.org.mozilla.javascript.Parser + * + * @author Mike McCabe + * @author Brendan Eich + */ + +class TokenStream +{ + /* + * For chars - because we need something out-of-range + * to check. (And checking EOF by exception is annoying.) + * Note distinction from EOF token type! + */ + private final static int + EOF_CHAR = -1; + + TokenStream(Parser parser, Reader sourceReader, String sourceString, + int lineno) + { + this.parser = parser; + this.lineno = lineno; + if (sourceReader != null) { + if (sourceString != null) Kit.codeBug(); + this.sourceReader = sourceReader; + this.sourceBuffer = new char[512]; + this.sourceEnd = 0; + } else { + if (sourceString == null) Kit.codeBug(); + this.sourceString = sourceString; + this.sourceEnd = sourceString.length(); + } + this.sourceCursor = 0; + } + + /* This function uses the cached op, string and number fields in + * TokenStream; if getToken has been called since the passed token + * was scanned, the op or string printed may be incorrect. + */ + String tokenToString(int token) + { + if (Token.printTrees) { + String name = Token.name(token); + + switch (token) { + case Token.STRING: + case Token.REGEXP: + case Token.NAME: + return name + " `" + this.string + "'"; + + case Token.NUMBER: + return "NUMBER " + this.number; + } + + return name; + } + return ""; + } + + static boolean isKeyword(String s) + { + return Token.EOF != stringToKeyword(s); + } + + private static int stringToKeyword(String name) + { +// #string_id_map# +// The following assumes that Token.EOF == 0 + final int + Id_break = Token.BREAK, + Id_case = Token.CASE, + Id_continue = Token.CONTINUE, + Id_default = Token.DEFAULT, + Id_delete = Token.DELPROP, + Id_do = Token.DO, + Id_else = Token.ELSE, + Id_export = Token.EXPORT, + Id_false = Token.FALSE, + Id_for = Token.FOR, + Id_function = Token.FUNCTION, + Id_if = Token.IF, + Id_in = Token.IN, + Id_new = Token.NEW, + Id_null = Token.NULL, + Id_return = Token.RETURN, + Id_switch = Token.SWITCH, + Id_this = Token.THIS, + Id_true = Token.TRUE, + Id_typeof = Token.TYPEOF, + Id_var = Token.VAR, + Id_void = Token.VOID, + Id_while = Token.WHILE, + Id_with = Token.WITH, + + // the following are #ifdef RESERVE_JAVA_KEYWORDS in jsscan.c + Id_abstract = Token.RESERVED, + Id_boolean = Token.RESERVED, + Id_byte = Token.RESERVED, + Id_catch = Token.CATCH, + Id_char = Token.RESERVED, + Id_class = Token.RESERVED, + Id_const = Token.CONST, + Id_debugger = Token.RESERVED, + Id_double = Token.RESERVED, + Id_enum = Token.RESERVED, + Id_extends = Token.RESERVED, + Id_final = Token.RESERVED, + Id_finally = Token.FINALLY, + Id_float = Token.RESERVED, + Id_goto = Token.RESERVED, + Id_implements = Token.RESERVED, + Id_import = Token.IMPORT, + Id_instanceof = Token.INSTANCEOF, + Id_int = Token.RESERVED, + Id_interface = Token.RESERVED, + Id_long = Token.RESERVED, + Id_native = Token.RESERVED, + Id_package = Token.RESERVED, + Id_private = Token.RESERVED, + Id_protected = Token.RESERVED, + Id_public = Token.RESERVED, + Id_short = Token.RESERVED, + Id_static = Token.RESERVED, + Id_super = Token.RESERVED, + Id_synchronized = Token.RESERVED, + Id_throw = Token.THROW, + Id_throws = Token.RESERVED, + Id_transient = Token.RESERVED, + Id_try = Token.TRY, + Id_volatile = Token.RESERVED; + + int id; + String s = name; +// #generated# Last update: 2001-06-01 17:45:01 CEST + L0: { id = 0; String X = null; int c; + L: switch (s.length()) { + case 2: c=s.charAt(1); + if (c=='f') { if (s.charAt(0)=='i') {id=Id_if; break L0;} } + else if (c=='n') { if (s.charAt(0)=='i') {id=Id_in; break L0;} } + else if (c=='o') { if (s.charAt(0)=='d') {id=Id_do; break L0;} } + break L; + case 3: switch (s.charAt(0)) { + case 'f': if (s.charAt(2)=='r' && s.charAt(1)=='o') {id=Id_for; break L0;} break L; + case 'i': if (s.charAt(2)=='t' && s.charAt(1)=='n') {id=Id_int; break L0;} break L; + case 'n': if (s.charAt(2)=='w' && s.charAt(1)=='e') {id=Id_new; break L0;} break L; + case 't': if (s.charAt(2)=='y' && s.charAt(1)=='r') {id=Id_try; break L0;} break L; + case 'v': if (s.charAt(2)=='r' && s.charAt(1)=='a') {id=Id_var; break L0;} break L; + } break L; + case 4: switch (s.charAt(0)) { + case 'b': X="byte";id=Id_byte; break L; + case 'c': c=s.charAt(3); + if (c=='e') { if (s.charAt(2)=='s' && s.charAt(1)=='a') {id=Id_case; break L0;} } + else if (c=='r') { if (s.charAt(2)=='a' && s.charAt(1)=='h') {id=Id_char; break L0;} } + break L; + case 'e': c=s.charAt(3); + if (c=='e') { if (s.charAt(2)=='s' && s.charAt(1)=='l') {id=Id_else; break L0;} } + else if (c=='m') { if (s.charAt(2)=='u' && s.charAt(1)=='n') {id=Id_enum; break L0;} } + break L; + case 'g': X="goto";id=Id_goto; break L; + case 'l': X="long";id=Id_long; break L; + case 'n': X="null";id=Id_null; break L; + case 't': c=s.charAt(3); + if (c=='e') { if (s.charAt(2)=='u' && s.charAt(1)=='r') {id=Id_true; break L0;} } + else if (c=='s') { if (s.charAt(2)=='i' && s.charAt(1)=='h') {id=Id_this; break L0;} } + break L; + case 'v': X="void";id=Id_void; break L; + case 'w': X="with";id=Id_with; break L; + } break L; + case 5: switch (s.charAt(2)) { + case 'a': X="class";id=Id_class; break L; + case 'e': X="break";id=Id_break; break L; + case 'i': X="while";id=Id_while; break L; + case 'l': X="false";id=Id_false; break L; + case 'n': c=s.charAt(0); + if (c=='c') { X="const";id=Id_const; } + else if (c=='f') { X="final";id=Id_final; } + break L; + case 'o': c=s.charAt(0); + if (c=='f') { X="float";id=Id_float; } + else if (c=='s') { X="short";id=Id_short; } + break L; + case 'p': X="super";id=Id_super; break L; + case 'r': X="throw";id=Id_throw; break L; + case 't': X="catch";id=Id_catch; break L; + } break L; + case 6: switch (s.charAt(1)) { + case 'a': X="native";id=Id_native; break L; + case 'e': c=s.charAt(0); + if (c=='d') { X="delete";id=Id_delete; } + else if (c=='r') { X="return";id=Id_return; } + break L; + case 'h': X="throws";id=Id_throws; break L; + case 'm': X="import";id=Id_import; break L; + case 'o': X="double";id=Id_double; break L; + case 't': X="static";id=Id_static; break L; + case 'u': X="public";id=Id_public; break L; + case 'w': X="switch";id=Id_switch; break L; + case 'x': X="export";id=Id_export; break L; + case 'y': X="typeof";id=Id_typeof; break L; + } break L; + case 7: switch (s.charAt(1)) { + case 'a': X="package";id=Id_package; break L; + case 'e': X="default";id=Id_default; break L; + case 'i': X="finally";id=Id_finally; break L; + case 'o': X="boolean";id=Id_boolean; break L; + case 'r': X="private";id=Id_private; break L; + case 'x': X="extends";id=Id_extends; break L; + } break L; + case 8: switch (s.charAt(0)) { + case 'a': X="abstract";id=Id_abstract; break L; + case 'c': X="continue";id=Id_continue; break L; + case 'd': X="debugger";id=Id_debugger; break L; + case 'f': X="function";id=Id_function; break L; + case 'v': X="volatile";id=Id_volatile; break L; + } break L; + case 9: c=s.charAt(0); + if (c=='i') { X="interface";id=Id_interface; } + else if (c=='p') { X="protected";id=Id_protected; } + else if (c=='t') { X="transient";id=Id_transient; } + break L; + case 10: c=s.charAt(1); + if (c=='m') { X="implements";id=Id_implements; } + else if (c=='n') { X="instanceof";id=Id_instanceof; } + break L; + case 12: X="synchronized";id=Id_synchronized; break L; + } + if (X!=null && X!=s && !X.equals(s)) id = 0; + } +// #/generated# +// #/string_id_map# + if (id == 0) { return Token.EOF; } + return id & 0xff; + } + + final int getLineno() { return lineno; } + + final String getString() { return string; } + + final double getNumber() { return number; } + + final boolean eof() { return hitEOF; } + + final int getToken() throws IOException + { + int c; + + retry: + for (;;) { + // Eat whitespace, possibly sensitive to newlines. + for (;;) { + c = getChar(); + if (c == EOF_CHAR) { + return Token.EOF; + } else if (c == '\n') { + dirtyLine = false; + return Token.EOL; + } else if (!isJSSpace(c)) { + if (c != '-') { + dirtyLine = true; + } + break; + } + } + + if (c == '@') return Token.XMLATTR; + + // identifier/keyword/instanceof? + // watch out for starting with a <backslash> + boolean identifierStart; + boolean isUnicodeEscapeStart = false; + if (c == '\\') { + c = getChar(); + if (c == 'u') { + identifierStart = true; + isUnicodeEscapeStart = true; + stringBufferTop = 0; + } else { + identifierStart = false; + ungetChar(c); + c = '\\'; + } + } else { + identifierStart = Character.isJavaIdentifierStart((char)c); + if (identifierStart) { + stringBufferTop = 0; + addToString(c); + } + } + + if (identifierStart) { + boolean containsEscape = isUnicodeEscapeStart; + for (;;) { + if (isUnicodeEscapeStart) { + // strictly speaking we should probably push-back + // all the bad characters if the <backslash>uXXXX + // sequence is malformed. But since there isn't a + // correct context(is there?) for a bad Unicode + // escape sequence in an identifier, we can report + // an error here. + int escapeVal = 0; + for (int i = 0; i != 4; ++i) { + c = getChar(); + escapeVal = Kit.xDigitToInt(c, escapeVal); + // Next check takes care about c < 0 and bad escape + if (escapeVal < 0) { break; } + } + if (escapeVal < 0) { + parser.addError("msg.invalid.escape"); + return Token.ERROR; + } + addToString(escapeVal); + isUnicodeEscapeStart = false; + } else { + c = getChar(); + if (c == '\\') { + c = getChar(); + if (c == 'u') { + isUnicodeEscapeStart = true; + containsEscape = true; + } else { + parser.addError("msg.illegal.character"); + return Token.ERROR; + } + } else { + if (c == EOF_CHAR + || !Character.isJavaIdentifierPart((char)c)) + { + break; + } + addToString(c); + } + } + } + ungetChar(c); + + String str = getStringFromBuffer(); + if (!containsEscape) { + // OPT we shouldn't have to make a string (object!) to + // check if it's a keyword. + + // Return the corresponding token if it's a keyword + int result = stringToKeyword(str); + if (result != Token.EOF) { + if (result != Token.RESERVED) { + return result; + } else if (!parser.compilerEnv. + isReservedKeywordAsIdentifier()) + { + return result; + } else { + // If implementation permits to use future reserved + // keywords in violation with the EcmaScript, + // treat it as name but issue warning + parser.addWarning("msg.reserved.keyword", str); + } + } + } + this.string = (String)allStrings.intern(str); + return Token.NAME; + } + + // is it a number? + if (isDigit(c) || (c == '.' && isDigit(peekChar()))) { + + stringBufferTop = 0; + int base = 10; + + if (c == '0') { + c = getChar(); + if (c == 'x' || c == 'X') { + base = 16; + c = getChar(); + } else if (isDigit(c)) { + base = 8; + } else { + addToString('0'); + } + } + + if (base == 16) { + while (0 <= Kit.xDigitToInt(c, 0)) { + addToString(c); + c = getChar(); + } + } else { + while ('0' <= c && c <= '9') { + /* + * We permit 08 and 09 as decimal numbers, which + * makes our behavior a superset of the ECMA + * numeric grammar. We might not always be so + * permissive, so we warn about it. + */ + if (base == 8 && c >= '8') { + parser.addWarning("msg.bad.octal.literal", + c == '8' ? "8" : "9"); + base = 10; + } + addToString(c); + c = getChar(); + } + } + + boolean isInteger = true; + + if (base == 10 && (c == '.' || c == 'e' || c == 'E')) { + isInteger = false; + if (c == '.') { + do { + addToString(c); + c = getChar(); + } while (isDigit(c)); + } + if (c == 'e' || c == 'E') { + addToString(c); + c = getChar(); + if (c == '+' || c == '-') { + addToString(c); + c = getChar(); + } + if (!isDigit(c)) { + parser.addError("msg.missing.exponent"); + return Token.ERROR; + } + do { + addToString(c); + c = getChar(); + } while (isDigit(c)); + } + } + ungetChar(c); + String numString = getStringFromBuffer(); + + double dval; + if (base == 10 && !isInteger) { + try { + // Use Java conversion to number from string... + dval = Double.valueOf(numString).doubleValue(); + } + catch (NumberFormatException ex) { + parser.addError("msg.caught.nfe"); + return Token.ERROR; + } + } else { + dval = ScriptRuntime.stringToNumber(numString, 0, base); + } + + this.number = dval; + return Token.NUMBER; + } + + // is it a string? + if (c == '"' || c == '\'') { + // We attempt to accumulate a string the fast way, by + // building it directly out of the reader. But if there + // are any escaped characters in the string, we revert to + // building it out of a StringBuffer. + + int quoteChar = c; + stringBufferTop = 0; + + c = getChar(); + while (c != quoteChar) { + if (c == '\n' || c == EOF_CHAR) { + ungetChar(c); + parser.addError("msg.unterminated.string.lit"); + return Token.ERROR; + } + + if (c == '\\') { + // We've hit an escaped character + + c = getChar(); + + switch (c) { + + case '\\': // backslash + case 'b': // backspace + case 'f': // form feed + case 'n': // line feed + case 'r': // carriage return + case 't': // horizontal tab + case 'v': // vertical tab + case 'd': // octal sequence + case 'u': // unicode sequence + case 'x': // hexadecimal sequence + // Only keep the '\' character for those + // characters that need to be escaped... + // Don't escape quoting characters... + addToString('\\'); + addToString(c); + break; + + case '\n': + // Remove line terminator after escape + break; + + default: + if (isDigit(c)) { + // Octal representation of a character. + // Preserve the escaping (see Y! bug #1637286) + addToString('\\'); + } + addToString(c); + break; + } + + } else { + + addToString(c); + } + + c = getChar(); + } + + String str = getStringFromBuffer(); + this.string = (String)allStrings.intern(str); + return Token.STRING; + } + + switch (c) { + case ';': return Token.SEMI; + case '[': return Token.LB; + case ']': return Token.RB; + case '{': return Token.LC; + case '}': return Token.RC; + case '(': return Token.LP; + case ')': return Token.RP; + case ',': return Token.COMMA; + case '?': return Token.HOOK; + case ':': + if (matchChar(':')) { + return Token.COLONCOLON; + } else { + return Token.COLON; + } + case '.': + if (matchChar('.')) { + return Token.DOTDOT; + } else if (matchChar('(')) { + return Token.DOTQUERY; + } else { + return Token.DOT; + } + + case '|': + if (matchChar('|')) { + return Token.OR; + } else if (matchChar('=')) { + return Token.ASSIGN_BITOR; + } else { + return Token.BITOR; + } + + case '^': + if (matchChar('=')) { + return Token.ASSIGN_BITXOR; + } else { + return Token.BITXOR; + } + + case '&': + if (matchChar('&')) { + return Token.AND; + } else if (matchChar('=')) { + return Token.ASSIGN_BITAND; + } else { + return Token.BITAND; + } + + case '=': + if (matchChar('=')) { + if (matchChar('=')) + return Token.SHEQ; + else + return Token.EQ; + } else { + return Token.ASSIGN; + } + + case '!': + if (matchChar('=')) { + if (matchChar('=')) + return Token.SHNE; + else + return Token.NE; + } else { + return Token.NOT; + } + + case '<': + /* NB:treat HTML begin-comment as comment-till-eol */ + if (matchChar('!')) { + if (matchChar('-')) { + if (matchChar('-')) { + skipLine(); + continue retry; + } + ungetChar('-'); + } + ungetChar('!'); + } + if (matchChar('<')) { + if (matchChar('=')) { + return Token.ASSIGN_LSH; + } else { + return Token.LSH; + } + } else { + if (matchChar('=')) { + return Token.LE; + } else { + return Token.LT; + } + } + + case '>': + if (matchChar('>')) { + if (matchChar('>')) { + if (matchChar('=')) { + return Token.ASSIGN_URSH; + } else { + return Token.URSH; + } + } else { + if (matchChar('=')) { + return Token.ASSIGN_RSH; + } else { + return Token.RSH; + } + } + } else { + if (matchChar('=')) { + return Token.GE; + } else { + return Token.GT; + } + } + + case '*': + if (matchChar('=')) { + return Token.ASSIGN_MUL; + } else { + return Token.MUL; + } + + case '/': + // is it a // comment? + if (matchChar('/')) { + skipLine(); + continue retry; + } + if (matchChar('*')) { + boolean lookForSlash = false; + StringBuffer sb = new StringBuffer(); + for (;;) { + c = getChar(); + if (c == EOF_CHAR) { + parser.addError("msg.unterminated.comment"); + return Token.ERROR; + } + sb.append((char) c); + if (c == '*') { + lookForSlash = true; + } else if (c == '/') { + if (lookForSlash) { + sb.delete(sb.length()-2, sb.length()); + String s = sb.toString(); + if (s.startsWith("!") || + s.startsWith("@cc_on") || + s.startsWith("@if") || + s.startsWith("@elif") || + s.startsWith("@else") || + s.startsWith("@end")) { + if (s.startsWith("!")) { + // Remove the leading '!' + this.string = s.substring(1); + } else { + this.string = s; + } + return Token.SPECIALCOMMENT; + } else { + continue retry; + } + } + } else { + lookForSlash = false; + } + } + } + + if (matchChar('=')) { + return Token.ASSIGN_DIV; + } else { + return Token.DIV; + } + + case '%': + if (matchChar('=')) { + return Token.ASSIGN_MOD; + } else { + return Token.MOD; + } + + case '~': + return Token.BITNOT; + + case '+': + if (matchChar('=')) { + return Token.ASSIGN_ADD; + } else if (matchChar('+')) { + return Token.INC; + } else { + return Token.ADD; + } + + case '-': + if (matchChar('=')) { + c = Token.ASSIGN_SUB; + } else if (matchChar('-')) { + if (!dirtyLine) { + // treat HTML end-comment after possible whitespace + // after line start as comment-utill-eol + if (matchChar('>')) { + skipLine(); + continue retry; + } + } + c = Token.DEC; + } else { + c = Token.SUB; + } + dirtyLine = true; + return c; + + default: + parser.addError("msg.illegal.character"); + return Token.ERROR; + } + } + } + + private static boolean isAlpha(int c) + { + // Use 'Z' < 'a' + if (c <= 'Z') { + return 'A' <= c; + } else { + return 'a' <= c && c <= 'z'; + } + } + + static boolean isDigit(int c) + { + return '0' <= c && c <= '9'; + } + + /* As defined in ECMA. jsscan.c uses C isspace() (which allows + * \v, I think.) note that code in getChar() implicitly accepts + * '\r' == \u000D as well. + */ + static boolean isJSSpace(int c) + { + if (c <= 127) { + return c == 0x20 || c == 0x9 || c == 0xC || c == 0xB; + } else { + return c == 0xA0 + || Character.getType((char)c) == Character.SPACE_SEPARATOR; + } + } + + private static boolean isJSFormatChar(int c) + { + return c > 127 && Character.getType((char)c) == Character.FORMAT; + } + + /** + * Parser calls the method when it gets / or /= in literal context. + */ + void readRegExp(int startToken) + throws IOException + { + stringBufferTop = 0; + if (startToken == Token.ASSIGN_DIV) { + // Miss-scanned /= + addToString('='); + } else { + if (startToken != Token.DIV) Kit.codeBug(); + } + + int c; + boolean inClass = false; + while ((c = getChar()) != '/' || inClass) { + if (c == '\n' || c == EOF_CHAR) { + ungetChar(c); + throw parser.reportError("msg.unterminated.re.lit"); + } + if (c == '\\') { + addToString(c); + c = getChar(); + } else if (c == '[') { + inClass = true; + } else if (c == ']') { + inClass = false; + } + + addToString(c); + } + int reEnd = stringBufferTop; + + while (true) { + if (matchChar('g')) + addToString('g'); + else if (matchChar('i')) + addToString('i'); + else if (matchChar('m')) + addToString('m'); + else + break; + } + + if (isAlpha(peekChar())) { + throw parser.reportError("msg.invalid.re.flag"); + } + + this.string = new String(stringBuffer, 0, reEnd); + this.regExpFlags = new String(stringBuffer, reEnd, + stringBufferTop - reEnd); + } + + boolean isXMLAttribute() + { + return xmlIsAttribute; + } + + int getFirstXMLToken() throws IOException + { + xmlOpenTagsCount = 0; + xmlIsAttribute = false; + xmlIsTagContent = false; + ungetChar('<'); + return getNextXMLToken(); + } + + int getNextXMLToken() throws IOException + { + stringBufferTop = 0; // remember the XML + + for (int c = getChar(); c != EOF_CHAR; c = getChar()) { + if (xmlIsTagContent) { + switch (c) { + case '>': + addToString(c); + xmlIsTagContent = false; + xmlIsAttribute = false; + break; + case '/': + addToString(c); + if (peekChar() == '>') { + c = getChar(); + addToString(c); + xmlIsTagContent = false; + xmlOpenTagsCount--; + } + break; + case '{': + ungetChar(c); + this.string = getStringFromBuffer(); + return Token.XML; + case '\'': + case '"': + addToString(c); + if (!readQuotedString(c)) return Token.ERROR; + break; + case '=': + addToString(c); + xmlIsAttribute = true; + break; + case ' ': + case '\t': + case '\r': + case '\n': + addToString(c); + break; + default: + addToString(c); + xmlIsAttribute = false; + break; + } + + if (!xmlIsTagContent && xmlOpenTagsCount == 0) { + this.string = getStringFromBuffer(); + return Token.XMLEND; + } + } else { + switch (c) { + case '<': + addToString(c); + c = peekChar(); + switch (c) { + case '!': + c = getChar(); // Skip ! + addToString(c); + c = peekChar(); + switch (c) { + case '-': + c = getChar(); // Skip - + addToString(c); + c = getChar(); + if (c == '-') { + addToString(c); + if(!readXmlComment()) return Token.ERROR; + } else { + // throw away the string in progress + stringBufferTop = 0; + this.string = null; + parser.addError("msg.XML.bad.form"); + return Token.ERROR; + } + break; + case '[': + c = getChar(); // Skip [ + addToString(c); + if (getChar() == 'C' && + getChar() == 'D' && + getChar() == 'A' && + getChar() == 'T' && + getChar() == 'A' && + getChar() == '[') + { + addToString('C'); + addToString('D'); + addToString('A'); + addToString('T'); + addToString('A'); + addToString('['); + if (!readCDATA()) return Token.ERROR; + + } else { + // throw away the string in progress + stringBufferTop = 0; + this.string = null; + parser.addError("msg.XML.bad.form"); + return Token.ERROR; + } + break; + default: + if(!readEntity()) return Token.ERROR; + break; + } + break; + case '?': + c = getChar(); // Skip ? + addToString(c); + if (!readPI()) return Token.ERROR; + break; + case '/': + // End tag + c = getChar(); // Skip / + addToString(c); + if (xmlOpenTagsCount == 0) { + // throw away the string in progress + stringBufferTop = 0; + this.string = null; + parser.addError("msg.XML.bad.form"); + return Token.ERROR; + } + xmlIsTagContent = true; + xmlOpenTagsCount--; + break; + default: + // Start tag + xmlIsTagContent = true; + xmlOpenTagsCount++; + break; + } + break; + case '{': + ungetChar(c); + this.string = getStringFromBuffer(); + return Token.XML; + default: + addToString(c); + break; + } + } + } + + stringBufferTop = 0; // throw away the string in progress + this.string = null; + parser.addError("msg.XML.bad.form"); + return Token.ERROR; + } + + /** + * + */ + private boolean readQuotedString(int quote) throws IOException + { + for (int c = getChar(); c != EOF_CHAR; c = getChar()) { + addToString(c); + if (c == quote) return true; + } + + stringBufferTop = 0; // throw away the string in progress + this.string = null; + parser.addError("msg.XML.bad.form"); + return false; + } + + /** + * + */ + private boolean readXmlComment() throws IOException + { + for (int c = getChar(); c != EOF_CHAR;) { + addToString(c); + if (c == '-' && peekChar() == '-') { + c = getChar(); + addToString(c); + if (peekChar() == '>') { + c = getChar(); // Skip > + addToString(c); + return true; + } else { + continue; + } + } + c = getChar(); + } + + stringBufferTop = 0; // throw away the string in progress + this.string = null; + parser.addError("msg.XML.bad.form"); + return false; + } + + /** + * + */ + private boolean readCDATA() throws IOException + { + for (int c = getChar(); c != EOF_CHAR;) { + addToString(c); + if (c == ']' && peekChar() == ']') { + c = getChar(); + addToString(c); + if (peekChar() == '>') { + c = getChar(); // Skip > + addToString(c); + return true; + } else { + continue; + } + } + c = getChar(); + } + + stringBufferTop = 0; // throw away the string in progress + this.string = null; + parser.addError("msg.XML.bad.form"); + return false; + } + + /** + * + */ + private boolean readEntity() throws IOException + { + int declTags = 1; + for (int c = getChar(); c != EOF_CHAR; c = getChar()) { + addToString(c); + switch (c) { + case '<': + declTags++; + break; + case '>': + declTags--; + if (declTags == 0) return true; + break; + } + } + + stringBufferTop = 0; // throw away the string in progress + this.string = null; + parser.addError("msg.XML.bad.form"); + return false; + } + + /** + * + */ + private boolean readPI() throws IOException + { + for (int c = getChar(); c != EOF_CHAR; c = getChar()) { + addToString(c); + if (c == '?' && peekChar() == '>') { + c = getChar(); // Skip > + addToString(c); + return true; + } + } + + stringBufferTop = 0; // throw away the string in progress + this.string = null; + parser.addError("msg.XML.bad.form"); + return false; + } + + private String getStringFromBuffer() + { + return new String(stringBuffer, 0, stringBufferTop); + } + + private void addToString(int c) + { + int N = stringBufferTop; + if (N == stringBuffer.length) { + char[] tmp = new char[stringBuffer.length * 2]; + System.arraycopy(stringBuffer, 0, tmp, 0, N); + stringBuffer = tmp; + } + stringBuffer[N] = (char)c; + stringBufferTop = N + 1; + } + + private void ungetChar(int c) + { + // can not unread past across line boundary + if (ungetCursor != 0 && ungetBuffer[ungetCursor - 1] == '\n') + Kit.codeBug(); + ungetBuffer[ungetCursor++] = c; + } + + private boolean matchChar(int test) throws IOException + { + int c = getChar(); + if (c == test) { + return true; + } else { + ungetChar(c); + return false; + } + } + + private int peekChar() throws IOException + { + int c = getChar(); + ungetChar(c); + return c; + } + + private int getChar() throws IOException + { + if (ungetCursor != 0) { + return ungetBuffer[--ungetCursor]; + } + + for(;;) { + int c; + if (sourceString != null) { + if (sourceCursor == sourceEnd) { + hitEOF = true; + return EOF_CHAR; + } + c = sourceString.charAt(sourceCursor++); + } else { + if (sourceCursor == sourceEnd) { + if (!fillSourceBuffer()) { + hitEOF = true; + return EOF_CHAR; + } + } + c = sourceBuffer[sourceCursor++]; + } + + if (lineEndChar >= 0) { + if (lineEndChar == '\r' && c == '\n') { + lineEndChar = '\n'; + continue; + } + lineEndChar = -1; + lineStart = sourceCursor - 1; + lineno++; + } + + if (c <= 127) { + if (c == '\n' || c == '\r') { + lineEndChar = c; + c = '\n'; + } + } else { + if (isJSFormatChar(c)) { + continue; + } + if (ScriptRuntime.isJSLineTerminator(c)) { + lineEndChar = c; + c = '\n'; + } + } + return c; + } + } + + private void skipLine() throws IOException + { + // skip to end of line + int c; + while ((c = getChar()) != EOF_CHAR && c != '\n') { } + ungetChar(c); + } + + final int getOffset() + { + int n = sourceCursor - lineStart; + if (lineEndChar >= 0) { --n; } + return n; + } + + final String getLine() + { + if (sourceString != null) { + // String case + int lineEnd = sourceCursor; + if (lineEndChar >= 0) { + --lineEnd; + } else { + for (; lineEnd != sourceEnd; ++lineEnd) { + int c = sourceString.charAt(lineEnd); + if (ScriptRuntime.isJSLineTerminator(c)) { + break; + } + } + } + return sourceString.substring(lineStart, lineEnd); + } else { + // Reader case + int lineLength = sourceCursor - lineStart; + if (lineEndChar >= 0) { + --lineLength; + } else { + // Read until the end of line + for (;; ++lineLength) { + int i = lineStart + lineLength; + if (i == sourceEnd) { + try { + if (!fillSourceBuffer()) { break; } + } catch (IOException ioe) { + // ignore it, we're already displaying an error... + break; + } + // i recalculuation as fillSourceBuffer can move saved + // line buffer and change lineStart + i = lineStart + lineLength; + } + int c = sourceBuffer[i]; + if (ScriptRuntime.isJSLineTerminator(c)) { + break; + } + } + } + return new String(sourceBuffer, lineStart, lineLength); + } + } + + private boolean fillSourceBuffer() throws IOException + { + if (sourceString != null) Kit.codeBug(); + if (sourceEnd == sourceBuffer.length) { + if (lineStart != 0) { + System.arraycopy(sourceBuffer, lineStart, sourceBuffer, 0, + sourceEnd - lineStart); + sourceEnd -= lineStart; + sourceCursor -= lineStart; + lineStart = 0; + } else { + char[] tmp = new char[sourceBuffer.length * 2]; + System.arraycopy(sourceBuffer, 0, tmp, 0, sourceEnd); + sourceBuffer = tmp; + } + } + int n = sourceReader.read(sourceBuffer, sourceEnd, + sourceBuffer.length - sourceEnd); + if (n < 0) { + return false; + } + sourceEnd += n; + return true; + } + + // stuff other than whitespace since start of line + private boolean dirtyLine; + + String regExpFlags; + + // Set this to an inital non-null value so that the Parser has + // something to retrieve even if an error has occured and no + // string is found. Fosters one class of error, but saves lots of + // code. + private String string = ""; + private double number; + + private char[] stringBuffer = new char[128]; + private int stringBufferTop; + private ObjToIntMap allStrings = new ObjToIntMap(50); + + // Room to backtrace from to < on failed match of the last - in <!-- + private final int[] ungetBuffer = new int[3]; + private int ungetCursor; + + private boolean hitEOF = false; + + private int lineStart = 0; + private int lineno; + private int lineEndChar = -1; + + private String sourceString; + private Reader sourceReader; + private char[] sourceBuffer; + private int sourceEnd; + private int sourceCursor; + + // for xml tokenizer + private boolean xmlIsAttribute; + private boolean xmlIsTagContent; + private int xmlOpenTagsCount; + + private Parser parser; +} diff --git a/trunk/infrastructure/yuicompressor/src/yuicompressor/org/mozilla/javascript/TokenStream.java.orig b/trunk/infrastructure/yuicompressor/src/yuicompressor/org/mozilla/javascript/TokenStream.java.orig new file mode 100644 index 0000000..c276894 --- /dev/null +++ b/trunk/infrastructure/yuicompressor/src/yuicompressor/org/mozilla/javascript/TokenStream.java.orig @@ -0,0 +1,1398 @@ +/* -*- Mode: java; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is Rhino code, released + * May 6, 1999. + * + * The Initial Developer of the Original Code is + * Netscape Communications Corporation. + * Portions created by the Initial Developer are Copyright (C) 1997-1999 + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * Roger Lawrence + * Mike McCabe + * Igor Bukanov + * Ethan Hugg + * Bob Jervis + * Terry Lucas + * Milen Nankov + * + * Alternatively, the contents of this file may be used under the terms of + * the GNU General Public License Version 2 or later (the "GPL"), in which + * case the provisions of the GPL are applicable instead of those above. If + * you wish to allow use of your version of this file only under the terms of + * the GPL and not to allow others to use your version of this file under the + * MPL, indicate your decision by deleting the provisions above and replacing + * them with the notice and other provisions required by the GPL. If you do + * not delete the provisions above, a recipient may use your version of this + * file under either the MPL or the GPL. + * + * ***** END LICENSE BLOCK ***** */ + +package org.mozilla.javascript; + +import java.io.*; + +/** + * This class implements the JavaScript scanner. + * + * It is based on the C source files jsscan.c and jsscan.h + * in the jsref package. + * + * @see org.mozilla.javascript.Parser + * + * @author Mike McCabe + * @author Brendan Eich + */ + +class TokenStream +{ + /* + * For chars - because we need something out-of-range + * to check. (And checking EOF by exception is annoying.) + * Note distinction from EOF token type! + */ + private final static int + EOF_CHAR = -1; + + TokenStream(Parser parser, Reader sourceReader, String sourceString, + int lineno) + { + this.parser = parser; + this.lineno = lineno; + if (sourceReader != null) { + if (sourceString != null) Kit.codeBug(); + this.sourceReader = sourceReader; + this.sourceBuffer = new char[512]; + this.sourceEnd = 0; + } else { + if (sourceString == null) Kit.codeBug(); + this.sourceString = sourceString; + this.sourceEnd = sourceString.length(); + } + this.sourceCursor = 0; + } + + /* This function uses the cached op, string and number fields in + * TokenStream; if getToken has been called since the passed token + * was scanned, the op or string printed may be incorrect. + */ + String tokenToString(int token) + { + if (Token.printTrees) { + String name = Token.name(token); + + switch (token) { + case Token.STRING: + case Token.REGEXP: + case Token.NAME: + return name + " `" + this.string + "'"; + + case Token.NUMBER: + return "NUMBER " + this.number; + } + + return name; + } + return ""; + } + + static boolean isKeyword(String s) + { + return Token.EOF != stringToKeyword(s); + } + + private static int stringToKeyword(String name) + { +// #string_id_map# +// The following assumes that Token.EOF == 0 + final int + Id_break = Token.BREAK, + Id_case = Token.CASE, + Id_continue = Token.CONTINUE, + Id_default = Token.DEFAULT, + Id_delete = Token.DELPROP, + Id_do = Token.DO, + Id_else = Token.ELSE, + Id_export = Token.EXPORT, + Id_false = Token.FALSE, + Id_for = Token.FOR, + Id_function = Token.FUNCTION, + Id_if = Token.IF, + Id_in = Token.IN, + Id_new = Token.NEW, + Id_null = Token.NULL, + Id_return = Token.RETURN, + Id_switch = Token.SWITCH, + Id_this = Token.THIS, + Id_true = Token.TRUE, + Id_typeof = Token.TYPEOF, + Id_var = Token.VAR, + Id_void = Token.VOID, + Id_while = Token.WHILE, + Id_with = Token.WITH, + + // the following are #ifdef RESERVE_JAVA_KEYWORDS in jsscan.c + Id_abstract = Token.RESERVED, + Id_boolean = Token.RESERVED, + Id_byte = Token.RESERVED, + Id_catch = Token.CATCH, + Id_char = Token.RESERVED, + Id_class = Token.RESERVED, + Id_const = Token.CONST, + Id_debugger = Token.RESERVED, + Id_double = Token.RESERVED, + Id_enum = Token.RESERVED, + Id_extends = Token.RESERVED, + Id_final = Token.RESERVED, + Id_finally = Token.FINALLY, + Id_float = Token.RESERVED, + Id_goto = Token.RESERVED, + Id_implements = Token.RESERVED, + Id_import = Token.IMPORT, + Id_instanceof = Token.INSTANCEOF, + Id_int = Token.RESERVED, + Id_interface = Token.RESERVED, + Id_long = Token.RESERVED, + Id_native = Token.RESERVED, + Id_package = Token.RESERVED, + Id_private = Token.RESERVED, + Id_protected = Token.RESERVED, + Id_public = Token.RESERVED, + Id_short = Token.RESERVED, + Id_static = Token.RESERVED, + Id_super = Token.RESERVED, + Id_synchronized = Token.RESERVED, + Id_throw = Token.THROW, + Id_throws = Token.RESERVED, + Id_transient = Token.RESERVED, + Id_try = Token.TRY, + Id_volatile = Token.RESERVED; + + int id; + String s = name; +// #generated# Last update: 2001-06-01 17:45:01 CEST + L0: { id = 0; String X = null; int c; + L: switch (s.length()) { + case 2: c=s.charAt(1); + if (c=='f') { if (s.charAt(0)=='i') {id=Id_if; break L0;} } + else if (c=='n') { if (s.charAt(0)=='i') {id=Id_in; break L0;} } + else if (c=='o') { if (s.charAt(0)=='d') {id=Id_do; break L0;} } + break L; + case 3: switch (s.charAt(0)) { + case 'f': if (s.charAt(2)=='r' && s.charAt(1)=='o') {id=Id_for; break L0;} break L; + case 'i': if (s.charAt(2)=='t' && s.charAt(1)=='n') {id=Id_int; break L0;} break L; + case 'n': if (s.charAt(2)=='w' && s.charAt(1)=='e') {id=Id_new; break L0;} break L; + case 't': if (s.charAt(2)=='y' && s.charAt(1)=='r') {id=Id_try; break L0;} break L; + case 'v': if (s.charAt(2)=='r' && s.charAt(1)=='a') {id=Id_var; break L0;} break L; + } break L; + case 4: switch (s.charAt(0)) { + case 'b': X="byte";id=Id_byte; break L; + case 'c': c=s.charAt(3); + if (c=='e') { if (s.charAt(2)=='s' && s.charAt(1)=='a') {id=Id_case; break L0;} } + else if (c=='r') { if (s.charAt(2)=='a' && s.charAt(1)=='h') {id=Id_char; break L0;} } + break L; + case 'e': c=s.charAt(3); + if (c=='e') { if (s.charAt(2)=='s' && s.charAt(1)=='l') {id=Id_else; break L0;} } + else if (c=='m') { if (s.charAt(2)=='u' && s.charAt(1)=='n') {id=Id_enum; break L0;} } + break L; + case 'g': X="goto";id=Id_goto; break L; + case 'l': X="long";id=Id_long; break L; + case 'n': X="null";id=Id_null; break L; + case 't': c=s.charAt(3); + if (c=='e') { if (s.charAt(2)=='u' && s.charAt(1)=='r') {id=Id_true; break L0;} } + else if (c=='s') { if (s.charAt(2)=='i' && s.charAt(1)=='h') {id=Id_this; break L0;} } + break L; + case 'v': X="void";id=Id_void; break L; + case 'w': X="with";id=Id_with; break L; + } break L; + case 5: switch (s.charAt(2)) { + case 'a': X="class";id=Id_class; break L; + case 'e': X="break";id=Id_break; break L; + case 'i': X="while";id=Id_while; break L; + case 'l': X="false";id=Id_false; break L; + case 'n': c=s.charAt(0); + if (c=='c') { X="const";id=Id_const; } + else if (c=='f') { X="final";id=Id_final; } + break L; + case 'o': c=s.charAt(0); + if (c=='f') { X="float";id=Id_float; } + else if (c=='s') { X="short";id=Id_short; } + break L; + case 'p': X="super";id=Id_super; break L; + case 'r': X="throw";id=Id_throw; break L; + case 't': X="catch";id=Id_catch; break L; + } break L; + case 6: switch (s.charAt(1)) { + case 'a': X="native";id=Id_native; break L; + case 'e': c=s.charAt(0); + if (c=='d') { X="delete";id=Id_delete; } + else if (c=='r') { X="return";id=Id_return; } + break L; + case 'h': X="throws";id=Id_throws; break L; + case 'm': X="import";id=Id_import; break L; + case 'o': X="double";id=Id_double; break L; + case 't': X="static";id=Id_static; break L; + case 'u': X="public";id=Id_public; break L; + case 'w': X="switch";id=Id_switch; break L; + case 'x': X="export";id=Id_export; break L; + case 'y': X="typeof";id=Id_typeof; break L; + } break L; + case 7: switch (s.charAt(1)) { + case 'a': X="package";id=Id_package; break L; + case 'e': X="default";id=Id_default; break L; + case 'i': X="finally";id=Id_finally; break L; + case 'o': X="boolean";id=Id_boolean; break L; + case 'r': X="private";id=Id_private; break L; + case 'x': X="extends";id=Id_extends; break L; + } break L; + case 8: switch (s.charAt(0)) { + case 'a': X="abstract";id=Id_abstract; break L; + case 'c': X="continue";id=Id_continue; break L; + case 'd': X="debugger";id=Id_debugger; break L; + case 'f': X="function";id=Id_function; break L; + case 'v': X="volatile";id=Id_volatile; break L; + } break L; + case 9: c=s.charAt(0); + if (c=='i') { X="interface";id=Id_interface; } + else if (c=='p') { X="protected";id=Id_protected; } + else if (c=='t') { X="transient";id=Id_transient; } + break L; + case 10: c=s.charAt(1); + if (c=='m') { X="implements";id=Id_implements; } + else if (c=='n') { X="instanceof";id=Id_instanceof; } + break L; + case 12: X="synchronized";id=Id_synchronized; break L; + } + if (X!=null && X!=s && !X.equals(s)) id = 0; + } +// #/generated# +// #/string_id_map# + if (id == 0) { return Token.EOF; } + return id & 0xff; + } + + final int getLineno() { return lineno; } + + final String getString() { return string; } + + final double getNumber() { return number; } + + final boolean eof() { return hitEOF; } + + final int getToken() throws IOException + { + int c; + + retry: + for (;;) { + // Eat whitespace, possibly sensitive to newlines. + for (;;) { + c = getChar(); + if (c == EOF_CHAR) { + return Token.EOF; + } else if (c == '\n') { + dirtyLine = false; + return Token.EOL; + } else if (!isJSSpace(c)) { + if (c != '-') { + dirtyLine = true; + } + break; + } + } + + if (c == '@') return Token.XMLATTR; + + // identifier/keyword/instanceof? + // watch out for starting with a <backslash> + boolean identifierStart; + boolean isUnicodeEscapeStart = false; + if (c == '\\') { + c = getChar(); + if (c == 'u') { + identifierStart = true; + isUnicodeEscapeStart = true; + stringBufferTop = 0; + } else { + identifierStart = false; + ungetChar(c); + c = '\\'; + } + } else { + identifierStart = Character.isJavaIdentifierStart((char)c); + if (identifierStart) { + stringBufferTop = 0; + addToString(c); + } + } + + if (identifierStart) { + boolean containsEscape = isUnicodeEscapeStart; + for (;;) { + if (isUnicodeEscapeStart) { + // strictly speaking we should probably push-back + // all the bad characters if the <backslash>uXXXX + // sequence is malformed. But since there isn't a + // correct context(is there?) for a bad Unicode + // escape sequence in an identifier, we can report + // an error here. + int escapeVal = 0; + for (int i = 0; i != 4; ++i) { + c = getChar(); + escapeVal = Kit.xDigitToInt(c, escapeVal); + // Next check takes care about c < 0 and bad escape + if (escapeVal < 0) { break; } + } + if (escapeVal < 0) { + parser.addError("msg.invalid.escape"); + return Token.ERROR; + } + addToString(escapeVal); + isUnicodeEscapeStart = false; + } else { + c = getChar(); + if (c == '\\') { + c = getChar(); + if (c == 'u') { + isUnicodeEscapeStart = true; + containsEscape = true; + } else { + parser.addError("msg.illegal.character"); + return Token.ERROR; + } + } else { + if (c == EOF_CHAR + || !Character.isJavaIdentifierPart((char)c)) + { + break; + } + addToString(c); + } + } + } + ungetChar(c); + + String str = getStringFromBuffer(); + if (!containsEscape) { + // OPT we shouldn't have to make a string (object!) to + // check if it's a keyword. + + // Return the corresponding token if it's a keyword + int result = stringToKeyword(str); + if (result != Token.EOF) { + if (result != Token.RESERVED) { + return result; + } else if (!parser.compilerEnv. + isReservedKeywordAsIdentifier()) + { + return result; + } else { + // If implementation permits to use future reserved + // keywords in violation with the EcmaScript, + // treat it as name but issue warning + parser.addWarning("msg.reserved.keyword", str); + } + } + } + this.string = (String)allStrings.intern(str); + return Token.NAME; + } + + // is it a number? + if (isDigit(c) || (c == '.' && isDigit(peekChar()))) { + + stringBufferTop = 0; + int base = 10; + + if (c == '0') { + c = getChar(); + if (c == 'x' || c == 'X') { + base = 16; + c = getChar(); + } else if (isDigit(c)) { + base = 8; + } else { + addToString('0'); + } + } + + if (base == 16) { + while (0 <= Kit.xDigitToInt(c, 0)) { + addToString(c); + c = getChar(); + } + } else { + while ('0' <= c && c <= '9') { + /* + * We permit 08 and 09 as decimal numbers, which + * makes our behavior a superset of the ECMA + * numeric grammar. We might not always be so + * permissive, so we warn about it. + */ + if (base == 8 && c >= '8') { + parser.addWarning("msg.bad.octal.literal", + c == '8' ? "8" : "9"); + base = 10; + } + addToString(c); + c = getChar(); + } + } + + boolean isInteger = true; + + if (base == 10 && (c == '.' || c == 'e' || c == 'E')) { + isInteger = false; + if (c == '.') { + do { + addToString(c); + c = getChar(); + } while (isDigit(c)); + } + if (c == 'e' || c == 'E') { + addToString(c); + c = getChar(); + if (c == '+' || c == '-') { + addToString(c); + c = getChar(); + } + if (!isDigit(c)) { + parser.addError("msg.missing.exponent"); + return Token.ERROR; + } + do { + addToString(c); + c = getChar(); + } while (isDigit(c)); + } + } + ungetChar(c); + String numString = getStringFromBuffer(); + + double dval; + if (base == 10 && !isInteger) { + try { + // Use Java conversion to number from string... + dval = Double.valueOf(numString).doubleValue(); + } + catch (NumberFormatException ex) { + parser.addError("msg.caught.nfe"); + return Token.ERROR; + } + } else { + dval = ScriptRuntime.stringToNumber(numString, 0, base); + } + + this.number = dval; + return Token.NUMBER; + } + + // is it a string? + if (c == '"' || c == '\'') { + // We attempt to accumulate a string the fast way, by + // building it directly out of the reader. But if there + // are any escaped characters in the string, we revert to + // building it out of a StringBuffer. + + int quoteChar = c; + stringBufferTop = 0; + + c = getChar(); + strLoop: while (c != quoteChar) { + if (c == '\n' || c == EOF_CHAR) { + ungetChar(c); + parser.addError("msg.unterminated.string.lit"); + return Token.ERROR; + } + + if (c == '\\') { + // We've hit an escaped character + int escapeVal; + + c = getChar(); + switch (c) { + case 'b': c = '\b'; break; + case 'f': c = '\f'; break; + case 'n': c = '\n'; break; + case 'r': c = '\r'; break; + case 't': c = '\t'; break; + + // \v a late addition to the ECMA spec, + // it is not in Java, so use 0xb + case 'v': c = 0xb; break; + + case 'u': + // Get 4 hex digits; if the u escape is not + // followed by 4 hex digits, use 'u' + the + // literal character sequence that follows. + int escapeStart = stringBufferTop; + addToString('u'); + escapeVal = 0; + for (int i = 0; i != 4; ++i) { + c = getChar(); + escapeVal = Kit.xDigitToInt(c, escapeVal); + if (escapeVal < 0) { + continue strLoop; + } + addToString(c); + } + // prepare for replace of stored 'u' sequence + // by escape value + stringBufferTop = escapeStart; + c = escapeVal; + break; + case 'x': + // Get 2 hex digits, defaulting to 'x'+literal + // sequence, as above. + c = getChar(); + escapeVal = Kit.xDigitToInt(c, 0); + if (escapeVal < 0) { + addToString('x'); + continue strLoop; + } else { + int c1 = c; + c = getChar(); + escapeVal = Kit.xDigitToInt(c, escapeVal); + if (escapeVal < 0) { + addToString('x'); + addToString(c1); + continue strLoop; + } else { + // got 2 hex digits + c = escapeVal; + } + } + break; + + case '\n': + // Remove line terminator after escape to follow + // SpiderMonkey and C/C++ + c = getChar(); + continue strLoop; + + default: + if ('0' <= c && c < '8') { + int val = c - '0'; + c = getChar(); + if ('0' <= c && c < '8') { + val = 8 * val + c - '0'; + c = getChar(); + if ('0' <= c && c < '8' && val <= 037) { + // c is 3rd char of octal sequence only + // if the resulting val <= 0377 + val = 8 * val + c - '0'; + c = getChar(); + } + } + ungetChar(c); + c = val; + } + } + } + addToString(c); + c = getChar(); + } + + String str = getStringFromBuffer(); + this.string = (String)allStrings.intern(str); + return Token.STRING; + } + + switch (c) { + case ';': return Token.SEMI; + case '[': return Token.LB; + case ']': return Token.RB; + case '{': return Token.LC; + case '}': return Token.RC; + case '(': return Token.LP; + case ')': return Token.RP; + case ',': return Token.COMMA; + case '?': return Token.HOOK; + case ':': + if (matchChar(':')) { + return Token.COLONCOLON; + } else { + return Token.COLON; + } + case '.': + if (matchChar('.')) { + return Token.DOTDOT; + } else if (matchChar('(')) { + return Token.DOTQUERY; + } else { + return Token.DOT; + } + + case '|': + if (matchChar('|')) { + return Token.OR; + } else if (matchChar('=')) { + return Token.ASSIGN_BITOR; + } else { + return Token.BITOR; + } + + case '^': + if (matchChar('=')) { + return Token.ASSIGN_BITXOR; + } else { + return Token.BITXOR; + } + + case '&': + if (matchChar('&')) { + return Token.AND; + } else if (matchChar('=')) { + return Token.ASSIGN_BITAND; + } else { + return Token.BITAND; + } + + case '=': + if (matchChar('=')) { + if (matchChar('=')) + return Token.SHEQ; + else + return Token.EQ; + } else { + return Token.ASSIGN; + } + + case '!': + if (matchChar('=')) { + if (matchChar('=')) + return Token.SHNE; + else + return Token.NE; + } else { + return Token.NOT; + } + + case '<': + /* NB:treat HTML begin-comment as comment-till-eol */ + if (matchChar('!')) { + if (matchChar('-')) { + if (matchChar('-')) { + skipLine(); + continue retry; + } + ungetChar('-'); + } + ungetChar('!'); + } + if (matchChar('<')) { + if (matchChar('=')) { + return Token.ASSIGN_LSH; + } else { + return Token.LSH; + } + } else { + if (matchChar('=')) { + return Token.LE; + } else { + return Token.LT; + } + } + + case '>': + if (matchChar('>')) { + if (matchChar('>')) { + if (matchChar('=')) { + return Token.ASSIGN_URSH; + } else { + return Token.URSH; + } + } else { + if (matchChar('=')) { + return Token.ASSIGN_RSH; + } else { + return Token.RSH; + } + } + } else { + if (matchChar('=')) { + return Token.GE; + } else { + return Token.GT; + } + } + + case '*': + if (matchChar('=')) { + return Token.ASSIGN_MUL; + } else { + return Token.MUL; + } + + case '/': + // is it a // comment? + if (matchChar('/')) { + skipLine(); + continue retry; + } + if (matchChar('*')) { + boolean lookForSlash = false; + for (;;) { + c = getChar(); + if (c == EOF_CHAR) { + parser.addError("msg.unterminated.comment"); + return Token.ERROR; + } else if (c == '*') { + lookForSlash = true; + } else if (c == '/') { + if (lookForSlash) { + continue retry; + } + } else { + lookForSlash = false; + } + } + } + + if (matchChar('=')) { + return Token.ASSIGN_DIV; + } else { + return Token.DIV; + } + + case '%': + if (matchChar('=')) { + return Token.ASSIGN_MOD; + } else { + return Token.MOD; + } + + case '~': + return Token.BITNOT; + + case '+': + if (matchChar('=')) { + return Token.ASSIGN_ADD; + } else if (matchChar('+')) { + return Token.INC; + } else { + return Token.ADD; + } + + case '-': + if (matchChar('=')) { + c = Token.ASSIGN_SUB; + } else if (matchChar('-')) { + if (!dirtyLine) { + // treat HTML end-comment after possible whitespace + // after line start as comment-utill-eol + if (matchChar('>')) { + skipLine(); + continue retry; + } + } + c = Token.DEC; + } else { + c = Token.SUB; + } + dirtyLine = true; + return c; + + default: + parser.addError("msg.illegal.character"); + return Token.ERROR; + } + } + } + + private static boolean isAlpha(int c) + { + // Use 'Z' < 'a' + if (c <= 'Z') { + return 'A' <= c; + } else { + return 'a' <= c && c <= 'z'; + } + } + + static boolean isDigit(int c) + { + return '0' <= c && c <= '9'; + } + + /* As defined in ECMA. jsscan.c uses C isspace() (which allows + * \v, I think.) note that code in getChar() implicitly accepts + * '\r' == \u000D as well. + */ + static boolean isJSSpace(int c) + { + if (c <= 127) { + return c == 0x20 || c == 0x9 || c == 0xC || c == 0xB; + } else { + return c == 0xA0 + || Character.getType((char)c) == Character.SPACE_SEPARATOR; + } + } + + private static boolean isJSFormatChar(int c) + { + return c > 127 && Character.getType((char)c) == Character.FORMAT; + } + + /** + * Parser calls the method when it gets / or /= in literal context. + */ + void readRegExp(int startToken) + throws IOException + { + stringBufferTop = 0; + if (startToken == Token.ASSIGN_DIV) { + // Miss-scanned /= + addToString('='); + } else { + if (startToken != Token.DIV) Kit.codeBug(); + } + + int c; + while ((c = getChar()) != '/') { + if (c == '\n' || c == EOF_CHAR) { + ungetChar(c); + throw parser.reportError("msg.unterminated.re.lit"); + } + if (c == '\\') { + addToString(c); + c = getChar(); + } + + addToString(c); + } + int reEnd = stringBufferTop; + + while (true) { + if (matchChar('g')) + addToString('g'); + else if (matchChar('i')) + addToString('i'); + else if (matchChar('m')) + addToString('m'); + else + break; + } + + if (isAlpha(peekChar())) { + throw parser.reportError("msg.invalid.re.flag"); + } + + this.string = new String(stringBuffer, 0, reEnd); + this.regExpFlags = new String(stringBuffer, reEnd, + stringBufferTop - reEnd); + } + + boolean isXMLAttribute() + { + return xmlIsAttribute; + } + + int getFirstXMLToken() throws IOException + { + xmlOpenTagsCount = 0; + xmlIsAttribute = false; + xmlIsTagContent = false; + ungetChar('<'); + return getNextXMLToken(); + } + + int getNextXMLToken() throws IOException + { + stringBufferTop = 0; // remember the XML + + for (int c = getChar(); c != EOF_CHAR; c = getChar()) { + if (xmlIsTagContent) { + switch (c) { + case '>': + addToString(c); + xmlIsTagContent = false; + xmlIsAttribute = false; + break; + case '/': + addToString(c); + if (peekChar() == '>') { + c = getChar(); + addToString(c); + xmlIsTagContent = false; + xmlOpenTagsCount--; + } + break; + case '{': + ungetChar(c); + this.string = getStringFromBuffer(); + return Token.XML; + case '\'': + case '"': + addToString(c); + if (!readQuotedString(c)) return Token.ERROR; + break; + case '=': + addToString(c); + xmlIsAttribute = true; + break; + case ' ': + case '\t': + case '\r': + case '\n': + addToString(c); + break; + default: + addToString(c); + xmlIsAttribute = false; + break; + } + + if (!xmlIsTagContent && xmlOpenTagsCount == 0) { + this.string = getStringFromBuffer(); + return Token.XMLEND; + } + } else { + switch (c) { + case '<': + addToString(c); + c = peekChar(); + switch (c) { + case '!': + c = getChar(); // Skip ! + addToString(c); + c = peekChar(); + switch (c) { + case '-': + c = getChar(); // Skip - + addToString(c); + c = getChar(); + if (c == '-') { + addToString(c); + if(!readXmlComment()) return Token.ERROR; + } else { + // throw away the string in progress + stringBufferTop = 0; + this.string = null; + parser.addError("msg.XML.bad.form"); + return Token.ERROR; + } + break; + case '[': + c = getChar(); // Skip [ + addToString(c); + if (getChar() == 'C' && + getChar() == 'D' && + getChar() == 'A' && + getChar() == 'T' && + getChar() == 'A' && + getChar() == '[') + { + addToString('C'); + addToString('D'); + addToString('A'); + addToString('T'); + addToString('A'); + addToString('['); + if (!readCDATA()) return Token.ERROR; + + } else { + // throw away the string in progress + stringBufferTop = 0; + this.string = null; + parser.addError("msg.XML.bad.form"); + return Token.ERROR; + } + break; + default: + if(!readEntity()) return Token.ERROR; + break; + } + break; + case '?': + c = getChar(); // Skip ? + addToString(c); + if (!readPI()) return Token.ERROR; + break; + case '/': + // End tag + c = getChar(); // Skip / + addToString(c); + if (xmlOpenTagsCount == 0) { + // throw away the string in progress + stringBufferTop = 0; + this.string = null; + parser.addError("msg.XML.bad.form"); + return Token.ERROR; + } + xmlIsTagContent = true; + xmlOpenTagsCount--; + break; + default: + // Start tag + xmlIsTagContent = true; + xmlOpenTagsCount++; + break; + } + break; + case '{': + ungetChar(c); + this.string = getStringFromBuffer(); + return Token.XML; + default: + addToString(c); + break; + } + } + } + + stringBufferTop = 0; // throw away the string in progress + this.string = null; + parser.addError("msg.XML.bad.form"); + return Token.ERROR; + } + + /** + * + */ + private boolean readQuotedString(int quote) throws IOException + { + for (int c = getChar(); c != EOF_CHAR; c = getChar()) { + addToString(c); + if (c == quote) return true; + } + + stringBufferTop = 0; // throw away the string in progress + this.string = null; + parser.addError("msg.XML.bad.form"); + return false; + } + + /** + * + */ + private boolean readXmlComment() throws IOException + { + for (int c = getChar(); c != EOF_CHAR;) { + addToString(c); + if (c == '-' && peekChar() == '-') { + c = getChar(); + addToString(c); + if (peekChar() == '>') { + c = getChar(); // Skip > + addToString(c); + return true; + } else { + continue; + } + } + c = getChar(); + } + + stringBufferTop = 0; // throw away the string in progress + this.string = null; + parser.addError("msg.XML.bad.form"); + return false; + } + + /** + * + */ + private boolean readCDATA() throws IOException + { + for (int c = getChar(); c != EOF_CHAR;) { + addToString(c); + if (c == ']' && peekChar() == ']') { + c = getChar(); + addToString(c); + if (peekChar() == '>') { + c = getChar(); // Skip > + addToString(c); + return true; + } else { + continue; + } + } + c = getChar(); + } + + stringBufferTop = 0; // throw away the string in progress + this.string = null; + parser.addError("msg.XML.bad.form"); + return false; + } + + /** + * + */ + private boolean readEntity() throws IOException + { + int declTags = 1; + for (int c = getChar(); c != EOF_CHAR; c = getChar()) { + addToString(c); + switch (c) { + case '<': + declTags++; + break; + case '>': + declTags--; + if (declTags == 0) return true; + break; + } + } + + stringBufferTop = 0; // throw away the string in progress + this.string = null; + parser.addError("msg.XML.bad.form"); + return false; + } + + /** + * + */ + private boolean readPI() throws IOException + { + for (int c = getChar(); c != EOF_CHAR; c = getChar()) { + addToString(c); + if (c == '?' && peekChar() == '>') { + c = getChar(); // Skip > + addToString(c); + return true; + } + } + + stringBufferTop = 0; // throw away the string in progress + this.string = null; + parser.addError("msg.XML.bad.form"); + return false; + } + + private String getStringFromBuffer() + { + return new String(stringBuffer, 0, stringBufferTop); + } + + private void addToString(int c) + { + int N = stringBufferTop; + if (N == stringBuffer.length) { + char[] tmp = new char[stringBuffer.length * 2]; + System.arraycopy(stringBuffer, 0, tmp, 0, N); + stringBuffer = tmp; + } + stringBuffer[N] = (char)c; + stringBufferTop = N + 1; + } + + private void ungetChar(int c) + { + // can not unread past across line boundary + if (ungetCursor != 0 && ungetBuffer[ungetCursor - 1] == '\n') + Kit.codeBug(); + ungetBuffer[ungetCursor++] = c; + } + + private boolean matchChar(int test) throws IOException + { + int c = getChar(); + if (c == test) { + return true; + } else { + ungetChar(c); + return false; + } + } + + private int peekChar() throws IOException + { + int c = getChar(); + ungetChar(c); + return c; + } + + private int getChar() throws IOException + { + if (ungetCursor != 0) { + return ungetBuffer[--ungetCursor]; + } + + for(;;) { + int c; + if (sourceString != null) { + if (sourceCursor == sourceEnd) { + hitEOF = true; + return EOF_CHAR; + } + c = sourceString.charAt(sourceCursor++); + } else { + if (sourceCursor == sourceEnd) { + if (!fillSourceBuffer()) { + hitEOF = true; + return EOF_CHAR; + } + } + c = sourceBuffer[sourceCursor++]; + } + + if (lineEndChar >= 0) { + if (lineEndChar == '\r' && c == '\n') { + lineEndChar = '\n'; + continue; + } + lineEndChar = -1; + lineStart = sourceCursor - 1; + lineno++; + } + + if (c <= 127) { + if (c == '\n' || c == '\r') { + lineEndChar = c; + c = '\n'; + } + } else { + if (isJSFormatChar(c)) { + continue; + } + if (ScriptRuntime.isJSLineTerminator(c)) { + lineEndChar = c; + c = '\n'; + } + } + return c; + } + } + + private void skipLine() throws IOException + { + // skip to end of line + int c; + while ((c = getChar()) != EOF_CHAR && c != '\n') { } + ungetChar(c); + } + + final int getOffset() + { + int n = sourceCursor - lineStart; + if (lineEndChar >= 0) { --n; } + return n; + } + + final String getLine() + { + if (sourceString != null) { + // String case + int lineEnd = sourceCursor; + if (lineEndChar >= 0) { + --lineEnd; + } else { + for (; lineEnd != sourceEnd; ++lineEnd) { + int c = sourceString.charAt(lineEnd); + if (ScriptRuntime.isJSLineTerminator(c)) { + break; + } + } + } + return sourceString.substring(lineStart, lineEnd); + } else { + // Reader case + int lineLength = sourceCursor - lineStart; + if (lineEndChar >= 0) { + --lineLength; + } else { + // Read until the end of line + for (;; ++lineLength) { + int i = lineStart + lineLength; + if (i == sourceEnd) { + try { + if (!fillSourceBuffer()) { break; } + } catch (IOException ioe) { + // ignore it, we're already displaying an error... + break; + } + // i recalculuation as fillSourceBuffer can move saved + // line buffer and change lineStart + i = lineStart + lineLength; + } + int c = sourceBuffer[i]; + if (ScriptRuntime.isJSLineTerminator(c)) { + break; + } + } + } + return new String(sourceBuffer, lineStart, lineLength); + } + } + + private boolean fillSourceBuffer() throws IOException + { + if (sourceString != null) Kit.codeBug(); + if (sourceEnd == sourceBuffer.length) { + if (lineStart != 0) { + System.arraycopy(sourceBuffer, lineStart, sourceBuffer, 0, + sourceEnd - lineStart); + sourceEnd -= lineStart; + sourceCursor -= lineStart; + lineStart = 0; + } else { + char[] tmp = new char[sourceBuffer.length * 2]; + System.arraycopy(sourceBuffer, 0, tmp, 0, sourceEnd); + sourceBuffer = tmp; + } + } + int n = sourceReader.read(sourceBuffer, sourceEnd, + sourceBuffer.length - sourceEnd); + if (n < 0) { + return false; + } + sourceEnd += n; + return true; + } + + // stuff other than whitespace since start of line + private boolean dirtyLine; + + String regExpFlags; + + // Set this to an inital non-null value so that the Parser has + // something to retrieve even if an error has occured and no + // string is found. Fosters one class of error, but saves lots of + // code. + private String string = ""; + private double number; + + private char[] stringBuffer = new char[128]; + private int stringBufferTop; + private ObjToIntMap allStrings = new ObjToIntMap(50); + + // Room to backtrace from to < on failed match of the last - in <!-- + private final int[] ungetBuffer = new int[3]; + private int ungetCursor; + + private boolean hitEOF = false; + + private int lineStart = 0; + private int lineno; + private int lineEndChar = -1; + + private String sourceString; + private Reader sourceReader; + private char[] sourceBuffer; + private int sourceEnd; + private int sourceCursor; + + // for xml tokenizer + private boolean xmlIsAttribute; + private boolean xmlIsTagContent; + private int xmlOpenTagsCount; + + private Parser parser; +} |