/* * @(#)BetterTokenizer.java 1.21g 99/03/04 * * Copyright 1995-1997 by Sun Microsystems, Inc., * 901 San Antonio Road, Palo Alto, California, 94303, U.S.A. * All rights reserved. * * This software is the confidential and proprietary information * of Sun Microsystems, Inc. ("Confidential Information"). You * shall not disclose such Confidential Information and shall use * it only in accordance with the terms of the license agreement * you entered into with Sun. */ package main; import java.io.*; /** * The BetterTokenizer class takes an input stream and * parses it into "tokens", allowing the tokens to be * read one at a time. The parsing process is controlled by a table * and a number of flags that can be set to various states. The * stream tokenizer can recognize identifiers, numbers, quoted * strings, and various comment styles. *

* Each byte read from the input stream is regarded as a character * in the range '\u0000' through '\u00FF'. * The character value is used to look up five possible attributes of * the character: white space, alphabetic, * numeric, string quote, and comment character. * Each character can have zero or more of these attributes. *

* In addition, an instance has four flags. These flags indicate: *

*

* A typical application first constructs an instance of this class, * sets up the syntax tables, and then repeatedly loops calling the * nextToken method in each iteration of the loop until * it returns the value TT_EOF. * * @author James Gosling, modified by GvonD * @version 1.21g, 03/04/99 * @see java.io.BetterTokenizer#nextToken() * @see java.io.BetterTokenizer#TT_EOF * @since JDK1.0 */ public class BetterTokenizer { /* Only one of these will be non-null */ private Reader reader = null; private InputStream input = null; private char buf[] = new char[20]; private int peekc; private boolean pushedBack; private boolean forceLower; /** The line number of the last token read */ private int LINENO = 1; private boolean eolIsSignificantP = false; private boolean slashSlashCommentsP = false; private boolean slashStarCommentsP = false; private byte ctype[] = new byte[256]; private static final byte CT_WHITESPACE = 1; private static final byte CT_DIGIT = 2; private static final byte CT_ALPHA = 4; private static final byte CT_QUOTE = 8; private static final byte CT_COMMENT = 16; /** * After a call to the nextToken method, this field * contains the type of the token just read. For a single character * token, its value is the single character, converted to an integer. * For a quoted string token (see , its value is the quote character. * Otherwise, its value is one of the following: *

* * @see java.io.BetterTokenizer#eolIsSignificant(boolean) * @see java.io.BetterTokenizer#nextToken() * @see java.io.BetterTokenizer#quoteChar(int) * @see java.io.BetterTokenizer#TT_EOF * @see java.io.BetterTokenizer#TT_EOL * @see java.io.BetterTokenizer#TT_NUMBER * @see java.io.BetterTokenizer#TT_WORD */ public int ttype = TT_NOTHING; /** * A constant indicating that the end of the stream has been read. */ public static final int TT_EOF = -1; /** * A constant indicating that the end of the line has been read. */ public static final int TT_EOL = '\n'; /** * A constant indicating that a number token has been read. */ public static final int TT_NUMBER = -2; /** * A constant indicating that a word token has been read. */ public static final int TT_WORD = -3; /* A constant indicating that no token has been read, used for * initializing ttype. FIXME This could be made public and * made available as the part of the API in a future release. */ private static final int TT_NOTHING = -4; /** * If the current token is a word token, this field contains a * string giving the characters of the word token. When the current * token is a quoted string token, this field contains the body of * the string. *

* The current token is a word when the value of the * ttype field is TT_WORD. The current token is * a quoted string token when the value of the ttype field is * a quote character. * * @see java.io.BetterTokenizer#quoteChar(int) * @see java.io.BetterTokenizer#TT_WORD * @see java.io.BetterTokenizer#ttype * @since JDK1.0 */ public String sval; /** * If the current token is a number, this field contains the value * of that number. The current token is a number when the value of * the ttype field is TT_NUMBER. * * @see java.io.BetterTokenizer#TT_NUMBER * @see java.io.BetterTokenizer#ttype */ public double nval; /** Private constructor that initializes everything except the streams. */ private BetterTokenizer() { wordChars('a', 'z'); wordChars('A', 'Z'); wordChars(128 + 32, 255); whitespaceChars(0, ' '); slashStarComments(true); slashSlashComments(true); //commentChar('/'); quoteChar('"'); quoteChar('\''); parseNumbers(); } /** * Creates a stream tokenizer that parses the specified input * stream. The stream tokenizer is initialized to the following * default state: *

* * @deprecated As of JDK version 1.1, the preferred way to tokenize an * input stream is to convert it into a character stream, for example: *

*

	 *   Reader r = new BufferedReader(new InputStreamReader(is));
	 *   BetterTokenizer st = new BetterTokenizer(r);
	 * 
* * @param is an input stream. * @see java.io.BufferedReader * @see java.io.InputStreamReader * @see java.io.BetterTokenizer#BetterTokenizer(java.io.Reader) */ public BetterTokenizer(InputStream is) { this(); input = is; } /** * Create a tokenizer that parses the given character stream. * @since JDK1.1 */ public BetterTokenizer(Reader r) { this(); reader = r; } /** * Resets this tokenizer's syntax table so that all characters are * "ordinary." See the ordinaryChar method * for more information on a character being ordinary. * * @see java.io.BetterTokenizer#ordinaryChar(int) */ public void resetSyntax() { for (int i = ctype.length; --i >= 0;) ctype[i] = 0; } /** * Specifies that all characters c in the range * low <= c <= high * are word constituents. A word token consists of a word constituent * followed by zero or more word constituents or number constituents. * * @param low the low end of the range. * @param hi the high end of the range. */ public void wordChars(int low, int hi) { if (low < 0) low = 0; if (hi >= ctype.length) hi = ctype.length - 1; while (low <= hi) ctype[low++] |= CT_ALPHA; } /** * Specifies that all characters c in the range * low <= c <= high * are white space characters. White space characters serve only to * separate tokens in the input stream. * * @param low the low end of the range. * @param hi the high end of the range. */ public void whitespaceChars(int low, int hi) { if (low < 0) low = 0; if (hi >= ctype.length) hi = ctype.length - 1; while (low <= hi) ctype[low++] = CT_WHITESPACE; } /** * Specifies that all characters c in the range * low <= c <= high * are "ordinary" in this tokenizer. See the * ordinaryChar method for more information on a * character being ordinary. * * @param low the low end of the range. * @param hi the high end of the range. * @see java.io.BetterTokenizer#ordinaryChar(int) */ public void ordinaryChars(int low, int hi) { if (low < 0) low = 0; if (hi >= ctype.length) hi = ctype.length - 1; while (low <= hi) ctype[low++] = 0; } /** * Specifies that the character argument is "ordinary" * in this tokenizer. It removes any special significance the * character has as a comment character, word component, string * delimiter, white space, or number character. When such a character * is encountered by the parser, the parser treates it as a * single-character token and sets ttype field to the * character value. * * @param ch the character. * @see java.io.BetterTokenizer#ttype */ public void ordinaryChar(int ch) { if (ch >= 0 && ch < ctype.length) ctype[ch] = 0; } /** * Specified that the character argument starts a single-line * comment. All characters from the comment character to the end of * the line are ignored by this stream tokenizer. * * @param ch the character. */ public void commentChar(int ch) { if (ch >= 0 && ch < ctype.length) ctype[ch] = CT_COMMENT; } /** * Specifies that matching pairs of this character delimit string * constants in this tokenizer. *

* When the nextToken method encounters a string * constant, the ttype field is set to the string * delimiter and the sval field is set to the body of * the string. *

* If a string quote character is encountered, then a string is * recognized, consisting of all characters after (but not including) * the string quote character, up to (but not including) the next * occurrence of that same string quote character, or a line * terminator, or end of file. The usual escape sequences such as * "\n" and "\t" are recognized and * converted to single characters as the string is parsed. * * @param ch the character. * @see java.io.BetterTokenizer#nextToken() * @see java.io.BetterTokenizer#sval * @see java.io.BetterTokenizer#ttype */ public void quoteChar(int ch) { if (ch >= 0 && ch < ctype.length) ctype[ch] = CT_QUOTE; } /** * Specifies that numbers should be parsed by this tokenizer. The * syntax table of this tokenizer is modified so that each of the twelve * characters: *

*

* has the "numeric" attribute. *

* When the parser encounters a word token that has the format of a * double precision floating-point number, it treats the token as a * number rather than a word, by setting the the ttype * field to the value TT_NUMBER and putting the numeric * value of the token into the nval field. * * @see java.io.BetterTokenizer#nval * @see java.io.BetterTokenizer#TT_NUMBER * @see java.io.BetterTokenizer#ttype */ public void parseNumbers() { for (int i = '0'; i <= '9'; i++) ctype[i] |= CT_DIGIT; ctype['.'] |= CT_DIGIT; ctype['-'] |= CT_DIGIT; } /** * Determines whether or not ends of line are treated as tokens. * If the flag argument is true, this tokenizer treats end of lines * as tokens; the nextToken method returns * TT_EOL and also sets the ttype field to * this value when an end of line is read. *

* A line is a sequence of characters ending with either a * carriage-return character ('\r') or a newline * character ('\n'). In addition, a carriage-return * character followed immediately by a newline character is treated * as a single end-of-line token. *

* If the flag is false, end-of-line characters are * treated as white space and serve only to separate tokens. * * @param flag true indicates that end-of-line characters * are separate tokens; false indicates that * end-of-line characters are white space. * @see java.io.BetterTokenizer#nextToken() * @see java.io.BetterTokenizer#ttype * @see java.io.BetterTokenizer#TT_EOL */ public void eolIsSignificant(boolean flag) { eolIsSignificantP = flag; } /** * Determines whether or not the tokenizer recognizes C-style comments. * If the flag argument is true, this stream tokenizer * recognizes C-style comments. All text between successive * occurrences of /* and */ are discarded. *

* If the flag argument is false, then C-style comments * are not treated specially. * * @param flag true indicates to recognize and ignore * C-style comments. */ public void slashStarComments(boolean flag) { slashStarCommentsP = flag; } /** * Determines whether or not the tokenizer recognizes C++-style comments. * If the flag argument is true, this stream tokenizer * recognizes C++-style comments. Any occurrence of two consecutive * slash characters ('/') is treated as the beginning of * a comment that extends to the end of the line. *

* If the flag argument is false, then C++-style * comments are not treated specially. * * @param flag true indicates to recognize and ignore * C++-style comments. */ public void slashSlashComments(boolean flag) { slashSlashCommentsP = flag; } /** * Determines whether or not word token are automatically lowercased. * If the flag argument is true, then the value in the * sval field is lowercased whenever a word token is * returned (the ttype field has the * value TT_WORD by the nextToken method * of this tokenizer. *

* If the flag argument is false, then the * sval field is not modified. * * @param fl true indicates that all word tokens should * be lowercased. * @see java.io.BetterTokenizer#nextToken() * @see java.io.BetterTokenizer#ttype * @see java.io.BetterTokenizer#TT_WORD */ public void lowerCaseMode(boolean fl) { forceLower = fl; } /** Read the next character */ private int read() throws IOException { if (reader != null) return reader.read(); else if (input != null) return input.read(); else throw new IllegalStateException(); } /** * Parses the next token from the input stream of this tokenizer. * The type of the next token is returned in the ttype * field. Additional information about the token may be in the * nval field or the sval field of this * tokenizer. *

* Typical clients of this * class first set up the syntax tables and then sit in a loop * calling nextToken to parse successive tokens until TT_EOF * is returned. * * @return the value of the ttype field. * @exception IOException if an I/O error occurs. * @see java.io.BetterTokenizer#nval * @see java.io.BetterTokenizer#sval * @see java.io.BetterTokenizer#ttype */ public int nextToken() throws IOException { if (pushedBack) { pushedBack = false; return ttype; } byte ct[] = ctype; int c; sval = null; if (ttype == TT_NOTHING) { c = read(); if (c >= 0) // ttype is surely overwritten below to its correct value. ttype = c; // for now we just make sure it isn't TT_NOTHING } else { c = peekc; } if (c < 0) return ttype = TT_EOF; int ctype = c < 256 ? ct[c] : CT_ALPHA; while ((ctype & CT_WHITESPACE) != 0) { if (c == '\r') { LINENO++; c = read(); if (c == '\n') c = read(); if (eolIsSignificantP) { peekc = c; return ttype = TT_EOL; } } else { if (c == '\n') { LINENO++; if (eolIsSignificantP) { peekc = read(); return ttype = TT_EOL; } } c = read(); } if (c < 0) return ttype = TT_EOF; ctype = c < 256 ? ct[c] : CT_ALPHA; } if ((ctype & CT_DIGIT) != 0) { // GvonD changed to handle scientific notation boolean neg = false; boolean exp = false; // GvonD added boolean negexp = false; // GvonD added if (c == '-') { c = read(); if (c != '.' && (c < '0' || c > '9')) { peekc = c; return ttype = '-'; } neg = true; } double v = 0; int e = 0; // GvonD added int decexp = 0; int seendot = 0; while (true) { if (c == '.' && seendot == 0) seendot = 1; else if ('0' <= c && c <= '9' && !exp) { v = v * 10 + (c - '0'); decexp += seendot; } else if ('0' <= c && c <= '9' && exp) { e = e * 10 + (c - '0'); } else if ((c == 'E' || c == 'e') && seendot == 1) { exp = true; } else if (c == '-' && exp) { negexp = true; } else if (seendot == 1 && (c == 'f' || c == 'F' || c == 'd' || c == 'D')) ; // this is here to swallow the possible trailing character correctly else break; c = read(); } peekc = c; if (decexp != 0) { double denom = 10; decexp--; while (decexp > 0) { denom *= 10; decexp--; } /* do one division of a likely-to-be-more-accurate number */ v = v / denom; } if (exp) { if (negexp) { for(int i = 0; i < e; i++) { v = v / 10; } } else { for(int i = 0; i < e; i++) { v = v * 10; } } } nval = neg ? -v : v; return ttype = TT_NUMBER; } if ((ctype & CT_ALPHA) != 0) { int i = 0; do { if (i >= buf.length) { char nb[] = new char[buf.length * 2]; System.arraycopy(buf, 0, nb, 0, buf.length); buf = nb; } buf[i++] = (char) c; c = read(); ctype = c < 0 ? CT_WHITESPACE : c < 256 ? ct[c] : CT_ALPHA; } while ((ctype & (CT_ALPHA | CT_DIGIT)) != 0); peekc = c; sval = String.copyValueOf(buf, 0, i); if (forceLower) sval = sval.toLowerCase(); return ttype = TT_WORD; } if ((ctype & CT_COMMENT) != 0) { while ((c = read()) != '\n' && c != '\r' && c >= 0); peekc = c; return nextToken(); } if ((ctype & CT_QUOTE) != 0) { ttype = c; int i = 0; // invariants (because \Octal needs a lookahead): // (i) c contains char value // (ii) peekc contains the lookahead peekc = read(); while (peekc >= 0 && peekc != ttype && peekc != '\n' && peekc != '\r') { if (peekc == '\\') { c = read(); int first = c; // to allow \377, but not \477 if (c >= '0' && c <= '7') { c = c - '0'; int c2 = read(); if ('0' <= c2 && c2 <= '7') { c = (c << 3) + (c2 - '0'); c2 = read(); if ('0' <= c2 && c2 <= '7' && first <= '3') { c = (c << 3) + (c2 - '0'); peekc = read(); } else peekc = c2; } else peekc = c2; } else { switch (c) { case 'a': c = 0x7; break; case 'b': c = '\b'; break; case 'f': c = 0xC; break; case 'n': c = '\n'; break; case 'r': c = '\r'; break; case 't': c = '\t'; break; case 'v': c = 0xB; break; } peekc = read(); } } else { c = peekc; peekc = read(); } if (i >= buf.length) { char nb[] = new char[buf.length * 2]; System.arraycopy(buf, 0, nb, 0, buf.length); buf = nb; } buf[i++] = (char) c; } if (peekc == ttype) // keep \n or \r intact in peekc peekc = read(); sval = String.copyValueOf(buf, 0, i); return ttype; } if (c == '/' && (slashSlashCommentsP || slashStarCommentsP)) { c = read(); if (c == '*' && slashStarCommentsP) { int prevc = 0; while ((c = read()) != '/' || prevc != '*') { if (c == '\r') { LINENO++; c = read(); if (c == '\n') { c = read(); } } else { if (c == '\n') { LINENO++; c = read(); } } if (c < 0) return ttype = TT_EOF; prevc = c; } peekc = read(); return nextToken(); } else if (c == '/' && slashSlashCommentsP) { while ((c = read()) != '\n' && c != '\r' && c >= 0); peekc = c; return nextToken(); } else { peekc = c; return ttype = '/'; } } peekc = read(); return ttype = c; } /** * Causes the next call to the nextToken method of this * tokenizer to return the current value in the ttype * field, and not to modify the value in the nval or * sval field. * * @see java.io.BetterTokenizer#nextToken() * @see java.io.BetterTokenizer#nval * @see java.io.BetterTokenizer#sval * @see java.io.BetterTokenizer#ttype */ public void pushBack() { if (ttype != TT_NOTHING) // no-op if nextToken() not called pushedBack = true; } /** * Return the current line number. * * @return the current line number of this stream tokenizer. */ public int lineno() { return LINENO; } /** * Returns the string representation of the current stream token. * * @return a string representation of the token specified by the * ttype, nval, and sval * fields. * @see java.io.BetterTokenizer#nval * @see java.io.BetterTokenizer#sval * @see java.io.BetterTokenizer#ttype */ public String toString() { String ret; switch (ttype) { case TT_EOF: ret = "EOF"; break; case TT_EOL: ret = "EOL"; break; case TT_WORD: ret = sval; break; case TT_NUMBER: ret = "n=" + nval; break; case TT_NOTHING: ret = "NOTHING"; break; default:{ char s[] = new char[3]; s[0] = s[2] = '\''; s[1] = (char) ttype; ret = new String(s); break; } } return "Token[" + ret + "], line " + LINENO; } }