1 // $Id: CQLLexer.java,v 1.6 2002-11-17 23:29:02 mike Exp $
3 package org.z3950.zing.cql;
4 import java.io.StreamTokenizer;
5 import java.io.StringReader;
6 import java.util.Hashtable;
9 // This is a semi-trivial subclass for java.io.StreamTokenizer that:
10 // * Has a halfDecentPushBack() method that actually works
11 // * Includes a render() method
12 // * Knows about the multi-character tokens "<=", ">=" and "<>"
13 // * Recognises a set of keywords as tokens in their own right
14 // * Includes some primitive debugging-output facilities
15 // It's used only by CQLParser.
17 class CQLLexer extends StreamTokenizer {
18 // New publicly visible token-types
19 static int TT_LE = 1000; // The "<=" relation
20 static int TT_GE = 1001; // The ">=" relation
21 static int TT_NE = 1002; // The "<>" relation
22 static int TT_AND = 1003; // The "and" boolean
23 static int TT_OR = 1004; // The "or" boolean
24 static int TT_NOT = 1005; // The "not" boolean
25 static int TT_PROX = 1006; // The "prox" boolean
26 static int TT_ANY = 1007; // The "any" relation
27 static int TT_ALL = 1008; // The "all" relation
28 static int TT_EXACT = 1009; // The "exact" relation
29 static int TT_pWORD = 1010; // The "word" proximity unit
30 static int TT_SENTENCE = 1011; // The "sentence" proximity unit
31 static int TT_PARAGRAPH = 1012; // The "paragraph" proximity unit
32 static int TT_ELEMENT = 1013; // The "element" proximity unit
33 static int TT_ORDERED = 1014; // The "ordered" proximity ordering
34 static int TT_UNORDERED = 1015; // The "unordered" proximity ordering
35 static int TT_RELEVANT = 1016; // The "relevant" relation modifier
36 static int TT_FUZZY = 1017; // The "fuzzy" relation modifier
37 static int TT_STEM = 1018; // The "stem" relation modifier
38 static int TT_SCR = 1019; // The server choice relation
39 static int TT_PHONETIC = 1020; // The "phonetic" relation modifier
41 // Support for keywords. It would be nice to compile this linear
42 // list into a Hashtable, but it's hard to store ints as hash
43 // values, and next to impossible to use them as hash keys. So
44 // we'll just scan the (very short) list every time we need to do
46 private class Keyword {
49 Keyword(int token, String keyword) {
51 this.keyword = keyword;
54 // This should logically be static, but Java won't allow it :-P
55 private Keyword[] keywords = {
56 new Keyword(TT_AND, "and"),
57 new Keyword(TT_OR, "or"),
58 new Keyword(TT_NOT, "not"),
59 new Keyword(TT_PROX, "prox"),
60 new Keyword(TT_ANY, "any"),
61 new Keyword(TT_ALL, "all"),
62 new Keyword(TT_EXACT, "exact"),
63 new Keyword(TT_pWORD, "word"),
64 new Keyword(TT_SENTENCE, "sentence"),
65 new Keyword(TT_PARAGRAPH, "paragraph"),
66 new Keyword(TT_ELEMENT, "element"),
67 new Keyword(TT_ORDERED, "ordered"),
68 new Keyword(TT_UNORDERED, "unordered"),
69 new Keyword(TT_RELEVANT, "relevant"),
70 new Keyword(TT_FUZZY, "fuzzy"),
71 new Keyword(TT_STEM, "stem"),
72 new Keyword(TT_SCR, "scr"),
73 new Keyword(TT_PHONETIC, "phonetic"),
76 // For halfDecentPushBack() and the code at the top of nextToken()
77 private static int TT_UNDEFINED = -1000;
78 private int saved_ttype = TT_UNDEFINED;
79 private double saved_nval;
80 private String saved_sval;
82 // Controls debugging output
83 private static boolean DEBUG;
85 CQLLexer(String cql, boolean lexdebug) {
86 super(new StringReader(cql));
87 wordChars('!', '?'); // ASCII-dependency!
88 wordChars('[', '`'); // ASCII-dependency!
96 wordChars('\'', '\''); // prevent this from introducing strings
101 private static void debug(String str) {
103 System.err.println("LEXDEBUG: " + str);
106 // I don't honestly understand why we need this, but the
107 // documentation for java.io.StreamTokenizer.pushBack() is pretty
108 // vague about its semantics, and it seems to me that they could
109 // be summed up as "it doesn't work". This version has the very
110 // clear semantics "pretend I didn't call nextToken() just then".
112 private void halfDecentPushBack() {
118 public int nextToken() throws java.io.IOException {
119 if (saved_ttype != TT_UNDEFINED) {
123 saved_ttype = TT_UNDEFINED;
124 debug("using saved ttype=" + ttype + ", " +
125 "nval=" + nval + ", sval='" + sval + "'");
129 underlyingNextToken();
131 debug("token starts with '<' ...");
132 underlyingNextToken();
134 debug("token continues with '=' - it's '<='");
136 } else if (ttype == '>') {
137 debug("token continues with '>' - it's '<>'");
140 debug("next token is " + render() + " (pushed back)");
141 halfDecentPushBack();
143 debug("AFTER: ttype is now " + ttype + " - " + render());
145 } else if (ttype == '>') {
146 debug("token starts with '>' ...");
147 underlyingNextToken();
149 debug("token continues with '=' - it's '>='");
152 debug("next token is " + render() + " (pushed back)");
153 halfDecentPushBack();
155 debug("AFTER: ttype is now " + ttype + " - " + render());
159 debug("done nextToken(): ttype=" + ttype + ", " +
160 "nval=" + nval + ", " + "sval='" + sval + "'" +
161 " (" + render() + ")");
166 // It's important to do keyword recognition here at the lowest
167 // level, otherwise when one of these words follows "<" or ">"
168 // (which can be the beginning of multi-character tokens) it gets
169 // pushed back as a string, and its keywordiness is not
172 public int underlyingNextToken() throws java.io.IOException {
174 if (ttype == TT_WORD)
175 for (int i = 0; i < keywords.length; i++)
176 if (sval.equalsIgnoreCase(keywords[i].keyword))
177 ttype = keywords[i].token;
182 // Simpler interface for the usual case: current token with quoting
184 return render(ttype, true);
187 String render(int token, boolean quoteChars) {
188 if (token == TT_EOF) {
190 } else if (token == TT_NUMBER) {
191 return new Integer((int) nval).toString();
192 } else if (token == TT_WORD) {
193 return "word: " + sval;
194 } else if (token == '"') {
195 return "string: \"" + sval + "\"";
196 } else if (token == TT_LE) {
198 } else if (token == TT_GE) {
200 } else if (token == TT_NE) {
204 // Check whether its associated with one of the keywords
205 for (int i = 0; i < keywords.length; i++)
206 if (token == keywords[i].token)
207 return keywords[i].keyword;
209 // Otherwise it must be a single character, such as '(' or '/'.
210 String res = String.valueOf((char) token);
211 if (quoteChars) res = "'" + res + "'";
215 public static void main(String[] args) throws Exception {
216 if (args.length > 1) {
217 System.err.println("Usage: CQLLexer [<CQL-query>]");
218 System.err.println("If unspecified, query is read from stdin");
223 if (args.length == 1) {
226 byte[] bytes = new byte[10000];
228 // Read in the whole of standard input in one go
229 int nbytes = System.in.read(bytes);
230 } catch (java.io.IOException ex) {
231 System.err.println("Can't read query: " + ex.getMessage());
234 cql = new String(bytes);
237 CQLLexer lexer = new CQLLexer(cql, true);
239 while ((token = lexer.nextToken()) != TT_EOF) {
240 // Nothing to do: debug() statements render tokens for us