From 70be8275aabe87c909bf11148bb279c3e1380bb2 Mon Sep 17 00:00:00 2001 From: mike Date: Wed, 30 Oct 2002 09:19:26 +0000 Subject: [PATCH] LOTS of changes. Biggies include proper support for relations, including multi-characters symbolics, word relations and relation modifiers. Also added the new CQLGenerator class, which generates a random CQL tree to be used in testing, most likely. Various changes involved in all this include moving CQLLexer and CQLParseException into their own source files (the former just for neatness, the latter for application visibility) --- Grammar | 35 +++ README | 54 ++-- etc/generate.properties | 12 + src/org/z3950/zing/cql/CQLAndNode.java | 6 +- src/org/z3950/zing/cql/CQLBooleanNode.java | 7 +- src/org/z3950/zing/cql/CQLGenerator.java | 306 ++++++++++++++++++++ src/org/z3950/zing/cql/CQLLexer.java | 185 ++++++++++++ src/org/z3950/zing/cql/CQLNode.java | 9 +- src/org/z3950/zing/cql/CQLNotNode.java | 6 +- src/org/z3950/zing/cql/CQLOrNode.java | 6 +- src/org/z3950/zing/cql/CQLParseException.java | 17 ++ src/org/z3950/zing/cql/CQLParser.java | 188 ++++-------- src/org/z3950/zing/cql/CQLRelation.java | 76 +++++ src/org/z3950/zing/cql/CQLTermNode.java | 53 ++-- src/org/z3950/zing/cql/Makefile | 8 +- .../z3950/zing/cql/ParameterMissingException.java | 16 + 16 files changed, 781 insertions(+), 203 deletions(-) create mode 100644 Grammar create mode 100644 etc/generate.properties create mode 100644 src/org/z3950/zing/cql/CQLGenerator.java create mode 100644 src/org/z3950/zing/cql/CQLLexer.java create mode 100644 src/org/z3950/zing/cql/CQLParseException.java create mode 100644 src/org/z3950/zing/cql/CQLRelation.java create mode 100644 src/org/z3950/zing/cql/ParameterMissingException.java diff --git a/Grammar b/Grammar new file mode 100644 index 0000000..bd51793 --- /dev/null +++ b/Grammar @@ -0,0 +1,35 @@ +$Id: Grammar,v 1.1 2002-10-30 09:19:26 mike Exp $ + +This is the CQL grammar, more or less as on the official Maintenance +Agency page (http://lcweb.loc.gov/z3950/agency/zing/srwu/cql.html) but +with a few tweaks described in my message of Tue, 29 Oct 2002 14:11:48 +which I hope will be integrated into the official grammar. + +-- + +cql-query ::= cql-query boolean search-clause + | search-clause +boolean ::= "and" | "or" | "not" | prox +search-clause ::= "(" cql-query ")" + | [ qualifier relation ] term + +relation ::= base-relation { "/" relation-modifier } +base-relation ::= numeric-relation | "exact" | "all" | "any" +relation-modifier ::= "relevant" | "fuzzy" | "stem" +numeric-relation ::= "<" | ">" | "<=" | ">=" | "<>" | "=" + +prox ::= "prox" [ "/" prox-parameters ] +prox-parameters ::= [ numeric-relation ] "/" [ distance ] "/" [ unit ] "/" ordering + | [ numeric-relation ] "/" [ distance ] "/" unit + | [ numeric-relation ] "/" distance + | numeric-relation +unit ::= "word" | "sentence" | "paragraph" | "element" +ordering ::= "ordered" | "unordered" +distance ::= non-negative-integer + +qualifier ::= [ qualifier-prefix "." ] qualifier-name +qualifier-prefix ::= identifier +qualifier-name ::= identifier +identifer ::= string +term ::= string | ""string"" +string ::= a character string diff --git a/README b/README index ad98b9c..e4e2320 100644 --- a/README +++ b/README @@ -1,13 +1,15 @@ -$Id: README,v 1.4 2002-10-29 10:15:58 mike Exp $ +$Id: README,v 1.5 2002-10-30 09:19:26 mike Exp $ cql-java -- a free CQL compiler for Java This project provides a set of classes for representing a CQL parse -tree (CQLBooleanNode, CQLTermNode, etc.) and a Compiler class which +tree (CQLBooleanNode, CQLTermNode, etc.) and a CQLCompiler class which builds a parse tree given a CQL query as input. It also provides -compiler back-ends to render out the parse tree either as XCQL or -Yaz-style Prefix Query Format (PQF). +compiler back-ends to render out the parse tree as XCQL (the XML +representation), as PQF (Yaz-style Prefix Query Format) and as CQL +(i.e. decompiling the parse-tree). Oh, and there's a random query +generator, too. CQL is "Common Query Language", a new query language designed under the umbrella of the ZING initiative (Z39.59-International Next @@ -75,30 +77,42 @@ SEE ALSO Adam Dickmeiss's CQL compiler, written in C. Rob Sanderson's CQL compiler, written in Python. -All the other free CQL compilers everyone's going to write. +All the other free CQL compilers everyone's going to write :-) TO DO ----- -### Finish the parser: -### * multi-character relations DONE but ### single "<" fails! -### * word relations -### * relation modifiers -### * proximity, +* Add proximity support to parser -### Finish the CXQL-rendering back end (mostly a matter of quoting - characters to be emitted as part of an XML document). - DONE +* Some niceties for the CQL-decompiling back-end: + * Don't emit redundant parentheses. + * Don't put spaces around relations that don't need them. -### Finish CQL-decompiling back end (mostly a matter of quoting) +* Write PQN-generating back-end (will need to be driven from a + configuation file specifying how to represent the qualifiers, + relations, relation modifiers and wildcard characters as Z39.50 + attributes.) -### Write PQN-generating back end (will need to be driven from a - configuation file specifying how to represent the qualifiers, - relations, relation modifiers and wildcard characters as Z39.50 - attributes.) +* Consider the utility of yet another back-end that translates a + CQLNode tree into a Type-1 query tree using the JZKit data + structures. That would be nice so that CQL could become a JZKit + query-type, but you could achieve the same effect by generating PQN, + and running that through JZKit's existing PQN-to-Type-1 compiler. -### Write stochastic query generator, driven off MA grammar. +* Refinements to random query generator: + * Fix to handle new, structured, relation representation + * Generate relation modifiers + * Proximity support + * Better selection of qualifier (configurable?) + * Better selection of terms (from a dictionary file?) + * Introduce wildcard characters into generated terms + * Generate multi-word terms -### Write "javadoc" comments. +* Write fuller "javadoc" comments. + +* Write generic test suite. + +* Fix CQLParser test harness to read query from command-line + arguments, if any, falling back to stdin if there are none. diff --git a/etc/generate.properties b/etc/generate.properties new file mode 100644 index 0000000..7ef2b65 --- /dev/null +++ b/etc/generate.properties @@ -0,0 +1,12 @@ +# $Id: generate.properties,v 1.1 2002-10-30 09:19:26 mike Exp $ +# +# Propeties file to drive the org.z3950.zing.cql.CQLGenerator +# test-harness. See that class's documentation for the semantics of +# these properties. +# +#seed=18398 +complexQuery=0.4 +complexClause=0.4 +equalsRelation=0.5 +numericRelation=0.7 +proxOp=0.0 diff --git a/src/org/z3950/zing/cql/CQLAndNode.java b/src/org/z3950/zing/cql/CQLAndNode.java index f46456d..1893a3e 100644 --- a/src/org/z3950/zing/cql/CQLAndNode.java +++ b/src/org/z3950/zing/cql/CQLAndNode.java @@ -1,13 +1,13 @@ -// $Id: CQLAndNode.java,v 1.2 2002-10-25 16:01:26 mike Exp $ +// $Id: CQLAndNode.java,v 1.3 2002-10-30 09:19:26 mike Exp $ package org.z3950.zing.cql; /** - * Represents an AND node in a CQL parse-tree ... + * Represents an AND node in a CQL parse-tree. * ### * - * @version $Id: CQLAndNode.java,v 1.2 2002-10-25 16:01:26 mike Exp $ + * @version $Id: CQLAndNode.java,v 1.3 2002-10-30 09:19:26 mike Exp $ */ public class CQLAndNode extends CQLBooleanNode { public CQLAndNode(CQLNode left, CQLNode right) { diff --git a/src/org/z3950/zing/cql/CQLBooleanNode.java b/src/org/z3950/zing/cql/CQLBooleanNode.java index 728768c..489e39b 100644 --- a/src/org/z3950/zing/cql/CQLBooleanNode.java +++ b/src/org/z3950/zing/cql/CQLBooleanNode.java @@ -1,13 +1,13 @@ -// $Id: CQLBooleanNode.java,v 1.4 2002-10-25 19:44:31 mike Exp $ +// $Id: CQLBooleanNode.java,v 1.5 2002-10-30 09:19:26 mike Exp $ package org.z3950.zing.cql; /** - * Represents a boolean node in a CQL parse-tree ... + * Represents a boolean node in a CQL parse-tree. * ### * - * @version $Id: CQLBooleanNode.java,v 1.4 2002-10-25 19:44:31 mike Exp $ + * @version $Id: CQLBooleanNode.java,v 1.5 2002-10-30 09:19:26 mike Exp $ */ public abstract class CQLBooleanNode extends CQLNode { protected CQLNode left; @@ -24,6 +24,7 @@ public abstract class CQLBooleanNode extends CQLNode { } String toCQL() { + // ### We don't always need parens around the operands return "(" + left.toCQL() + ") " + op() + " (" + right.toCQL() + ")"; } } diff --git a/src/org/z3950/zing/cql/CQLGenerator.java b/src/org/z3950/zing/cql/CQLGenerator.java new file mode 100644 index 0000000..bc732e2 --- /dev/null +++ b/src/org/z3950/zing/cql/CQLGenerator.java @@ -0,0 +1,306 @@ +// $Id: CQLGenerator.java,v 1.1 2002-10-30 09:19:26 mike Exp $ + +package org.z3950.zing.cql; +import java.util.Properties; +import java.util.Random; +import java.io.InputStream; +import java.io.FileInputStream; +import java.io.FileNotFoundException; + + +/** + * A generator that produces random CQL queries. + *

+ * Why is that useful? Mainly to produce test-cases for CQL parsers + * (including the CQLParser class in this package): you can + * generate a random search tree, render it to XCQL and remember the + * result. Then decompile the tree to CQL, feed the generated CQL to + * the parser of your choice, and check that the XCQL it comes up with + * is the same what you got from your initial rendering. + *

+ * This code is based on the grammar in the Grammar file of + * this distribution - there is a generate_x() method + * for each grammar element X. + * + * @version $Id: CQLGenerator.java,v 1.1 2002-10-30 09:19:26 mike Exp $ + * @see http://zing.z3950.org/cql/index.html + */ +public class CQLGenerator { + Properties params; + Random rnd; + static private boolean DEBUG = false; + + /** + * Creates a new CQL generator with the specified parameters. + *

+ * @param params + * A Properties table containing configuration + * parameters for the queries to be generated by this generator. + * Recognised parameters are: + *

+ *

+ *
seed
+ *
+ * If specified, this is a long used to seed the + * random number generator, so that the CQL generator can be + * run repeatably, giving the same results each time. If it's + * omitted, then no seed is explicitly specified, and the + * results of each run will be different (so long as you don't + * run it more that 2^32 times :-) + *

+ *

+ *
complexQuery
+ *
+ * [mandatory] A floating-point number between 0.0 and 1.0, + * indicating the probability for each cql-query node + * that it will be expanded into a ``complex query'' + * (cql-query boolean search-clause) rather + * than a search-clause. + *

+ *

+ *
complexClause
+ *
+ * [mandatory] A floating-point number between 0.0 and 1.0, + * indicating the probability for each search-clause + * node that it will be expanded into a full sub-query rather + * than a [ qualifier relation ] term triplet. + *

+ *

+ *
proxOp
+ *
+ * [mandatory] A floating-point number between 0.0 and 1.0, + * indicating the probability that each boolean operator will + * be chosen to be proximity operation; otherwise, the three + * simpler boolean operations (and, or and + * not) are chosen with equal probability. + *

+ *

+ *
equalsRelation
+ *
+ * [mandatory] A floating-point number between 0.0 and 1.0, + * indicating the probability that each relation will be chosen + * to be = - this is treated as a special case, since + * it's likely to be by far the most common relation in + * ``real life'' searches. + *

+ *

+ *
numericRelation
+ *
+ * [mandatory] A floating-point number between 0.0 and 1.0, + * indicating the probability that a relation, having chosen + * not to be =, is instead chosen to be one of the six + * numeric relations (<, >, + * <=, >=, <> and + * =). + *

+ *

+ *
+ */ + public CQLGenerator(Properties params) { + this.params = params; + String seed = params.getProperty("seed"); + if (seed != null) + rnd = new Random(new Long(seed).longValue()); + else + rnd = new Random(); + } + + private static void debug(String str) { + if (DEBUG) + System.err.println("DEBUG: " + str); + } + + /** + * Generates a single random CQL query. + *

+ * Uses the parameters that were associated with the generator + * when it was created. You are free to create as many random + * queries as you wish from a single generator; each of them will + * use the same parameters. + *

+ * @return + * A CQLNode that is the root of the generated tree. + * That tree may be rendered in XCQL using its toXCQL() + * method, or decompiled into CQL using its toCQL + * method. + */ + public CQLNode generate() throws ParameterMissingException { + return generate_cql_query(); + } + + private CQLNode generate_cql_query() throws ParameterMissingException { + if (!maybe("complexQuery")) { + return generate_search_clause(); + } + + CQLNode node1 = generate_cql_query(); + CQLNode node2 = generate_search_clause(); + if (maybe("proxOp")) { + // ### generate proximity nodes + } else { + switch (rnd.nextInt(3)) { + case 0: return new CQLAndNode(node1, node2); + case 1: return new CQLOrNode(node1, node2); + case 2: return new CQLNotNode(node1, node2); + } + } + + return generate_search_clause(); + } + + private CQLNode generate_search_clause() throws ParameterMissingException { + if (maybe("complexClause")) { + return generate_cql_query(); + } + + String qualifier = generate_qualifier(); + String relation = generate_relation(); + String term = generate_term(); + + return new CQLTermNode(qualifier, relation, term); + } + + // ### Should probably be more configurable + private String generate_qualifier() { + String qualifier = ""; // shut up compiler warning + if (rnd.nextInt(2) == 0) { + switch (rnd.nextInt(3)) { + case 0: qualifier = "dc.author"; break; + case 1: qualifier = "dc.title"; break; + case 2: qualifier = "dc.subject"; break; + } + } else { + switch (rnd.nextInt(4)) { + case 0: qualifier = "bath.author"; break; + case 1: qualifier = "bath.title"; break; + case 2: qualifier = "bath.subject"; break; + case 3: qualifier = "foo>bar"; break; + } + } + + return qualifier; + } + + // ### Representation of relations will change when we handle modifiers + private String generate_relation() throws ParameterMissingException { + return generate_base_relation(); + // ### should generate modifiers too + } + + private String generate_base_relation() throws ParameterMissingException { + if (maybe("equalsRelation")) { + return "="; + } else if (maybe("numericRelation")) { + return generate_numeric_relation(); + } else { + switch (rnd.nextInt(3)) { + case 0: return "exact"; + case 1: return "all"; + case 2: return "any"; + } + } + + // NOTREACHED + return ""; // shut up compiler warning + } + + // ### could read candidate terms from /usr/dict/words + // ### should introduce wildcard characters + // ### should generate multi-word terms + private String generate_term() { + switch (rnd.nextInt(10)) { + case 0: return "cat"; + case 1: return "\"cat\""; + case 2: return "comp.os.linux"; + case 3: return "xml:element"; + case 4: return ""; + case 5: return "prox/word/>=/5"; + case 6: return ""; + case 7: return "frog fish"; + case 8: return "the complete dinosaur"; + case 9: return "foo*bar"; + } + + // NOTREACHED + return ""; // shut up compiler warning + } + + private String generate_numeric_relation() { + switch (rnd.nextInt(6)) { + case 0: return "<"; + case 1: return ">"; + case 2: return "<="; + case 3: return ">="; + case 4: return "<>"; + case 5: return "="; + } + + // NOTREACHED + return ""; // shut up compiler warning + } + + boolean maybe(String param) throws ParameterMissingException { + String probability = params.getProperty(param); + if (probability == null) + throw new ParameterMissingException(param); + + double dice = rnd.nextDouble(); + double threshhold = new Double(probability).doubleValue(); + boolean res = dice < threshhold; + debug("dice=" + String.valueOf(dice).substring(0, 8) + + " vs. " + threshhold + "='" + param + "': " + res); + return res; + } + + + /** + * A simple test-harness for the generator. + *

+ * It generates a single random query using the parameters + * specified in a nominated properties file, and decompiles it + * into CQL which is written to standard output. + *

+ * For example, + * java org.z3950.zing.cql.CQLGenerator etc/generate.properties + * where the file generate.properties contains:

+     *	seed=18398
+     *	complexQuery=0.4
+     *	complexClause=0.4
+     *	equalsRelation=0.5
+     *	numericRelation=0.7
+     *	proxOp=0.0
+     *
+ * yields:
+     *	((dc.author = "<xml.element>") or (bath.title = cat)) and
+     *		(dc.subject >= "the complete dinosaur")
+     *
+ *

+ * @param configFile + * The name of a properties file from which to read the + * configuration parameters (see above). + * @return + * A CQL query expressed in a form that should be comprehensible + * to all conformant CQL compilers. + */ + public static void main (String[] args) throws Exception { + if (args.length != 1) { + System.err.println("Usage: CQLGenerator "); + System.exit(1); + } + + String configFile = args[0]; + InputStream f = new FileInputStream(configFile); + if (f == null) + throw new FileNotFoundException("getResourceAsStream(" + + configFile + ")"); + + Properties params = new Properties(); + params.load(f); + f.close(); + + CQLGenerator generator = new CQLGenerator(params); + CQLNode tree = generator.generate(); + System.out.println(tree.toCQL()); + } +} diff --git a/src/org/z3950/zing/cql/CQLLexer.java b/src/org/z3950/zing/cql/CQLLexer.java new file mode 100644 index 0000000..1dc580e --- /dev/null +++ b/src/org/z3950/zing/cql/CQLLexer.java @@ -0,0 +1,185 @@ +// $Id: CQLLexer.java,v 1.1 2002-10-30 09:19:26 mike Exp $ + +package org.z3950.zing.cql; +import java.io.StreamTokenizer; +import java.io.StringReader; + + +// This is a trivial subclass for java.io.StreamTokenizer which knows +// about the multi-character tokens "<=", ">=" and "<>", and includes +// a render() method. Used only by CQLParser. +// +class CQLLexer extends StreamTokenizer { + private static boolean DEBUG; + static int TT_LE = 1000; // The "<=" relation + static int TT_GE = 1001; // The ">=" relation + static int TT_NE = 1002; // The "<>" relation + static int TT_AND = 1003; // The "and" boolean + static int TT_OR = 1004; // The "or" boolean + static int TT_NOT = 1005; // The "not" boolean + static int TT_PROX = 1006; // The "prox" boolean + static int TT_ANY = 1007; // The "any" relation + static int TT_ALL = 1008; // The "all" relation + static int TT_EXACT = 1009; // The "exact" relation + + // For halfDecentPushBack() and the code at the top of nextToken() + private static int TT_UNDEFINED = -1000; + int saved_ttype = TT_UNDEFINED; + double saved_nval; + String saved_sval; + + CQLLexer(String cql, boolean lexdebug) { + super(new StringReader(cql)); + ordinaryChar('='); + ordinaryChar('<'); + ordinaryChar('>'); + ordinaryChar('/'); + ordinaryChar('('); + ordinaryChar(')'); + wordChars('\'', '\''); // prevent this from introducing strings + DEBUG = lexdebug; + } + + private static void debug(String str) { + if (DEBUG) + System.err.println("LEXDEBUG: " + str); + } + + // I don't honestly understand why we need this, but the + // documentation for java.io.StreamTokenizer.pushBack() is pretty + // vague about its semantics, and it seems to me that they could + // be summed up as "it doesn't work". This version has the very + // clear semantics "pretend I didn't call nextToken() just then". + // + private void halfDecentPushBack() { + saved_ttype = ttype; + saved_nval = nval; + saved_sval = sval; + } + + public int nextToken() throws java.io.IOException { + if (saved_ttype != TT_UNDEFINED) { + ttype = saved_ttype; + nval = saved_nval; + sval = saved_sval; + saved_ttype = TT_UNDEFINED; + debug("using saved ttype=" + ttype + ", " + + "nval=" + nval + ", sval='" + sval + "'"); + return ttype; + } + + underlyingNextToken(); + if (ttype == '<') { + debug("token starts with '<' ..."); + underlyingNextToken(); + if (ttype == '=') { + debug("token continues with '=' - it's '<='"); + ttype = TT_LE; + } else if (ttype == '>') { + debug("token continues with '>' - it's '<>'"); + ttype = TT_NE; + } else { + debug("next token is " + render() + " (pushed back)"); + halfDecentPushBack(); + ttype = '<'; + debug("AFTER: ttype is now " + ttype + " - " + render()); + } + } else if (ttype == '>') { + debug("token starts with '>' ..."); + underlyingNextToken(); + if (ttype == '=') { + debug("token continues with '=' - it's '>='"); + ttype = TT_GE; + } else { + debug("next token is " + render() + " (pushed back)"); + halfDecentPushBack(); + ttype = '>'; + debug("AFTER: ttype is now " + ttype + " - " + render()); + } + } + + debug("done nextToken(): ttype=" + ttype + ", " + + "nval=" + nval + ", " + "sval='" + sval + "'" + + " (" + render() + ")"); + + return ttype; + } + + // It's important to do keyword recognition here at the lowest + // level, otherwise when one of these words follows "<" or ">" + // (which can be the beginning of multi-character tokens) it gets + // pushed back as a string, and its keywordiness is not + // recognised. + // + public int underlyingNextToken() throws java.io.IOException { + super.nextToken(); + if (ttype == TT_WORD) { + if (sval.equalsIgnoreCase("and")) { + ttype = TT_AND; + } else if (sval.equalsIgnoreCase("or")) { + ttype = TT_OR; + } else if (sval.equalsIgnoreCase("not")) { + ttype = TT_NOT; + } else if (sval.equalsIgnoreCase("prox")) { + ttype = TT_PROX; + } else if (sval.equalsIgnoreCase("any")) { + ttype = TT_ANY; + } else if (sval.equalsIgnoreCase("all")) { + ttype = TT_ALL; + } else if (sval.equalsIgnoreCase("exact")) { + ttype = TT_EXACT; + } + } + return ttype; + } + + // Simpler interface for the usual case: current token with quoting + String render() { + return render(ttype, true); + } + + String render(int token, boolean quoteChars) { + if (token == TT_EOF) { + return "EOF"; + } else if (token == TT_NUMBER) { + return "number: " + nval; + } else if (token == TT_WORD) { + return "word: " + sval; + } else if (token == '"') { + return "string: \"" + sval + "\""; + } else if (token == TT_LE) { + return "<="; + } else if (token == TT_GE) { + return ">="; + } else if (token == TT_NE) { + return "<>"; + } else if (token == TT_AND) { + return "and"; + } else if (token == TT_OR) { + return "or"; + } else if (token == TT_NOT) { + return "not"; + } else if (token == TT_PROX) { + return "prox"; + } else if (token == TT_ANY) { + return "any"; + } else if (token == TT_ALL) { + return "all"; + } else if (token == TT_EXACT) { + return "exact"; + } + + String res = String.valueOf((char) token); + if (quoteChars) res = "'" + res + "'"; + return res; + } + + public static void main(String[] args) throws Exception { + CQLLexer lexer = new CQLLexer(args[0], true); + int token; + + while ((token = lexer.nextToken()) != TT_EOF) { + // Nothing to do: debug() statements render tokens for us + } + } +} diff --git a/src/org/z3950/zing/cql/CQLNode.java b/src/org/z3950/zing/cql/CQLNode.java index 3d16a37..3267432 100644 --- a/src/org/z3950/zing/cql/CQLNode.java +++ b/src/org/z3950/zing/cql/CQLNode.java @@ -1,13 +1,13 @@ -// $Id: CQLNode.java,v 1.6 2002-10-29 10:15:58 mike Exp $ +// $Id: CQLNode.java,v 1.7 2002-10-30 09:19:26 mike Exp $ package org.z3950.zing.cql; /** - * Represents a node in a CQL parse-tree ... + * Represents a node in a CQL parse-tree. * ### * - * @version $Id: CQLNode.java,v 1.6 2002-10-29 10:15:58 mike Exp $ + * @version $Id: CQLNode.java,v 1.7 2002-10-30 09:19:26 mike Exp $ */ public abstract class CQLNode { abstract String toXCQL(int level); @@ -35,7 +35,8 @@ public abstract class CQLNode { return str; } - String replace(String str, String from, String to) { + // I can't _believe_ I have to write this by hand in 2002 ... + protected static String replace(String str, String from, String to) { StringBuffer sb = new StringBuffer(); int ix; // index of next `from' int offset = 0; // index of previous `from' + length(from) diff --git a/src/org/z3950/zing/cql/CQLNotNode.java b/src/org/z3950/zing/cql/CQLNotNode.java index 3aa3ced..312cf94 100644 --- a/src/org/z3950/zing/cql/CQLNotNode.java +++ b/src/org/z3950/zing/cql/CQLNotNode.java @@ -1,13 +1,13 @@ -// $Id: CQLNotNode.java,v 1.2 2002-10-25 16:01:26 mike Exp $ +// $Id: CQLNotNode.java,v 1.3 2002-10-30 09:19:26 mike Exp $ package org.z3950.zing.cql; /** - * Represents a NOT node in a CQL parse-tree ... + * Represents a NOT node in a CQL parse-tree. * ### * - * @version $Id: CQLNotNode.java,v 1.2 2002-10-25 16:01:26 mike Exp $ + * @version $Id: CQLNotNode.java,v 1.3 2002-10-30 09:19:26 mike Exp $ */ public class CQLNotNode extends CQLBooleanNode { public CQLNotNode(CQLNode left, CQLNode right) { diff --git a/src/org/z3950/zing/cql/CQLOrNode.java b/src/org/z3950/zing/cql/CQLOrNode.java index 39e8a52..e24bb04 100644 --- a/src/org/z3950/zing/cql/CQLOrNode.java +++ b/src/org/z3950/zing/cql/CQLOrNode.java @@ -1,13 +1,13 @@ -// $Id: CQLOrNode.java,v 1.2 2002-10-25 16:01:26 mike Exp $ +// $Id: CQLOrNode.java,v 1.3 2002-10-30 09:19:26 mike Exp $ package org.z3950.zing.cql; /** - * Represents an OR node in a CQL parse-tree ... + * Represents an OR node in a CQL parse-tree. * ### * - * @version $Id: CQLOrNode.java,v 1.2 2002-10-25 16:01:26 mike Exp $ + * @version $Id: CQLOrNode.java,v 1.3 2002-10-30 09:19:26 mike Exp $ */ public class CQLOrNode extends CQLBooleanNode { public CQLOrNode(CQLNode left, CQLNode right) { diff --git a/src/org/z3950/zing/cql/CQLParseException.java b/src/org/z3950/zing/cql/CQLParseException.java new file mode 100644 index 0000000..ed0e0c6 --- /dev/null +++ b/src/org/z3950/zing/cql/CQLParseException.java @@ -0,0 +1,17 @@ +// $Id: CQLParseException.java,v 1.1 2002-10-30 09:19:26 mike Exp $ + +package org.z3950.zing.cql; +import java.lang.Exception; + + +/** + * Exception indicating that an error ocurred parsing CQL. + * + * @version $Id: CQLParseException.java,v 1.1 2002-10-30 09:19:26 mike Exp $ + */ +public class CQLParseException extends Exception { + CQLParseException(String s) { + super(s); + } +} + diff --git a/src/org/z3950/zing/cql/CQLParser.java b/src/org/z3950/zing/cql/CQLParser.java index 8c5d871..e463576 100644 --- a/src/org/z3950/zing/cql/CQLParser.java +++ b/src/org/z3950/zing/cql/CQLParser.java @@ -1,32 +1,24 @@ -// $Id: CQLParser.java,v 1.9 2002-10-29 10:15:58 mike Exp $ +// $Id: CQLParser.java,v 1.10 2002-10-30 09:19:26 mike Exp $ package org.z3950.zing.cql; -import java.util.Properties; -import java.io.InputStream; import java.io.IOException; -import java.io.StringReader; -import java.io.StreamTokenizer; /** - * Compiles a CQL string into a parse tree ... + * Compiles a CQL string into a parse tree. * ### * - * @version $Id: CQLParser.java,v 1.9 2002-10-29 10:15:58 mike Exp $ + * @version $Id: CQLParser.java,v 1.10 2002-10-30 09:19:26 mike Exp $ * @see http://zing.z3950.org/cql/index.html */ public class CQLParser { private CQLLexer lexer; - static private boolean PARSEDEBUG = false; - static private boolean LEXDEBUG = true; + static private boolean DEBUG = false; + static private boolean LEXDEBUG = false; - private class CQLParseException extends Exception { - CQLParseException(String s) { super(s); } - } - - static void debug(String str) { - if (PARSEDEBUG) + private static void debug(String str) { + if (DEBUG) System.err.println("PARSEDEBUG: " + str); } @@ -36,38 +28,37 @@ public class CQLParser { lexer.nextToken(); debug("about to parse_query()"); - CQLNode root = parse_query("srw.serverChoice", "="); + CQLNode root = parse_query("srw.serverChoice", new CQLRelation("=")); if (lexer.ttype != lexer.TT_EOF) throw new CQLParseException("junk after end: " + lexer.render()); return root; } - private CQLNode parse_query(String qualifier, String relation) + private CQLNode parse_query(String qualifier, CQLRelation relation) throws CQLParseException, IOException { debug("in parse_query()"); CQLNode term = parse_term(qualifier, relation); - while (lexer.ttype == lexer.TT_WORD) { - String op = lexer.sval.toLowerCase(); - debug("checking op '" + op + "'"); - if (lexer.sval.equals("and")) { - match(lexer.TT_WORD); + while (lexer.ttype != lexer.TT_EOF && + lexer.ttype != ')') { + if (lexer.ttype == lexer.TT_AND) { + match(lexer.TT_AND); CQLNode term2 = parse_term(qualifier, relation); term = new CQLAndNode(term, term2); - } else if (lexer.sval.equals("or")) { - match(lexer.TT_WORD); + } else if (lexer.ttype == lexer.TT_OR) { + match(lexer.TT_OR); CQLNode term2 = parse_term(qualifier, relation); term = new CQLOrNode(term, term2); - } else if (lexer.sval.equals("not")) { - match(lexer.TT_WORD); + } else if (lexer.ttype == lexer.TT_NOT) { + match(lexer.TT_NOT); CQLNode term2 = parse_term(qualifier, relation); term = new CQLNotNode(term, term2); - } else if (lexer.sval.equals("prox")) { + } else if (lexer.ttype == lexer.TT_PROX) { // ### Handle "prox" } else { - throw new CQLParseException("unrecognised boolean: '" + - lexer.sval + "'"); + throw new CQLParseException("expected boolean, got " + + lexer.render()); } } @@ -75,7 +66,7 @@ public class CQLParser { return term; } - private CQLNode parse_term(String qualifier, String relation) + private CQLNode parse_term(String qualifier, CQLRelation relation) throws CQLParseException, IOException { debug("in parse_term()"); @@ -95,39 +86,57 @@ public class CQLParser { debug("non-parenthesised term"); word = lexer.sval; match(lexer.ttype); - if (!isRelation()) + if (!isBaseRelation()) break; qualifier = word; - relation = lexer.render(false); - debug("got relation '" + relation + "'"); + relation = new CQLRelation(lexer.render(lexer.ttype, false)); match(lexer.ttype); - debug("qualifier='" + qualifier + ", relation='" + relation + "'"); + + while (lexer.ttype == '/') { + match('/'); + // ### could insist on known modifiers only + if (lexer.ttype != lexer.TT_WORD) + throw new CQLParseException("expected relation modifier, " + + "got " + lexer.render()); + relation.addModifier(lexer.sval); + match(lexer.TT_WORD); + } + + debug("qualifier='" + qualifier + ", " + + "relation='" + relation.toCQL() + "'"); } CQLTermNode node = new CQLTermNode(qualifier, relation, word); - debug("made term node " + node); + debug("made term node " + node.toCQL()); return node; } - boolean isRelation() { - // ### Handle any, all and exact + boolean isBaseRelation() { + debug("isBaseRelation: checking ttype=" + lexer.ttype + + " (" + lexer.render() + ")"); return (lexer.ttype == '<' || lexer.ttype == '>' || lexer.ttype == '=' || lexer.ttype == lexer.TT_LE || lexer.ttype == lexer.TT_GE || - lexer.ttype == lexer.TT_NE); + lexer.ttype == lexer.TT_NE || + lexer.ttype == lexer.TT_ANY || + lexer.ttype == lexer.TT_ALL || + lexer.ttype == lexer.TT_EXACT); } private void match(int token) throws CQLParseException, IOException { - debug("in match(" + lexer.render(token, null, true) + ")"); + debug("in match(" + lexer.render(token, true) + ")"); if (lexer.ttype != token) throw new CQLParseException("expected " + - lexer.render(token, null, true) + + lexer.render(token, true) + ", " + "got " + lexer.render()); - lexer.nextToken(); + int tmp = lexer.nextToken(); + debug("match() got token=" + lexer.ttype + ", " + + "nval=" + lexer.nval + ", sval='" + lexer.sval + "'" + + " (tmp=" + tmp + ")"); } @@ -188,100 +197,3 @@ public class CQLParser { } } } - - -// This is a trivial subclass for java.io.StreamTokenizer which knows -// about the multi-character tokens "<=", ">=" and "<>", and included -// a render() method. Used only by CQLParser. -// -class CQLLexer extends StreamTokenizer { - private static boolean LEXDEBUG; - static int TT_LE = 1000; // The token "<=" - static int TT_GE = 1001; // The token ">=" - static int TT_NE = 1002; // The token "<>" - - static void debug(String str) { - if (LEXDEBUG) - System.err.println("LEXDEBUG: " + str); - } - - CQLLexer(String cql, boolean lexdebug) { - super(new StringReader(cql)); - this.ordinaryChar('='); - this.ordinaryChar('<'); - this.ordinaryChar('>'); - this.ordinaryChar('/'); - this.ordinaryChar('('); - this.ordinaryChar(')'); - this.wordChars('\'', '\''); // prevent this from introducing strings - this.LEXDEBUG = lexdebug; - } - - public int nextToken() throws java.io.IOException { - int token = super.nextToken(); - - if (token == '<') { - debug("token starts with '<' ..."); - int t2 = super.nextToken(); - if (t2 == '=') { - debug("token continues with '=' - it's '<='"); - this.ttype = token = TT_LE; - } else if (t2 == '>') { - debug("token continues with '>' - it's '<>'"); - this.ttype = token = TT_NE; - } else { - debug("next token is " + token + " (pushed back)"); - //this.pushBack(); - } - } else if (token == '>') { - debug("token starts with '>' ..."); - int t2 = super.nextToken(); - if (t2 == '=') { - debug("token continues with '=' - it's '>='"); - this.ttype = token = TT_GE; - } else { - debug("next token is " + token + " (pushed back)"); - //this.pushBack(); - } - } - - debug("token=" + token + ", " + - "nval=" + this.nval + ", " + "sval=" + this.sval); - - return token; - } - - String render() { - return this.render(this.ttype, null, true); - } - - String render(boolean quoteChars) { - return this.render(this.ttype, null, quoteChars); - } - - String render(int token, String str, boolean quoteChars) { - String ret; - - if (token == this.TT_EOF) { - return "EOF"; - } else if (token == this.TT_EOL) { - return "EOL"; - } else if (token == this.TT_NUMBER) { - return "number: " + this.nval; - } else if (token == this.TT_WORD) { - return "word: " + this.sval; - } else if (token == '"') { - return "string: \"" + this.sval + "\""; - } else if (token == TT_LE) { - return "<="; - } else if (token == TT_GE) { - return ">="; - } else if (token == TT_NE) { - return "<>"; - } - - String res = String.valueOf((char) token); - if (quoteChars) res = "'" + res + "'"; - return res; - } -} diff --git a/src/org/z3950/zing/cql/CQLRelation.java b/src/org/z3950/zing/cql/CQLRelation.java new file mode 100644 index 0000000..3de9f57 --- /dev/null +++ b/src/org/z3950/zing/cql/CQLRelation.java @@ -0,0 +1,76 @@ +// $Id: CQLRelation.java,v 1.1 2002-10-30 09:19:26 mike Exp $ + +package org.z3950.zing.cql; +import java.util.Vector; +import java.lang.StringBuffer; + +/** + * Represents a relation between a CQL qualifier and term. + * ### + * + * @version $Id: CQLRelation.java,v 1.1 2002-10-30 09:19:26 mike Exp $ + */ +public class CQLRelation extends CQLNode { + String base; + Vector modifiers; + + public CQLRelation(String base) { + this.base = base; + modifiers = new Vector(); + } + + public void addModifier(String modifier) { + modifiers.add(modifier); + } + + public String[] getModifiers() { + int n = modifiers.size(); + String[] res = new String[n]; + for (int i = 0; i < n; i++) { + res[i] = (String) modifiers.get(i); + } + + return res; + } + + public String toXCQL(int level) { + StringBuffer buf = new StringBuffer(); + buf.append (indent(level) + "\n" + + indent(level+1) + "" + xq(base) + "\n"); + String[] mods = getModifiers(); + if (mods.length > 0) { + buf.append(indent(level+1) + "\n"); + for (int i = 0; i < mods.length; i++) + buf.append(indent(level+2)). + append(""). append(mods[i]). + append("\n"); + buf.append(indent(level+1) + "\n"); + } + buf.append(indent(level) + "\n"); + return buf.toString(); + } + + public String toCQL() { + StringBuffer buf = new StringBuffer(base); + String[] mods = getModifiers(); + for (int i = 0; i < mods.length; i++) { + buf.append("/").append(mods[i]); + } + + return buf.toString(); + } + + public static void main(String[] args) { + if (args.length < 1) { + System.err.println("Usage: CQLRelation ..."); + System.exit(1); + } + + CQLRelation res = new CQLRelation(args[0]); + for (int i = 1; i < args.length; i++) { + res.addModifier(args[i]); + } + + System.out.println(res.toCQL()); + } +} diff --git a/src/org/z3950/zing/cql/CQLTermNode.java b/src/org/z3950/zing/cql/CQLTermNode.java index da6c75d..5c149a9 100644 --- a/src/org/z3950/zing/cql/CQLTermNode.java +++ b/src/org/z3950/zing/cql/CQLTermNode.java @@ -1,20 +1,20 @@ -// $Id: CQLTermNode.java,v 1.4 2002-10-27 00:46:25 mike Exp $ +// $Id: CQLTermNode.java,v 1.5 2002-10-30 09:19:26 mike Exp $ package org.z3950.zing.cql; /** - * Represents a terminal node in a CQL parse-tree ... + * Represents a terminal node in a CQL parse-tree. * ### * - * @version $Id: CQLTermNode.java,v 1.4 2002-10-27 00:46:25 mike Exp $ + * @version $Id: CQLTermNode.java,v 1.5 2002-10-30 09:19:26 mike Exp $ */ public class CQLTermNode extends CQLNode { private String qualifier; - private String relation; + private CQLRelation relation; private String term; - public CQLTermNode(String qualifier, String relation, String term) { + public CQLTermNode(String qualifier, CQLRelation relation, String term) { this.qualifier = qualifier; this.relation = relation; this.term = term; @@ -22,34 +22,35 @@ public class CQLTermNode extends CQLNode { String toXCQL(int level) { return (indent(level) + "\n" + - indent(level+1) + "" + xq(qualifier) + "\n" + - indent(level+1) + "" + xq(relation) + "\n"+ - indent(level+1) + "" + xq(term) + "\n" + + indent(level+1) + "" + xq(qualifier) + "\n" + + relation.toXCQL(level+1) + + indent(level+1) + "" + xq(term) + "\n" + indent(level) + "\n"); } String toCQL() { - String quotedTerm = term; + String quotedQualifier = maybeQuote(qualifier); + String quotedTerm = maybeQuote(term); - if (quotedTerm.indexOf('"') != -1) { - // ### precede each '"' with a '/' - } + // ### We don't always need spaces around `relation'. + return quotedQualifier + " " + relation.toCQL() + " " + quotedTerm; + } - // ### There must be a better way ... - if (quotedTerm.indexOf('"') != -1 || - quotedTerm.indexOf(' ') != -1 || - quotedTerm.indexOf('\t') != -1 || - quotedTerm.indexOf('=') != -1 || - quotedTerm.indexOf('<') != -1 || - quotedTerm.indexOf('>') != -1 || - quotedTerm.indexOf('/') != -1 || - quotedTerm.indexOf('(') != -1 || - quotedTerm.indexOf(')') != -1) { - quotedTerm = '"' + quotedTerm + '"'; + static String maybeQuote(String str) { + // There _must_ be a better way to make this test ... + if (str.length() == 0 || + str.indexOf('"') != -1 || + str.indexOf(' ') != -1 || + str.indexOf('\t') != -1 || + str.indexOf('=') != -1 || + str.indexOf('<') != -1 || + str.indexOf('>') != -1 || + str.indexOf('/') != -1 || + str.indexOf('(') != -1 || + str.indexOf(')') != -1) { + str = '"' + replace(str, "\"", "\\\"") + '"'; } - // ### The qualifier may need quoting. - // ### We don't always need spaces around `relation'. - return qualifier + " " + relation + " " + quotedTerm; + return str; } } diff --git a/src/org/z3950/zing/cql/Makefile b/src/org/z3950/zing/cql/Makefile index acd7ccc..703de72 100644 --- a/src/org/z3950/zing/cql/Makefile +++ b/src/org/z3950/zing/cql/Makefile @@ -1,10 +1,12 @@ -# $Id: Makefile,v 1.1 2002-10-25 07:38:17 mike Exp $ +# $Id: Makefile,v 1.2 2002-10-30 09:19:26 mike Exp $ all: CQLNode.class CQLTermNode.class CQLBooleanNode.class \ CQLAndNode.class CQLOrNode.class CQLNotNode.class \ - CQLParser.class + CQLRelation.class \ + CQLParser.class CQLLexer.class CQLParseException.class \ + CQLGenerator.class ParameterMissingException.class -javadocs: +docs: nice javadoc -d ../../../../../docs -author -version \ -windowtitle cql-java org.z3950.zing.cql diff --git a/src/org/z3950/zing/cql/ParameterMissingException.java b/src/org/z3950/zing/cql/ParameterMissingException.java new file mode 100644 index 0000000..cfd549d --- /dev/null +++ b/src/org/z3950/zing/cql/ParameterMissingException.java @@ -0,0 +1,16 @@ +// $Id: ParameterMissingException.java,v 1.1 2002-10-30 09:19:26 mike Exp $ + +package org.z3950.zing.cql; +import java.lang.Exception; + + +/** + * Exception indicating that a required property was not specified. + * + * @version $Id: ParameterMissingException.java,v 1.1 2002-10-30 09:19:26 mike Exp $ + */ +public class ParameterMissingException extends Exception { + ParameterMissingException(String s) { + super(s); + } +} -- 1.7.10.4