2 * Copyright (C) 1995-1999, Index Data
4 * Sebastian Hammer, Adam Dickmeiss
7 * Revision 1.103 2000-03-20 19:08:36 adam
8 * Added remote record import using Z39.50 extended services and Segment
11 * Revision 1.102 2000/03/15 15:00:31 adam
12 * First work on threaded version.
14 * Revision 1.101 2000/03/02 14:35:03 adam
15 * Fixed proximity handling.
17 * Revision 1.100 1999/12/28 15:48:12 adam
20 * Revision 1.99 1999/12/23 09:03:32 adam
21 * Changed behaviour of trunc=105 so that * is regular .* and ! is regular .
23 * Revision 1.98 1999/11/30 13:48:04 adam
24 * Improved installation. Updated for inclusion of YAZ header files.
26 * Revision 1.97 1999/10/14 14:33:50 adam
27 * Added truncation 5=106.
29 * Revision 1.96 1999/09/23 10:05:05 adam
30 * Implemented structure=105 searching.
32 * Revision 1.95 1999/09/07 07:19:21 adam
33 * Work on character mapping. Implemented replace rules.
35 * Revision 1.94 1999/07/20 13:59:18 adam
36 * Fixed bug that occurred when phrases had 0 hits.
38 * Revision 1.93 1999/06/17 14:38:40 adam
39 * Bug fix: Scan SEGV'ed when getting unknown use attribute.
41 * Revision 1.92 1999/05/26 07:49:13 adam
44 * Revision 1.91 1999/02/02 14:51:13 adam
45 * Updated WIN32 code specific sections. Changed header.
47 * Revision 1.90 1998/11/16 16:03:43 adam
48 * Moved loggin utilities to Yaz. Was implemented in file zlogs.c.
50 * Revision 1.89 1998/11/16 10:11:55 adam
51 * Added addtional info for error 114 - unsupported use attribute.
53 * Revision 1.88 1998/10/18 07:54:52 adam
54 * Additional info added for diagnostics 114 (Unsupported use attribute) and
55 * 121 (Unsupported attribute set).
57 * Revision 1.87 1998/09/28 11:19:12 adam
58 * Fix for Compiled ASN.1.
60 * Revision 1.86 1998/09/22 10:48:20 adam
61 * Minor changes in search API.
63 * Revision 1.85 1998/09/22 10:03:43 adam
64 * Changed result sets to be persistent in the sense that they can
65 * be re-searched if needed.
66 * Fixed memory leak in rsm_or.
68 * Revision 1.84 1998/09/18 12:41:00 adam
69 * Fixed bug with numerical relations.
71 * Revision 1.83 1998/09/02 13:53:19 adam
72 * Extra parameter decode added to search routines to implement
75 * Revision 1.82 1998/06/26 11:16:40 quinn
76 * Added support (un-optimised) for left and left/right truncation
78 * Revision 1.81 1998/06/24 12:16:14 adam
79 * Support for relations on text operands. Open range support in
80 * DFA module (i.e. [-j], [g-]).
82 * Revision 1.80 1998/06/23 15:33:34 adam
83 * Added feature to specify sort criteria in query (type 7 specifies
86 * Revision 1.79 1998/06/22 11:35:09 adam
89 * Revision 1.78 1998/06/08 14:43:17 adam
90 * Added suport for EXPLAIN Proxy servers - added settings databasePath
91 * and explainDatabase to facilitate this. Increased maximum number
92 * of databases and attributes in one register.
94 * Revision 1.77 1998/05/20 10:12:22 adam
95 * Implemented automatic EXPLAIN database maintenance.
96 * Modified Zebra to work with ASN.1 compiled version of YAZ.
98 * Revision 1.76 1998/04/02 14:35:29 adam
99 * First version of Zebra that works with compiled ASN.1.
101 * Revision 1.75 1998/03/05 08:45:13 adam
102 * New result set model and modular ranking system. Moved towards
103 * descent server API. System information stored as "SGML" records.
105 * Revision 1.74 1998/02/10 12:03:06 adam
108 * Revision 1.73 1998/01/29 13:40:11 adam
109 * Better logging for scan service.
111 * Revision 1.72 1998/01/07 13:53:41 adam
112 * Queries using simple ranked operands returns right number of hits.
114 * Revision 1.71 1997/12/18 10:54:24 adam
115 * New method result set method rs_hits that returns the number of
116 * hits in result-set (if known). The ranked result set returns real
117 * number of hits but only when not combined with other operands.
119 * Revision 1.70 1997/10/31 12:34:43 adam
120 * Changed a few log statements.
122 * Revision 1.69 1997/10/29 12:05:02 adam
123 * Server produces diagnostic "Unsupported Attribute Set" when appropriate.
125 * Revision 1.68 1997/10/27 14:33:06 adam
126 * Moved towards generic character mapping depending on "structure"
127 * field in abstract syntax file. Fixed a few memory leaks. Fixed
128 * bug with negative integers when doing searches with relational
131 * Revision 1.67 1997/09/29 09:06:10 adam
132 * Removed one static var in order to make this module thread safe.
134 * Revision 1.66 1997/09/25 14:58:03 adam
137 * Revision 1.65 1997/09/22 12:39:06 adam
138 * Added get_pos method for the ranked result sets.
140 * Revision 1.64 1997/09/18 08:59:20 adam
141 * Extra generic handle for the character mapping routines.
143 * Revision 1.63 1997/09/17 12:19:18 adam
144 * Zebra version corresponds to YAZ version 1.4.
145 * Changed Zebra server so that it doesn't depend on global common_resource.
147 * Revision 1.62 1997/09/05 15:30:09 adam
148 * Changed prototype for chr_map_input - added const.
149 * Added support for C++, headers uses extern "C" for public definitions.
151 * Revision 1.61 1997/02/10 10:21:14 adam
152 * Bug fix: in search terms character (^) wasn't observed.
154 * Revision 1.60 1997/01/31 11:10:34 adam
155 * Bug fix: Leading and trailing white space weren't removed in scan tokens.
157 * Revision 1.59 1997/01/17 11:31:46 adam
158 * Bug fix: complete phrase search didn't work.
160 * Revision 1.58 1996/12/23 15:30:45 adam
161 * Work on truncation.
162 * Bug fix: result sets weren't deleted after server shut down.
164 * Revision 1.57 1996/11/11 13:38:02 adam
165 * Added proximity support in search.
167 * Revision 1.56 1996/11/08 11:10:32 adam
168 * Buffers used during file match got bigger.
169 * Compressed ISAM support everywhere.
170 * Bug fixes regarding masking characters in queries.
171 * Redesigned Regexp-2 queries.
173 * Revision 1.55 1996/11/04 14:07:44 adam
174 * Moved truncation code to trunc.c.
176 * Revision 1.54 1996/10/29 14:09:52 adam
177 * Use of cisam system - enabled if setting isamc is 1.
179 * Revision 1.53 1996/06/26 09:21:43 adam
180 * Bug fix: local attribute set wasn't obeyed in scan.
182 * Revision 1.52 1996/06/17 14:26:20 adam
183 * Function gen_regular_rel changed to handle negative numbers.
185 * Revision 1.51 1996/06/11 10:54:15 quinn
188 * Revision 1.50 1996/06/07 08:51:53 adam
189 * Bug fix: Character mapping was broken (introducued by last revision).
191 * Revision 1.49 1996/06/04 10:18:11 adam
192 * Search/scan uses character mapping module.
194 * Revision 1.48 1996/05/28 15:15:01 adam
195 * Bug fix: Didn't handle unknown database correctly.
197 * Revision 1.47 1996/05/15 18:36:28 adam
198 * Function trans_term transforms unsearchable characters to blanks.
200 * Revision 1.46 1996/05/15 11:57:56 adam
201 * Fixed bug introduced by set/field mapping in search operations.
203 * Revision 1.45 1996/05/14 11:34:00 adam
204 * Scan support in multiple registers/databases.
206 * Revision 1.44 1996/05/14 06:16:44 adam
207 * Compact use/set bytes used in search service.
209 * Revision 1.43 1996/05/09 09:54:43 adam
210 * Server supports maps from one logical attributes to a list of physical
212 * The extraction process doesn't make space consuming 'any' keys.
214 * Revision 1.42 1996/05/09 07:28:56 quinn
215 * Work towards phrases and multiple registers
217 * Revision 1.41 1996/03/20 09:36:43 adam
218 * Function dict_lookup_grep got extra parameter, init_pos, which marks
219 * from which position in pattern approximate pattern matching should occur.
220 * Approximate pattern matching is used in relevance=re-2.
222 * Revision 1.40 1996/02/02 13:44:44 adam
223 * The public dictionary functions simply use char instead of Dict_char
224 * to represent search strings. Dict_char is used internally only.
226 * Revision 1.39 1996/01/03 16:22:13 quinn
227 * operator->roperator
229 * Revision 1.38 1995/12/11 09:12:55 adam
230 * The rec_get function returns NULL if record doesn't exist - will
231 * happen in the server if the result set records have been deleted since
232 * the creation of the set (i.e. the search).
233 * The server saves a result temporarily if it is 'volatile', i.e. the
234 * set is register dependent.
236 * Revision 1.37 1995/12/06 15:05:28 adam
237 * More verbose in count_set.
239 * Revision 1.36 1995/12/06 12:41:27 adam
240 * New command 'stat' for the index program.
241 * Filenames can be read from stdin by specifying '-'.
242 * Bug fix/enhancement of the transformation from terms to regular
243 * expressons in the search engine.
245 * Revision 1.35 1995/11/27 09:29:00 adam
246 * Bug fixes regarding conversion to regular expressions.
248 * Revision 1.34 1995/11/16 17:00:56 adam
249 * Better logging of rpn query.
251 * Revision 1.33 1995/11/01 13:58:28 quinn
252 * Moving data1 to yaz/retrieval
254 * Revision 1.32 1995/10/27 14:00:11 adam
255 * Implemented detection of database availability.
257 * Revision 1.31 1995/10/17 18:02:10 adam
258 * New feature: databases. Implemented as prefix to words in dictionary.
260 * Revision 1.30 1995/10/16 09:32:38 adam
261 * More work on relational op.
263 * Revision 1.29 1995/10/13 16:01:49 adam
266 * Revision 1.28 1995/10/13 12:26:43 adam
267 * Optimization of truncation.
269 * Revision 1.27 1995/10/12 17:07:22 adam
272 * Revision 1.26 1995/10/12 12:40:54 adam
273 * Bug fixes in rpn_prox.
275 * Revision 1.25 1995/10/10 13:59:24 adam
276 * Function rset_open changed its wflag parameter to general flags.
278 * Revision 1.24 1995/10/09 16:18:37 adam
279 * Function dict_lookup_grep got extra client data parameter.
281 * Revision 1.23 1995/10/06 16:33:37 adam
282 * Use attribute mappings.
284 * Revision 1.22 1995/10/06 15:07:39 adam
285 * Structure 'local-number' handled.
287 * Revision 1.21 1995/10/06 13:52:06 adam
288 * Bug fixes. Handler may abort further scanning.
290 * Revision 1.20 1995/10/06 11:06:33 adam
291 * Scan entries include 'occurrences' now.
293 * Revision 1.19 1995/10/06 10:43:56 adam
294 * Scan added. 'occurrences' in scan entries not set yet.
296 * Revision 1.18 1995/10/04 16:57:20 adam
297 * Key input and merge sort in one pass.
299 * Revision 1.17 1995/10/04 12:55:17 adam
300 * Bug fix in ranked search. Use=Any keys inserted.
302 * Revision 1.16 1995/10/02 16:24:40 adam
303 * Use attribute actually used in search requests.
305 * Revision 1.15 1995/10/02 15:18:52 adam
306 * New member in recRetrieveCtrl: diagnostic.
308 * Revision 1.14 1995/09/28 12:10:32 adam
309 * Bug fixes. Field prefix used in queries.
311 * Revision 1.13 1995/09/18 14:17:50 adam
314 * Revision 1.12 1995/09/15 14:45:21 adam
316 * Work on truncation.
318 * Revision 1.11 1995/09/14 11:53:27 adam
319 * First work on regular expressions/truncations.
321 * Revision 1.10 1995/09/11 15:23:26 adam
322 * More work on relevance search.
324 * Revision 1.9 1995/09/11 13:09:35 adam
325 * More work on relevance feedback.
327 * Revision 1.8 1995/09/08 14:52:27 adam
328 * Minor changes. Dictionary is lower case now.
330 * Revision 1.7 1995/09/07 13:58:36 adam
331 * New parameter: result-set file descriptor (RSFD) to support multiple
332 * positions within the same result-set.
333 * Boolean operators: and, or, not implemented.
334 * Result-set references.
336 * Revision 1.6 1995/09/06 16:11:18 adam
337 * Option: only one word key per file.
339 * Revision 1.5 1995/09/06 10:33:04 adam
340 * More work on present. Some log messages removed.
342 * Revision 1.4 1995/09/05 15:28:40 adam
343 * More work on search engine.
345 * Revision 1.3 1995/09/04 15:20:22 adam
348 * Revision 1.2 1995/09/04 12:33:43 adam
349 * Various cleanup. YAZ util used instead.
351 * Revision 1.1 1995/09/04 09:10:40 adam
352 * More work on index add/del/update.
353 * Merge sort implemented.
354 * Initial work on z39 server.
373 struct rpn_char_map_info {
378 static const char **rpn_char_map_handler (void *vp, const char **from, int len)
380 struct rpn_char_map_info *p = (struct rpn_char_map_info *) vp;
381 return zebra_maps_input (p->zm, p->reg_type, from, len);
384 static void rpn_char_map_prepare (ZebraHandle zh, int reg_type,
385 struct rpn_char_map_info *map_info)
387 map_info->zm = zh->service->zebra_maps;
388 map_info->reg_type = reg_type;
389 dict_grep_cmap (zh->service->dict, map_info, rpn_char_map_handler);
396 Z_AttributesPlusTerm *zapt;
399 static int attr_find (AttrType *src, oid_value *attributeSetP)
404 num_attributes = src->zapt->attributes->num_attributes;
406 num_attributes = src->zapt->num_attributes;
408 while (src->major < num_attributes)
410 Z_AttributeElement *element;
413 element = src->zapt->attributes->attributes[src->major];
415 element = src->zapt->attributeList[src->major];
417 if (src->type == *element->attributeType)
419 switch (element->which)
421 case Z_AttributeValue_numeric:
423 if (element->attributeSet && attributeSetP)
427 attrset = oid_getentbyoid (element->attributeSet);
428 *attributeSetP = attrset->value;
430 return *element->value.numeric;
432 case Z_AttributeValue_complex:
433 if (src->minor >= element->value.complex->num_list ||
434 element->value.complex->list[src->minor]->which !=
435 Z_StringOrNumeric_numeric)
438 if (element->attributeSet && attributeSetP)
442 attrset = oid_getentbyoid (element->attributeSet);
443 *attributeSetP = attrset->value;
445 return *element->value.complex->list[src->minor-1]->u.numeric;
455 static void attr_init (AttrType *src, Z_AttributesPlusTerm *zapt,
477 static void term_untrans (ZebraHandle zh, int reg_type,
478 char *dst, const char *src)
482 const char *cp = zebra_maps_output (zh->service->zebra_maps,
493 static void add_isam_p (const char *name, const char *info,
496 if (p->isam_p_indx == p->isam_p_size)
498 ISAMS_P *new_isam_p_buf;
502 p->isam_p_size = 2*p->isam_p_size + 100;
503 new_isam_p_buf = (ISAMS_P *) xmalloc (sizeof(*new_isam_p_buf) *
507 memcpy (new_isam_p_buf, p->isam_p_buf,
508 p->isam_p_indx * sizeof(*p->isam_p_buf));
509 xfree (p->isam_p_buf);
511 p->isam_p_buf = new_isam_p_buf;
514 new_term_no = (int *) xmalloc (sizeof(*new_term_no) *
518 memcpy (new_term_no, p->isam_p_buf,
519 p->isam_p_indx * sizeof(*p->term_no));
522 p->term_no = new_term_no;
525 assert (*info == sizeof(*p->isam_p_buf));
526 memcpy (p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
529 term_untrans (p->zh, p->reg_type, term_tmp, name+2);
530 logf (LOG_DEBUG, "grep: %s", term_tmp);
535 static int grep_handle (char *name, const char *info, void *p)
537 add_isam_p (name, info, (struct grep_info *) p);
541 static int term_pre (ZebraMaps zebra_maps, int reg_type, const char **src,
542 const char *ct1, const char *ct2)
544 const char *s1, *s0 = *src;
547 /* skip white space */
550 if (ct1 && strchr (ct1, *s0))
552 if (ct2 && strchr (ct2, *s0))
555 map = zebra_maps_input (zebra_maps, reg_type, &s1, strlen(s1));
556 if (**map != *CHR_SPACE)
564 /* term_100: handle term, where trunc=none (no operators at all) */
565 static int term_100 (ZebraMaps zebra_maps, int reg_type,
566 const char **src, char *dst, int space_split,
574 if (!term_pre (zebra_maps, reg_type, src, NULL, NULL))
580 map = zebra_maps_input (zebra_maps, reg_type, &s0, strlen(s0));
581 if (space_split && **map == *CHR_SPACE)
585 if (!isalnum (*s1) && *s1 != '-')
597 /* term_101: handle term, where trunc=Process # */
598 static int term_101 (ZebraMaps zebra_maps, int reg_type,
599 const char **src, char *dst, int space_split,
607 if (!term_pre (zebra_maps, reg_type, src, "#", "#"))
616 dst_term[j++] = *s0++;
621 map = zebra_maps_input (zebra_maps, reg_type, &s0, strlen(s0));
622 if (space_split && **map == *CHR_SPACE)
634 dst_term[j++] = '\0';
639 /* term_103: handle term, where trunc=re-2 (regular expressions) */
640 static int term_103 (ZebraMaps zebra_maps, int reg_type, const char **src,
641 char *dst, int *errors, int space_split,
649 if (!term_pre (zebra_maps, reg_type, src, "^\\()[].*+?|", "("))
652 if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
655 *errors = s0[1] - '0';
662 if (strchr ("^\\()[].*+?|-", *s0))
670 map = zebra_maps_input (zebra_maps, reg_type, &s0, strlen(s0));
671 if (**map == *CHR_SPACE)
688 /* term_103: handle term, where trunc=re-1 (regular expressions) */
689 static int term_102 (ZebraMaps zebra_maps, int reg_type, const char **src,
690 char *dst, int space_split, char *dst_term)
692 return term_103 (zebra_maps, reg_type, src, dst, NULL, space_split,
697 /* term_104: handle term, where trunc=Process # and ! */
698 static int term_104 (ZebraMaps zebra_maps, int reg_type,
699 const char **src, char *dst, int space_split,
707 if (!term_pre (zebra_maps, reg_type, src, "#!", "#!"))
716 dst_term[j++] = *s0++;
721 dst_term[j++] = *s0++;
725 map = zebra_maps_input (zebra_maps, reg_type, &s0, strlen(s0));
726 if (space_split && **map == *CHR_SPACE)
738 dst_term[j++] = '\0';
743 /* term_105/106: handle term, where trunc=Process * and ! and right trunc */
744 static int term_105 (ZebraMaps zebra_maps, int reg_type,
745 const char **src, char *dst, int space_split,
746 char *dst_term, int right_truncate)
753 if (!term_pre (zebra_maps, reg_type, src, "*!", "*!"))
762 dst_term[j++] = *s0++;
767 dst_term[j++] = *s0++;
771 map = zebra_maps_input (zebra_maps, reg_type, &s0, strlen(s0));
772 if (space_split && **map == *CHR_SPACE)
790 dst_term[j++] = '\0';
796 /* gen_regular_rel - generate regular expression from relation
797 * val: border value (inclusive)
798 * islt: 1 if <=; 0 if >=.
800 static void gen_regular_rel (char *dst, int val, int islt)
807 logf (LOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
811 strcpy (dst, "(-[0-9]+|(");
819 strcpy (dst, "([0-9]+|-(");
820 dst_p = strlen (dst);
830 dst_p = strlen (dst);
831 sprintf (numstr, "%d", val);
832 for (w = strlen(numstr); --w >= 0; pos++)
851 strcpy (dst + dst_p, numstr);
852 dst_p = strlen(dst) - pos - 1;
880 for (i = 0; i<pos; i++)
893 /* match everything less than 10^(pos-1) */
895 for (i=1; i<pos; i++)
896 strcat (dst, "[0-9]?");
900 /* match everything greater than 10^pos */
901 for (i = 0; i <= pos; i++)
902 strcat (dst, "[0-9]");
903 strcat (dst, "[0-9]*");
908 void string_rel_add_char (char **term_p, const char *src, int *indx)
910 if (src[*indx] == '\\')
911 *(*term_p)++ = src[(*indx)++];
912 *(*term_p)++ = src[(*indx)++];
916 * > abc ([b-].*|a[c-].*|ab[d-].*|abc.+)
917 * ([^-a].*|a[^-b].*ab[^-c].*|abc.+)
918 * >= abc ([b-].*|a[c-].*|ab[c-].*)
919 * ([^-a].*|a[^-b].*|ab[c-].*)
920 * < abc ([-0].*|a[-a].*|ab[-b].*)
921 * ([^a-].*|a[^b-].*|ab[^c-].*)
922 * <= abc ([-0].*|a[-a].*|ab[-b].*|abc)
923 * ([^a-].*|a[^b-].*|ab[^c-].*|abc)
925 static int string_relation (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
926 const char **term_sub, char *term_dict,
927 oid_value attributeSet,
928 int reg_type, int space_split, char *term_dst)
933 char *term_tmp = term_dict + strlen(term_dict);
934 char term_component[256];
936 attr_init (&relation, zapt, 2);
937 relation_value = attr_find (&relation, NULL);
939 logf (LOG_DEBUG, "string relation value=%d", relation_value);
940 switch (relation_value)
943 if (!term_100 (zh->service->zebra_maps, reg_type,
944 term_sub, term_component,
945 space_split, term_dst))
947 logf (LOG_DEBUG, "Relation <");
950 for (i = 0; term_component[i]; )
957 string_rel_add_char (&term_tmp, term_component, &j);
962 string_rel_add_char (&term_tmp, term_component, &i);
973 if (!term_100 (zh->service->zebra_maps, reg_type,
974 term_sub, term_component,
975 space_split, term_dst))
977 logf (LOG_DEBUG, "Relation <=");
980 for (i = 0; term_component[i]; )
985 string_rel_add_char (&term_tmp, term_component, &j);
989 string_rel_add_char (&term_tmp, term_component, &i);
998 for (i = 0; term_component[i]; )
999 string_rel_add_char (&term_tmp, term_component, &i);
1004 if (!term_100 (zh->service->zebra_maps, reg_type,
1005 term_sub, term_component, space_split, term_dst))
1007 logf (LOG_DEBUG, "Relation >");
1010 for (i = 0; term_component[i];)
1015 string_rel_add_char (&term_tmp, term_component, &j);
1020 string_rel_add_char (&term_tmp, term_component, &i);
1028 for (i = 0; term_component[i];)
1029 string_rel_add_char (&term_tmp, term_component, &i);
1036 if (!term_100 (zh->service->zebra_maps, reg_type, term_sub,
1037 term_component, space_split, term_dst))
1039 logf (LOG_DEBUG, "Relation >=");
1042 for (i = 0; term_component[i];)
1049 string_rel_add_char (&term_tmp, term_component, &j);
1052 if (term_component[i+1])
1056 string_rel_add_char (&term_tmp, term_component, &i);
1060 string_rel_add_char (&term_tmp, term_component, &i);
1072 logf (LOG_DEBUG, "Relation =");
1073 if (!term_100 (zh->service->zebra_maps, reg_type, term_sub,
1074 term_component, space_split, term_dst))
1076 strcat (term_tmp, "(");
1077 strcat (term_tmp, term_component);
1078 strcat (term_tmp, ")");
1083 static int string_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1084 const char **term_sub,
1085 oid_value attributeSet, NMEM stream,
1086 struct grep_info *grep_info,
1087 int reg_type, int complete_flag,
1088 int num_bases, char **basenames,
1091 char term_dict[2*IT_MAX_WORD+4000];
1093 AttrType truncation;
1094 int truncation_value;
1097 oid_value curAttributeSet = attributeSet;
1099 struct rpn_char_map_info rcmi;
1100 int space_split = complete_flag ? 0 : 1;
1102 rpn_char_map_prepare (zh, reg_type, &rcmi);
1103 attr_init (&use, zapt, 1);
1104 use_value = attr_find (&use, &curAttributeSet);
1105 logf (LOG_DEBUG, "string_term, use value %d", use_value);
1106 attr_init (&truncation, zapt, 5);
1107 truncation_value = attr_find (&truncation, NULL);
1108 logf (LOG_DEBUG, "truncation value %d", truncation_value);
1110 if (use_value == -1)
1113 for (base_no = 0; base_no < num_bases; base_no++)
1116 data1_local_attribute *local_attr;
1117 int max_pos, prefix_len = 0;
1120 if ((r=att_getentbyatt (zh, &attp, curAttributeSet, use_value)))
1122 logf (LOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1123 curAttributeSet, use_value, r);
1127 sprintf (val_str, "%d", use_value);
1129 zh->errString = nmem_strdup (stream, val_str);
1134 struct oident oident;
1136 oident.proto = PROTO_Z3950;
1137 oident.oclass = CLASS_ATTSET;
1138 oident.value = curAttributeSet;
1139 oid_ent_to_oid (&oident, oid);
1142 zh->errString = nmem_strdup (stream, oident.desc);
1146 if (zebraExplain_curDatabase (zh->service->zei, basenames[base_no]))
1148 zh->errCode = 109; /* Database unavailable */
1149 zh->errString = basenames[base_no];
1152 for (local_attr = attp.local_attributes; local_attr;
1153 local_attr = local_attr->next)
1159 ord = zebraExplain_lookupSU (zh->service->zei, attp.attset_ordinal,
1164 term_dict[prefix_len++] = '|';
1166 term_dict[prefix_len++] = '(';
1168 ord_len = key_SU_code (ord, ord_buf);
1169 for (i = 0; i<ord_len; i++)
1171 term_dict[prefix_len++] = 1;
1172 term_dict[prefix_len++] = ord_buf[i];
1178 sprintf (val_str, "%d", use_value);
1180 zh->errString = nmem_strdup (stream, val_str);
1183 term_dict[prefix_len++] = ')';
1184 term_dict[prefix_len++] = 1;
1185 term_dict[prefix_len++] = reg_type;
1186 logf (LOG_DEBUG, "reg_type = %d", term_dict[prefix_len-1]);
1187 term_dict[prefix_len] = '\0';
1189 switch (truncation_value)
1191 case -1: /* not specified */
1192 case 100: /* do not truncate */
1193 if (!string_relation (zh, zapt, &termp, term_dict,
1195 reg_type, space_split, term_dst))
1197 logf (LOG_DEBUG, "dict_lookup_grep: %s", term_dict+prefix_len);
1198 r = dict_lookup_grep (zh->service->dict, term_dict, 0,
1199 grep_info, &max_pos, 0, grep_handle);
1201 logf (LOG_WARN, "dict_lookup_grep fail, rel=gt: %d", r);
1203 case 1: /* right truncation */
1204 term_dict[j++] = '(';
1205 if (!term_100 (zh->service->zebra_maps, reg_type,
1206 &termp, term_dict + j, space_split, term_dst))
1208 strcat (term_dict, ".*)");
1209 dict_lookup_grep (zh->service->dict, term_dict, 0, grep_info,
1210 &max_pos, 0, grep_handle);
1212 case 2: /* keft truncation */
1213 term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1214 if (!term_100 (zh->service->zebra_maps, reg_type,
1215 &termp, term_dict + j, space_split, term_dst))
1217 strcat (term_dict, ")");
1218 dict_lookup_grep (zh->service->dict, term_dict, 0, grep_info,
1219 &max_pos, 0, grep_handle);
1221 case 3: /* left&right truncation */
1222 term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1223 if (!term_100 (zh->service->zebra_maps, reg_type,
1224 &termp, term_dict + j, space_split, term_dst))
1226 strcat (term_dict, ".*)");
1227 dict_lookup_grep (zh->service->dict, term_dict, 0, grep_info,
1228 &max_pos, 0, grep_handle);
1232 case 101: /* process # in term */
1233 term_dict[j++] = '(';
1234 if (!term_101 (zh->service->zebra_maps, reg_type,
1235 &termp, term_dict + j, space_split, term_dst))
1237 strcat (term_dict, ")");
1238 r = dict_lookup_grep (zh->service->dict, term_dict, 0, grep_info,
1239 &max_pos, 0, grep_handle);
1241 logf (LOG_WARN, "dict_lookup_grep err, trunc=#: %d", r);
1243 case 102: /* Regexp-1 */
1244 term_dict[j++] = '(';
1245 if (!term_102 (zh->service->zebra_maps, reg_type,
1246 &termp, term_dict + j, space_split, term_dst))
1248 strcat (term_dict, ")");
1249 logf (LOG_DEBUG, "Regexp-1 tolerance=%d", r);
1250 r = dict_lookup_grep (zh->service->dict, term_dict, 0, grep_info,
1251 &max_pos, 0, grep_handle);
1253 logf (LOG_WARN, "dict_lookup_grep err, trunc=regular: %d",
1256 case 103: /* Regexp-2 */
1258 term_dict[j++] = '(';
1259 if (!term_103 (zh->service->zebra_maps, reg_type,
1260 &termp, term_dict + j, &r, space_split, term_dst))
1262 strcat (term_dict, ")");
1263 logf (LOG_DEBUG, "Regexp-2 tolerance=%d", r);
1264 r = dict_lookup_grep (zh->service->dict, term_dict, r, grep_info,
1265 &max_pos, 2, grep_handle);
1267 logf (LOG_WARN, "dict_lookup_grep err, trunc=eregular: %d",
1270 case 104: /* process # and ! in term */
1271 term_dict[j++] = '(';
1272 if (!term_104 (zh->service->zebra_maps, reg_type,
1273 &termp, term_dict + j, space_split, term_dst))
1275 strcat (term_dict, ")");
1276 r = dict_lookup_grep (zh->service->dict, term_dict, 0, grep_info,
1277 &max_pos, 0, grep_handle);
1279 logf (LOG_WARN, "dict_lookup_grep err, trunc=#/!: %d", r);
1281 case 105: /* process * and ! in term */
1282 term_dict[j++] = '(';
1283 if (!term_105 (zh->service->zebra_maps, reg_type,
1284 &termp, term_dict + j, space_split, term_dst, 1))
1286 strcat (term_dict, ")");
1287 r = dict_lookup_grep (zh->service->dict, term_dict, 0, grep_info,
1288 &max_pos, 0, grep_handle);
1290 logf (LOG_WARN, "dict_lookup_grep err, trunc=*/!: %d", r);
1292 case 106: /* process * and ! in term */
1293 term_dict[j++] = '(';
1294 if (!term_105 (zh->service->zebra_maps, reg_type,
1295 &termp, term_dict + j, space_split, term_dst, 0))
1297 strcat (term_dict, ")");
1298 r = dict_lookup_grep (zh->service->dict, term_dict, 0, grep_info,
1299 &max_pos, 0, grep_handle);
1301 logf (LOG_WARN, "dict_lookup_grep err, trunc=*/!: %d", r);
1306 logf (LOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1310 static void trans_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1314 Z_Term *term = zapt->term;
1316 sizez = term->u.general->len;
1317 if (sizez > IT_MAX_WORD-1)
1318 sizez = IT_MAX_WORD-1;
1319 memcpy (termz, term->u.general->buf, sizez);
1320 termz[sizez] = '\0';
1323 static void trans_scan_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1324 char *termz, int reg_type)
1326 Z_Term *term = zapt->term;
1328 const char *cp = (const char *) term->u.general->buf;
1329 const char *cp_end = cp + term->u.general->len;
1332 const char *space_map = NULL;
1335 while ((len = (cp_end - cp)) > 0)
1337 map = zebra_maps_input (zh->service->zebra_maps, reg_type, &cp, len);
1338 if (**map == *CHR_SPACE)
1343 for (src = space_map; *src; src++)
1346 for (src = *map; *src; src++)
1353 static RSET rpn_prox (ZebraHandle zh, RSET *rset, int rset_no,
1354 int ordered, int exclusion, int relation, int distance)
1359 struct it_key **buf;
1361 char prox_term[1024];
1362 int length_prox_term = 0;
1363 int min_nn = 10000000;
1365 const char *flags = NULL;
1367 rsfd = (RSFD *) xmalloc (sizeof(*rsfd)*rset_no);
1368 more = (int *) xmalloc (sizeof(*more)*rset_no);
1369 buf = (struct it_key **) xmalloc (sizeof(*buf)*rset_no);
1372 for (i = 0; i<rset_no; i++)
1375 for (j = 0; j<rset[i]->no_rset_terms; j++)
1377 const char *nflags = rset[i]->rset_terms[j]->flags;
1378 char *term = rset[i]->rset_terms[j]->name;
1379 int lterm = strlen(term);
1380 if (lterm + length_prox_term < sizeof(prox_term)-1)
1382 if (length_prox_term)
1383 prox_term[length_prox_term++] = ' ';
1384 strcpy (prox_term + length_prox_term, term);
1385 length_prox_term += lterm;
1387 if (min_nn > rset[i]->rset_terms[j]->nn)
1388 min_nn = rset[i]->rset_terms[j]->nn;
1392 for (i = 0; i<rset_no; i++)
1397 for (i = 0; i<rset_no; i++)
1399 buf[i] = (struct it_key *) xmalloc (sizeof(**buf));
1400 rsfd[i] = rset_open (rset[i], RSETF_READ);
1401 if (!(more[i] = rset_read (rset[i], rsfd[i], buf[i], &term_index)))
1406 /* at least one is empty ... return null set */
1407 rset_null_parms parms;
1409 parms.rset_term = rset_term_create (prox_term, length_prox_term,
1411 parms.rset_term->nn = 0;
1412 result = rset_create (rset_kind_null, &parms);
1414 else if (ordered && relation == 3 && exclusion == 0 && distance == 1)
1416 /* special proximity case = phrase search ... */
1417 rset_temp_parms parms;
1420 parms.rset_term = rset_term_create (prox_term, length_prox_term,
1422 parms.rset_term->nn = min_nn;
1423 parms.key_size = sizeof (struct it_key);
1424 parms.temp_path = res_get (zh->service->res, "setTmpDir");
1425 result = rset_create (rset_kind_temp, &parms);
1426 rsfd_result = rset_open (result, RSETF_WRITE);
1430 for (i = 1; i<rset_no; i++)
1439 cmp = key_compare_it (buf[i], buf[i-1]);
1442 more[i-1] = rset_read (rset[i-1], rsfd[i-1],
1443 buf[i-1], &term_index);
1448 if (buf[i-1]->seqno+1 != buf[i]->seqno)
1450 more[i-1] = rset_read (rset[i-1], rsfd[i-1],
1451 buf[i-1], &term_index);
1457 more[i] = rset_read (rset[i], rsfd[i], buf[i],
1464 rset_write (result, rsfd_result, buf[0]);
1465 more[0] = rset_read (*rset, *rsfd, *buf, &term_index);
1468 rset_close (result, rsfd_result);
1470 else if (rset_no == 2)
1472 /* generic proximity case (two input sets only) ... */
1473 rset_temp_parms parms;
1476 logf (LOG_LOG, "generic prox, dist = %d, relation = %d, ordered =%d, exclusion=%d",
1477 distance, relation, ordered, exclusion);
1478 parms.rset_term = rset_term_create (prox_term, length_prox_term,
1480 parms.rset_term->nn = min_nn;
1481 parms.key_size = sizeof (struct it_key);
1482 parms.temp_path = res_get (zh->service->res, "setTmpDir");
1483 result = rset_create (rset_kind_temp, &parms);
1484 rsfd_result = rset_open (result, RSETF_WRITE);
1486 while (more[0] && more[1])
1488 int cmp = key_compare_it (buf[0], buf[1]);
1490 more[0] = rset_read (rset[0], rsfd[0], buf[0], &term_index);
1492 more[1] = rset_read (rset[1], rsfd[1], buf[1], &term_index);
1495 int sysno = buf[0]->sysno;
1499 seqno[n++] = buf[0]->seqno;
1500 while ((more[0] = rset_read (rset[0], rsfd[0], buf[0],
1502 sysno == buf[0]->sysno)
1504 seqno[n++] = buf[0]->seqno;
1507 for (i = 0; i<n; i++)
1509 int diff = buf[1]->seqno - seqno[i];
1510 int excl = exclusion;
1511 if (!ordered && diff < 0)
1516 if (diff < distance && diff >= 0)
1520 if (diff <= distance && diff >= 0)
1524 if (diff == distance && diff >= 0)
1528 if (diff >= distance && diff >= 0)
1532 if (diff > distance && diff >= 0)
1536 if (diff != distance && diff >= 0)
1542 rset_write (result, rsfd_result, buf[1]);
1546 } while ((more[1] = rset_read (rset[1], rsfd[1], buf[1],
1548 sysno == buf[1]->sysno);
1551 rset_close (result, rsfd_result);
1555 rset_null_parms parms;
1557 parms.rset_term = rset_term_create (prox_term, length_prox_term,
1559 parms.rset_term->nn = 0;
1560 result = rset_create (rset_kind_null, &parms);
1562 for (i = 0; i<rset_no; i++)
1565 rset_close (rset[i], rsfd[i]);
1575 char *normalize_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1576 const char *termz, NMEM stream, unsigned reg_id)
1579 AttrType truncation;
1580 int truncation_value;
1583 attr_init (&truncation, zapt, 5);
1584 truncation_value = attr_find (&truncation, NULL);
1586 switch (truncation_value)
1606 wrbuf = zebra_replace(zh->service->zebra_maps, reg_id, ex_list,
1607 termz, strlen(termz));
1609 return nmem_strdup(stream, termz);
1612 char *buf = (char*) nmem_malloc (stream, wrbuf_len(wrbuf)+1);
1613 memcpy (buf, wrbuf_buf(wrbuf), wrbuf_len(wrbuf));
1614 buf[wrbuf_len(wrbuf)] = '\0';
1619 static RSET rpn_search_APT_phrase (ZebraHandle zh,
1620 Z_AttributesPlusTerm *zapt,
1621 const char *termz_org,
1622 oid_value attributeSet,
1624 int reg_type, int complete_flag,
1625 const char *rank_type,
1626 int num_bases, char **basenames)
1628 char term_dst[IT_MAX_WORD+1];
1629 RSET rset[60], result;
1630 int i, r, rset_no = 0;
1631 struct grep_info grep_info;
1632 char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1633 const char *termp = termz;
1636 grep_info.term_no = 0;
1638 grep_info.isam_p_size = 0;
1639 grep_info.isam_p_buf = NULL;
1641 grep_info.reg_type = reg_type;
1645 logf (LOG_DEBUG, "APT_phrase termp=%s", termp);
1646 grep_info.isam_p_indx = 0;
1647 r = string_term (zh, zapt, &termp, attributeSet, stream, &grep_info,
1648 reg_type, complete_flag, num_bases, basenames,
1652 logf (LOG_DEBUG, "term: %s", term_dst);
1653 rset[rset_no] = rset_trunc (zh, grep_info.isam_p_buf,
1654 grep_info.isam_p_indx, term_dst,
1655 strlen(term_dst), rank_type);
1656 assert (rset[rset_no]);
1657 if (++rset_no >= (int) (sizeof(rset)/sizeof(*rset)))
1661 xfree(grep_info.term_no);
1663 xfree (grep_info.isam_p_buf);
1666 rset_null_parms parms;
1668 parms.rset_term = rset_term_create (term_dst, -1, rank_type);
1669 return rset_create (rset_kind_null, &parms);
1671 else if (rset_no == 1)
1673 result = rpn_prox (zh, rset, rset_no, 1, 0, 3, 1);
1674 for (i = 0; i<rset_no; i++)
1675 rset_delete (rset[i]);
1679 static RSET rpn_search_APT_or_list (ZebraHandle zh,
1680 Z_AttributesPlusTerm *zapt,
1681 const char *termz_org,
1682 oid_value attributeSet,
1684 int reg_type, int complete_flag,
1685 const char *rank_type,
1686 int num_bases, char **basenames)
1688 char term_dst[IT_MAX_WORD+1];
1689 RSET rset[60], result;
1690 int i, r, rset_no = 0;
1691 struct grep_info grep_info;
1692 char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1693 const char *termp = termz;
1695 grep_info.term_no = 0;
1697 grep_info.isam_p_size = 0;
1698 grep_info.isam_p_buf = NULL;
1700 grep_info.reg_type = reg_type;
1704 logf (LOG_DEBUG, "APT_or_list termp=%s", termp);
1705 grep_info.isam_p_indx = 0;
1706 r = string_term (zh, zapt, &termp, attributeSet, stream, &grep_info,
1707 reg_type, complete_flag, num_bases, basenames,
1711 logf (LOG_DEBUG, "term: %s", term_dst);
1712 rset[rset_no] = rset_trunc (zh, grep_info.isam_p_buf,
1713 grep_info.isam_p_indx, term_dst,
1714 strlen(term_dst), rank_type);
1715 assert (rset[rset_no]);
1716 if (++rset_no >= (int) (sizeof(rset)/sizeof(*rset)))
1720 xfree(grep_info.term_no);
1722 xfree (grep_info.isam_p_buf);
1725 rset_null_parms parms;
1727 parms.rset_term = rset_term_create (term_dst, -1, rank_type);
1728 return rset_create (rset_kind_null, &parms);
1731 for (i = 1; i<rset_no; i++)
1733 rset_bool_parms bool_parms;
1735 bool_parms.rset_l = result;
1736 bool_parms.rset_r = rset[i];
1737 bool_parms.key_size = sizeof(struct it_key);
1738 bool_parms.cmp = key_compare_it;
1739 result = rset_create (rset_kind_or, &bool_parms);
1744 static RSET rpn_search_APT_and_list (ZebraHandle zh,
1745 Z_AttributesPlusTerm *zapt,
1746 const char *termz_org,
1747 oid_value attributeSet,
1749 int reg_type, int complete_flag,
1750 const char *rank_type,
1751 int num_bases, char **basenames)
1753 char term_dst[IT_MAX_WORD+1];
1754 RSET rset[60], result;
1755 int i, r, rset_no = 0;
1756 struct grep_info grep_info;
1757 char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1758 const char *termp = termz;
1761 grep_info.term_no = 0;
1763 grep_info.isam_p_size = 0;
1764 grep_info.isam_p_buf = NULL;
1766 grep_info.reg_type = reg_type;
1770 logf (LOG_DEBUG, "APT_and_list termp=%s", termp);
1771 grep_info.isam_p_indx = 0;
1772 r = string_term (zh, zapt, &termp, attributeSet, stream, &grep_info,
1773 reg_type, complete_flag, num_bases, basenames,
1777 logf (LOG_DEBUG, "term: %s", term_dst);
1778 rset[rset_no] = rset_trunc (zh, grep_info.isam_p_buf,
1779 grep_info.isam_p_indx, term_dst,
1780 strlen(term_dst), rank_type);
1781 assert (rset[rset_no]);
1782 if (++rset_no >= (int) (sizeof(rset)/sizeof(*rset)))
1786 xfree(grep_info.term_no);
1788 xfree (grep_info.isam_p_buf);
1791 rset_null_parms parms;
1793 parms.rset_term = rset_term_create (term_dst, -1, rank_type);
1794 return rset_create (rset_kind_null, &parms);
1797 for (i = 1; i<rset_no; i++)
1799 rset_bool_parms bool_parms;
1801 bool_parms.rset_l = result;
1802 bool_parms.rset_r = rset[i];
1803 bool_parms.key_size = sizeof(struct it_key);
1804 bool_parms.cmp = key_compare_it;
1805 result = rset_create (rset_kind_and, &bool_parms);
1810 static int numeric_relation (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1811 const char **term_sub,
1813 oid_value attributeSet,
1814 struct grep_info *grep_info,
1823 char *term_tmp = term_dict + strlen(term_dict);
1825 attr_init (&relation, zapt, 2);
1826 relation_value = attr_find (&relation, NULL);
1828 logf (LOG_DEBUG, "numeric relation value=%d", relation_value);
1830 if (!term_100 (zh->service->zebra_maps, reg_type, term_sub, term_tmp, 1,
1833 term_value = atoi (term_tmp);
1834 switch (relation_value)
1837 logf (LOG_DEBUG, "Relation <");
1838 gen_regular_rel (term_tmp, term_value-1, 1);
1841 logf (LOG_DEBUG, "Relation <=");
1842 gen_regular_rel (term_tmp, term_value, 1);
1845 logf (LOG_DEBUG, "Relation >=");
1846 gen_regular_rel (term_tmp, term_value, 0);
1849 logf (LOG_DEBUG, "Relation >");
1850 gen_regular_rel (term_tmp, term_value+1, 0);
1854 logf (LOG_DEBUG, "Relation =");
1855 sprintf (term_tmp, "(0*%d)", term_value);
1857 logf (LOG_DEBUG, "dict_lookup_grep: %s", term_tmp);
1858 r = dict_lookup_grep (zh->service->dict, term_dict, 0, grep_info, max_pos,
1861 logf (LOG_WARN, "dict_lookup_grep fail, rel=gt: %d", r);
1862 logf (LOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1866 static int numeric_term (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1867 const char **term_sub,
1868 oid_value attributeSet, struct grep_info *grep_info,
1869 int reg_type, int complete_flag,
1870 int num_bases, char **basenames,
1873 char term_dict[2*IT_MAX_WORD+2];
1877 oid_value curAttributeSet = attributeSet;
1879 struct rpn_char_map_info rcmi;
1881 rpn_char_map_prepare (zh, reg_type, &rcmi);
1882 attr_init (&use, zapt, 1);
1883 use_value = attr_find (&use, &curAttributeSet);
1884 logf (LOG_DEBUG, "numeric_term, use value %d", use_value);
1886 if (use_value == -1)
1889 for (base_no = 0; base_no < num_bases; base_no++)
1892 data1_local_attribute *local_attr;
1893 int max_pos, prefix_len = 0;
1896 if ((r=att_getentbyatt (zh, &attp, curAttributeSet, use_value)))
1898 logf (LOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1899 curAttributeSet, use_value, r);
1906 if (zebraExplain_curDatabase (zh->service->zei, basenames[base_no]))
1908 zh->errCode = 109; /* Database unavailable */
1909 zh->errString = basenames[base_no];
1912 for (local_attr = attp.local_attributes; local_attr;
1913 local_attr = local_attr->next)
1919 ord = zebraExplain_lookupSU (zh->service->zei, attp.attset_ordinal,
1924 term_dict[prefix_len++] = '|';
1926 term_dict[prefix_len++] = '(';
1928 ord_len = key_SU_code (ord, ord_buf);
1929 for (i = 0; i<ord_len; i++)
1931 term_dict[prefix_len++] = 1;
1932 term_dict[prefix_len++] = ord_buf[i];
1940 term_dict[prefix_len++] = ')';
1941 term_dict[prefix_len++] = 1;
1942 term_dict[prefix_len++] = reg_type;
1943 logf (LOG_DEBUG, "reg_type = %d", term_dict[prefix_len-1]);
1944 term_dict[prefix_len] = '\0';
1945 if (!numeric_relation (zh, zapt, &termp, term_dict,
1946 attributeSet, grep_info, &max_pos, reg_type,
1951 logf (LOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1955 static RSET rpn_search_APT_numeric (ZebraHandle zh,
1956 Z_AttributesPlusTerm *zapt,
1958 oid_value attributeSet,
1960 int reg_type, int complete_flag,
1961 const char *rank_type,
1962 int num_bases, char **basenames)
1964 char term_dst[IT_MAX_WORD+1];
1965 const char *termp = termz;
1966 RSET rset[60], result;
1967 int i, r, rset_no = 0;
1968 struct grep_info grep_info;
1971 grep_info.term_no = 0;
1973 grep_info.isam_p_size = 0;
1974 grep_info.isam_p_buf = NULL;
1976 grep_info.reg_type = reg_type;
1980 logf (LOG_DEBUG, "APT_numeric termp=%s", termp);
1981 grep_info.isam_p_indx = 0;
1982 r = numeric_term (zh, zapt, &termp, attributeSet, &grep_info,
1983 reg_type, complete_flag, num_bases, basenames,
1987 logf (LOG_DEBUG, "term: %s", term_dst);
1988 rset[rset_no] = rset_trunc (zh, grep_info.isam_p_buf,
1989 grep_info.isam_p_indx, term_dst,
1990 strlen(term_dst), rank_type);
1991 assert (rset[rset_no]);
1992 if (++rset_no >= (int) (sizeof(rset)/sizeof(*rset)))
1996 xfree(grep_info.term_no);
1998 xfree (grep_info.isam_p_buf);
2001 rset_null_parms parms;
2003 parms.rset_term = rset_term_create (term_dst, -1, rank_type);
2004 return rset_create (rset_kind_null, &parms);
2007 for (i = 1; i<rset_no; i++)
2009 rset_bool_parms bool_parms;
2011 bool_parms.rset_l = result;
2012 bool_parms.rset_r = rset[i];
2013 bool_parms.key_size = sizeof(struct it_key);
2014 bool_parms.cmp = key_compare_it;
2015 result = rset_create (rset_kind_and, &bool_parms);
2020 static RSET rpn_search_APT_local (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2022 oid_value attributeSet,
2024 const char *rank_type)
2029 rset_temp_parms parms;
2031 parms.rset_term = rset_term_create (termz, -1, rank_type);
2032 parms.key_size = sizeof (struct it_key);
2033 parms.temp_path = res_get (zh->service->res, "setTmpDir");
2034 result = rset_create (rset_kind_temp, &parms);
2035 rsfd = rset_open (result, RSETF_WRITE);
2037 key.sysno = atoi (termz);
2041 rset_write (result, rsfd, &key);
2042 rset_close (result, rsfd);
2046 static RSET rpn_sort_spec (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2047 oid_value attributeSet, NMEM stream,
2048 Z_SortKeySpecList *sort_sequence,
2049 const char *rank_type)
2051 rset_null_parms parms;
2053 int sort_relation_value;
2054 AttrType sort_relation_type;
2059 Z_AttributeElement *ae;
2063 attr_init (&sort_relation_type, zapt, 7);
2064 sort_relation_value = attr_find (&sort_relation_type, &attributeSet);
2066 attr_init (&use_type, zapt, 1);
2067 use_value = attr_find (&use_type, &attributeSet);
2069 if (!sort_sequence->specs)
2071 sort_sequence->num_specs = 10;
2072 sort_sequence->specs = (Z_SortKeySpec **)
2073 nmem_malloc (stream, sort_sequence->num_specs *
2074 sizeof(*sort_sequence->specs));
2075 for (i = 0; i<sort_sequence->num_specs; i++)
2076 sort_sequence->specs[i] = 0;
2078 if (zapt->term->which != Z_Term_general)
2081 i = atoi_n ((char *) zapt->term->u.general->buf,
2082 zapt->term->u.general->len);
2083 if (i >= sort_sequence->num_specs)
2086 oe.proto = PROTO_Z3950;
2087 oe.oclass = CLASS_ATTSET;
2088 oe.value = attributeSet;
2089 if (!oid_ent_to_oid (&oe, oid))
2092 sks = (Z_SortKeySpec *) nmem_malloc (stream, sizeof(*sks));
2093 sks->sortElement = (Z_SortElement *)
2094 nmem_malloc (stream, sizeof(*sks->sortElement));
2095 sks->sortElement->which = Z_SortElement_generic;
2096 sk = sks->sortElement->u.generic = (Z_SortKey *)
2097 nmem_malloc (stream, sizeof(*sk));
2098 sk->which = Z_SortKey_sortAttributes;
2099 sk->u.sortAttributes = (Z_SortAttributes *)
2100 nmem_malloc (stream, sizeof(*sk->u.sortAttributes));
2102 sk->u.sortAttributes->id = oid;
2103 sk->u.sortAttributes->list = (Z_AttributeList *)
2104 nmem_malloc (stream, sizeof(*sk->u.sortAttributes->list));
2105 sk->u.sortAttributes->list->num_attributes = 1;
2106 sk->u.sortAttributes->list->attributes = (Z_AttributeElement **)
2107 nmem_malloc (stream, sizeof(*sk->u.sortAttributes->list->attributes));
2108 ae = *sk->u.sortAttributes->list->attributes = (Z_AttributeElement *)
2109 nmem_malloc (stream, sizeof(**sk->u.sortAttributes->list->attributes));
2110 ae->attributeSet = 0;
2111 ae->attributeType = (int *)
2112 nmem_malloc (stream, sizeof(*ae->attributeType));
2113 *ae->attributeType = 1;
2114 ae->which = Z_AttributeValue_numeric;
2115 ae->value.numeric = (int *)
2116 nmem_malloc (stream, sizeof(*ae->value.numeric));
2117 *ae->value.numeric = use_value;
2119 sks->sortRelation = (int *)
2120 nmem_malloc (stream, sizeof(*sks->sortRelation));
2121 if (sort_relation_value == 1)
2122 *sks->sortRelation = Z_SortRelation_ascending;
2123 else if (sort_relation_value == 2)
2124 *sks->sortRelation = Z_SortRelation_descending;
2126 *sks->sortRelation = Z_SortRelation_ascending;
2128 sks->caseSensitivity = (int *)
2129 nmem_malloc (stream, sizeof(*sks->caseSensitivity));
2130 *sks->caseSensitivity = 0;
2133 sks->which = Z_SortKeySpec_null;
2134 sks->u.null = odr_nullval ();
2136 sks->missingValueAction = 0;
2139 sort_sequence->specs[i] = sks;
2141 parms.rset_term = rset_term_create ("", -1, rank_type);
2142 return rset_create (rset_kind_null, &parms);
2146 static RSET rpn_search_APT (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2147 oid_value attributeSet, NMEM stream,
2148 Z_SortKeySpecList *sort_sequence,
2149 int num_bases, char **basenames)
2152 char *search_type = NULL;
2153 char *rank_type = NULL;
2156 char termz[IT_MAX_WORD+1];
2158 zebra_maps_attr (zh->service->zebra_maps, zapt, ®_id, &search_type,
2159 &rank_type, &complete_flag, &sort_flag);
2161 logf (LOG_DEBUG, "reg_id=%c", reg_id);
2162 logf (LOG_DEBUG, "complete_flag=%d", complete_flag);
2163 logf (LOG_DEBUG, "search_type=%s", search_type);
2164 logf (LOG_DEBUG, "rank_type=%s", rank_type);
2166 if (zapt->term->which != Z_Term_general)
2171 trans_term (zh, zapt, termz);
2174 return rpn_sort_spec (zh, zapt, attributeSet, stream, sort_sequence,
2177 if (!strcmp (search_type, "phrase"))
2179 return rpn_search_APT_phrase (zh, zapt, termz, attributeSet, stream,
2180 reg_id, complete_flag, rank_type,
2181 num_bases, basenames);
2183 else if (!strcmp (search_type, "and-list"))
2185 return rpn_search_APT_and_list (zh, zapt, termz, attributeSet, stream,
2186 reg_id, complete_flag, rank_type,
2187 num_bases, basenames);
2189 else if (!strcmp (search_type, "or-list"))
2191 return rpn_search_APT_or_list (zh, zapt, termz, attributeSet, stream,
2192 reg_id, complete_flag, rank_type,
2193 num_bases, basenames);
2195 else if (!strcmp (search_type, "local"))
2197 return rpn_search_APT_local (zh, zapt, termz, attributeSet, stream,
2200 else if (!strcmp (search_type, "numeric"))
2202 return rpn_search_APT_numeric (zh, zapt, termz, attributeSet, stream,
2203 reg_id, complete_flag, rank_type,
2204 num_bases, basenames);
2210 static RSET rpn_search_structure (ZebraHandle zh, Z_RPNStructure *zs,
2211 oid_value attributeSet, NMEM stream,
2212 Z_SortKeySpecList *sort_sequence,
2213 int num_bases, char **basenames)
2216 if (zs->which == Z_RPNStructure_complex)
2218 Z_Operator *zop = zs->u.complex->roperator;
2219 rset_bool_parms bool_parms;
2221 bool_parms.rset_l = rpn_search_structure (zh, zs->u.complex->s1,
2222 attributeSet, stream,
2224 num_bases, basenames);
2225 if (bool_parms.rset_l == NULL)
2227 bool_parms.rset_r = rpn_search_structure (zh, zs->u.complex->s2,
2228 attributeSet, stream,
2230 num_bases, basenames);
2231 if (bool_parms.rset_r == NULL)
2233 rset_delete (bool_parms.rset_l);
2236 bool_parms.key_size = sizeof(struct it_key);
2237 bool_parms.cmp = key_compare_it;
2241 case Z_Operator_and:
2242 r = rset_create (rset_kind_and, &bool_parms);
2245 r = rset_create (rset_kind_or, &bool_parms);
2247 case Z_Operator_and_not:
2248 r = rset_create (rset_kind_not, &bool_parms);
2250 case Z_Operator_prox:
2252 if (zop->u.prox->which != Z_ProximityOperator_known)
2258 if (zop->u.prox->which != Z_ProxCode_known)
2266 if (*zop->u.prox->u.known != Z_ProxUnit_word)
2268 char *val = (char *) nmem_malloc (stream, 16);
2270 zh->errString = val;
2271 sprintf (val, "%d", *zop->u.prox->u.known);
2275 if (*zop->u.prox->proximityUnitCode != Z_ProxUnit_word)
2277 char *val = (char *) nmem_malloc (stream, 16);
2279 zh->errString = val;
2280 sprintf (val, "%d", *zop->u.prox->proximityUnitCode);
2288 rsets[0] = bool_parms.rset_l;
2289 rsets[1] = bool_parms.rset_r;
2291 r = rpn_prox (zh, rsets, 2,
2292 *zop->u.prox->ordered,
2293 (!zop->u.prox->exclusion ? 0 :
2294 *zop->u.prox->exclusion),
2295 *zop->u.prox->relationType,
2296 *zop->u.prox->distance);
2297 rset_delete (rsets[0]);
2298 rset_delete (rsets[1]);
2306 else if (zs->which == Z_RPNStructure_simple)
2308 if (zs->u.simple->which == Z_Operand_APT)
2310 logf (LOG_DEBUG, "rpn_search_APT");
2311 r = rpn_search_APT (zh, zs->u.simple->u.attributesPlusTerm,
2312 attributeSet, stream, sort_sequence,
2313 num_bases, basenames);
2315 else if (zs->u.simple->which == Z_Operand_resultSetId)
2317 logf (LOG_DEBUG, "rpn_search_ref");
2318 r = resultSetRef (zh, zs->u.simple->u.resultSetId);
2320 r = rset_create (rset_kind_null, NULL);
2337 RSET rpn_search (ZebraHandle zh, NMEM nmem,
2338 Z_RPNQuery *rpn, int num_bases, char **basenames,
2339 const char *setname,
2344 oid_value attributeSet;
2345 Z_SortKeySpecList *sort_sequence;
2349 zh->errString = NULL;
2352 sort_sequence = (Z_SortKeySpecList *)
2353 nmem_malloc (nmem, sizeof(*sort_sequence));
2354 sort_sequence->num_specs = 10;
2355 sort_sequence->specs = (Z_SortKeySpec **)
2356 nmem_malloc (nmem, sort_sequence->num_specs *
2357 sizeof(*sort_sequence->specs));
2358 for (i = 0; i<sort_sequence->num_specs; i++)
2359 sort_sequence->specs[i] = 0;
2361 attrset = oid_getentbyoid (rpn->attributeSetId);
2362 attributeSet = attrset->value;
2363 rset = rpn_search_structure (zh, rpn->RPNStructure, attributeSet,
2364 nmem, sort_sequence, num_bases, basenames);
2369 logf (LOG_DEBUG, "search error: %d", zh->errCode);
2371 for (i = 0; sort_sequence->specs[i]; i++)
2373 sort_sequence->num_specs = i;
2375 resultSetRank (zh, sset, rset);
2378 logf (LOG_DEBUG, "resultSetSortSingle in rpn_search");
2379 resultSetSortSingle (zh, nmem, sset, rset,
2380 sort_sequence, &sort_status);
2383 logf (LOG_DEBUG, "resultSetSortSingle status = %d", zh->errCode);
2389 struct scan_info_entry {
2395 struct scan_info_entry *list;
2401 static int scan_handle (char *name, const char *info, int pos, void *client)
2403 int len_prefix, idx;
2404 struct scan_info *scan_info = (struct scan_info *) client;
2406 len_prefix = strlen(scan_info->prefix);
2407 if (memcmp (name, scan_info->prefix, len_prefix))
2409 if (pos > 0) idx = scan_info->after - pos + scan_info->before;
2412 scan_info->list[idx].term = (char *)
2413 odr_malloc (scan_info->odr, strlen(name + len_prefix)+1);
2414 strcpy (scan_info->list[idx].term, name + len_prefix);
2415 assert (*info == sizeof(ISAMS_P));
2416 memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAMS_P));
2420 static void scan_term_untrans (ZebraHandle zh, NMEM stream, int reg_type,
2421 char **dst, const char *src)
2423 char term_dst[1024];
2425 term_untrans (zh, reg_type, term_dst, src);
2427 *dst = (char *) nmem_malloc (stream, strlen(term_dst)+1);
2428 strcpy (*dst, term_dst);
2431 static void count_set (RSET r, int *count)
2439 logf (LOG_DEBUG, "count_set");
2442 rfd = rset_open (r, RSETF_READ);
2443 while (rset_read (r, rfd, &key, &term_index))
2445 if (key.sysno != psysno)
2452 rset_close (r, rfd);
2453 logf (LOG_DEBUG, "%d keys, %d records", kno, *count);
2456 void rpn_scan (ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
2457 oid_value attributeset,
2458 int num_bases, char **basenames,
2459 int *position, int *num_entries, ZebraScanEntry **list,
2463 int pos = *position;
2464 int num = *num_entries;
2468 char termz[IT_MAX_WORD+20];
2471 struct scan_info *scan_info_array;
2472 ZebraScanEntry *glist;
2473 int ords[32], ord_no = 0;
2477 char *search_type = NULL;
2478 char *rank_type = NULL;
2483 if (attributeset == VAL_NONE)
2484 attributeset = VAL_BIB1;
2486 logf (LOG_DEBUG, "position = %d, num = %d", pos, num);
2488 attr_init (&use, zapt, 1);
2489 use_value = attr_find (&use, &attributeset);
2491 if (zebra_maps_attr (zh->service->zebra_maps, zapt, ®_id, &search_type,
2492 &rank_type, &complete_flag, &sort_flag))
2498 if (use_value == -1)
2500 for (base_no = 0; base_no < num_bases && ord_no < 32; base_no++)
2504 data1_local_attribute *local_attr;
2506 if ((r=att_getentbyatt (zh, &attp, attributeset, use_value)))
2508 logf (LOG_DEBUG, "att_getentbyatt fail. set=%d use=%d",
2509 attributeset, use_value);
2517 if (zebraExplain_curDatabase (zh->service->zei, basenames[base_no]))
2519 zh->errString = basenames[base_no];
2520 zh->errCode = 109; /* Database unavailable */
2523 for (local_attr = attp.local_attributes; local_attr && ord_no < 32;
2524 local_attr = local_attr->next)
2528 ord = zebraExplain_lookupSU (zh->service->zei, attp.attset_ordinal,
2531 ords[ord_no++] = ord;
2539 /* prepare dictionary scanning */
2542 scan_info_array = (struct scan_info *)
2543 odr_malloc (stream, ord_no * sizeof(*scan_info_array));
2544 for (i = 0; i < ord_no; i++)
2546 int j, prefix_len = 0;
2547 int before_tmp = before, after_tmp = after;
2548 struct scan_info *scan_info = scan_info_array + i;
2549 struct rpn_char_map_info rcmi;
2551 rpn_char_map_prepare (zh, reg_id, &rcmi);
2553 scan_info->before = before;
2554 scan_info->after = after;
2555 scan_info->odr = stream;
2557 scan_info->list = (struct scan_info_entry *)
2558 odr_malloc (stream, (before+after) * sizeof(*scan_info->list));
2559 for (j = 0; j<before+after; j++)
2560 scan_info->list[j].term = NULL;
2562 prefix_len += key_SU_code (ords[i], termz + prefix_len);
2563 termz[prefix_len++] = reg_id;
2564 termz[prefix_len] = 0;
2565 strcpy (scan_info->prefix, termz);
2567 trans_scan_term (zh, zapt, termz+prefix_len, reg_id);
2569 dict_scan (zh->service->dict, termz, &before_tmp, &after_tmp,
2570 scan_info, scan_handle);
2572 glist = (ZebraScanEntry *)
2573 odr_malloc (stream, (before+after)*sizeof(*glist));
2575 /* consider terms after main term */
2576 for (i = 0; i < ord_no; i++)
2580 for (i = 0; i<after; i++)
2583 const char *mterm = NULL;
2587 for (j = 0; j < ord_no; j++)
2589 if (ptr[j] < before+after &&
2590 (tst=scan_info_array[j].list[ptr[j]].term) &&
2591 (!mterm || strcmp (tst, mterm) < 0))
2599 scan_term_untrans (zh, stream->mem, reg_id,
2600 &glist[i+before].term, mterm);
2601 rset = rset_trunc (zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
2602 glist[i+before].term, strlen(glist[i+before].term),
2606 for (j = j0+1; j<ord_no; j++)
2608 if (ptr[j] < before+after &&
2609 (tst=scan_info_array[j].list[ptr[j]].term) &&
2610 !strcmp (tst, mterm))
2612 rset_bool_parms bool_parms;
2616 rset_trunc (zh, &scan_info_array[j].list[ptr[j]].isam_p, 1,
2617 glist[i+before].term,
2618 strlen(glist[i+before].term), NULL);
2620 bool_parms.key_size = sizeof(struct it_key);
2621 bool_parms.cmp = key_compare_it;
2622 bool_parms.rset_l = rset;
2623 bool_parms.rset_r = rset2;
2625 rset = rset_create (rset_kind_or, &bool_parms);
2630 count_set (rset, &glist[i+before].occurrences);
2635 *num_entries -= (after-i);
2639 /* consider terms before main term */
2640 for (i = 0; i<ord_no; i++)
2643 for (i = 0; i<before; i++)
2646 const char *mterm = NULL;
2650 for (j = 0; j <ord_no; j++)
2652 if (ptr[j] < before &&
2653 (tst=scan_info_array[j].list[before-1-ptr[j]].term) &&
2654 (!mterm || strcmp (tst, mterm) > 0))
2663 scan_term_untrans (zh, stream->mem, reg_id,
2664 &glist[before-1-i].term, mterm);
2667 (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
2668 glist[before-1-i].term, strlen(glist[before-1-i].term),
2673 for (j = j0+1; j<ord_no; j++)
2675 if (ptr[j] < before &&
2676 (tst=scan_info_array[j].list[before-1-ptr[j]].term) &&
2677 !strcmp (tst, mterm))
2679 rset_bool_parms bool_parms;
2682 rset2 = rset_trunc (zh,
2683 &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
2684 glist[before-1-i].term,
2685 strlen(glist[before-1-i].term), NULL);
2687 bool_parms.key_size = sizeof(struct it_key);
2688 bool_parms.cmp = key_compare_it;
2689 bool_parms.rset_l = rset;
2690 bool_parms.rset_r = rset2;
2692 rset = rset_create (rset_kind_or, &bool_parms);
2697 count_set (rset, &glist[before-1-i].occurrences);
2707 *list = glist + i; /* list is set to first 'real' entry */
2709 logf (LOG_DEBUG, "position = %d, num_entries = %d",
2710 *position, *num_entries);
2712 logf (LOG_DEBUG, "scan error: %d", zh->errCode);