2 * Copyright (C) 1995-1998, Index Data I/S
4 * Sebastian Hammer, Adam Dickmeiss
7 * Revision 1.73 1998-01-29 13:40:11 adam
8 * Better logging for scan service.
10 * Revision 1.72 1998/01/07 13:53:41 adam
11 * Queries using simple ranked operands returns right number of hits.
13 * Revision 1.71 1997/12/18 10:54:24 adam
14 * New method result set method rs_hits that returns the number of
15 * hits in result-set (if known). The ranked result set returns real
16 * number of hits but only when not combined with other operands.
18 * Revision 1.70 1997/10/31 12:34:43 adam
19 * Changed a few log statements.
21 * Revision 1.69 1997/10/29 12:05:02 adam
22 * Server produces diagnostic "Unsupported Attribute Set" when appropriate.
24 * Revision 1.68 1997/10/27 14:33:06 adam
25 * Moved towards generic character mapping depending on "structure"
26 * field in abstract syntax file. Fixed a few memory leaks. Fixed
27 * bug with negative integers when doing searches with relational
30 * Revision 1.67 1997/09/29 09:06:10 adam
31 * Removed one static var in order to make this module thread safe.
33 * Revision 1.66 1997/09/25 14:58:03 adam
36 * Revision 1.65 1997/09/22 12:39:06 adam
37 * Added get_pos method for the ranked result sets.
39 * Revision 1.64 1997/09/18 08:59:20 adam
40 * Extra generic handle for the character mapping routines.
42 * Revision 1.63 1997/09/17 12:19:18 adam
43 * Zebra version corresponds to YAZ version 1.4.
44 * Changed Zebra server so that it doesn't depend on global common_resource.
46 * Revision 1.62 1997/09/05 15:30:09 adam
47 * Changed prototype for chr_map_input - added const.
48 * Added support for C++, headers uses extern "C" for public definitions.
50 * Revision 1.61 1997/02/10 10:21:14 adam
51 * Bug fix: in search terms character (^) wasn't observed.
53 * Revision 1.60 1997/01/31 11:10:34 adam
54 * Bug fix: Leading and trailing white space weren't removed in scan tokens.
56 * Revision 1.59 1997/01/17 11:31:46 adam
57 * Bug fix: complete phrase search didn't work.
59 * Revision 1.58 1996/12/23 15:30:45 adam
61 * Bug fix: result sets weren't deleted after server shut down.
63 * Revision 1.57 1996/11/11 13:38:02 adam
64 * Added proximity support in search.
66 * Revision 1.56 1996/11/08 11:10:32 adam
67 * Buffers used during file match got bigger.
68 * Compressed ISAM support everywhere.
69 * Bug fixes regarding masking characters in queries.
70 * Redesigned Regexp-2 queries.
72 * Revision 1.55 1996/11/04 14:07:44 adam
73 * Moved truncation code to trunc.c.
75 * Revision 1.54 1996/10/29 14:09:52 adam
76 * Use of cisam system - enabled if setting isamc is 1.
78 * Revision 1.53 1996/06/26 09:21:43 adam
79 * Bug fix: local attribute set wasn't obeyed in scan.
81 * Revision 1.52 1996/06/17 14:26:20 adam
82 * Function gen_regular_rel changed to handle negative numbers.
84 * Revision 1.51 1996/06/11 10:54:15 quinn
87 * Revision 1.50 1996/06/07 08:51:53 adam
88 * Bug fix: Character mapping was broken (introducued by last revision).
90 * Revision 1.49 1996/06/04 10:18:11 adam
91 * Search/scan uses character mapping module.
93 * Revision 1.48 1996/05/28 15:15:01 adam
94 * Bug fix: Didn't handle unknown database correctly.
96 * Revision 1.47 1996/05/15 18:36:28 adam
97 * Function trans_term transforms unsearchable characters to blanks.
99 * Revision 1.46 1996/05/15 11:57:56 adam
100 * Fixed bug introduced by set/field mapping in search operations.
102 * Revision 1.45 1996/05/14 11:34:00 adam
103 * Scan support in multiple registers/databases.
105 * Revision 1.44 1996/05/14 06:16:44 adam
106 * Compact use/set bytes used in search service.
108 * Revision 1.43 1996/05/09 09:54:43 adam
109 * Server supports maps from one logical attributes to a list of physical
111 * The extraction process doesn't make space consuming 'any' keys.
113 * Revision 1.42 1996/05/09 07:28:56 quinn
114 * Work towards phrases and multiple registers
116 * Revision 1.41 1996/03/20 09:36:43 adam
117 * Function dict_lookup_grep got extra parameter, init_pos, which marks
118 * from which position in pattern approximate pattern matching should occur.
119 * Approximate pattern matching is used in relevance=re-2.
121 * Revision 1.40 1996/02/02 13:44:44 adam
122 * The public dictionary functions simply use char instead of Dict_char
123 * to represent search strings. Dict_char is used internally only.
125 * Revision 1.39 1996/01/03 16:22:13 quinn
126 * operator->roperator
128 * Revision 1.38 1995/12/11 09:12:55 adam
129 * The rec_get function returns NULL if record doesn't exist - will
130 * happen in the server if the result set records have been deleted since
131 * the creation of the set (i.e. the search).
132 * The server saves a result temporarily if it is 'volatile', i.e. the
133 * set is register dependent.
135 * Revision 1.37 1995/12/06 15:05:28 adam
136 * More verbose in count_set.
138 * Revision 1.36 1995/12/06 12:41:27 adam
139 * New command 'stat' for the index program.
140 * Filenames can be read from stdin by specifying '-'.
141 * Bug fix/enhancement of the transformation from terms to regular
142 * expressons in the search engine.
144 * Revision 1.35 1995/11/27 09:29:00 adam
145 * Bug fixes regarding conversion to regular expressions.
147 * Revision 1.34 1995/11/16 17:00:56 adam
148 * Better logging of rpn query.
150 * Revision 1.33 1995/11/01 13:58:28 quinn
151 * Moving data1 to yaz/retrieval
153 * Revision 1.32 1995/10/27 14:00:11 adam
154 * Implemented detection of database availability.
156 * Revision 1.31 1995/10/17 18:02:10 adam
157 * New feature: databases. Implemented as prefix to words in dictionary.
159 * Revision 1.30 1995/10/16 09:32:38 adam
160 * More work on relational op.
162 * Revision 1.29 1995/10/13 16:01:49 adam
165 * Revision 1.28 1995/10/13 12:26:43 adam
166 * Optimization of truncation.
168 * Revision 1.27 1995/10/12 17:07:22 adam
171 * Revision 1.26 1995/10/12 12:40:54 adam
172 * Bug fixes in rpn_prox.
174 * Revision 1.25 1995/10/10 13:59:24 adam
175 * Function rset_open changed its wflag parameter to general flags.
177 * Revision 1.24 1995/10/09 16:18:37 adam
178 * Function dict_lookup_grep got extra client data parameter.
180 * Revision 1.23 1995/10/06 16:33:37 adam
181 * Use attribute mappings.
183 * Revision 1.22 1995/10/06 15:07:39 adam
184 * Structure 'local-number' handled.
186 * Revision 1.21 1995/10/06 13:52:06 adam
187 * Bug fixes. Handler may abort further scanning.
189 * Revision 1.20 1995/10/06 11:06:33 adam
190 * Scan entries include 'occurrences' now.
192 * Revision 1.19 1995/10/06 10:43:56 adam
193 * Scan added. 'occurrences' in scan entries not set yet.
195 * Revision 1.18 1995/10/04 16:57:20 adam
196 * Key input and merge sort in one pass.
198 * Revision 1.17 1995/10/04 12:55:17 adam
199 * Bug fix in ranked search. Use=Any keys inserted.
201 * Revision 1.16 1995/10/02 16:24:40 adam
202 * Use attribute actually used in search requests.
204 * Revision 1.15 1995/10/02 15:18:52 adam
205 * New member in recRetrieveCtrl: diagnostic.
207 * Revision 1.14 1995/09/28 12:10:32 adam
208 * Bug fixes. Field prefix used in queries.
210 * Revision 1.13 1995/09/18 14:17:50 adam
213 * Revision 1.12 1995/09/15 14:45:21 adam
215 * Work on truncation.
217 * Revision 1.11 1995/09/14 11:53:27 adam
218 * First work on regular expressions/truncations.
220 * Revision 1.10 1995/09/11 15:23:26 adam
221 * More work on relevance search.
223 * Revision 1.9 1995/09/11 13:09:35 adam
224 * More work on relevance feedback.
226 * Revision 1.8 1995/09/08 14:52:27 adam
227 * Minor changes. Dictionary is lower case now.
229 * Revision 1.7 1995/09/07 13:58:36 adam
230 * New parameter: result-set file descriptor (RSFD) to support multiple
231 * positions within the same result-set.
232 * Boolean operators: and, or, not implemented.
233 * Result-set references.
235 * Revision 1.6 1995/09/06 16:11:18 adam
236 * Option: only one word key per file.
238 * Revision 1.5 1995/09/06 10:33:04 adam
239 * More work on present. Some log messages removed.
241 * Revision 1.4 1995/09/05 15:28:40 adam
242 * More work on search engine.
244 * Revision 1.3 1995/09/04 15:20:22 adam
247 * Revision 1.2 1995/09/04 12:33:43 adam
248 * Various cleanup. YAZ util used instead.
250 * Revision 1.1 1995/09/04 09:10:40 adam
251 * More work on index add/del/update.
252 * Merge sort implemented.
253 * Initial work on z39 server.
273 struct rpn_char_map_info {
278 static const char **rpn_char_map_handler (void *vp, const char **from, int len)
280 struct rpn_char_map_info *p = vp;
281 return zebra_maps_input (p->zm, p->reg_type, from, len);
284 static void rpn_char_map_prepare (ZServerInfo *zi, int reg_type,
285 struct rpn_char_map_info *map_info)
287 map_info->zm = zi->zebra_maps;
288 map_info->reg_type = reg_type;
289 dict_grep_cmap (zi->dict, map_info, rpn_char_map_handler);
296 Z_AttributesPlusTerm *zapt;
299 static int attr_find (AttrType *src, oid_value *attributeSetP)
301 while (src->major < src->zapt->num_attributes)
303 Z_AttributeElement *element;
305 element = src->zapt->attributeList[src->major];
306 if (src->type == *element->attributeType)
308 switch (element->which)
310 case Z_AttributeValue_numeric:
312 if (element->attributeSet && attributeSetP)
316 attrset = oid_getentbyoid (element->attributeSet);
317 *attributeSetP = attrset->value;
319 return *element->value.numeric;
321 case Z_AttributeValue_complex:
322 if (src->minor >= element->value.complex->num_list ||
323 element->value.complex->list[src->minor]->which !=
324 Z_StringOrNumeric_numeric)
327 if (element->attributeSet && attributeSetP)
331 attrset = oid_getentbyoid (element->attributeSet);
332 *attributeSetP = attrset->value;
334 return *element->value.complex->list[src->minor-1]->u.numeric;
344 static void attr_init (AttrType *src, Z_AttributesPlusTerm *zapt,
364 static void add_isam_p (const char *info, struct grep_info *p)
366 if (p->isam_p_indx == p->isam_p_size)
368 ISAM_P *new_isam_p_buf;
372 p->isam_p_size = 2*p->isam_p_size + 100;
373 new_isam_p_buf = xmalloc (sizeof(*new_isam_p_buf) *
377 memcpy (new_isam_p_buf, p->isam_p_buf,
378 p->isam_p_indx * sizeof(*p->isam_p_buf));
379 xfree (p->isam_p_buf);
381 p->isam_p_buf = new_isam_p_buf;
384 new_term_no = xmalloc (sizeof(*new_term_no) *
388 memcpy (new_term_no, p->isam_p_buf,
389 p->isam_p_indx * sizeof(*p->term_no));
392 p->term_no = new_term_no;
395 assert (*info == sizeof(*p->isam_p_buf));
396 memcpy (p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
400 static int grep_handle (char *name, const char *info, void *p)
402 add_isam_p (info, p);
406 static int term_pre (ZebraMaps zebra_maps, int reg_type, const char **src,
407 const char *ct1, const char *ct2)
409 const char *s1, *s0 = *src;
412 /* skip white space */
415 if (ct1 && strchr (ct1, *s0))
417 if (ct2 && strchr (ct2, *s0))
420 map = zebra_maps_input (zebra_maps, reg_type, &s1, strlen(s1));
421 if (**map != *CHR_SPACE)
429 static int term_100 (ZebraMaps zebra_maps, int reg_type,
430 const char **src, char *dst, int space_split)
436 if (!term_pre (zebra_maps, reg_type, src, NULL, NULL))
442 map = zebra_maps_input (zebra_maps, reg_type, &s0, strlen(s0));
443 if (space_split && **map == *CHR_SPACE)
447 if (!isalnum (*s1) && *s1 != '-')
457 static int term_101 (ZebraMaps zebra_maps, int reg_type,
458 const char **src, char *dst, int space_split)
464 if (!term_pre (zebra_maps, reg_type, src, "#", "#"))
478 map = zebra_maps_input (zebra_maps, reg_type, &s0, strlen(s0));
479 if (space_split && **map == *CHR_SPACE)
495 static int term_103 (ZebraMaps zebra_maps, int reg_type, const char **src,
496 char *dst, int *errors, int space_split)
502 if (!term_pre (zebra_maps, reg_type, src, "^\\()[].*+?|", "("))
505 if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
508 *errors = s0[1] - '0';
515 if (strchr ("^\\()[].*+?|-", *s0))
520 map = zebra_maps_input (zebra_maps, reg_type, &s0, strlen(s0));
521 if (**map == *CHR_SPACE)
536 static int term_102 (ZebraMaps zebra_maps, int reg_type, const char **src,
537 char *dst, int space_split)
539 return term_103 (zebra_maps, reg_type, src, dst, NULL, space_split);
542 /* gen_regular_rel - generate regular expression from relation
543 * val: border value (inclusive)
544 * islt: 1 if <=; 0 if >=.
546 static void gen_regular_rel (char *dst, int val, int islt)
553 logf (LOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
557 strcpy (dst, "(-[0-9]+|(");
565 strcpy (dst, "([0-9]+|-(");
566 dst_p = strlen (dst);
576 dst_p = strlen (dst);
577 sprintf (numstr, "%d", val);
578 for (w = strlen(numstr); --w >= 0; pos++)
597 strcpy (dst + dst_p, numstr);
598 dst_p = strlen(dst) - pos - 1;
626 for (i = 0; i<pos; i++)
639 for (i=1; i<pos; i++)
640 strcat (dst, "[0-9]?");
644 for (i = 0; i <= pos; i++)
645 strcat (dst, "[0-9]");
646 strcat (dst, "[0-9]*");
651 static int relational_term (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
652 const char **term_sub,
654 oid_value attributeSet,
655 struct grep_info *grep_info,
663 char *term_tmp = term_dict + strlen(term_dict);
665 attr_init (&relation, zapt, 2);
666 relation_value = attr_find (&relation, NULL);
668 logf (LOG_DEBUG, "relation value=%d", relation_value);
669 switch (relation_value)
672 if (!term_100 (zi->zebra_maps, reg_type, term_sub, term_tmp, 1))
674 term_value = atoi (term_tmp);
675 logf (LOG_DEBUG, "Relation <");
676 gen_regular_rel (term_tmp, term_value-1, 1);
679 if (!term_100 (zi->zebra_maps, reg_type, term_sub, term_tmp, 1))
681 term_value = atoi (term_tmp);
682 logf (LOG_DEBUG, "Relation <=");
683 gen_regular_rel (term_tmp, term_value, 1);
686 if (!term_100 (zi->zebra_maps, reg_type, term_sub, term_tmp, 1))
688 term_value = atoi (term_tmp);
689 logf (LOG_DEBUG, "Relation >=");
690 gen_regular_rel (term_tmp, term_value, 0);
693 if (!term_100 (zi->zebra_maps, reg_type, term_sub, term_tmp, 1))
695 term_value = atoi (term_tmp);
696 logf (LOG_DEBUG, "Relation >");
697 gen_regular_rel (term_tmp, term_value+1, 0);
702 logf (LOG_DEBUG, "dict_lookup_grep: %s", term_tmp);
703 r = dict_lookup_grep (zi->dict, term_dict, 0, grep_info, max_pos,
706 logf (LOG_WARN, "dict_lookup_grep fail, rel=gt: %d", r);
707 logf (LOG_DEBUG, "%d positions", grep_info->isam_p_indx);
711 static int field_term (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
712 const char **term_sub,
713 oid_value attributeSet, struct grep_info *grep_info,
714 int reg_type, int complete_flag,
715 int num_bases, char **basenames)
717 char term_dict[2*IT_MAX_WORD+2];
720 int truncation_value;
723 oid_value curAttributeSet = attributeSet;
725 struct rpn_char_map_info rcmi;
726 int space_split = complete_flag ? 0 : 1;
728 rpn_char_map_prepare (zi, reg_type, &rcmi);
729 attr_init (&use, zapt, 1);
730 use_value = attr_find (&use, &curAttributeSet);
731 logf (LOG_DEBUG, "field_term, use value %d", use_value);
732 attr_init (&truncation, zapt, 5);
733 truncation_value = attr_find (&truncation, NULL);
734 logf (LOG_DEBUG, "truncation value %d", truncation_value);
739 for (base_no = 0; base_no < num_bases; base_no++)
742 data1_local_attribute *local_attr;
743 int max_pos, prefix_len = 0;
746 if ((r=att_getentbyatt (zi, &attp, curAttributeSet, use_value)))
748 logf (LOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
749 curAttributeSet, use_value, r);
756 if (zebTargetInfo_curDatabase (zi->zti, basenames[base_no]))
758 zi->errCode = 109; /* Database unavailable */
759 zi->errString = basenames[base_no];
762 for (local_attr = attp.local_attributes; local_attr;
763 local_attr = local_attr->next)
767 ord = zebTargetInfo_lookupSU (zi->zti, attp.attset_ordinal,
772 term_dict[prefix_len++] = '|';
774 term_dict[prefix_len++] = '(';
775 term_dict[prefix_len++] = 1;
776 term_dict[prefix_len++] = ord;
783 term_dict[prefix_len++] = ')';
784 term_dict[prefix_len++] = 1;
785 term_dict[prefix_len++] = reg_type;
786 logf (LOG_DEBUG, "reg_type = %d", term_dict[prefix_len-1]);
787 term_dict[prefix_len] = '\0';
788 if (!relational_term (zi, zapt, &termp, term_dict,
789 attributeSet, grep_info, &max_pos, reg_type))
792 switch (truncation_value)
794 case -1: /* not specified */
795 case 100: /* do not truncate */
796 term_dict[j++] = '(';
797 if (!term_100 (zi->zebra_maps, reg_type,
798 &termp, term_dict + j, space_split))
800 strcat (term_dict, ")");
801 r = dict_lookup_grep (zi->dict, term_dict, 0, grep_info,
802 &max_pos, 0, grep_handle);
804 logf (LOG_WARN, "dict_lookup_grep err, trunc=none:%d", r);
806 case 1: /* right truncation */
807 term_dict[j++] = '(';
808 if (!term_100 (zi->zebra_maps, reg_type,
809 &termp, term_dict + j, space_split))
811 strcat (term_dict, ".*)");
812 dict_lookup_grep (zi->dict, term_dict, 0, grep_info,
813 &max_pos, 0, grep_handle);
815 case 2: /* left truncation */
816 case 3: /* left&right truncation */
819 case 101: /* process # in term */
820 term_dict[j++] = '(';
821 if (!term_101 (zi->zebra_maps, reg_type,
822 &termp, term_dict + j, space_split))
824 strcat (term_dict, ")");
825 r = dict_lookup_grep (zi->dict, term_dict, 0, grep_info,
826 &max_pos, 0, grep_handle);
828 logf (LOG_WARN, "dict_lookup_grep err, trunc=#: %d", r);
830 case 102: /* Regexp-1 */
831 term_dict[j++] = '(';
832 if (!term_102 (zi->zebra_maps, reg_type,
833 &termp, term_dict + j, space_split))
835 strcat (term_dict, ")");
836 logf (LOG_DEBUG, "Regexp-1 tolerance=%d", r);
837 r = dict_lookup_grep (zi->dict, term_dict, 0, grep_info,
838 &max_pos, 0, grep_handle);
840 logf (LOG_WARN, "dict_lookup_grep err, trunc=regular: %d",
843 case 103: /* Regexp-2 */
845 term_dict[j++] = '(';
846 if (!term_103 (zi->zebra_maps, reg_type,
847 &termp, term_dict + j, &r, space_split))
849 strcat (term_dict, ")");
850 logf (LOG_DEBUG, "Regexp-2 tolerance=%d", r);
851 r = dict_lookup_grep (zi->dict, term_dict, r, grep_info,
852 &max_pos, 2, grep_handle);
854 logf (LOG_WARN, "dict_lookup_grep err, trunc=eregular: %d",
861 logf (LOG_DEBUG, "%d positions", grep_info->isam_p_indx);
865 static void trans_term (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
869 Z_Term *term = zapt->term;
871 sizez = term->u.general->len;
872 if (sizez > IT_MAX_WORD-1)
873 sizez = IT_MAX_WORD-1;
874 memcpy (termz, term->u.general->buf, sizez);
878 static void trans_scan_term (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
879 char *termz, int reg_type)
881 Z_Term *term = zapt->term;
883 const char *cp = (const char *) term->u.general->buf;
884 const char *cp_end = cp + term->u.general->len;
887 const char *space_map = NULL;
890 while ((len = (cp_end - cp)) > 0)
892 map = zebra_maps_input (zi->zebra_maps, reg_type, &cp, len);
893 if (**map == *CHR_SPACE)
898 for (src = space_map; *src; src++)
901 for (src = *map; *src; src++)
908 static RSET rpn_search_APT_relevance (ZServerInfo *zi,
909 Z_AttributesPlusTerm *zapt,
910 oid_value attributeSet,
911 int reg_type, int complete_flag,
912 int num_bases, char **basenames)
914 rset_relevance_parms parms;
915 char termz[IT_MAX_WORD+1];
916 const char *termp = termz;
917 struct grep_info grep_info;
922 parms.key_size = sizeof(struct it_key);
924 parms.cmp = key_compare_it;
925 parms.get_pos = key_get_pos;
927 parms.isc = zi->isamc;
929 parms.method = RSREL_METHOD_A;
931 if (zapt->term->which != Z_Term_general)
936 trans_term (zi, zapt, termz);
939 grep_info.term_no = 0;
941 grep_info.isam_p_indx = 0;
942 grep_info.isam_p_size = 0;
943 grep_info.isam_p_buf = NULL;
946 r = field_term (zi, zapt, &termp, attributeSet, &grep_info,
947 reg_type, complete_flag, num_bases, basenames);
951 for (; term_index < grep_info.isam_p_indx; term_index++)
952 grep_info.term_no[term_index] = parms.no_terms;
956 parms.term_no = grep_info.term_no;
957 parms.isam_positions = grep_info.isam_p_buf;
958 parms.no_isam_positions = grep_info.isam_p_indx;
959 if (grep_info.isam_p_indx > 0)
960 result = rset_create (rset_kind_relevance, &parms);
962 result = rset_create (rset_kind_null, NULL);
964 xfree(grep_info.term_no);
966 xfree (grep_info.isam_p_buf);
970 static RSET rpn_proximity (ZServerInfo *zi, RSET rset1, RSET rset2,
972 int exclusion, int relation, int distance)
977 struct it_key buf1, buf2;
980 rset_temp_parms parms;
982 rsfd1 = rset_open (rset1, RSETF_READ|RSETF_SORT_SYSNO);
983 more1 = rset_read (rset1, rsfd1, &buf1);
985 rsfd2 = rset_open (rset2, RSETF_READ|RSETF_SORT_SYSNO);
986 more2 = rset_read (rset2, rsfd2, &buf2);
988 parms.key_size = sizeof (struct it_key);
989 parms.temp_path = res_get (zi->res, "setTmpDir");
990 result = rset_create (rset_kind_temp, &parms);
991 rsfd_result = rset_open (result, RSETF_WRITE|RSETF_SORT_SYSNO);
993 logf (LOG_DEBUG, "rpn_proximity excl=%d ord=%d rel=%d dis=%d",
994 exclusion, ordered, relation, distance);
995 while (more1 && more2)
997 int cmp = key_compare_it (&buf1, &buf2);
999 more1 = rset_read (rset1, rsfd1, &buf1);
1001 more2 = rset_read (rset2, rsfd2, &buf2);
1004 int sysno = buf1.sysno;
1008 seqno[n++] = buf1.seqno;
1009 while ((more1 = rset_read (rset1, rsfd1, &buf1)) &&
1010 sysno == buf1.sysno)
1012 seqno[n++] = buf1.seqno;
1015 for (i = 0; i<n; i++)
1017 int diff = buf2.seqno - seqno[i];
1018 int excl = exclusion;
1019 if (!ordered && diff < 0)
1024 if (diff < distance)
1028 if (diff <= distance)
1032 if (diff == distance)
1036 if (diff >= distance)
1040 if (diff > distance)
1044 if (diff != distance)
1049 rset_write (result, rsfd_result, &buf2);
1051 } while ((more2 = rset_read (rset2, rsfd2, &buf2)) &&
1052 sysno == buf2.sysno);
1055 rset_close (result, rsfd_result);
1056 rset_close (rset1, rsfd1);
1057 rset_close (rset2, rsfd2);
1061 static RSET rpn_prox (ZServerInfo *zi, RSET *rset, int rset_no)
1066 struct it_key **buf;
1069 rset_temp_parms parms;
1071 rsfd = xmalloc (sizeof(*rsfd)*rset_no);
1072 more = xmalloc (sizeof(*more)*rset_no);
1073 buf = xmalloc (sizeof(*buf)*rset_no);
1075 for (i = 0; i<rset_no; i++)
1077 buf[i] = xmalloc (sizeof(**buf));
1078 rsfd[i] = rset_open (rset[i], RSETF_READ|RSETF_SORT_SYSNO);
1079 if (!(more[i] = rset_read (rset[i], rsfd[i], buf[i])))
1083 rset_close (rset[i], rsfd[i]);
1090 return rset_create (rset_kind_null, NULL);
1093 parms.key_size = sizeof (struct it_key);
1094 parms.temp_path = res_get (zi->res, "setTmpDir");
1095 result = rset_create (rset_kind_temp, &parms);
1096 rsfd_result = rset_open (result, RSETF_WRITE|RSETF_SORT_SYSNO);
1100 for (i = 1; i<rset_no; i++)
1109 cmp = key_compare_it (buf[i], buf[i-1]);
1112 more[i-1] = rset_read (rset[i-1], rsfd[i-1], buf[i-1]);
1117 if (buf[i-1]->seqno+1 != buf[i]->seqno)
1119 more[i-1] = rset_read (rset[i-1], rsfd[i-1], buf[i-1]);
1125 more[i] = rset_read (rset[i], rsfd[i], buf[i]);
1131 rset_write (result, rsfd_result, buf[0]);
1132 more[0] = rset_read (*rset, *rsfd, *buf);
1136 for (i = 0; i<rset_no; i++)
1138 rset_close (rset[i], rsfd[i]);
1141 rset_close (result, rsfd_result);
1148 static RSET rpn_search_APT_phrase (ZServerInfo *zi,
1149 Z_AttributesPlusTerm *zapt,
1150 oid_value attributeSet,
1151 int reg_type, int complete_flag,
1152 int num_bases, char **basenames)
1154 char termz[IT_MAX_WORD+1];
1155 const char *termp = termz;
1156 RSET rset[60], result;
1157 int i, r, rset_no = 0;
1158 struct grep_info grep_info;
1160 if (zapt->term->which != Z_Term_general)
1165 trans_term (zi, zapt, termz);
1168 grep_info.term_no = 0;
1170 grep_info.isam_p_size = 0;
1171 grep_info.isam_p_buf = NULL;
1175 logf (LOG_DEBUG, "APT_phrase termp=%s", termp);
1176 grep_info.isam_p_indx = 0;
1177 r = field_term (zi, zapt, &termp, attributeSet, &grep_info,
1178 reg_type, complete_flag, num_bases, basenames);
1181 rset[rset_no] = rset_trunc (zi, grep_info.isam_p_buf,
1182 grep_info.isam_p_indx);
1183 assert (rset[rset_no]);
1184 if (++rset_no >= sizeof(rset)/sizeof(*rset))
1188 xfree(grep_info.term_no);
1190 xfree (grep_info.isam_p_buf);
1192 return rset_create (rset_kind_null, NULL);
1193 else if (rset_no == 1)
1195 result = rpn_prox (zi, rset, rset_no);
1196 for (i = 0; i<rset_no; i++)
1197 rset_delete (rset[i]);
1201 static RSET rpn_search_APT_local (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
1202 oid_value attributeSet)
1207 rset_temp_parms parms;
1208 char termz[IT_MAX_WORD+1];
1210 if (zapt->term->which != Z_Term_general)
1215 parms.key_size = sizeof (struct it_key);
1216 parms.temp_path = res_get (zi->res, "setTmpDir");
1217 result = rset_create (rset_kind_temp, &parms);
1218 rsfd = rset_open (result, RSETF_WRITE|RSETF_SORT_SYSNO);
1220 trans_term (zi, zapt, termz);
1222 key.sysno = atoi (termz);
1225 rset_write (result, rsfd, &key);
1226 rset_close (result, rsfd);
1230 static RSET rpn_search_APT (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
1231 oid_value attributeSet,
1232 int num_bases, char **basenames)
1235 char *search_type = NULL;
1238 zebra_maps_attr (zi->zebra_maps, zapt, ®_type, &search_type,
1241 logf (LOG_DEBUG, "reg_type=%c", reg_type);
1242 logf (LOG_DEBUG, "complete_flag=%d", complete_flag);
1243 logf (LOG_DEBUG, "search_type=%s", search_type);
1244 if (!strcmp (search_type, "phrase"))
1246 return rpn_search_APT_phrase (zi, zapt, attributeSet,
1247 reg_type, complete_flag,
1248 num_bases, basenames);
1250 else if (!strcmp (search_type, "ranked"))
1252 return rpn_search_APT_relevance (zi, zapt, attributeSet,
1253 reg_type, complete_flag,
1254 num_bases, basenames);
1256 else if (!strcmp (search_type, "local"))
1258 return rpn_search_APT_local (zi, zapt, attributeSet);
1264 static RSET rpn_search_ref (ZServerInfo *zi, Z_ResultSetId *resultSetId)
1268 if (!(s = resultSetGet (zi, resultSetId)))
1269 return rset_create (rset_kind_null, NULL);
1273 static RSET rpn_search_structure (ZServerInfo *zi, Z_RPNStructure *zs,
1274 oid_value attributeSet,
1275 int num_bases, char **basenames)
1278 if (zs->which == Z_RPNStructure_complex)
1280 Z_Operator *zop = zs->u.complex->roperator;
1281 rset_bool_parms bool_parms;
1285 bool_parms.rset_l = rpn_search_structure (zi, zs->u.complex->s1,
1287 num_bases, basenames);
1288 if (bool_parms.rset_l == NULL)
1290 if (rset_is_ranked(bool_parms.rset_l))
1292 bool_parms.rset_r = rpn_search_structure (zi, zs->u.complex->s2,
1294 num_bases, basenames);
1295 if (bool_parms.rset_r == NULL)
1297 rset_delete (bool_parms.rset_l);
1300 if (rset_is_ranked(bool_parms.rset_r))
1302 bool_parms.key_size = sizeof(struct it_key);
1303 bool_parms.cmp = key_compare_it;
1307 case Z_Operator_and:
1308 r = rset_create (soft ? rset_kind_sand:rset_kind_and, &bool_parms);
1311 r = rset_create (soft ? rset_kind_sor:rset_kind_or, &bool_parms);
1313 case Z_Operator_and_not:
1314 r = rset_create (soft ? rset_kind_snot:rset_kind_not, &bool_parms);
1316 case Z_Operator_prox:
1317 if (zop->u.prox->which != Z_ProxCode_known)
1322 if (*zop->u.prox->proximityUnitCode != Z_ProxUnit_word)
1324 char *val = odr_malloc (zi->odr, 16);
1326 zi->errString = val;
1327 sprintf (val, "%d", *zop->u.prox->proximityUnitCode);
1330 r = rpn_proximity (zi, bool_parms.rset_l, bool_parms.rset_r,
1331 *zop->u.prox->ordered,
1332 (!zop->u.prox->exclusion ? 0 :
1333 *zop->u.prox->exclusion),
1334 *zop->u.prox->relationType,
1335 *zop->u.prox->distance);
1342 else if (zs->which == Z_RPNStructure_simple)
1344 if (zs->u.simple->which == Z_Operand_APT)
1346 logf (LOG_DEBUG, "rpn_search_APT");
1347 r = rpn_search_APT (zi, zs->u.simple->u.attributesPlusTerm,
1348 attributeSet, num_bases, basenames);
1350 else if (zs->u.simple->which == Z_Operand_resultSetId)
1352 logf (LOG_DEBUG, "rpn_search_ref");
1353 r = rpn_search_ref (zi, zs->u.simple->u.resultSetId);
1369 void count_set_save (ZServerInfo *zi, RSET *r, int *count)
1376 rset_temp_parms parms;
1377 int maxResultSetSize = atoi (res_get_def (zi->res,
1378 "maxResultSetSize", "400"));
1379 logf (LOG_DEBUG, "count_set_save");
1381 parms.key_size = sizeof(struct it_key);
1382 parms.temp_path = res_get (zi->res, "setTmpDir");
1383 w = rset_create (rset_kind_temp, &parms);
1384 wfd = rset_open (w, RSETF_WRITE|RSETF_SORT_SYSNO);
1385 rfd = rset_open (*r, RSETF_READ|RSETF_SORT_SYSNO);
1386 while (rset_read (*r, rfd, &key))
1388 if (key.sysno != psysno)
1390 if (*count < maxResultSetSize)
1391 rset_write (w, wfd, &key);
1397 rset_close (*r, rfd);
1399 rset_close (w, wfd);
1401 logf (LOG_DEBUG, "%d keys, %d distinct sysnos", kno, *count);
1404 static void count_set (RSET r, int *count)
1411 logf (LOG_DEBUG, "count_set");
1414 rfd = rset_open (r, RSETF_READ|RSETF_SORT_SYSNO);
1415 while (rset_read (r, rfd, &key))
1417 if (key.sysno != psysno)
1424 rset_close (r, rfd);
1425 logf (LOG_DEBUG, "%d keys, %d distinct sysnos", kno, *count);
1428 int rpn_search (ZServerInfo *zi,
1429 Z_RPNQuery *rpn, int num_bases, char **basenames,
1430 const char *setname, int *hits)
1435 oid_value attributeSet;
1440 zi->errString = NULL;
1442 attrset = oid_getentbyoid (rpn->attributeSetId);
1443 attributeSet = attrset->value;
1444 rset = rpn_search_structure (zi, rpn->RPNStructure, attributeSet,
1445 num_bases, basenames);
1448 if (rset_is_volatile(rset))
1449 count_set_save(zi, &rset, hits);
1450 else if ((i = rset_hits (rset)) >= 0)
1453 count_set (rset, hits);
1454 resultSetAdd (zi, setname, 1, rset);
1456 logf (LOG_DEBUG, "search error: %d", zi->errCode);
1460 struct scan_info_entry {
1466 struct scan_info_entry *list;
1472 static int scan_handle (char *name, const char *info, int pos, void *client)
1474 int len_prefix, idx;
1475 struct scan_info *scan_info = client;
1477 len_prefix = strlen(scan_info->prefix);
1478 if (memcmp (name, scan_info->prefix, len_prefix))
1481 idx = scan_info->after - pos + scan_info->before;
1484 scan_info->list[idx].term = odr_malloc (scan_info->odr,
1485 strlen(name + len_prefix)+1);
1486 strcpy (scan_info->list[idx].term, name + len_prefix);
1487 assert (*info == sizeof(ISAM_P));
1488 memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAM_P));
1493 static void scan_term_untrans (ZServerInfo *zi, int reg_type,
1494 char **dstp, const char *src)
1496 char *dst = odr_malloc (zi->odr, strlen(src)*2+1);
1501 const char *cp = zebra_maps_output (zi->zebra_maps, reg_type, &src);
1508 int rpn_scan (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
1509 oid_value attributeset,
1510 int num_bases, char **basenames,
1511 int *position, int *num_entries, struct scan_entry **list,
1515 int pos = *position;
1516 int num = *num_entries;
1520 char termz[IT_MAX_WORD+20];
1523 struct scan_info *scan_info_array;
1524 struct scan_entry *glist;
1525 int ords[32], ord_no = 0;
1529 char *search_type = NULL;
1532 if (attributeset == VAL_NONE)
1533 attributeset = VAL_BIB1;
1535 zlog_scan (zapt, attributeset);
1536 logf (LOG_DEBUG, "position = %d, num = %d", pos, num);
1538 attr_init (&use, zapt, 1);
1539 use_value = attr_find (&use, &attributeset);
1541 if (zebra_maps_attr (zi->zebra_maps, zapt, ®_type, &search_type,
1548 if (use_value == -1)
1550 for (base_no = 0; base_no < num_bases && ord_no < 32; base_no++)
1554 data1_local_attribute *local_attr;
1556 if ((r=att_getentbyatt (zi, &attp, attributeset, use_value)))
1558 logf (LOG_DEBUG, "att_getentbyatt fail. set=%d use=%d",
1559 attributeset, use_value);
1565 if (zebTargetInfo_curDatabase (zi->zti, basenames[base_no]))
1567 zi->errString = basenames[base_no];
1568 return zi->errCode = 109; /* Database unavailable */
1570 for (local_attr = attp.local_attributes; local_attr && ord_no < 32;
1571 local_attr = local_attr->next)
1575 ord = zebTargetInfo_lookupSU (zi->zti, attp.attset_ordinal,
1578 ords[ord_no++] = ord;
1582 return zi->errCode = 113;
1585 scan_info_array = odr_malloc (zi->odr, ord_no * sizeof(*scan_info_array));
1586 for (i = 0; i < ord_no; i++)
1588 int j, prefix_len = 0;
1589 int before_tmp = before, after_tmp = after;
1590 struct scan_info *scan_info = scan_info_array + i;
1591 struct rpn_char_map_info rcmi;
1593 rpn_char_map_prepare (zi, reg_type, &rcmi);
1595 scan_info->before = before;
1596 scan_info->after = after;
1597 scan_info->odr = zi->odr;
1599 scan_info->list = odr_malloc (zi->odr, (before+after)*
1600 sizeof(*scan_info->list));
1601 for (j = 0; j<before+after; j++)
1602 scan_info->list[j].term = NULL;
1603 termz[prefix_len++] = ords[i];
1604 termz[prefix_len++] = reg_type;
1605 termz[prefix_len] = 0;
1606 strcpy (scan_info->prefix, termz);
1608 trans_scan_term (zi, zapt, termz+prefix_len, reg_type);
1610 dict_scan (zi->dict, termz, &before_tmp, &after_tmp, scan_info,
1613 glist = odr_malloc (zi->odr, (before+after)*sizeof(*glist));
1614 for (i = 0; i < ord_no; i++)
1617 *status = BEND_SCAN_SUCCESS;
1618 for (i = 0; i<after; i++)
1621 const char *mterm = NULL;
1625 for (j = 0; j < ord_no; j++)
1627 if (ptr[j] < before+after &&
1628 (tst=scan_info_array[j].list[ptr[j]].term) &&
1629 (!mterm || strcmp (tst, mterm) < 0))
1637 scan_term_untrans (zi, reg_type, &glist[i+before].term, mterm);
1638 rset = rset_trunc (zi, &scan_info_array[j0].list[ptr[j0]].isam_p, 1);
1641 for (j = j0+1; j<ord_no; j++)
1643 if (ptr[j] < before+after &&
1644 (tst=scan_info_array[j].list[ptr[j]].term) &&
1645 !strcmp (tst, mterm))
1647 rset_bool_parms bool_parms;
1651 rset_trunc (zi, &scan_info_array[j].list[ptr[j]].isam_p, 1);
1653 bool_parms.key_size = sizeof(struct it_key);
1654 bool_parms.cmp = key_compare_it;
1655 bool_parms.rset_l = rset;
1656 bool_parms.rset_r = rset2;
1658 rset = rset_create (rset_kind_or, &bool_parms);
1663 count_set (rset, &glist[i+before].occurrences);
1668 *num_entries -= (after-i);
1669 *status = BEND_SCAN_PARTIAL;
1672 for (i = 0; i<ord_no; i++)
1675 for (i = 0; i<before; i++)
1678 const char *mterm = NULL;
1682 for (j = 0; j <ord_no; j++)
1684 if (ptr[j] < before &&
1685 (tst=scan_info_array[j].list[before-1-ptr[j]].term) &&
1686 (!mterm || strcmp (tst, mterm) > 0))
1695 scan_term_untrans (zi, reg_type, &glist[before-1-i].term, mterm);
1698 (zi, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1);
1702 for (j = j0+1; j<ord_no; j++)
1704 if (ptr[j] < before &&
1705 (tst=scan_info_array[j].list[before-1-ptr[j]].term) &&
1706 !strcmp (tst, mterm))
1708 rset_bool_parms bool_parms;
1711 rset2 = rset_trunc (zi,
1712 &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1);
1714 bool_parms.key_size = sizeof(struct it_key);
1715 bool_parms.cmp = key_compare_it;
1716 bool_parms.rset_l = rset;
1717 bool_parms.rset_r = rset2;
1719 rset = rset_create (rset_kind_or, &bool_parms);
1724 count_set (rset, &glist[before-1-i].occurrences);
1730 *status = BEND_SCAN_PARTIAL;
1734 *list = glist + i; /* list is set to first 'real' entry */
1736 logf (LOG_DEBUG, "position = %d, num_entries = %d",
1737 *position, *num_entries);
1739 logf (LOG_DEBUG, "scan error: %d", zi->errCode);