2 * Copyright (C) 1994-1996, Index Data I/S
4 * Sebastian Hammer, Adam Dickmeiss
7 * Revision 1.56 1996-11-08 11:10:32 adam
8 * Buffers used during file match got bigger.
9 * Compressed ISAM support everywhere.
10 * Bug fixes regarding masking characters in queries.
11 * Redesigned Regexp-2 queries.
13 * Revision 1.55 1996/11/04 14:07:44 adam
14 * Moved truncation code to trunc.c.
16 * Revision 1.54 1996/10/29 14:09:52 adam
17 * Use of cisam system - enabled if setting isamc is 1.
19 * Revision 1.53 1996/06/26 09:21:43 adam
20 * Bug fix: local attribute set wasn't obeyed in scan.
22 * Revision 1.52 1996/06/17 14:26:20 adam
23 * Function gen_regular_rel changed to handle negative numbers.
25 * Revision 1.51 1996/06/11 10:54:15 quinn
28 * Revision 1.50 1996/06/07 08:51:53 adam
29 * Bug fix: Character mapping was broken (introducued by last revision).
31 * Revision 1.49 1996/06/04 10:18:11 adam
32 * Search/scan uses character mapping module.
34 * Revision 1.48 1996/05/28 15:15:01 adam
35 * Bug fix: Didn't handle unknown database correctly.
37 * Revision 1.47 1996/05/15 18:36:28 adam
38 * Function trans_term transforms unsearchable characters to blanks.
40 * Revision 1.46 1996/05/15 11:57:56 adam
41 * Fixed bug introduced by set/field mapping in search operations.
43 * Revision 1.45 1996/05/14 11:34:00 adam
44 * Scan support in multiple registers/databases.
46 * Revision 1.44 1996/05/14 06:16:44 adam
47 * Compact use/set bytes used in search service.
49 * Revision 1.43 1996/05/09 09:54:43 adam
50 * Server supports maps from one logical attributes to a list of physical
52 * The extraction process doesn't make space consuming 'any' keys.
54 * Revision 1.42 1996/05/09 07:28:56 quinn
55 * Work towards phrases and multiple registers
57 * Revision 1.41 1996/03/20 09:36:43 adam
58 * Function dict_lookup_grep got extra parameter, init_pos, which marks
59 * from which position in pattern approximate pattern matching should occur.
60 * Approximate pattern matching is used in relevance=re-2.
62 * Revision 1.40 1996/02/02 13:44:44 adam
63 * The public dictionary functions simply use char instead of Dict_char
64 * to represent search strings. Dict_char is used internally only.
66 * Revision 1.39 1996/01/03 16:22:13 quinn
69 * Revision 1.38 1995/12/11 09:12:55 adam
70 * The rec_get function returns NULL if record doesn't exist - will
71 * happen in the server if the result set records have been deleted since
72 * the creation of the set (i.e. the search).
73 * The server saves a result temporarily if it is 'volatile', i.e. the
74 * set is register dependent.
76 * Revision 1.37 1995/12/06 15:05:28 adam
77 * More verbose in count_set.
79 * Revision 1.36 1995/12/06 12:41:27 adam
80 * New command 'stat' for the index program.
81 * Filenames can be read from stdin by specifying '-'.
82 * Bug fix/enhancement of the transformation from terms to regular
83 * expressons in the search engine.
85 * Revision 1.35 1995/11/27 09:29:00 adam
86 * Bug fixes regarding conversion to regular expressions.
88 * Revision 1.34 1995/11/16 17:00:56 adam
89 * Better logging of rpn query.
91 * Revision 1.33 1995/11/01 13:58:28 quinn
92 * Moving data1 to yaz/retrieval
94 * Revision 1.32 1995/10/27 14:00:11 adam
95 * Implemented detection of database availability.
97 * Revision 1.31 1995/10/17 18:02:10 adam
98 * New feature: databases. Implemented as prefix to words in dictionary.
100 * Revision 1.30 1995/10/16 09:32:38 adam
101 * More work on relational op.
103 * Revision 1.29 1995/10/13 16:01:49 adam
106 * Revision 1.28 1995/10/13 12:26:43 adam
107 * Optimization of truncation.
109 * Revision 1.27 1995/10/12 17:07:22 adam
112 * Revision 1.26 1995/10/12 12:40:54 adam
113 * Bug fixes in rpn_prox.
115 * Revision 1.25 1995/10/10 13:59:24 adam
116 * Function rset_open changed its wflag parameter to general flags.
118 * Revision 1.24 1995/10/09 16:18:37 adam
119 * Function dict_lookup_grep got extra client data parameter.
121 * Revision 1.23 1995/10/06 16:33:37 adam
122 * Use attribute mappings.
124 * Revision 1.22 1995/10/06 15:07:39 adam
125 * Structure 'local-number' handled.
127 * Revision 1.21 1995/10/06 13:52:06 adam
128 * Bug fixes. Handler may abort further scanning.
130 * Revision 1.20 1995/10/06 11:06:33 adam
131 * Scan entries include 'occurrences' now.
133 * Revision 1.19 1995/10/06 10:43:56 adam
134 * Scan added. 'occurrences' in scan entries not set yet.
136 * Revision 1.18 1995/10/04 16:57:20 adam
137 * Key input and merge sort in one pass.
139 * Revision 1.17 1995/10/04 12:55:17 adam
140 * Bug fix in ranked search. Use=Any keys inserted.
142 * Revision 1.16 1995/10/02 16:24:40 adam
143 * Use attribute actually used in search requests.
145 * Revision 1.15 1995/10/02 15:18:52 adam
146 * New member in recRetrieveCtrl: diagnostic.
148 * Revision 1.14 1995/09/28 12:10:32 adam
149 * Bug fixes. Field prefix used in queries.
151 * Revision 1.13 1995/09/18 14:17:50 adam
154 * Revision 1.12 1995/09/15 14:45:21 adam
156 * Work on truncation.
158 * Revision 1.11 1995/09/14 11:53:27 adam
159 * First work on regular expressions/truncations.
161 * Revision 1.10 1995/09/11 15:23:26 adam
162 * More work on relevance search.
164 * Revision 1.9 1995/09/11 13:09:35 adam
165 * More work on relevance feedback.
167 * Revision 1.8 1995/09/08 14:52:27 adam
168 * Minor changes. Dictionary is lower case now.
170 * Revision 1.7 1995/09/07 13:58:36 adam
171 * New parameter: result-set file descriptor (RSFD) to support multiple
172 * positions within the same result-set.
173 * Boolean operators: and, or, not implemented.
174 * Result-set references.
176 * Revision 1.6 1995/09/06 16:11:18 adam
177 * Option: only one word key per file.
179 * Revision 1.5 1995/09/06 10:33:04 adam
180 * More work on present. Some log messages removed.
182 * Revision 1.4 1995/09/05 15:28:40 adam
183 * More work on search engine.
185 * Revision 1.3 1995/09/04 15:20:22 adam
188 * Revision 1.2 1995/09/04 12:33:43 adam
189 * Various cleanup. YAZ util used instead.
191 * Revision 1.1 1995/09/04 09:10:40 adam
192 * More work on index add/del/update.
193 * Merge sort implemented.
194 * Initial work on z39 server.
203 #include "attribute.h"
215 Z_AttributesPlusTerm *zapt;
218 static int attr_find (AttrType *src, oid_value *attributeSetP)
220 while (src->major < src->zapt->num_attributes)
222 Z_AttributeElement *element;
224 element = src->zapt->attributeList[src->major];
225 if (src->type == *element->attributeType)
227 switch (element->which)
229 case Z_AttributeValue_numeric:
231 if (element->attributeSet && attributeSetP)
235 attrset = oid_getentbyoid (element->attributeSet);
236 *attributeSetP = attrset->value;
238 return *element->value.numeric;
240 case Z_AttributeValue_complex:
241 if (src->minor >= element->value.complex->num_list ||
242 element->value.complex->list[src->minor]->which !=
243 Z_StringOrNumeric_numeric)
246 if (element->attributeSet && attributeSetP)
250 attrset = oid_getentbyoid (element->attributeSet);
251 *attributeSetP = attrset->value;
253 return *element->value.complex->list[src->minor-1]->u.numeric;
263 static void attr_init (AttrType *src, Z_AttributesPlusTerm *zapt,
283 static void add_isam_p (const char *info, struct grep_info *p)
285 if (p->isam_p_indx == p->isam_p_size)
287 ISAM_P *new_isam_p_buf;
292 p->isam_p_size = 2*p->isam_p_size + 100;
293 new_isam_p_buf = xmalloc (sizeof(*new_isam_p_buf) *
297 memcpy (new_isam_p_buf, p->isam_p_buf,
298 p->isam_p_indx * sizeof(*p->isam_p_buf));
299 xfree (p->isam_p_buf);
301 p->isam_p_buf = new_isam_p_buf;
304 new_term_no = xmalloc (sizeof(*new_term_no) *
308 memcpy (new_term_no, p->isam_p_buf,
309 p->isam_p_indx * sizeof(*p->term_no));
312 p->term_no = new_term_no;
315 assert (*info == sizeof(*p->isam_p_buf));
316 memcpy (p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
320 static int grep_handle (char *name, const char *info, void *p)
322 add_isam_p (info, p);
326 static int term_pre (char **src, const char *ct1, const char *ct2)
328 char *s1, *s0 = *src;
331 /* skip white space */
334 if (ct1 && strchr (ct1, *s0))
336 if (ct2 && strchr (ct2, *s0))
339 map = map_chrs_input (&s1, strlen(s1));
340 if (**map != *CHR_SPACE)
348 static int term_100 (char **src, char *dst)
350 char *s0, *s1, **map;
353 if (!term_pre (src, NULL, NULL))
359 map = map_chrs_input (&s0, strlen(s0));
360 if (**map == *CHR_SPACE)
374 static int term_101 (char **src, char *dst)
376 char *s0, *s1, **map;
379 if (!term_pre (src, "#", "#"))
393 map = map_chrs_input (&s0, strlen(s0));
394 if (**map == *CHR_SPACE)
410 static int term_103 (char **src, char *dst, int *errors)
413 char *s0, *s1, **map;
415 if (!term_pre (src, "\\()[].*+?|", "("))
418 if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
421 *errors = s0[1] - '0';
428 if (strchr ("\\()[].*+?|-", *s0))
433 map = map_chrs_input (&s0, strlen(s0));
434 if (**map == *CHR_SPACE)
449 static int term_102 (char **src, char *dst)
451 return term_103 (src, dst, NULL);
454 /* gen_regular_rel - generate regular expression from relation
455 * val: border value (inclusive)
456 * islt: 1 if <=; 0 if >=.
458 static void gen_regular_rel (char *dst, int val, int islt)
465 logf (LOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
469 strcpy (dst, "(-[0-9]+|");
477 strcpy (dst, "([0-9]+|-");
478 dst_p = strlen (dst);
488 dst_p = strlen (dst);
489 sprintf (numstr, "%d", val);
490 for (w = strlen(numstr); --w >= 0; pos++)
509 strcpy (dst + dst_p, numstr);
510 dst_p = strlen(dst) - pos - 1;
538 for (i = 0; i<pos; i++)
551 for (i=1; i<pos; i++)
552 strcat (dst, "[0-9]?");
556 for (i = 0; i <= pos; i++)
557 strcat (dst, "[0-9]");
558 strcat (dst, "[0-9]*");
563 static int relational_term (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
566 oid_value attributeSet,
567 struct grep_info *grep_info,
575 attr_init (&relation, zapt, 2);
576 relation_value = attr_find (&relation, NULL);
578 switch (relation_value)
581 if (!term_100 (term_sub, term_dict))
583 term_value = atoi (term_dict);
586 logf (LOG_DEBUG, "Relation <");
587 gen_regular_rel (term_dict + strlen(term_dict), term_value-1, 1);
590 if (!term_100 (term_sub, term_dict))
592 term_value = atoi (term_dict);
595 logf (LOG_DEBUG, "Relation <=");
596 gen_regular_rel (term_dict + strlen(term_dict), term_value, 1);
599 if (!term_100 (term_sub, term_dict))
601 term_value = atoi (term_dict);
604 logf (LOG_DEBUG, "Relation >=");
605 gen_regular_rel (term_dict + strlen(term_dict), term_value, 0);
608 if (!term_100 (term_sub, term_dict))
610 term_value = atoi (term_dict);
613 logf (LOG_DEBUG, "Relation >");
614 gen_regular_rel (term_dict + strlen(term_dict), term_value+1, 0);
619 logf (LOG_DEBUG, "dict_lookup_grep: %s", term_dict);
620 r = dict_lookup_grep (zi->dict, term_dict, 0, grep_info, max_pos,
623 logf (LOG_WARN, "dict_lookup_grep fail, rel=gt: %d", r);
624 logf (LOG_DEBUG, "%d positions", grep_info->isam_p_indx);
628 static int field_term (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
629 char **term_sub, int regType,
630 oid_value attributeSet, struct grep_info *grep_info,
631 int num_bases, char **basenames)
633 char term_dict[2*IT_MAX_WORD+2];
636 int truncation_value;
639 oid_value curAttributeSet = attributeSet;
642 attr_init (&use, zapt, 1);
643 use_value = attr_find (&use, &curAttributeSet);
644 logf (LOG_DEBUG, "use value %d", use_value);
645 attr_init (&truncation, zapt, 5);
646 truncation_value = attr_find (&truncation, NULL);
647 logf (LOG_DEBUG, "truncation value %d", truncation_value);
652 for (base_no = 0; base_no < num_bases; base_no++)
655 data1_local_attribute *local_attr;
656 int max_pos, prefix_len = 0;
659 attp = att_getentbyatt (curAttributeSet, use_value);
662 logf (LOG_DEBUG, "att_getentbyatt fail. set=%d use=%d",
663 curAttributeSet, use_value);
667 if (zebTargetInfo_curDatabase (zi->zti, basenames[base_no]))
669 zi->errCode = 109; /* Database unavailable */
670 zi->errString = basenames[base_no];
673 for (local_attr = attp->local_attributes; local_attr;
674 local_attr = local_attr->next)
678 ord = zebTargetInfo_lookupSU (zi->zti, attp->attset_ordinal,
683 term_dict[prefix_len++] = '|';
685 term_dict[prefix_len++] = '(';
686 term_dict[prefix_len++] = 1;
687 term_dict[prefix_len++] = ord;
694 term_dict[prefix_len++] = ')';
695 term_dict[prefix_len++] = 1;
696 term_dict[prefix_len++] = regType;
697 term_dict[prefix_len] = '\0';
698 if (!relational_term (zi, zapt, &termp, term_dict,
699 attributeSet, grep_info, &max_pos))
702 switch (truncation_value)
704 case -1: /* not specified */
705 case 100: /* do not truncate */
706 term_dict[j++] = '(';
707 if (!term_100 (&termp, term_dict + j))
709 strcat (term_dict, ")");
710 r = dict_lookup_grep (zi->dict, term_dict, 0, grep_info,
711 &max_pos, 0, grep_handle);
713 logf (LOG_WARN, "dict_lookup_grep err, trunc=none:%d", r);
715 case 1: /* right truncation */
716 term_dict[j++] = '(';
717 if (!term_100 (&termp, term_dict + j))
719 strcat (term_dict, ".*)");
720 dict_lookup_grep (zi->dict, term_dict, 0, grep_info,
721 &max_pos, 0, grep_handle);
723 case 2: /* left truncation */
724 case 3: /* left&right truncation */
727 case 101: /* process # in term */
728 term_dict[j++] = '(';
729 if (!term_101 (&termp, term_dict + j))
731 strcat (term_dict, ")");
732 r = dict_lookup_grep (zi->dict, term_dict, 0, grep_info,
733 &max_pos, 0, grep_handle);
735 logf (LOG_WARN, "dict_lookup_grep err, trunc=#: %d", r);
737 case 102: /* Regexp-1 */
738 term_dict[j++] = '(';
739 if (!term_102 (&termp, term_dict + j))
741 strcat (term_dict, ")");
742 logf (LOG_DEBUG, "Regexp-1 tolerance=%d", r);
743 r = dict_lookup_grep (zi->dict, term_dict, 0, grep_info,
744 &max_pos, 0, grep_handle);
746 logf (LOG_WARN, "dict_lookup_grep err, trunc=regular: %d",
749 case 103: /* Regexp-1 */
751 term_dict[j++] = '(';
752 if (!term_103 (&termp, term_dict + j, &r))
754 strcat (term_dict, ")");
755 logf (LOG_DEBUG, "Regexp-2 tolerance=%d", r);
756 r = dict_lookup_grep (zi->dict, term_dict, r, grep_info,
757 &max_pos, 2, grep_handle);
759 logf (LOG_WARN, "dict_lookup_grep err, trunc=eregular: %d",
766 logf (LOG_DEBUG, "%d positions", grep_info->isam_p_indx);
770 static void trans_term (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
774 Z_Term *term = zapt->term;
776 sizez = term->u.general->len;
777 if (sizez > IT_MAX_WORD-1)
778 sizez = IT_MAX_WORD-1;
779 memcpy (termz, term->u.general->buf, sizez);
783 static void trans_scan_term (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
786 Z_Term *term = zapt->term;
788 char *cp = (char*) term->u.general->buf;
789 const char *cp_end = cp + term->u.general->len;
795 while ((len = (cp_end - cp)) > 0)
797 map = map_chrs_input (&cp, len);
798 if (**map == *CHR_SPACE)
806 for (src = *map; *src; src++)
812 static RSET rpn_search_APT_relevance (ZServerInfo *zi,
813 Z_AttributesPlusTerm *zapt,
814 oid_value attributeSet,
815 int num_bases, char **basenames)
817 rset_relevance_parms parms;
818 char termz[IT_MAX_WORD+1];
820 struct grep_info grep_info;
825 parms.key_size = sizeof(struct it_key);
827 parms.cmp = key_compare;
829 parms.isc = zi->isamc;
832 if (zapt->term->which != Z_Term_general)
837 trans_term (zi, zapt, termz);
840 grep_info.term_no = 0;
842 grep_info.isam_p_indx = 0;
843 grep_info.isam_p_size = 0;
844 grep_info.isam_p_buf = NULL;
847 r = field_term (zi, zapt, &termp, 'w', attributeSet, &grep_info,
848 num_bases, basenames);
852 for (; term_index < grep_info.isam_p_indx; term_index++)
853 grep_info.term_no[term_index] = parms.no_terms;
857 parms.term_no = grep_info.term_no;
858 parms.isam_positions = grep_info.isam_p_buf;
859 parms.no_isam_positions = grep_info.isam_p_indx;
860 if (grep_info.isam_p_indx > 0)
861 result = rset_create (rset_kind_relevance, &parms);
863 result = rset_create (rset_kind_null, NULL);
865 xfree(grep_info.term_no);
867 xfree (grep_info.isam_p_buf);
871 static RSET rpn_search_APT_cphrase (ZServerInfo *zi,
872 Z_AttributesPlusTerm *zapt,
873 oid_value attributeSet,
874 int num_bases, char **basenames)
876 char termz[IT_MAX_WORD+1];
877 struct grep_info grep_info;
882 if (zapt->term->which != Z_Term_general)
887 trans_term (zi, zapt, termz);
890 grep_info.term_no = 0;
892 grep_info.isam_p_indx = 0;
893 grep_info.isam_p_size = 0;
894 grep_info.isam_p_buf = NULL;
896 r = field_term (zi, zapt, &termp, 'p', attributeSet, &grep_info,
897 num_bases, basenames);
898 result = rset_trunc (zi, grep_info.isam_p_buf, grep_info.isam_p_indx);
900 xfree(grep_info.term_no);
902 xfree (grep_info.isam_p_buf);
906 static RSET rpn_prox (RSET *rset, int rset_no)
914 rset_temp_parms parms;
916 rsfd = xmalloc (sizeof(*rsfd)*rset_no);
917 more = xmalloc (sizeof(*more)*rset_no);
918 buf = xmalloc (sizeof(*buf)*rset_no);
920 for (i = 0; i<rset_no; i++)
922 buf[i] = xmalloc (sizeof(**buf));
923 rsfd[i] = rset_open (rset[i], RSETF_READ|RSETF_SORT_SYSNO);
924 if (!(more[i] = rset_read (rset[i], rsfd[i], buf[i])))
928 rset_close (rset[i], rsfd[i]);
935 return rset_create (rset_kind_null, NULL);
938 parms.key_size = sizeof (struct it_key);
939 result = rset_create (rset_kind_temp, &parms);
940 rsfd_result = rset_open (result, RSETF_WRITE|RSETF_SORT_SYSNO);
944 for (i = 1; i<rset_no; i++)
953 cmp = key_compare (buf[i], buf[i-1]);
956 more[i-1] = rset_read (rset[i-1], rsfd[i-1], buf[i-1]);
961 if (buf[i-1]->seqno+1 != buf[i]->seqno)
963 more[i-1] = rset_read (rset[i-1], rsfd[i-1], buf[i-1]);
969 more[i] = rset_read (rset[i], rsfd[i], buf[i]);
975 rset_write (result, rsfd_result, buf[0]);
976 more[0] = rset_read (*rset, *rsfd, *buf);
980 for (i = 0; i<rset_no; i++)
982 rset_close (rset[i], rsfd[i]);
985 rset_close (result, rsfd_result);
992 static RSET rpn_search_APT_phrase (ZServerInfo *zi,
993 Z_AttributesPlusTerm *zapt,
994 oid_value attributeSet,
995 int num_bases, char **basenames)
997 char termz[IT_MAX_WORD+1];
999 RSET rset[60], result;
1000 int i, r, rset_no = 0;
1001 struct grep_info grep_info;
1003 if (zapt->term->which != Z_Term_general)
1008 trans_term (zi, zapt, termz);
1011 grep_info.term_no = 0;
1013 grep_info.isam_p_size = 0;
1014 grep_info.isam_p_buf = NULL;
1018 grep_info.isam_p_indx = 0;
1019 r = field_term (zi, zapt, &termp, 'w', attributeSet, &grep_info,
1020 num_bases, basenames);
1023 rset[rset_no] = rset_trunc (zi, grep_info.isam_p_buf,
1024 grep_info.isam_p_indx);
1025 assert (rset[rset_no]);
1026 if (++rset_no >= sizeof(rset)/sizeof(*rset))
1030 xfree(grep_info.term_no);
1032 xfree (grep_info.isam_p_buf);
1034 return rset_create (rset_kind_null, NULL);
1035 else if (rset_no == 1)
1037 result = rpn_prox (rset, rset_no);
1038 for (i = 0; i<rset_no; i++)
1039 rset_delete (rset[i]);
1043 static RSET rpn_search_APT_local (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
1044 oid_value attributeSet)
1049 rset_temp_parms parms;
1050 char termz[IT_MAX_WORD+1];
1052 if (zapt->term->which != Z_Term_general)
1057 parms.key_size = sizeof (struct it_key);
1058 result = rset_create (rset_kind_temp, &parms);
1059 rsfd = rset_open (result, RSETF_WRITE|RSETF_SORT_SYSNO);
1061 trans_term (zi, zapt, termz);
1063 key.sysno = atoi (termz);
1066 rset_write (result, rsfd, &key);
1067 rset_close (result, rsfd);
1071 static RSET rpn_search_APT (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
1072 oid_value attributeSet,
1073 int num_bases, char **basenames)
1077 AttrType completeness;
1078 int relation_value, structure_value, completeness_value;
1080 attr_init (&relation, zapt, 2);
1081 attr_init (&structure, zapt, 4);
1082 attr_init (&completeness, zapt, 6);
1084 relation_value = attr_find (&relation, NULL);
1085 structure_value = attr_find (&structure, NULL);
1086 completeness_value = attr_find (&completeness, NULL);
1087 switch (structure_value)
1090 if (relation_value == 102) /* relevance relation */
1091 return rpn_search_APT_relevance (zi, zapt, attributeSet,
1092 num_bases, basenames);
1093 if (completeness_value == 2 || completeness_value == 3)
1094 return rpn_search_APT_cphrase (zi, zapt, attributeSet,
1095 num_bases, basenames);
1096 return rpn_search_APT_phrase (zi, zapt, attributeSet,
1097 num_bases, basenames);
1098 case 1: /* phrase */
1099 if (relation_value == 102) /* relevance relation */
1100 return rpn_search_APT_relevance (zi, zapt, attributeSet,
1101 num_bases, basenames);
1102 if (completeness_value == 2 || completeness_value == 3)
1103 return rpn_search_APT_cphrase (zi, zapt, attributeSet,
1104 num_bases, basenames);
1105 return rpn_search_APT_phrase (zi, zapt, attributeSet,
1106 num_bases, basenames);
1109 if (relation_value == 102) /* relevance relation */
1110 return rpn_search_APT_relevance (zi, zapt, attributeSet,
1111 num_bases, basenames);
1112 if (completeness_value == 2 || completeness_value == 3)
1113 return rpn_search_APT_cphrase (zi, zapt, attributeSet,
1114 num_bases, basenames);
1115 return rpn_search_APT_phrase (zi, zapt, attributeSet,
1116 num_bases, basenames);
1121 case 5: /* date - normalized */
1123 case 6: /* word list */
1124 return rpn_search_APT_relevance (zi, zapt, attributeSet,
1125 num_bases, basenames);
1126 case 100: /* date - un-normalized */
1128 case 101: /* name - normalized */
1130 case 102: /* date - un-normalized */
1132 case 103: /* structure */
1136 case 105: /* free-form-text */
1137 return rpn_search_APT_relevance (zi, zapt, attributeSet,
1138 num_bases, basenames);
1139 case 106: /* document-text */
1140 return rpn_search_APT_relevance (zi, zapt, attributeSet,
1141 num_bases, basenames);
1142 case 107: /* local-number */
1143 return rpn_search_APT_local (zi, zapt, attributeSet);
1144 case 108: /* string */
1145 return rpn_search_APT_phrase (zi, zapt, attributeSet,
1146 num_bases, basenames);
1147 case 109: /* numeric string */
1154 static RSET rpn_search_ref (ZServerInfo *zi, Z_ResultSetId *resultSetId)
1158 if (!(s = resultSetGet (zi, resultSetId)))
1159 return rset_create (rset_kind_null, NULL);
1163 static RSET rpn_search_structure (ZServerInfo *zi, Z_RPNStructure *zs,
1164 oid_value attributeSet,
1165 int num_bases, char **basenames)
1168 if (zs->which == Z_RPNStructure_complex)
1170 rset_bool_parms bool_parms;
1173 bool_parms.rset_l = rpn_search_structure (zi, zs->u.complex->s1,
1175 num_bases, basenames);
1176 if (bool_parms.rset_l == NULL)
1178 if (rset_is_ranked(bool_parms.rset_l))
1180 bool_parms.rset_r = rpn_search_structure (zi, zs->u.complex->s2,
1182 num_bases, basenames);
1183 if (bool_parms.rset_r == NULL)
1185 rset_delete (bool_parms.rset_l);
1188 if (rset_is_ranked(bool_parms.rset_r))
1190 bool_parms.key_size = sizeof(struct it_key);
1191 bool_parms.cmp = key_compare;
1193 switch (zs->u.complex->roperator->which)
1195 case Z_Operator_and:
1196 r = rset_create (soft ? rset_kind_sand:rset_kind_and, &bool_parms);
1199 r = rset_create (soft ? rset_kind_sor:rset_kind_or, &bool_parms);
1201 case Z_Operator_and_not:
1202 r = rset_create (soft ? rset_kind_snot:rset_kind_not, &bool_parms);
1208 else if (zs->which == Z_RPNStructure_simple)
1210 if (zs->u.simple->which == Z_Operand_APT)
1212 logf (LOG_DEBUG, "rpn_search_APT");
1213 r = rpn_search_APT (zi, zs->u.simple->u.attributesPlusTerm,
1214 attributeSet, num_bases, basenames);
1216 else if (zs->u.simple->which == Z_Operand_resultSetId)
1218 logf (LOG_DEBUG, "rpn_search_ref");
1219 r = rpn_search_ref (zi, zs->u.simple->u.resultSetId);
1233 void count_set_save (RSET *r, int *count)
1240 rset_temp_parms parms;
1242 logf (LOG_DEBUG, "count_set_save");
1244 parms.key_size = sizeof(struct it_key);
1245 w = rset_create (rset_kind_temp, &parms);
1246 wfd = rset_open (w, RSETF_WRITE|RSETF_SORT_SYSNO);
1247 rfd = rset_open (*r, RSETF_READ|RSETF_SORT_SYSNO);
1248 while (rset_read (*r, rfd, &key))
1250 logf (LOG_DEBUG, "sysno=%-7d seqno=%d", key.sysno, key.seqno);
1251 if (key.sysno != psysno)
1253 rset_write (w, wfd, &key);
1259 rset_close (*r, rfd);
1261 rset_close (w, wfd);
1263 logf (LOG_DEBUG, "%d keys, %d distinct sysnos", kno, *count);
1266 static void count_set (RSET r, int *count)
1273 logf (LOG_DEBUG, "count_set");
1275 rfd = rset_open (r, RSETF_READ|RSETF_SORT_SYSNO);
1276 while (rset_read (r, rfd, &key))
1278 if (key.sysno != psysno)
1285 rset_close (r, rfd);
1286 logf (LOG_DEBUG, "%d keys, %d distinct sysnos", kno, *count);
1289 int rpn_search (ZServerInfo *zi,
1290 Z_RPNQuery *rpn, int num_bases, char **basenames,
1291 const char *setname, int *hits)
1295 oid_value attributeSet;
1297 dict_grep_cmap (zi->dict, map_chrs_input);
1301 zi->errString = NULL;
1303 attrset = oid_getentbyoid (rpn->attributeSetId);
1304 attributeSet = attrset->value;
1305 rset = rpn_search_structure (zi, rpn->RPNStructure, attributeSet,
1306 num_bases, basenames);
1309 if (rset_is_volatile(rset))
1310 count_set_save(&rset,hits);
1312 count_set (rset, hits);
1313 resultSetAdd (zi, setname, 1, rset);
1315 logf (LOG_DEBUG, "search error: %d", zi->errCode);
1319 struct scan_info_entry {
1325 struct scan_info_entry *list;
1331 static int scan_handle (char *name, const char *info, int pos, void *client)
1333 int len_prefix, idx;
1334 struct scan_info *scan_info = client;
1336 len_prefix = strlen(scan_info->prefix);
1337 if (memcmp (name, scan_info->prefix, len_prefix))
1340 idx = scan_info->after - pos + scan_info->before;
1343 logf (LOG_DEBUG, "%-3d %s", idx, name+len_prefix);
1344 scan_info->list[idx].term = odr_malloc (scan_info->odr,
1345 strlen(name + len_prefix)+1);
1346 strcpy (scan_info->list[idx].term, name + len_prefix);
1347 assert (*info == sizeof(ISAM_P));
1348 memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAM_P));
1353 static void scan_term_untrans (ODR odr, char **dstp, const char *src)
1355 char *dst = odr_malloc (odr, strlen(src)*2+1);
1360 const char *cp = map_chrs_output (&src);
1367 int rpn_scan (ZServerInfo *zi, Z_AttributesPlusTerm *zapt,
1368 oid_value attributeset,
1369 int num_bases, char **basenames,
1370 int *position, int *num_entries, struct scan_entry **list,
1374 int pos = *position;
1375 int num = *num_entries;
1379 char termz[IT_MAX_WORD+20];
1382 AttrType completeness;
1383 int completeness_value;
1384 struct scan_info *scan_info_array;
1385 struct scan_entry *glist;
1386 int ords[32], ord_no = 0;
1389 logf (LOG_DEBUG, "scan, position = %d, num = %d", pos, num);
1391 if (attributeset == VAL_NONE)
1392 attributeset = VAL_BIB1;
1394 attr_init (&use, zapt, 1);
1395 use_value = attr_find (&use, &attributeset);
1396 logf (LOG_DEBUG, "use value %d", use_value);
1398 attr_init (&completeness, zapt, 6);
1399 completeness_value = attr_find (&completeness, NULL);
1400 logf (LOG_DEBUG, "completeness value %d", completeness_value);
1402 if (use_value == -1)
1404 for (base_no = 0; base_no < num_bases && ord_no < 32; base_no++)
1407 data1_local_attribute *local_attr;
1409 attp = att_getentbyatt (attributeset, use_value);
1412 logf (LOG_DEBUG, "att_getentbyatt fail. set=%d use=%d",
1413 attributeset, use_value);
1414 return zi->errCode = 114;
1416 if (zebTargetInfo_curDatabase (zi->zti, basenames[base_no]))
1418 zi->errString = basenames[base_no];
1419 return zi->errCode = 109; /* Database unavailable */
1421 for (local_attr = attp->local_attributes; local_attr && ord_no < 32;
1422 local_attr = local_attr->next)
1426 ord = zebTargetInfo_lookupSU (zi->zti, attp->attset_ordinal,
1429 ords[ord_no++] = ord;
1433 return zi->errCode = 113;
1436 scan_info_array = odr_malloc (zi->odr, ord_no * sizeof(*scan_info_array));
1437 for (i = 0; i < ord_no; i++)
1439 int j, prefix_len = 0;
1440 int before_tmp = before, after_tmp = after;
1441 struct scan_info *scan_info = scan_info_array + i;
1443 scan_info->before = before;
1444 scan_info->after = after;
1445 scan_info->odr = zi->odr;
1447 scan_info->list = odr_malloc (zi->odr, (before+after)*
1448 sizeof(*scan_info->list));
1449 for (j = 0; j<before+after; j++)
1450 scan_info->list[j].term = NULL;
1451 termz[prefix_len++] = ords[i];
1452 termz[prefix_len++] =
1453 (completeness_value==2 || completeness_value==3) ? 'p': 'w';
1454 termz[prefix_len] = 0;
1455 strcpy (scan_info->prefix, termz);
1457 trans_scan_term (zi, zapt, termz+prefix_len);
1459 dict_scan (zi->dict, termz, &before_tmp, &after_tmp, scan_info,
1462 glist = odr_malloc (zi->odr, (before+after)*sizeof(*glist));
1463 for (i = 0; i < ord_no; i++)
1466 *status = BEND_SCAN_SUCCESS;
1467 for (i = 0; i<after; i++)
1470 const char *mterm = NULL;
1474 for (j = 0; j < ord_no; j++)
1476 if (ptr[j] < before+after &&
1477 (tst=scan_info_array[j].list[ptr[j]].term) &&
1478 (!mterm || strcmp (tst, mterm) < 0))
1486 scan_term_untrans (zi->odr, &glist[i+before].term, mterm);
1487 rset = rset_trunc (zi, &scan_info_array[j0].list[ptr[j0]].isam_p, 1);
1490 for (j = j0+1; j<ord_no; j++)
1492 if (ptr[j] < before+after &&
1493 (tst=scan_info_array[j].list[ptr[j]].term) &&
1494 !strcmp (tst, mterm))
1496 rset_bool_parms bool_parms;
1500 rset_trunc (zi, &scan_info_array[j].list[ptr[j]].isam_p, 1);
1502 bool_parms.key_size = sizeof(struct it_key);
1503 bool_parms.cmp = key_compare;
1504 bool_parms.rset_l = rset;
1505 bool_parms.rset_r = rset2;
1507 rset = rset_create (rset_kind_or, &bool_parms);
1512 count_set (rset, &glist[i+before].occurrences);
1517 *num_entries -= (after-i);
1518 *status = BEND_SCAN_PARTIAL;
1521 for (i = 0; i<ord_no; i++)
1524 for (i = 0; i<before; i++)
1527 const char *mterm = NULL;
1531 for (j = 0; j <ord_no; j++)
1533 if (ptr[j] < before &&
1534 (tst=scan_info_array[j].list[before-1-ptr[j]].term) &&
1535 (!mterm || strcmp (tst, mterm) > 0))
1544 scan_term_untrans (zi->odr, &glist[before-1-i].term, mterm);
1547 (zi, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1);
1551 for (j = j0+1; j<ord_no; j++)
1553 if (ptr[j] < before &&
1554 (tst=scan_info_array[j].list[before-1-ptr[j]].term) &&
1555 !strcmp (tst, mterm))
1557 rset_bool_parms bool_parms;
1560 rset2 = rset_trunc (zi,
1561 &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1);
1563 bool_parms.key_size = sizeof(struct it_key);
1564 bool_parms.cmp = key_compare;
1565 bool_parms.rset_l = rset;
1566 bool_parms.rset_r = rset2;
1568 rset = rset_create (rset_kind_or, &bool_parms);
1573 count_set (rset, &glist[before-1-i].occurrences);
1579 *status = BEND_SCAN_PARTIAL;
1583 *list = glist + i; /* list is set to first 'real' entry */
1585 logf (LOG_DEBUG, "position = %d, num_entries = %d",
1586 *position, *num_entries);
1588 logf (LOG_DEBUG, "scan error: %d", zi->errCode);