2 * Copyright (C) 1994-1998, Index Data
4 * Sebastian Hammer, Adam Dickmeiss
7 * Revision 1.18 1998-09-22 10:03:45 adam
8 * Changed result sets to be persistent in the sense that they can
9 * be re-searched if needed.
10 * Fixed memory leak in rsm_or.
12 * Revision 1.17 1998/06/23 15:33:36 adam
13 * Added feature to specify sort criteria in query (type 7 specifies
16 * Revision 1.16 1998/05/20 10:12:24 adam
17 * Implemented automatic EXPLAIN database maintenance.
18 * Modified Zebra to work with ASN.1 compiled version of YAZ.
20 * Revision 1.15 1998/03/05 08:45:14 adam
21 * New result set model and modular ranking system. Moved towards
22 * descent server API. System information stored as "SGML" records.
24 * Revision 1.14 1998/02/10 16:39:15 adam
27 * Revision 1.13 1998/02/10 12:03:06 adam
30 * Revision 1.12 1997/09/25 14:57:36 adam
33 * Revision 1.11 1996/12/23 15:30:46 adam
35 * Bug fix: result sets weren't deleted after server shut down.
37 * Revision 1.10 1995/10/30 15:08:08 adam
40 * Revision 1.9 1995/10/17 18:02:14 adam
41 * New feature: databases. Implemented as prefix to words in dictionary.
43 * Revision 1.8 1995/10/10 13:59:25 adam
44 * Function rset_open changed its wflag parameter to general flags.
46 * Revision 1.7 1995/10/06 14:38:01 adam
47 * New result set method: r_score.
48 * Local no (sysno) and score is transferred to retrieveCtrl.
50 * Revision 1.6 1995/09/28 09:19:49 adam
51 * xfree/xmalloc used everywhere.
52 * Extract/retrieve method seems to work for text records.
54 * Revision 1.5 1995/09/27 16:17:32 adam
55 * More work on retrieve.
57 * Revision 1.4 1995/09/07 13:58:36 adam
58 * New parameter: result-set file descriptor (RSFD) to support multiple
59 * positions within the same result-set.
60 * Boolean operators: and, or, not implemented.
61 * Result-set references.
63 * Revision 1.3 1995/09/06 16:11:19 adam
64 * Option: only one word key per file.
66 * Revision 1.2 1995/09/06 10:33:04 adam
67 * More work on present. Some log messages removed.
69 * Revision 1.1 1995/09/05 15:28:40 adam
70 * More work on search engine.
84 #define SORT_IDX_ENTRYSIZE 64
85 #define ZSET_SORT_MAX_LEVEL 3
95 struct zset_sort_info *sort_info;
96 struct zebra_set *next;
99 struct zset_sort_entry {
102 char buf[ZSET_SORT_MAX_LEVEL][SORT_IDX_ENTRYSIZE];
105 struct zset_sort_info {
108 struct zset_sort_entry *all_entries;
109 struct zset_sort_entry **entries;
112 ZebraSet resultSetAddRPN (ZebraHandle zh, ODR stream, ODR decode,
113 Z_RPNQuery *rpn, int num_bases, char **basenames,
121 zh->errString = NULL;
124 zebraSet = resultSetAdd (zh, setname, 1);
128 zebraSet->num_bases = num_bases;
129 zebraSet->basenames = basenames;
130 zebraSet->nmem = odr_extract_mem (decode);
132 zebraSet->rset = rpn_search (zh, stream->mem, rpn,
134 zebraSet->basenames, zebraSet->name,
136 zh->hits = zebraSet->hits;
142 ZebraSet resultSetAdd (ZebraHandle zh, const char *name, int ov)
147 for (s = zh->sets; s; s = s->next)
148 if (!strcmp (s->name, name))
152 logf (LOG_DEBUG, "updating result set %s", name);
156 rset_delete (s->rset);
158 nmem_destroy (s->nmem);
162 logf (LOG_DEBUG, "adding result set %s", name);
163 s = xmalloc (sizeof(*s));
166 s->name = xmalloc (strlen(name)+1);
167 strcpy (s->name, name);
169 s->sort_info = xmalloc (sizeof(*s->sort_info));
170 s->sort_info->max_entries = 1000;
171 s->sort_info->entries =
172 xmalloc (sizeof(*s->sort_info->entries) *
173 s->sort_info->max_entries);
174 s->sort_info->all_entries =
175 xmalloc (sizeof(*s->sort_info->all_entries) *
176 s->sort_info->max_entries);
177 for (i = 0; i < s->sort_info->max_entries; i++)
178 s->sort_info->entries[i] = s->sort_info->all_entries + i;
185 ZebraSet resultSetGet (ZebraHandle zh, const char *name)
189 for (s = zh->sets; s; s = s->next)
190 if (!strcmp (s->name, name))
192 if (!s->rset && s->rpn)
194 NMEM nmem = nmem_create ();
196 rpn_search (zh, nmem, s->rpn, s->num_bases,
197 s->basenames, s->name, s);
207 void resultSetDestroy (ZebraHandle zh)
211 for (s = zh->sets; s; s = s1)
215 xfree (s->sort_info->all_entries);
216 xfree (s->sort_info->entries);
217 xfree (s->sort_info);
220 nmem_destroy (s->nmem);
221 rset_delete (s->rset);
228 ZebraPosSet zebraPosSetCreate (ZebraHandle zh, const char *name,
229 int num, int *positions)
235 struct zset_sort_info *sort_info;
237 if (!(sset = resultSetGet (zh, name)))
239 if (!(rset = sset->rset))
241 sr = xmalloc (sizeof(*sr) * num);
242 for (i = 0; i<num; i++)
247 sort_info = sset->sort_info;
252 for (i = 0; i<num; i++)
254 position = positions[i];
255 if (position <= sort_info->num_entries)
257 logf (LOG_DEBUG, "got pos=%d (sorted)", position);
258 sr[i].sysno = sort_info->entries[position-1]->sysno;
259 sr[i].score = sort_info->entries[position-1]->score;
263 /* did we really get all entries using sort ? */
264 for (i = 0; i<num; i++)
269 if (i < num) /* nope, get the rest, unsorted - sorry */
279 position = sort_info->num_entries;
280 while (num_i < num && positions[num_i] < position)
282 rfd = rset_open (rset, RSETF_READ);
283 while (num_i < num && rset_read (rset, rfd, &key, &term_index))
285 if (key.sysno != psysno)
290 /* determine we alreay have this in our set */
291 for (i = sort_info->num_entries; --i >= 0; )
292 if (psysno == sort_info->entries[i]->sysno)
298 assert (num_i < num);
299 if (position == positions[num_i])
301 sr[num_i].sysno = psysno;
302 logf (LOG_DEBUG, "got pos=%d (unsorted)", position);
303 sr[num_i].score = -1;
308 rset_close (rset, rfd);
313 void zebraPosSetDestroy (ZebraHandle zh, ZebraPosSet records, int num)
323 void resultSetInsertSort (ZebraHandle zh, ZebraSet sset,
324 struct sortKey *criteria, int num_criteria,
327 struct zset_sort_entry this_entry;
328 struct zset_sort_entry *new_entry = NULL;
329 struct zset_sort_info *sort_info = sset->sort_info;
332 sortIdx_sysno (zh->sortIdx, sysno);
333 for (i = 0; i<num_criteria; i++)
335 sortIdx_type (zh->sortIdx, criteria[i].attrUse);
336 sortIdx_read (zh->sortIdx, this_entry.buf[i]);
338 i = sort_info->num_entries;
342 for (j = 0; j<num_criteria; j++)
344 rel = memcmp (this_entry.buf[j], sort_info->entries[i]->buf[j],
351 if (criteria[j].relation == 'A')
356 else if (criteria[j].relation == 'D')
362 j = sort_info->max_entries-1;
366 new_entry = sort_info->entries[j];
369 sort_info->entries[j] = sort_info->entries[j-1];
372 sort_info->entries[j] = new_entry;
374 if (sort_info->num_entries != sort_info->max_entries)
375 (sort_info->num_entries)++;
376 for (i = 0; i<num_criteria; i++)
377 memcpy (new_entry->buf[i], this_entry.buf[i], SORT_IDX_ENTRYSIZE);
378 new_entry->sysno = sysno;
379 new_entry->score = -1;
382 void resultSetInsertRank (ZebraHandle zh, struct zset_sort_info *sort_info,
383 int sysno, int score, int relation)
385 struct zset_sort_entry *new_entry = NULL;
388 i = sort_info->num_entries;
393 rel = score - sort_info->entries[i]->score;
400 else if (relation == 'A')
406 j = sort_info->max_entries-1;
410 new_entry = sort_info->entries[j];
413 sort_info->entries[j] = sort_info->entries[j-1];
416 sort_info->entries[j] = new_entry;
418 if (sort_info->num_entries != sort_info->max_entries)
419 (sort_info->num_entries)++;
420 new_entry->sysno = sysno;
421 new_entry->score = score;
424 void resultSetSort (ZebraHandle zh, NMEM nmem,
425 int num_input_setnames, const char **input_setnames,
426 const char *output_setname,
427 Z_SortKeySpecList *sort_sequence, int *sort_status)
432 if (num_input_setnames == 0)
437 if (num_input_setnames > 1)
442 logf (LOG_DEBUG, "result set sort input=%s output=%s",
443 *input_setnames, output_setname);
444 sset = resultSetGet (zh, input_setnames[0]);
448 zh->errString = nmem_strdup (nmem, input_setnames[0]);
451 if (!(rset = sset->rset))
454 zh->errString = nmem_strdup (nmem, input_setnames[0]);
457 if (strcmp (output_setname, input_setnames[0]))
459 rset = rset_dup (rset);
460 sset = resultSetAdd (zh, output_setname, 1);
463 resultSetSortSingle (zh, nmem, sset, rset, sort_sequence, sort_status);
466 void resultSetSortSingle (ZebraHandle zh, NMEM nmem,
467 ZebraSet sset, RSET rset,
468 Z_SortKeySpecList *sort_sequence, int *sort_status)
472 struct sortKey sort_criteria[3];
477 logf (LOG_DEBUG, "resultSetSortSingle start");
478 sset->sort_info->num_entries = 0;
481 num_criteria = sort_sequence->num_specs;
482 if (num_criteria > 3)
484 for (i = 0; i < num_criteria; i++)
486 Z_SortKeySpec *sks = sort_sequence->specs[i];
489 if (*sks->sortRelation == Z_SortRelation_ascending)
490 sort_criteria[i].relation = 'A';
491 else if (*sks->sortRelation == Z_SortRelation_descending)
492 sort_criteria[i].relation = 'D';
498 if (sks->sortElement->which == Z_SortElement_databaseSpecific)
503 else if (sks->sortElement->which != Z_SortElement_generic)
508 sk = sks->sortElement->u.generic;
511 case Z_SortKey_sortField:
512 logf (LOG_DEBUG, "Sort: key %d is of type sortField", i+1);
515 case Z_SortKey_elementSpec:
516 logf (LOG_DEBUG, "Sort: key %d is of type elementSpec", i+1);
519 case Z_SortKey_sortAttributes:
520 logf (LOG_DEBUG, "Sort: key %d is of type sortAttributes", i+1);
521 sort_criteria[i].attrUse =
522 zebra_maps_sort (zh->zebra_maps, sk->u.sortAttributes);
523 logf (LOG_DEBUG, "use value = %d", sort_criteria[i].attrUse);
524 if (sort_criteria[i].attrUse == -1)
529 if (sortIdx_type (zh->sortIdx, sort_criteria[i].attrUse))
537 rfd = rset_open (rset, RSETF_READ);
538 while (rset_read (rset, rfd, &key, &term_index))
540 if (key.sysno != psysno)
544 resultSetInsertSort (zh, sset,
545 sort_criteria, num_criteria, psysno);
548 rset_close (rset, rfd);
551 *sort_status = Z_SortStatus_success;
552 logf (LOG_DEBUG, "resultSetSortSingle end");
555 RSET resultSetRef (ZebraHandle zh, Z_ResultSetId *resultSetId)
559 if ((s = resultSetGet (zh, resultSetId)))
564 void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset)
570 ZebraRankClass rank_class;
571 struct rank_control *rc;
572 struct zset_sort_info *sort_info;
574 sort_info = zebraSet->sort_info;
575 sort_info->num_entries = 0;
577 rfd = rset_open (rset, RSETF_READ);
579 logf (LOG_DEBUG, "resultSetRank");
580 for (i = 0; i < rset->no_rset_terms; i++)
581 logf (LOG_DEBUG, "term=\"%s\" cnt=%d type=%s",
582 rset->rset_terms[i]->name,
583 rset->rset_terms[i]->nn,
584 rset->rset_terms[i]->flags);
586 rank_class = zebraRankLookup (zh, "rank-1");
587 rc = rank_class->control;
589 if (rset_read (rset, rfd, &key, &term_index))
591 int psysno = key.sysno;
594 (*rc->begin) (zh, rank_class->class_handle, rset);
599 if (key.sysno != psysno)
601 score = (*rc->calc) (handle, psysno);
603 resultSetInsertRank (zh, sort_info, psysno, score, 'A');
607 (*rc->add) (handle, key.seqno, term_index);
609 while (rset_read (rset, rfd, &key, &term_index));
610 score = (*rc->calc) (handle, psysno);
611 resultSetInsertRank (zh, sort_info, psysno, score, 'A');
612 (*rc->end) (zh, handle);
614 rset_close (rset, rfd);
615 logf (LOG_DEBUG, "%d keys, %d distinct sysnos", kno, zebraSet->hits);
618 ZebraRankClass zebraRankLookup (ZebraHandle zh, const char *name)
620 ZebraRankClass p = zh->rank_classes;
621 while (p && strcmp (p->control->name, name))
623 if (p && !p->init_flag)
625 if (p->control->create)
626 p->class_handle = (*p->control->create)(zh);
632 void zebraRankInstall (ZebraHandle zh, struct rank_control *ctrl)
634 ZebraRankClass p = xmalloc (sizeof(*p));
635 p->control = xmalloc (sizeof(*p->control));
636 memcpy (p->control, ctrl, sizeof(*p->control));
637 p->control->name = xstrdup (ctrl->name);
639 p->next = zh->rank_classes;
640 zh->rank_classes = p;
643 void zebraRankDestroy (ZebraHandle zh)
645 ZebraRankClass p = zh->rank_classes;
648 ZebraRankClass p_next = p->next;
649 if (p->init_flag && p->control->destroy)
650 (*p->control->destroy)(zh, p->class_handle);
651 xfree (p->control->name);
656 zh->rank_classes = NULL;