2 * Copyright (C) 1994-1999, Index Data
4 * Sebastian Hammer, Adam Dickmeiss
7 * Revision 1.26 2000-03-20 19:08:36 adam
8 * Added remote record import using Z39.50 extended services and Segment
11 * Revision 1.25 2000/03/15 15:00:31 adam
12 * First work on threaded version.
14 * Revision 1.24 1999/11/04 15:00:45 adam
15 * Implemented delete result set(s).
17 * Revision 1.23 1999/05/26 07:49:13 adam
20 * Revision 1.22 1999/02/02 14:51:15 adam
21 * Updated WIN32 code specific sections. Changed header.
23 * Revision 1.21 1998/11/16 16:03:46 adam
24 * Moved loggin utilities to Yaz. Was implemented in file zlogs.c.
26 * Revision 1.20 1998/11/16 10:10:53 adam
27 * Fixed problem with zebraPosSetCreate that occurred when positions were
30 * Revision 1.19 1998/09/22 10:48:22 adam
31 * Minor changes in search API.
33 * Revision 1.18 1998/09/22 10:03:45 adam
34 * Changed result sets to be persistent in the sense that they can
35 * be re-searched if needed.
36 * Fixed memory leak in rsm_or.
38 * Revision 1.17 1998/06/23 15:33:36 adam
39 * Added feature to specify sort criteria in query (type 7 specifies
42 * Revision 1.16 1998/05/20 10:12:24 adam
43 * Implemented automatic EXPLAIN database maintenance.
44 * Modified Zebra to work with ASN.1 compiled version of YAZ.
46 * Revision 1.15 1998/03/05 08:45:14 adam
47 * New result set model and modular ranking system. Moved towards
48 * descent server API. System information stored as "SGML" records.
50 * Revision 1.14 1998/02/10 16:39:15 adam
53 * Revision 1.13 1998/02/10 12:03:06 adam
56 * Revision 1.12 1997/09/25 14:57:36 adam
59 * Revision 1.11 1996/12/23 15:30:46 adam
61 * Bug fix: result sets weren't deleted after server shut down.
63 * Revision 1.10 1995/10/30 15:08:08 adam
66 * Revision 1.9 1995/10/17 18:02:14 adam
67 * New feature: databases. Implemented as prefix to words in dictionary.
69 * Revision 1.8 1995/10/10 13:59:25 adam
70 * Function rset_open changed its wflag parameter to general flags.
72 * Revision 1.7 1995/10/06 14:38:01 adam
73 * New result set method: r_score.
74 * Local no (sysno) and score is transferred to retrieveCtrl.
76 * Revision 1.6 1995/09/28 09:19:49 adam
77 * xfree/xmalloc used everywhere.
78 * Extract/retrieve method seems to work for text records.
80 * Revision 1.5 1995/09/27 16:17:32 adam
81 * More work on retrieve.
83 * Revision 1.4 1995/09/07 13:58:36 adam
84 * New parameter: result-set file descriptor (RSFD) to support multiple
85 * positions within the same result-set.
86 * Boolean operators: and, or, not implemented.
87 * Result-set references.
89 * Revision 1.3 1995/09/06 16:11:19 adam
90 * Option: only one word key per file.
92 * Revision 1.2 1995/09/06 10:33:04 adam
93 * More work on present. Some log messages removed.
95 * Revision 1.1 1995/09/05 15:28:40 adam
96 * More work on search engine.
110 #define SORT_IDX_ENTRYSIZE 64
111 #define ZSET_SORT_MAX_LEVEL 3
121 struct zset_sort_info *sort_info;
122 struct zebra_set *next;
125 struct zset_sort_entry {
128 char buf[ZSET_SORT_MAX_LEVEL][SORT_IDX_ENTRYSIZE];
131 struct zset_sort_info {
134 struct zset_sort_entry *all_entries;
135 struct zset_sort_entry **entries;
138 ZebraSet resultSetAddRPN (ZebraHandle zh, ODR input, ODR output,
139 Z_RPNQuery *rpn, int num_bases, char **basenames,
145 zh->errString = NULL;
148 zebraSet = resultSetAdd (zh, setname, 1);
152 zebraSet->num_bases = num_bases;
153 zebraSet->basenames = basenames;
154 zebraSet->nmem = odr_extract_mem (input);
156 zebraSet->rset = rpn_search (zh, output->mem, rpn,
158 zebraSet->basenames, zebraSet->name,
160 zh->hits = zebraSet->hits;
166 ZebraSet resultSetAdd (ZebraHandle zh, const char *name, int ov)
171 for (s = zh->sets; s; s = s->next)
172 if (!strcmp (s->name, name))
176 logf (LOG_DEBUG, "updating result set %s", name);
180 rset_delete (s->rset);
182 nmem_destroy (s->nmem);
186 logf (LOG_DEBUG, "adding result set %s", name);
187 s = (ZebraSet) xmalloc (sizeof(*s));
190 s->name = (char *) xmalloc (strlen(name)+1);
191 strcpy (s->name, name);
193 s->sort_info = (struct zset_sort_info *)
194 xmalloc (sizeof(*s->sort_info));
195 s->sort_info->max_entries = 1000;
196 s->sort_info->entries = (struct zset_sort_entry **)
197 xmalloc (sizeof(*s->sort_info->entries) *
198 s->sort_info->max_entries);
199 s->sort_info->all_entries = (struct zset_sort_entry *)
200 xmalloc (sizeof(*s->sort_info->all_entries) *
201 s->sort_info->max_entries);
202 for (i = 0; i < s->sort_info->max_entries; i++)
203 s->sort_info->entries[i] = s->sort_info->all_entries + i;
210 ZebraSet resultSetGet (ZebraHandle zh, const char *name)
214 for (s = zh->sets; s; s = s->next)
215 if (!strcmp (s->name, name))
217 if (!s->rset && s->rpn)
219 NMEM nmem = nmem_create ();
221 rpn_search (zh, nmem, s->rpn, s->num_bases,
222 s->basenames, s->name, s);
231 void resultSetDestroy (ZebraHandle zh, int num, char **names,int *statuses)
233 ZebraSet *ss = &zh->sets;
237 for (i = 0; i<num; i++)
238 statuses[i] = Z_DeleteStatus_resultSetDidNotExist;
240 zh->errString = NULL;
247 for (i = 0; i<num; i++)
248 if (!strcmp (s->name, names[i]))
251 statuses[i] = Z_DeleteStatus_success;
260 xfree (s->sort_info->all_entries);
261 xfree (s->sort_info->entries);
262 xfree (s->sort_info);
265 nmem_destroy (s->nmem);
266 rset_delete (s->rset);
275 ZebraPosSet zebraPosSetCreate (ZebraHandle zh, const char *name,
276 int num, int *positions)
282 struct zset_sort_info *sort_info;
284 if (!(sset = resultSetGet (zh, name)))
286 if (!(rset = sset->rset))
288 sr = (ZebraPosSet) xmalloc (sizeof(*sr) * num);
289 for (i = 0; i<num; i++)
294 sort_info = sset->sort_info;
299 for (i = 0; i<num; i++)
301 position = positions[i];
302 if (position > 0 && position <= sort_info->num_entries)
304 logf (LOG_DEBUG, "got pos=%d (sorted)", position);
305 sr[i].sysno = sort_info->entries[position-1]->sysno;
306 sr[i].score = sort_info->entries[position-1]->score;
310 /* did we really get all entries using sort ? */
311 for (i = 0; i<num; i++)
316 if (i < num) /* nope, get the rest, unsorted - sorry */
326 position = sort_info->num_entries;
327 while (num_i < num && positions[num_i] < position)
329 rfd = rset_open (rset, RSETF_READ);
330 while (num_i < num && rset_read (rset, rfd, &key, &term_index))
332 if (key.sysno != psysno)
337 /* determine we alreay have this in our set */
338 for (i = sort_info->num_entries; --i >= 0; )
339 if (psysno == sort_info->entries[i]->sysno)
345 assert (num_i < num);
346 if (position == positions[num_i])
348 sr[num_i].sysno = psysno;
349 logf (LOG_DEBUG, "got pos=%d (unsorted)", position);
350 sr[num_i].score = -1;
355 rset_close (rset, rfd);
360 void zebraPosSetDestroy (ZebraHandle zh, ZebraPosSet records, int num)
370 void resultSetInsertSort (ZebraHandle zh, ZebraSet sset,
371 struct sortKeyInfo *criteria, int num_criteria,
374 struct zset_sort_entry this_entry;
375 struct zset_sort_entry *new_entry = NULL;
376 struct zset_sort_info *sort_info = sset->sort_info;
379 sortIdx_sysno (zh->service->sortIdx, sysno);
380 for (i = 0; i<num_criteria; i++)
382 sortIdx_type (zh->service->sortIdx, criteria[i].attrUse);
383 sortIdx_read (zh->service->sortIdx, this_entry.buf[i]);
385 i = sort_info->num_entries;
389 for (j = 0; j<num_criteria; j++)
391 rel = memcmp (this_entry.buf[j], sort_info->entries[i]->buf[j],
398 if (criteria[j].relation == 'A')
403 else if (criteria[j].relation == 'D')
409 j = sort_info->max_entries-1;
413 new_entry = sort_info->entries[j];
416 sort_info->entries[j] = sort_info->entries[j-1];
419 sort_info->entries[j] = new_entry;
421 if (sort_info->num_entries != sort_info->max_entries)
422 (sort_info->num_entries)++;
423 for (i = 0; i<num_criteria; i++)
424 memcpy (new_entry->buf[i], this_entry.buf[i], SORT_IDX_ENTRYSIZE);
425 new_entry->sysno = sysno;
426 new_entry->score = -1;
429 void resultSetInsertRank (ZebraHandle zh, struct zset_sort_info *sort_info,
430 int sysno, int score, int relation)
432 struct zset_sort_entry *new_entry = NULL;
435 i = sort_info->num_entries;
440 rel = score - sort_info->entries[i]->score;
447 else if (relation == 'A')
453 j = sort_info->max_entries-1;
457 new_entry = sort_info->entries[j];
460 sort_info->entries[j] = sort_info->entries[j-1];
463 sort_info->entries[j] = new_entry;
465 if (sort_info->num_entries != sort_info->max_entries)
466 (sort_info->num_entries)++;
467 new_entry->sysno = sysno;
468 new_entry->score = score;
471 void resultSetSort (ZebraHandle zh, NMEM nmem,
472 int num_input_setnames, const char **input_setnames,
473 const char *output_setname,
474 Z_SortKeySpecList *sort_sequence, int *sort_status)
479 if (num_input_setnames == 0)
484 if (num_input_setnames > 1)
489 logf (LOG_DEBUG, "result set sort input=%s output=%s",
490 *input_setnames, output_setname);
491 sset = resultSetGet (zh, input_setnames[0]);
495 zh->errString = nmem_strdup (nmem, input_setnames[0]);
498 if (!(rset = sset->rset))
501 zh->errString = nmem_strdup (nmem, input_setnames[0]);
504 if (strcmp (output_setname, input_setnames[0]))
506 rset = rset_dup (rset);
507 sset = resultSetAdd (zh, output_setname, 1);
510 resultSetSortSingle (zh, nmem, sset, rset, sort_sequence, sort_status);
513 void resultSetSortSingle (ZebraHandle zh, NMEM nmem,
514 ZebraSet sset, RSET rset,
515 Z_SortKeySpecList *sort_sequence, int *sort_status)
519 struct sortKeyInfo sort_criteria[3];
524 logf (LOG_DEBUG, "resultSetSortSingle start");
525 sset->sort_info->num_entries = 0;
528 num_criteria = sort_sequence->num_specs;
529 if (num_criteria > 3)
531 for (i = 0; i < num_criteria; i++)
533 Z_SortKeySpec *sks = sort_sequence->specs[i];
536 if (*sks->sortRelation == Z_SortRelation_ascending)
537 sort_criteria[i].relation = 'A';
538 else if (*sks->sortRelation == Z_SortRelation_descending)
539 sort_criteria[i].relation = 'D';
545 if (sks->sortElement->which == Z_SortElement_databaseSpecific)
550 else if (sks->sortElement->which != Z_SortElement_generic)
555 sk = sks->sortElement->u.generic;
558 case Z_SortKey_sortField:
559 logf (LOG_DEBUG, "Sort: key %d is of type sortField", i+1);
562 case Z_SortKey_elementSpec:
563 logf (LOG_DEBUG, "Sort: key %d is of type elementSpec", i+1);
566 case Z_SortKey_sortAttributes:
567 logf (LOG_DEBUG, "Sort: key %d is of type sortAttributes", i+1);
568 sort_criteria[i].attrUse =
569 zebra_maps_sort (zh->service->zebra_maps,
570 sk->u.sortAttributes);
571 logf (LOG_DEBUG, "use value = %d", sort_criteria[i].attrUse);
572 if (sort_criteria[i].attrUse == -1)
577 if (sortIdx_type (zh->service->sortIdx, sort_criteria[i].attrUse))
585 rfd = rset_open (rset, RSETF_READ);
586 while (rset_read (rset, rfd, &key, &term_index))
588 if (key.sysno != psysno)
592 resultSetInsertSort (zh, sset,
593 sort_criteria, num_criteria, psysno);
596 rset_close (rset, rfd);
599 *sort_status = Z_SortStatus_success;
600 logf (LOG_DEBUG, "resultSetSortSingle end");
603 RSET resultSetRef (ZebraHandle zh, Z_ResultSetId *resultSetId)
607 if ((s = resultSetGet (zh, resultSetId)))
612 void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset)
618 ZebraRankClass rank_class;
619 struct rank_control *rc;
620 struct zset_sort_info *sort_info;
622 sort_info = zebraSet->sort_info;
623 sort_info->num_entries = 0;
625 rfd = rset_open (rset, RSETF_READ);
627 logf (LOG_DEBUG, "resultSetRank");
628 for (i = 0; i < rset->no_rset_terms; i++)
629 logf (LOG_DEBUG, "term=\"%s\" cnt=%d type=%s",
630 rset->rset_terms[i]->name,
631 rset->rset_terms[i]->nn,
632 rset->rset_terms[i]->flags);
634 rank_class = zebraRankLookup (zh, "rank-1");
635 rc = rank_class->control;
637 if (rset_read (rset, rfd, &key, &term_index))
639 int psysno = key.sysno;
642 (*rc->begin) (zh, rank_class->class_handle, rset);
647 if (key.sysno != psysno)
649 score = (*rc->calc) (handle, psysno);
651 resultSetInsertRank (zh, sort_info, psysno, score, 'A');
655 (*rc->add) (handle, key.seqno, term_index);
657 while (rset_read (rset, rfd, &key, &term_index));
658 score = (*rc->calc) (handle, psysno);
659 resultSetInsertRank (zh, sort_info, psysno, score, 'A');
660 (*rc->end) (zh, handle);
662 rset_close (rset, rfd);
663 logf (LOG_DEBUG, "%d keys, %d distinct sysnos", kno, zebraSet->hits);
666 ZebraRankClass zebraRankLookup (ZebraHandle zh, const char *name)
668 ZebraRankClass p = zh->service->rank_classes;
669 while (p && strcmp (p->control->name, name))
671 if (p && !p->init_flag)
673 if (p->control->create)
674 p->class_handle = (*p->control->create)(zh->service);
680 void zebraRankInstall (ZebraService zh, struct rank_control *ctrl)
682 ZebraRankClass p = (ZebraRankClass) xmalloc (sizeof(*p));
683 p->control = (struct rank_control *) xmalloc (sizeof(*p->control));
684 memcpy (p->control, ctrl, sizeof(*p->control));
685 p->control->name = xstrdup (ctrl->name);
687 p->next = zh->rank_classes;
688 zh->rank_classes = p;
691 void zebraRankDestroy (ZebraService zh)
693 ZebraRankClass p = zh->rank_classes;
696 ZebraRankClass p_next = p->next;
697 if (p->init_flag && p->control->destroy)
698 (*p->control->destroy)(zh, p->class_handle);
699 xfree (p->control->name);
704 zh->rank_classes = NULL;