1 /* $Id: ranksimilarity.c,v 1.5 2006-05-08 14:03:16 marc Exp $
2 Copyright (C) 1995-2005
5 This file is part of the Zebra server.
7 Zebra is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with Zebra; see the file LICENSE.zebra. If not, write to the
19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
36 static int log_level = 0;
37 static int log_initialized = 0;
39 struct ranksimilarity_class_info {
43 /** term specific info and statistics to be used under ranking */
44 struct ranksimilarity_term_info {
46 /** frequency of term within document field */
47 int freq_term_docfield;
49 /** frequency of term within result set of given term */
50 zint freq_term_resset;
52 /** number of docs within result set */
55 /** number of terms in this field */
58 /** number of docs with this field in database */
61 /** size of all docs with this field in database (in bytes ir terms ??) */
64 /** rank flag is one if term is to be included in ranking */
67 /** relative ranking weight of term field */
70 /** term id used to access term name and other info */
73 /** index number in terms[i] array */
77 struct ranksimilarity_set_info {
80 /** number of terms in query */
83 /** number of terms in query which are included in ranking */
84 int no_ranked_terms_query;
86 /** number of documents in entire collection */
87 zint no_docs_database;
89 /** size of all documents in entire collection (in bytes or terms ??)*/
90 zint size_docs_database;
92 /** array of size no_terms_query with statistics gathered per term */
93 struct ranksimilarity_term_info *entries;
99 /* local clean-up function */
100 static void ranksimilar_rec_reset(struct ranksimilarity_set_info *si)
104 for (i = 0; i < si->no_terms_query; i++)
106 si->entries[i].freq_term_docfield = 0;
112 * create: Creates/Initialises this rank handler. This routine is
113 * called exactly once. The routine returns the class_handle.
115 static void *create (ZebraHandle zh)
117 struct ranksimilarity_class_info *ci =
118 (struct ranksimilarity_class_info *) xmalloc (sizeof(*ci));
120 if (!log_initialized)
122 log_level = yaz_log_module_level("rank-similarity");
125 yaz_log(log_level, "create()");
130 * destroy: Destroys this rank handler. This routine is called
131 * when the handler is no longer needed - i.e. when the server
132 * dies. The class_handle was previously returned by create.
134 static void destroy (struct zebra_register *reg, void *class_handle)
136 struct ranksimilarity_class_info *ci
137 = (struct ranksimilarity_class_info *) class_handle;
138 yaz_log(log_level, "destroy()");
144 * begin: Prepares beginning of "real" ranking. Called once for
145 * each result set. The returned handle is a "set handle" and
146 * will be used in each of the handlers below.
148 static void *begin (struct zebra_register *reg,
149 void *class_handle, RSET rset, NMEM nmem,
150 TERMID *terms, int numterms)
152 struct ranksimilarity_set_info *si =
153 (struct ranksimilarity_set_info *) nmem_malloc (nmem, sizeof(*si));
156 yaz_log(log_level, "begin() numterms=%d", numterms);
158 /* setting database global statistics */
159 si->no_docs_database = -1; /* TODO */
160 si->size_docs_database = -1; /* TODO */
162 /* setting query statistics */
163 si->no_terms_query = numterms;
164 si->no_ranked_terms_query = 0;
166 /* setting internal data structures */
168 si->entries = (struct ranksimilarity_term_info *)
169 nmem_malloc (si->nmem, sizeof(*si->entries)*numterms);
171 /* reset the counts for the next term */
172 ranksimilar_rec_reset(si);
175 /* looping all terms in a specific field of query */
176 for (i = 0; i < numterms; i++)
178 struct ord_list *ol = NULL;
181 /* adding to number of rank entries */
182 if (strncmp (terms[i]->flags, "rank,", 5))
184 si->entries[i].rank_flag = 0;
185 yaz_log(log_level, "begin() terms[%d]: '%s' flags=%s not ranked",
186 i, terms[i]->name, terms[i]->flags);
190 const char *cp = strstr(terms[i]->flags+4, ",w=");
192 yaz_log(log_level, "begin() terms[%d]: '%s' flags=%s",
193 i, terms[i]->name, terms[i]->flags);
195 (si->no_ranked_terms_query)++;
198 si->entries[i].rank_flag = 1;
199 /* notice that the call to rset_count(rset) has he side-effect
200 of setting rset->hits_limit = rset_count(rset) ??? */
201 si->entries[i].freq_term_resset = rset_count(terms[i]->rset);
202 si->entries[i].no_docs_resset = terms[i]->rset->hits_count;
203 si->entries[i].no_docs_field = -1; /*TODO*/
204 si->entries[i].size_docs_field = -1; /*TODO*/
205 si->entries[i].no_terms_field = -1; /*TODO*/
208 si->entries[i].field_weight = atoi (cp+3);
210 si->entries[i].field_weight = 34; /* sqrroot of 1000 */
214 yaz_log(log_level, "begin() rset_count(terms[%d]->rset) = "
215 ZINT_FORMAT, i, rset_count(terms[i]->rset));
216 yaz_log(log_level, "begin() terms[%d]->rset->hits_limit = "
217 ZINT_FORMAT, i, terms[i]->rset->hits_limit);
218 yaz_log(log_level, "begin() terms[%d]->rset->hits_count = "
219 ZINT_FORMAT, i, terms[i]->rset->hits_count);
220 yaz_log(log_level, "begin() terms[%d]->rset->hits_round = "
221 ZINT_FORMAT, i, terms[i]->rset->hits_round);
222 yaz_log(log_level, "begin() terms[%d]->rset->hits_approx = %d",
223 i, terms[i]->rset->hits_approx);
226 /* looping indexes where term terms[i] is found */
227 for (; ol; ol = ol->next)
231 const char *string_index = 0;
235 zebraExplain_lookup_ord(reg->zei,
236 ol->ord, &index_type, &db, &set, &use,
241 "begin() index: ord=%d type=%c db=%s str-index=%s",
242 ol->ord, index_type, db, string_index);
245 "begin() index: ord=%d type=%c db=%s set=%d use=%d",
246 ol->ord, index_type, db, set, use);
251 si->entries[i].term = terms[i];
252 si->entries[i].term_index=i;
254 /* setting next entry in term */
255 terms[i]->rankpriv = &(si->entries[i]);
262 * end: Terminates ranking process. Called after a result set
265 static void end (struct zebra_register *reg, void *set_handle)
267 yaz_log(log_level, "end()");
273 * add: Called for each word occurence in a result set. This routine
274 * should be as fast as possible. This routine should "incrementally"
277 static void add (void *set_handle, int seqno, TERMID term)
279 struct ranksimilarity_set_info *si
280 = (struct ranksimilarity_set_info *) set_handle;
281 struct ranksimilarity_term_info *ti;
285 /* yaz_log(log_level, "add() seqno=%d NULL term", seqno); */
289 ti= (struct ranksimilarity_term_info *) term->rankpriv;
291 si->last_pos = seqno;
292 ti->freq_term_docfield++;
293 /* yaz_log(log_level, "add() seqno=%d term=%s freq_term_docfield=%d",
294 seqno, term->name, ti->freq_term_docfield); */
298 * calc: Called for each document in a result. This handler should
299 * produce a score based on previous call(s) to the add handler. The
300 * score should be between 0 and 1000. If score cannot be obtained
301 * -1 should be returned.
303 static int calc (void *set_handle, zint sysno, zint staticrank,
307 struct ranksimilarity_set_info *si
308 = (struct ranksimilarity_set_info *) set_handle;
311 yaz_log(log_level, "calc() sysno = " ZINT_FORMAT, sysno);
312 yaz_log(log_level, "calc() staticrank = " ZINT_FORMAT, staticrank);
314 yaz_log(log_level, "calc() si->no_terms_query = %d",
316 yaz_log(log_level, "calc() si->no_ranked_terms_query = %d",
317 si->no_ranked_terms_query);
318 yaz_log(log_level, "calc() si->no_docs_database = " ZINT_FORMAT,
319 si->no_docs_database);
320 yaz_log(log_level, "calc() si->size_docs_database = " ZINT_FORMAT,
321 si->size_docs_database);
324 if (!si->no_ranked_terms_query)
325 return -1; /* ranking not enabled for any terms */
328 /* if we set *stop_flag = 1, we stop processing (of result set list) */
331 /* here goes your formula to compute a scoring function */
332 /* you may use all the gathered statistics here */
333 for (i = 0; i < si->no_terms_query; i++)
335 yaz_log(log_level, "calc() entries[%d] termid %p",
336 i, si->entries[i].term);
337 if (si->entries[i].term){
338 yaz_log(log_level, "calc() entries[%d] term '%s' flags=%s",
339 i, si->entries[i].term->name, si->entries[i].term->flags);
340 yaz_log(log_level, "calc() entries[%d] rank_flag %d",
341 i, si->entries[i].rank_flag );
342 yaz_log(log_level, "calc() entries[%d] field_weight %d",
343 i, si->entries[i].field_weight );
344 yaz_log(log_level, "calc() entries[%d] freq_term_docfield %d",
345 i, si->entries[i].freq_term_docfield );
346 yaz_log(log_level, "calc() entries[%d] freq_term_resset " ZINT_FORMAT,
347 i, si->entries[i].freq_term_resset );
348 yaz_log(log_level, "calc() entries[%d] no_docs_resset " ZINT_FORMAT,
349 i, si->entries[i].no_docs_resset );
350 yaz_log(log_level, "calc() entries[%d] no_docs_field " ZINT_FORMAT,
351 i, si->entries[i].no_docs_field );
352 yaz_log(log_level, "calc() entries[%d] size_docs_field " ZINT_FORMAT,
353 i, si->entries[i].size_docs_field );
354 yaz_log(log_level, "calc() entries[%d] no_terms_field " ZINT_FORMAT,
355 i, si->entries[i].no_terms_field );
360 /* reset the counts for the next term */
361 ranksimilar_rec_reset(si);
364 /* staticrank = 0 is highest, MAXINT lowest */
365 score = INT_MAX - staticrank; /* but score is reverse (logical) */
368 /* debugging statistics output */
369 yaz_log(log_level, "calc() statistics: score = %d", score);
375 * Pseudo-meta code with sequence of calls as they occur in a
376 * server. Handlers are prefixed by --:
392 static struct rank_control rank_control = {
402 struct rank_control *rank_similarity_class = &rank_control;