-/*
- * $Id: relevance.c,v 1.4 2007-01-08 12:43:41 adam Exp $
+/* $Id: relevance.c,v 1.11 2007-05-01 05:04:53 quinn Exp $
+ Copyright (c) 2006-2007, Index Data.
+
+This file is part of Pazpar2.
+
+Pazpar2 is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 2, or (at your option) any later
+version.
+
+Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with Pazpar2; see the file LICENSE. If not, write to the
+Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
+02111-1307, USA.
*/
#include <ctype.h>
static void word_trie_addterm(NMEM nmem, struct word_trie *n, const char *term, int num)
{
+
while (*term) {
int c = tolower(*term);
if (c < 'a' || c > 'z')
;
res->vec_len = ++i;
res->doc_frequency_vec = nmem_malloc(nmem, res->vec_len * sizeof(int));
- bzero(res->doc_frequency_vec, res->vec_len * sizeof(int));
+ memset(res->doc_frequency_vec, 0, res->vec_len * sizeof(int));
res->nmem = nmem;
res->wt = build_word_trie(nmem, terms);
return res;
}
-void relevance_newrec(struct relevance *r, struct record *rec)
+void relevance_newrec(struct relevance *r, struct record_cluster *rec)
{
if (!rec->term_frequency_vec)
{
rec->term_frequency_vec = nmem_malloc(r->nmem, r->vec_len * sizeof(int));
- bzero(rec->term_frequency_vec, r->vec_len * sizeof(int));
+ memset(rec->term_frequency_vec, 0, r->vec_len * sizeof(int));
}
}
// FIXME. The definition of a word is crude here.. should support
// some form of localization mechanism?
-void relevance_countwords(struct relevance *r, struct record *head,
+void relevance_countwords(struct relevance *r, struct record_cluster *cluster,
const char *words, int multiplier)
{
while (*words)
if ((res = word_trie_match(r->wt, words, &skipped)))
{
words += skipped;
- head->term_frequency_vec[res] += multiplier;
+ cluster->term_frequency_vec[res] += multiplier;
}
else
{
while (*words && (c = raw_char(tolower(*words))) >= 0)
words++;
}
- head->term_frequency_vec[0]++;
+ cluster->term_frequency_vec[0]++;
}
}
-void relevance_donerecord(struct relevance *r, struct record *head)
+void relevance_donerecord(struct relevance *r, struct record_cluster *cluster)
{
int i;
for (i = 1; i < r->vec_len; i++)
- if (head->term_frequency_vec[i] > 0)
+ if (cluster->term_frequency_vec[i] > 0)
r->doc_frequency_vec[i]++;
r->doc_frequency_vec[0]++;
}
+#ifdef GAGA
#ifdef FLOAT_REL
static int comp(const void *p1, const void *p2)
{
#else
static int comp(const void *p1, const void *p2)
{
- struct record **r1 = (struct record **) p1;
- struct record **r2 = (struct record **) p2;
+ struct record_cluster **r1 = (struct record_cluster **) p1;
+ struct record_cluster **r2 = (struct record_cluster **) p2;
return (*r2)->relevance - (*r1)->relevance;
}
#endif
+#endif
-// Prepare for a relevance-sorted read of up to num entries
+// Prepare for a relevance-sorted read
void relevance_prepare_read(struct relevance *rel, struct reclist *reclist)
{
int i;
if (!rel->doc_frequency_vec[i])
idfvec[i] = 0;
else
- idfvec[i] = log((float) rel->doc_frequency_vec[0] / rel->doc_frequency_vec[i]);
+ {
+ // This conditional may be terribly wrong
+ // It was there to address the situation where vec[0] == vec[i]
+ // which leads to idfvec[i] == 0... not sure about this
+ // Traditional TF-IDF may assume that a word that occurs in every
+ // record is irrelevant, but this is actually something we will
+ // see a lot
+ if ((idfvec[i] = log((float) rel->doc_frequency_vec[0] /
+ rel->doc_frequency_vec[i])) < 0.0000001)
+ idfvec[i] = 1;
+ }
}
// Calculate relevance for each document
for (i = 0; i < reclist->num_records; i++)
{
int t;
- struct record *rec = reclist->flatlist[i];
+ struct record_cluster *rec = reclist->flatlist[i];
float relevance;
relevance = 0;
for (t = 1; t < rel->vec_len; t++)
}
rec->relevance = (int) (relevance * 100000);
}
+#ifdef GAGA
qsort(reclist->flatlist, reclist->num_records, sizeof(struct record*), comp);
+#endif
reclist->pointer = 0;
xfree(idfvec);
}