MULTILINE_CPP_IS_BRIEF = NO
-# If the DETAILS_AT_TOP tag is set to YES then Doxygen
-# will output the detailed description near the top, like JavaDoc.
-# If set to NO, the detailed description appears after the member
-# documentation.
-
-DETAILS_AT_TOP = NO
-
# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented
# member inherits the documentation from any documented member that it
# re-implements.
+--- 2.0.39 2009/07/09
+
+Improved speed of record fetching by using a different block sizes
+for recd{A,B}.
+
+Added zlib-compression of recd-storage (record data). This is enabled
+by using recordCompression: zlib in zebra.cfg.
+
+Dictionary delete more agressive (dict_del_string).
+
--- 2.0.38 2009/06/19
Improved estimated hits for AND operation (bug #2907)
dnl Copyright (C) 1994-2009 Index Data
dnl
AC_PREREQ(2.60)
-AC_INIT([idzebra],[2.0.38],[zebra-help@indexdata.dk])
+AC_INIT([idzebra],[2.0.39],[zebra-help@indexdata.dk])
AC_CONFIG_SRCDIR(configure.ac)
AC_CONFIG_AUX_DIR(config)
AM_INIT_AUTOMAKE([1.9])
fi
fi
dnl
+dnl ------- zlib
+AC_CHECK_LIB([z],[compress2])
+if test "$ac_cv_lib_z_compress2" = "yes"; then
+ AC_CHECK_HEADERS([zlib.h])
+fi
dnl ------ -lm
AC_CHECK_LIB(m,sqrt)
dnl ------ -ldl
+idzebra (2.0.39-1indexdata) unstable; urgency=low
+
+ * Upstream.
+
+ -- Adam Dickmeiss <adam@indexdata.dk> Thu, 09 Jul 2009 10:32:04 +0200
+
idzebra (2.0.38-1indexdata) unstable; urgency=low
* Upstream.
Priority: optional
Maintainer: Adam Dickmeiss <adam@indexdata.dk>
Standards-Version: 3.6.0
-Build-Depends: debhelper (>= 4.0.0), libyaz3-dev (>= 3.0.30), libexpat1-dev, tcl8.4-dev, libbz2-dev, libxslt1-dev
+Build-Depends: debhelper (>= 4.0.0), libyaz3-dev (>= 3.0.30), libexpat1-dev, tcl8.4-dev, libbz2-dev, libxslt1-dev, zlib1g-dev
Package: idzebra-2.0
Section: text
# Flags for the non-authenticated user. w=write (allows ES Updates)
perm.anonymous: rw
+# Record compression (none, bzip2 or zlib)
+recordcompression: none
+
#rank: zvrank
recordId: (bib-1,title)
BuildRoot: %{_tmppath}/idzebra-%{version}-root
Packager: Adam Dickmeiss <adam@indexdata.dk>
URL: http://www.indexdata.dk/zebra/
-BuildRequires: libyaz3-devel expat-devel bzip2-devel tcl
+BuildRequires: libyaz3-devel expat-devel bzip2-devel tcl zlib-devel
Summary: High-performance, structured text indexing and retrival engine.
%description
*/
Records rec_open(BFiles bfs, int rw, int compression_method);
+/** \brief check whether a compression method is supported
+ \param compression_method (REC_COMPRESS_..)
+ \retval 0 if method is unsupported
+ \retval 1 if method is supported
+*/
+int rec_check_compression_method(int compression_method);
+
char *rec_strdup(const char *s, size_t *len);
void rec_prstat(Records p, int verbose);
zint rec_sysno_to_int(zint sysno);
-/** \brief compression types */
+
+/** \brief No compression ("none") */
#define REC_COMPRESS_NONE 0
+/** \brief BZIP2 compression (slow and requires big chunks) */
#define REC_COMPRESS_BZIP2 1
+/** \brief zlib compression (faster and works off small chunks) */
+#define REC_COMPRESS_ZLIB 2
+
enum {
recInfo_fileType,
#if HAVE_BZLIB_H
#include <bzlib.h>
#endif
+#if HAVE_ZLIB_H
+#include <zlib.h>
+#endif
#define REC_BLOCK_TYPES 2
#define REC_HEAD_MAGIC "recindex"
return ZEBRA_OK;
}
+int rec_check_compression_method(int compression_method)
+{
+ switch(compression_method)
+ {
+ case REC_COMPRESS_ZLIB:
+#if HAVE_ZLIB_H
+ return 1;
+#else
+ return 0;
+#endif
+ case REC_COMPRESS_BZIP2:
+#if HAVE_BZLIB_H
+ return 1;
+#else
+ return 0;
+#endif
+ case REC_COMPRESS_NONE:
+ return 1;
+ }
+ return 0;
+}
+
Records rec_open(BFiles bfs, int rw, int compression_method)
{
Records p;
char compression_method = p->compression_method;
switch (compression_method)
{
+ case REC_COMPRESS_ZLIB:
+#if HAVE_ZLIB_H
+ csize = out_offset + (out_offset >> 6) + 620;
+ while (1)
+ {
+ int r;
+ uLongf destLen = csize;
+ rec_tmp_expand(p, csize);
+ r = compress((Bytef *) p->tmp_buf+sizeof(zint)+sizeof(short)+
+ sizeof(char),
+ &destLen, (const Bytef *) out_buf, out_offset);
+ csize = destLen;
+ if (r == Z_OK)
+ {
+ break;
+ }
+ if (r != Z_MEM_ERROR)
+ {
+ yaz_log(YLOG_WARN, "compress error: %d", r);
+ csize = 0;
+ break;
+ }
+ csize = csize * 2;
+ }
+#endif
+ break;
case REC_COMPRESS_BZIP2:
#if HAVE_BZLIB_H
csize = out_offset + (out_offset >> 6) + 620;
yaz_log(YLOG_WARN, "bzBuffToBuffCompress error code=%d", i);
csize = 0;
}
- yaz_log(YLOG_LOG, "compress %4d %5d %5d", ref_count, out_offset,
- csize);
#endif
break;
case REC_COMPRESS_NONE:
char *nptr, *cptr;
char *in_buf = 0;
char *bz_buf = 0;
-#if HAVE_BZLIB_H
- unsigned int bz_size;
-#endif
char compression_method;
assert(sysno > 0);
in_size = entry.size - sizeof(short) - sizeof(char);
switch (compression_method)
{
+ case REC_COMPRESS_ZLIB:
+#if HAVE_ZLIB_H
+ if (1)
+ {
+ unsigned int bz_size = entry.size * 20 + 100;
+ while (1)
+ {
+ uLongf destLen = bz_size;
+ bz_buf = (char *) xmalloc(bz_size);
+ i = uncompress((Bytef *) bz_buf, &destLen,
+ (const Bytef *) in_buf, in_size);
+ if (i == Z_OK)
+ {
+ bz_size = destLen;
+ break;
+ }
+ yaz_log(YLOG_LOG, "failed");
+ xfree(bz_buf);
+ bz_size *= 2;
+ }
+ in_buf = bz_buf;
+ in_size = bz_size;
+ }
+#else
+ yaz_log(YLOG_FATAL, "cannot decompress record(s) in ZLIB format");
+ return 0;
+#endif
+ break;
case REC_COMPRESS_BZIP2:
#if HAVE_BZLIB_H
- bz_size = entry.size * 20 + 100;
- while (1)
- {
- bz_buf = (char *) xmalloc(bz_size);
+ if (1)
+ {
+ unsigned int bz_size = entry.size * 20 + 100;
+ while (1)
+ {
+ bz_buf = (char *) xmalloc(bz_size);
#ifdef BZ_CONFIG_ERROR
- i = BZ2_bzBuffToBuffDecompress
+ i = BZ2_bzBuffToBuffDecompress
#else
- i = bzBuffToBuffDecompress
+ i = bzBuffToBuffDecompress
#endif
- (bz_buf, &bz_size, in_buf, in_size, 0, 0);
- yaz_log(YLOG_LOG, "decompress %5d %5d", in_size, bz_size);
- if (i == BZ_OK)
- break;
- yaz_log(YLOG_LOG, "failed");
- xfree(bz_buf);
- bz_size *= 2;
- }
- in_buf = bz_buf;
- in_size = bz_size;
+ (bz_buf, &bz_size, in_buf, in_size, 0, 0);
+ if (i == BZ_OK)
+ break;
+ yaz_log(YLOG_LOG, "failed");
+ xfree(bz_buf);
+ bz_size *= 2;
+ }
+ in_buf = bz_buf;
+ in_size = bz_size;
+ }
#else
yaz_log(YLOG_FATAL, "cannot decompress record(s) in BZIP2 format");
return 0;
{
struct zebra_register *reg;
int record_compression = REC_COMPRESS_NONE;
- const char *recordCompression = 0;
+ const char *compression_str = 0;
const char *profilePath;
int sort_type = ZEBRA_SORT_TYPE_FLAT;
ZEBRA_RES ret = ZEBRA_OK;
zebraRankInstall(reg, rank_similarity_class);
zebraRankInstall(reg, rank_static_class);
- recordCompression = res_get_def(res, "recordCompression", "none");
- if (!strcmp(recordCompression, "none"))
+ compression_str = res_get_def(res, "recordCompression", "none");
+ if (!strcmp(compression_str, "none"))
record_compression = REC_COMPRESS_NONE;
- if (!strcmp(recordCompression, "bzip2"))
+ else if (!strcmp(compression_str, "bzip2"))
record_compression = REC_COMPRESS_BZIP2;
+ else if (!strcmp(compression_str, "zlib"))
+ record_compression = REC_COMPRESS_ZLIB;
+ else
+ {
+ yaz_log(YLOG_FATAL, "invalid recordCompression: %s", compression_str);
+ ret = ZEBRA_FAIL;
+ }
+
+ if (!rec_check_compression_method(record_compression))
+ {
+ yaz_log(YLOG_FATAL, "unsupported recordCompression: %s",
+ compression_str);
+ ret = ZEBRA_FAIL;
+ }
{
const char *index_fname = res_get_def(res, "index", "default.idx");