1 /* $Id: cfile.c,v 1.39 2006-11-14 08:12:06 adam Exp $
2 Copyright (C) 1995-2006
5 This file is part of the Zebra server.
7 Zebra is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with this program; if not, write to the Free Software
19 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
27 #include <idzebra/util.h>
28 #include <yaz/yaz-util.h>
32 static int write_head(CFile cf)
34 int left = cf->head.hash_size * sizeof(zint);
37 const char *tab = (char*) cf->array;
41 while (left >= (int) HASH_BSIZE)
43 r = mf_write(cf->hash_mf, bno++, 0, 0, tab);
50 r = mf_write(cf->hash_mf, bno, 0, left, tab);
54 static int read_head(CFile cf)
56 int left = cf->head.hash_size * sizeof(zint);
58 char *tab = (char*) cf->array;
62 while (left >= (int) HASH_BSIZE)
64 if (mf_read(cf->hash_mf, bno++, 0, 0, tab) == -1)
71 if (mf_read(cf->hash_mf, bno, 0, left, tab) == -1)
78 CFile cf_open(MFile mf, MFile_area area, const char *fname,
79 int block_size, int wflag, int *firstp)
83 CFile cf = (CFile) xmalloc(sizeof(*cf));
86 /* avoid valgrind warnings, but set to something nasty */
87 memset(cf, 'Z', sizeof(*cf));
89 yaz_log(YLOG_DEBUG, "cf: open %s %s", cf->rmf->name,
90 wflag ? "rdwr" : "rd");
98 cf->bucket_lru_front = cf->bucket_lru_back = NULL;
99 cf->bucket_in_memory = 0;
100 cf->max_bucket_in_memory = 100;
102 cf->iobuf = (char *) xmalloc(block_size);
103 memset(cf->iobuf, 0, block_size);
111 zebra_mutex_init(&cf->mutex);
113 sprintf(path, "%s-b", fname);
114 if (!(cf->block_mf = mf_open(area, path, block_size, wflag)))
119 sprintf(path, "%s-i", fname);
120 if (!(cf->hash_mf = mf_open(area, path, HASH_BSIZE, wflag)))
125 ret = mf_read(cf->hash_mf, 0, 0, sizeof(cf->head), &cf->head);
132 if (ret == 0 || !cf->head.state)
136 cf->head.block_size = block_size;
137 cf->head.hash_size = 199;
138 hash_bytes = cf->head.hash_size * sizeof(zint);
139 cf->head.flat_bucket = cf->head.next_bucket = cf->head.first_bucket =
140 (hash_bytes+sizeof(cf->head))/HASH_BSIZE + 2;
141 cf->head.next_block = 1;
142 cf->array = (zint *) xmalloc(hash_bytes);
143 for (i = 0; i<cf->head.hash_size; i++)
147 if (mf_write(cf->hash_mf, 0, 0, sizeof(cf->head), &cf->head))
162 assert(cf->head.block_size == block_size);
163 assert(cf->head.hash_size > 2);
164 hash_bytes = cf->head.hash_size * sizeof(zint);
165 assert(cf->head.next_bucket > 0);
166 assert(cf->head.next_block > 0);
167 if (cf->head.state == 1)
168 cf->array = (zint *) xmalloc(hash_bytes);
171 if (read_head(cf) == -1)
177 if (cf->head.state == 1)
179 cf->parray = (struct CFile_hash_bucket **)
180 xmalloc(cf->head.hash_size * sizeof(*cf->parray));
181 for (i = 0; i<cf->head.hash_size; i++)
182 cf->parray[i] = NULL;
187 static int cf_hash(CFile cf, zint no)
189 return (int) (((no >> 3) % cf->head.hash_size));
192 static void release_bucket(CFile cf, struct CFile_hash_bucket *p)
195 p->lru_prev->lru_next = p->lru_next;
197 cf->bucket_lru_back = p->lru_next;
199 p->lru_next->lru_prev = p->lru_prev;
201 cf->bucket_lru_front = p->lru_prev;
203 *p->h_prev = p->h_next;
205 p->h_next->h_prev = p->h_prev;
207 --(cf->bucket_in_memory);
211 static int flush_bucket(CFile cf, int no_to_flush)
215 struct CFile_hash_bucket *p;
217 for (i = 0; i != no_to_flush; i++)
219 p = cf->bucket_lru_back;
226 if (mf_write(cf->hash_mf, p->ph.this_bucket, 0, 0, &p->ph))
231 release_bucket(cf, p);
236 static struct CFile_hash_bucket *alloc_bucket(CFile cf, zint block_no, int hno)
238 struct CFile_hash_bucket *p, **pp;
240 if (cf->bucket_in_memory == cf->max_bucket_in_memory)
242 if (flush_bucket(cf, 1))
245 assert(cf->bucket_in_memory < cf->max_bucket_in_memory);
246 ++(cf->bucket_in_memory);
247 p = (struct CFile_hash_bucket *) xmalloc(sizeof(*p));
250 p->lru_prev = cf->bucket_lru_front;
251 if (cf->bucket_lru_front)
252 cf->bucket_lru_front->lru_next = p;
254 cf->bucket_lru_back = p;
255 cf->bucket_lru_front = p;
257 pp = cf->parray + hno;
261 (*pp)->h_prev = &p->h_next;
266 static struct CFile_hash_bucket *get_bucket(CFile cf, zint block_no, int hno)
268 struct CFile_hash_bucket *p;
270 p = alloc_bucket(cf, block_no, hno);
274 if (mf_read(cf->hash_mf, block_no, 0, 0, &p->ph) != 1)
276 yaz_log(YLOG_FATAL, "read get_bucket");
277 release_bucket(cf, p);
280 assert(p->ph.this_bucket == block_no);
284 static struct CFile_hash_bucket *new_bucket(CFile cf, zint *block_nop, int hno)
286 struct CFile_hash_bucket *p;
290 block_no = *block_nop = cf->head.next_bucket++;
291 p = alloc_bucket(cf, block_no, hno);
296 for (i = 0; i<HASH_BUCKET; i++)
301 p->ph.next_bucket = 0;
302 p->ph.this_bucket = block_no;
306 static int cf_lookup_flat(CFile cf, zint no, zint *vno)
308 zint hno = (no*sizeof(zint))/HASH_BSIZE;
309 int off = (int) ((no*sizeof(zint)) - hno*HASH_BSIZE);
312 if (mf_read(cf->hash_mf, hno+cf->head.next_bucket, off, sizeof(zint), vno)
320 static int cf_lookup_hash(CFile cf, zint no, zint *vno)
322 int hno = cf_hash(cf, no);
323 struct CFile_hash_bucket *hb;
327 for (hb = cf->parray[hno]; hb; hb = hb->h_next)
329 for (i = 0; i<HASH_BUCKET && hb->ph.vno[i]; i++)
330 if (hb->ph.no[i] == no)
333 *vno = hb->ph.vno[i];
337 for (block_no = cf->array[hno]; block_no; block_no = hb->ph.next_bucket)
339 for (hb = cf->parray[hno]; hb; hb = hb->h_next)
341 if (hb->ph.this_bucket == block_no)
347 /* extra check ... */
348 for (hb = cf->bucket_lru_back; hb; hb = hb->lru_next)
350 if (hb->ph.this_bucket == block_no)
352 yaz_log(YLOG_FATAL, "Found hash bucket on other chain(1)");
355 for (i = 0; i<HASH_BUCKET && hb->ph.vno[i]; i++)
356 if (hb->ph.no[i] == no)
358 yaz_log(YLOG_FATAL, "Found hash bucket on other chain (2)");
364 hb = get_bucket(cf, block_no, hno);
367 for (i = 0; i<HASH_BUCKET && hb->ph.vno[i]; i++)
368 if (hb->ph.no[i] == no)
370 *vno = hb->ph.vno[i];
377 static int cf_write_flat(CFile cf, zint no, zint vno)
379 zint hno = (no*sizeof(zint))/HASH_BSIZE;
380 int off = (int) ((no*sizeof(zint)) - hno*HASH_BSIZE);
382 hno += cf->head.next_bucket;
383 if (hno >= cf->head.flat_bucket)
384 cf->head.flat_bucket = hno+1;
386 return mf_write(cf->hash_mf, hno, off, sizeof(zint), &vno);
389 static int cf_moveto_flat(CFile cf)
391 struct CFile_hash_bucket *p;
395 yaz_log(YLOG_DEBUG, "cf: Moving to flat shadow: %s", cf->rmf->name);
396 yaz_log(YLOG_DEBUG, "cf: hits=%d miss=%d bucket_in_memory=" ZINT_FORMAT " total="
398 cf->no_hits, cf->no_miss, cf->bucket_in_memory,
399 cf->head.next_bucket - cf->head.first_bucket);
400 assert(cf->head.state == 1);
401 if (flush_bucket(cf, -1))
403 assert(cf->bucket_in_memory == 0);
404 p = (struct CFile_hash_bucket *) xmalloc(sizeof(*p));
405 for (i = cf->head.first_bucket; i < cf->head.next_bucket; i++)
407 if (mf_read(cf->hash_mf, i, 0, 0, &p->ph) != 1)
409 yaz_log(YLOG_FATAL|YLOG_ERRNO, "read bucket moveto flat");
413 for (j = 0; j < HASH_BUCKET && p->ph.vno[j]; j++)
415 if (cf_write_flat(cf, p->ph.no[j], p->ph.vno[j]))
432 static int cf_lookup(CFile cf, zint no, zint *vno)
434 if (cf->head.state > 1)
435 return cf_lookup_flat(cf, no, vno);
436 return cf_lookup_hash(cf, no, vno);
439 static zint cf_new_flat(CFile cf, zint no)
441 zint vno = (cf->head.next_block)++;
443 cf_write_flat(cf, no, vno);
447 static zint cf_new_hash(CFile cf, zint no)
449 int hno = cf_hash(cf, no);
450 struct CFile_hash_bucket *hbprev = NULL, *hb = cf->parray[hno];
451 zint *bucketpp = &cf->array[hno];
453 zint vno = (cf->head.next_block)++;
455 for (hb = cf->parray[hno]; hb; hb = hb->h_next)
456 if (!hb->ph.vno[HASH_BUCKET-1])
457 for (i = 0; i<HASH_BUCKET; i++)
469 for (hb = cf->parray[hno]; hb; hb = hb->h_next)
470 if (hb->ph.this_bucket == *bucketpp)
472 bucketpp = &hb->ph.next_bucket;
480 /* extra check ... */
481 for (hb = cf->bucket_lru_back; hb; hb = hb->lru_next)
483 if (hb->ph.this_bucket == *bucketpp)
485 yaz_log(YLOG_FATAL, "Found hash bucket on other chain");
491 hb = get_bucket(cf, *bucketpp, hno);
494 for (i = 0; i<HASH_BUCKET; i++)
502 bucketpp = &hb->ph.next_bucket;
507 hb = new_bucket(cf, bucketpp, hno);
516 zint cf_new(CFile cf, zint no)
518 if (cf->head.state > 1)
519 return cf_new_flat(cf, no);
520 if (cf->no_miss*2 > cf->no_hits)
522 if (cf_moveto_flat(cf))
524 assert(cf->head.state > 1);
525 return cf_new_flat(cf, no);
527 return cf_new_hash(cf, no);
531 /** \brief reads block from commit area
532 \param cf commit file
533 \param no block number
534 \param offset offset in block
535 \param nbytes number of bytes to read
536 \param buf buffer for content (if read was succesful)
537 \retval 0 block could not be fully read
538 \retval 1 block could be read
541 int cf_read(CFile cf, zint no, int offset, int nbytes, void *buf)
547 zebra_mutex_lock(&cf->mutex);
548 ret = cf_lookup(cf, no, &block);
549 zebra_mutex_unlock(&cf->mutex);
553 yaz_log(YLOG_FATAL, "cf_lookup failed");
558 /* block could not be read */
561 else if (mf_read(cf->block_mf, block, offset, nbytes, buf) != 1)
563 yaz_log(YLOG_FATAL|YLOG_ERRNO, "mf_read no=" ZINT_FORMAT " block=" ZINT_FORMAT, no, block);
569 /** \brief writes block to commit area
570 \param cf commit file
571 \param no block number
572 \param offset offset in block
573 \param nbytes number of bytes to be written
574 \param buf buffer to be written
575 \retval 0 block written
578 int cf_write(CFile cf, zint no, int offset, int nbytes, const void *buf)
584 zebra_mutex_lock(&cf->mutex);
586 ret = cf_lookup(cf, no, &block);
590 zebra_mutex_unlock(&cf->mutex);
595 block = cf_new(cf, no);
598 zebra_mutex_unlock(&cf->mutex);
601 if (offset || nbytes)
603 if (mf_read(cf->rmf, no, 0, 0, cf->iobuf) == -1)
605 memcpy(cf->iobuf + offset, buf, nbytes);
611 zebra_mutex_unlock(&cf->mutex);
612 return mf_write(cf->block_mf, block, offset, nbytes, buf);
615 int cf_close(CFile cf)
618 yaz_log(YLOG_DEBUG, "cf: close hits=%d miss=%d bucket_in_memory=" ZINT_FORMAT
619 " total=" ZINT_FORMAT,
620 cf->no_hits, cf->no_miss, cf->bucket_in_memory,
621 cf->head.next_bucket - cf->head.first_bucket);
622 if (flush_bucket(cf, -1))
628 if (mf_write(cf->hash_mf, 0, 0, sizeof(cf->head), &cf->head))
633 mf_close(cf->hash_mf);
636 mf_close(cf->block_mf);
640 zebra_mutex_destroy(&cf->mutex);
648 * indent-tabs-mode: nil
650 * vim: shiftwidth=4 tabstop=8 expandtab