1 /* $Id: cfile.c,v 1.35 2005-04-15 10:47:47 adam Exp $
2 Copyright (C) 1995-2005
5 This file is part of the Zebra server.
7 Zebra is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with Zebra; see the file LICENSE.zebra. If not, write to the
19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
27 #include <idzebra/util.h>
28 #include <yaz/yaz-util.h>
32 static int write_head (CFile cf)
34 int left = cf->head.hash_size * sizeof(zint);
36 const char *tab = (char*) cf->array;
40 while (left >= (int) HASH_BSIZE)
42 mf_write (cf->hash_mf, bno++, 0, 0, tab);
47 mf_write (cf->hash_mf, bno, 0, left, tab);
51 static int read_head (CFile cf)
53 int left = cf->head.hash_size * sizeof(zint);
55 char *tab = (char*) cf->array;
59 while (left >= (int) HASH_BSIZE)
61 mf_read (cf->hash_mf, bno++, 0, 0, tab);
66 mf_read (cf->hash_mf, bno, 0, left, tab);
71 CFile cf_open (MFile mf, MFile_area area, const char *fname,
72 int block_size, int wflag, int *firstp)
76 CFile cf = (CFile) xmalloc (sizeof(*cf));
80 yaz_log (YLOG_DEBUG, "cf: open %s %s", cf->rmf->name, wflag ? "rdwr" : "rd");
81 sprintf (path, "%s-b", fname);
82 if (!(cf->block_mf = mf_open (area, path, block_size, wflag)))
84 yaz_log (YLOG_FATAL|YLOG_ERRNO, "Failed to open %s", path);
87 sprintf (path, "%s-i", fname);
88 if (!(cf->hash_mf = mf_open (area, path, HASH_BSIZE, wflag)))
90 yaz_log (YLOG_FATAL|YLOG_ERRNO, "Failed to open %s", path);
94 if (!mf_read (cf->hash_mf, 0, 0, sizeof(cf->head), &cf->head) ||
99 cf->head.block_size = block_size;
100 cf->head.hash_size = 199;
101 hash_bytes = cf->head.hash_size * sizeof(zint);
102 cf->head.flat_bucket = cf->head.next_bucket = cf->head.first_bucket =
103 (hash_bytes+sizeof(cf->head))/HASH_BSIZE + 2;
104 cf->head.next_block = 1;
106 mf_write (cf->hash_mf, 0, 0, sizeof(cf->head), &cf->head);
107 cf->array = (zint *) xmalloc (hash_bytes);
108 for (i = 0; i<cf->head.hash_size; i++)
116 assert (cf->head.block_size == block_size);
117 assert (cf->head.hash_size > 2);
118 hash_bytes = cf->head.hash_size * sizeof(zint);
119 assert (cf->head.next_bucket > 0);
120 assert (cf->head.next_block > 0);
121 if (cf->head.state == 1)
122 cf->array = (zint *) xmalloc (hash_bytes);
127 if (cf->head.state == 1)
129 cf->parray = (struct CFile_hash_bucket **)
130 xmalloc (cf->head.hash_size * sizeof(*cf->parray));
131 for (i = 0; i<cf->head.hash_size; i++)
132 cf->parray[i] = NULL;
136 cf->bucket_lru_front = cf->bucket_lru_back = NULL;
137 cf->bucket_in_memory = 0;
138 cf->max_bucket_in_memory = 100;
140 cf->iobuf = (char *) xmalloc (cf->head.block_size);
141 memset (cf->iobuf, 0, cf->head.block_size);
144 zebra_mutex_init (&cf->mutex);
148 static int cf_hash (CFile cf, zint no)
150 return (int) (((no >> 3) % cf->head.hash_size));
153 static void release_bucket (CFile cf, struct CFile_hash_bucket *p)
156 p->lru_prev->lru_next = p->lru_next;
158 cf->bucket_lru_back = p->lru_next;
160 p->lru_next->lru_prev = p->lru_prev;
162 cf->bucket_lru_front = p->lru_prev;
164 *p->h_prev = p->h_next;
166 p->h_next->h_prev = p->h_prev;
168 --(cf->bucket_in_memory);
172 static void flush_bucket (CFile cf, int no_to_flush)
175 struct CFile_hash_bucket *p;
177 for (i = 0; i != no_to_flush; i++)
179 p = cf->bucket_lru_back;
184 mf_write (cf->hash_mf, p->ph.this_bucket, 0, 0, &p->ph);
187 release_bucket (cf, p);
191 static struct CFile_hash_bucket *alloc_bucket (CFile cf, zint block_no, int hno)
193 struct CFile_hash_bucket *p, **pp;
195 if (cf->bucket_in_memory == cf->max_bucket_in_memory)
196 flush_bucket (cf, 1);
197 assert (cf->bucket_in_memory < cf->max_bucket_in_memory);
198 ++(cf->bucket_in_memory);
199 p = (struct CFile_hash_bucket *) xmalloc (sizeof(*p));
202 p->lru_prev = cf->bucket_lru_front;
203 if (cf->bucket_lru_front)
204 cf->bucket_lru_front->lru_next = p;
206 cf->bucket_lru_back = p;
207 cf->bucket_lru_front = p;
209 pp = cf->parray + hno;
213 (*pp)->h_prev = &p->h_next;
218 static struct CFile_hash_bucket *get_bucket (CFile cf, zint block_no, int hno)
220 struct CFile_hash_bucket *p;
222 p = alloc_bucket (cf, block_no, hno);
223 if (!mf_read (cf->hash_mf, block_no, 0, 0, &p->ph))
225 yaz_log (YLOG_FATAL|YLOG_ERRNO, "read get_bucket");
228 assert (p->ph.this_bucket == block_no);
233 static struct CFile_hash_bucket *new_bucket (CFile cf, zint *block_nop, int hno)
235 struct CFile_hash_bucket *p;
239 block_no = *block_nop = cf->head.next_bucket++;
240 p = alloc_bucket (cf, block_no, hno);
242 for (i = 0; i<HASH_BUCKET; i++)
247 p->ph.next_bucket = 0;
248 p->ph.this_bucket = block_no;
253 static zint cf_lookup_flat (CFile cf, zint no)
255 zint hno = (no*sizeof(zint))/HASH_BSIZE;
256 int off = (int) ((no*sizeof(zint)) - hno*HASH_BSIZE);
259 mf_read (cf->hash_mf, hno+cf->head.next_bucket, off, sizeof(zint), &vno);
263 static zint cf_lookup_hash (CFile cf, zint no)
265 int hno = cf_hash (cf, no);
266 struct CFile_hash_bucket *hb;
270 for (hb = cf->parray[hno]; hb; hb = hb->h_next)
272 for (i = 0; i<HASH_BUCKET && hb->ph.vno[i]; i++)
273 if (hb->ph.no[i] == no)
276 return hb->ph.vno[i];
279 for (block_no = cf->array[hno]; block_no; block_no = hb->ph.next_bucket)
281 for (hb = cf->parray[hno]; hb; hb = hb->h_next)
283 if (hb->ph.this_bucket == block_no)
289 /* extra check ... */
290 for (hb = cf->bucket_lru_back; hb; hb = hb->lru_next)
292 if (hb->ph.this_bucket == block_no)
294 yaz_log (YLOG_FATAL, "Found hash bucket on other chain (1)");
297 for (i = 0; i<HASH_BUCKET && hb->ph.vno[i]; i++)
298 if (hb->ph.no[i] == no)
300 yaz_log (YLOG_FATAL, "Found hash bucket on other chain (2)");
306 hb = get_bucket (cf, block_no, hno);
307 for (i = 0; i<HASH_BUCKET && hb->ph.vno[i]; i++)
308 if (hb->ph.no[i] == no)
309 return hb->ph.vno[i];
314 static void cf_write_flat (CFile cf, zint no, zint vno)
316 zint hno = (no*sizeof(zint))/HASH_BSIZE;
317 int off = (int) ((no*sizeof(zint)) - hno*HASH_BSIZE);
319 hno += cf->head.next_bucket;
320 if (hno >= cf->head.flat_bucket)
321 cf->head.flat_bucket = hno+1;
323 mf_write (cf->hash_mf, hno, off, sizeof(zint), &vno);
326 static void cf_moveto_flat (CFile cf)
328 struct CFile_hash_bucket *p;
332 yaz_log (YLOG_DEBUG, "cf: Moving to flat shadow: %s", cf->rmf->name);
333 yaz_log (YLOG_DEBUG, "cf: hits=%d miss=%d bucket_in_memory=" ZINT_FORMAT " total="
335 cf->no_hits, cf->no_miss, cf->bucket_in_memory,
336 cf->head.next_bucket - cf->head.first_bucket);
337 assert (cf->head.state == 1);
338 flush_bucket (cf, -1);
339 assert (cf->bucket_in_memory == 0);
340 p = (struct CFile_hash_bucket *) xmalloc (sizeof(*p));
341 for (i = cf->head.first_bucket; i < cf->head.next_bucket; i++)
343 if (!mf_read (cf->hash_mf, i, 0, 0, &p->ph))
345 yaz_log (YLOG_FATAL|YLOG_ERRNO, "read bucket moveto flat");
348 for (j = 0; j < HASH_BUCKET && p->ph.vno[j]; j++)
349 cf_write_flat (cf, p->ph.no[j], p->ph.vno[j]);
360 static zint cf_lookup (CFile cf, zint no)
362 if (cf->head.state > 1)
363 return cf_lookup_flat (cf, no);
364 return cf_lookup_hash (cf, no);
367 static zint cf_new_flat (CFile cf, zint no)
369 zint vno = (cf->head.next_block)++;
371 cf_write_flat (cf, no, vno);
375 static zint cf_new_hash (CFile cf, zint no)
377 int hno = cf_hash (cf, no);
378 struct CFile_hash_bucket *hbprev = NULL, *hb = cf->parray[hno];
379 zint *bucketpp = &cf->array[hno];
381 zint vno = (cf->head.next_block)++;
383 for (hb = cf->parray[hno]; hb; hb = hb->h_next)
384 if (!hb->ph.vno[HASH_BUCKET-1])
385 for (i = 0; i<HASH_BUCKET; i++)
397 for (hb = cf->parray[hno]; hb; hb = hb->h_next)
398 if (hb->ph.this_bucket == *bucketpp)
400 bucketpp = &hb->ph.next_bucket;
408 /* extra check ... */
409 for (hb = cf->bucket_lru_back; hb; hb = hb->lru_next)
411 if (hb->ph.this_bucket == *bucketpp)
413 yaz_log (YLOG_FATAL, "Found hash bucket on other chain");
419 hb = get_bucket (cf, *bucketpp, hno);
421 for (i = 0; i<HASH_BUCKET; i++)
429 bucketpp = &hb->ph.next_bucket;
434 hb = new_bucket (cf, bucketpp, hno);
440 zint cf_new (CFile cf, zint no)
442 if (cf->head.state > 1)
443 return cf_new_flat (cf, no);
444 if (cf->no_miss*2 > cf->no_hits)
447 assert (cf->head.state > 1);
448 return cf_new_flat (cf, no);
450 return cf_new_hash (cf, no);
454 int cf_read (CFile cf, zint no, int offset, int nbytes, void *buf)
459 zebra_mutex_lock (&cf->mutex);
460 if (!(block = cf_lookup (cf, no)))
462 zebra_mutex_unlock (&cf->mutex);
465 zebra_mutex_unlock (&cf->mutex);
466 if (!mf_read (cf->block_mf, block, offset, nbytes, buf))
468 yaz_log (YLOG_FATAL|YLOG_ERRNO, "cf_read no=" ZINT_FORMAT " block=" ZINT_FORMAT, no, block);
474 int cf_write (CFile cf, zint no, int offset, int nbytes, const void *buf)
479 zebra_mutex_lock (&cf->mutex);
480 if (!(block = cf_lookup (cf, no)))
482 block = cf_new (cf, no);
483 if (offset || nbytes)
485 mf_read (cf->rmf, no, 0, 0, cf->iobuf);
486 memcpy (cf->iobuf + offset, buf, nbytes);
492 zebra_mutex_unlock (&cf->mutex);
493 if (mf_write (cf->block_mf, block, offset, nbytes, buf))
495 yaz_log (YLOG_FATAL|YLOG_ERRNO, "cf_write no=" ZINT_FORMAT
496 " block=" ZINT_FORMAT, no, block);
502 int cf_close (CFile cf)
504 yaz_log (YLOG_DEBUG, "cf: close hits=%d miss=%d bucket_in_memory=" ZINT_FORMAT
505 " total=" ZINT_FORMAT,
506 cf->no_hits, cf->no_miss, cf->bucket_in_memory,
507 cf->head.next_bucket - cf->head.first_bucket);
508 flush_bucket (cf, -1);
511 mf_write (cf->hash_mf, 0, 0, sizeof(cf->head), &cf->head);
514 mf_close (cf->hash_mf);
515 mf_close (cf->block_mf);
519 zebra_mutex_destroy (&cf->mutex);