1 /* $Id: extract.c,v 1.180 2005-04-28 09:32:09 adam Exp $
2 Copyright (C) 1995-2005
5 This file is part of the Zebra server.
7 Zebra is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with Zebra; see the file LICENSE.zebra. If not, write to the
19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
37 #if _FILE_OFFSET_BITS == 64
38 #define PRINTF_OFF_T "%Ld"
40 #define PRINTF_OFF_T "%ld"
43 #define USE_SHELLSORT 0
46 static void shellsort(void *ar, int r, size_t s,
47 int (*cmp)(const void *a, const void *b))
52 static const int incs[16] = { 1391376, 463792, 198768, 86961, 33936,
53 13776, 4592, 1968, 861, 336,
54 112, 48, 21, 7, 3, 1 };
55 for ( k = 0; k < 16; k++)
56 for (h = incs[k], i = h; i < r; i++)
60 while (j > h && (*cmp)(a + s*(j-h), v) > 0)
62 memcpy (a + s*j, a + s*(j-h), s);
70 static void logRecord (ZebraHandle zh)
72 ++zh->records_processed;
73 if (!(zh->records_processed % 1000))
75 yaz_log (YLOG_LOG, "Records: "ZINT_FORMAT" i/u/d "
76 ZINT_FORMAT"/"ZINT_FORMAT"/"ZINT_FORMAT,
77 zh->records_processed, zh->records_inserted, zh->records_updated,
82 static void extract_set_store_data_prepare(struct recExtractCtrl *p);
84 static void extract_init (struct recExtractCtrl *p, RecWord *w)
86 w->zebra_maps = p->zebra_maps;
88 w->attrSet = VAL_BIB1;
97 static const char **searchRecordKey (ZebraHandle zh,
98 struct recKeys *reckeys,
99 int attrSetS, int attrUseS)
101 static const char *ws[32];
102 void *decode_handle = iscz1_start();
108 for (i = 0; i<32; i++)
111 while (off < reckeys->buf_used)
113 const char *src = reckeys->buf + off;
115 char *dst = (char*) &key;
116 int attrSet, attrUse;
118 iscz1_decode(decode_handle, &dst, &src);
119 assert(key.len < 4 && key.len > 2);
121 attrSet = (int) key.mem[0] >> 16;
122 attrUse = (int) key.mem[0] & 65535;
123 seqno = (int) key.mem[key.len-1];
125 if (attrUseS == attrUse && attrSetS == attrSet)
131 woff = seqno - startSeq;
132 if (woff >= 0 && woff < 31)
138 off = src - reckeys->buf;
140 iscz1_stop(decode_handle);
141 assert (off == reckeys->buf_used);
145 struct file_read_info {
146 off_t file_max; /* maximum offset so far */
147 off_t file_offset; /* current offset */
148 off_t file_moffset; /* offset of rec/rec boundary */
153 static struct file_read_info *file_read_start (int fd)
155 struct file_read_info *fi = (struct file_read_info *)
156 xmalloc (sizeof(*fi));
160 fi->file_moffset = 0;
164 static void file_read_stop (struct file_read_info *fi)
169 static off_t file_seek (void *handle, off_t offset)
171 struct file_read_info *p = (struct file_read_info *) handle;
172 p->file_offset = offset;
173 return lseek (p->fd, offset, SEEK_SET);
176 static off_t file_tell (void *handle)
178 struct file_read_info *p = (struct file_read_info *) handle;
179 return p->file_offset;
182 static int file_read (void *handle, char *buf, size_t count)
184 struct file_read_info *p = (struct file_read_info *) handle;
187 r = read (fd, buf, count);
191 if (p->file_offset > p->file_max)
192 p->file_max = p->file_offset;
197 static void file_begin (void *handle)
199 struct file_read_info *p = (struct file_read_info *) handle;
201 p->file_offset = p->file_moffset;
203 lseek (p->fd, p->file_moffset, SEEK_SET);
207 static void file_end (void *handle, off_t offset)
209 struct file_read_info *p = (struct file_read_info *) handle;
211 assert (p->file_more == 0);
213 p->file_moffset = offset;
216 static char *fileMatchStr (ZebraHandle zh,
217 struct recKeys *reckeys,
218 const char *fname, const char *spec)
220 static char dstBuf[2048]; /* static here ??? */
222 const char *s = spec;
223 static const char **w;
227 while (*s == ' ' || *s == '\t')
233 char attset_str[64], attname_str[64];
234 data1_attset *attset;
237 int attSet = 1, attUse = 1;
241 for (i = 0; *s && *s != ',' && *s != ')'; s++)
243 attset_str[i++] = *s;
244 attset_str[i] = '\0';
249 for (i = 0; *s && *s != ')'; s++)
251 attname_str[i++] = *s;
252 attname_str[i] = '\0';
255 if ((attset = data1_get_attset (zh->reg->dh, attset_str)))
258 attSet = attset->reference;
259 att = data1_getattbyname(zh->reg->dh, attset, attname_str);
263 attUse = atoi (attname_str);
265 w = searchRecordKey (zh, reckeys, attSet, attUse);
270 for (i = 0; i<32; i++)
275 yaz_log (YLOG_WARN, "Missing ) in match criteria %s in group %s",
276 spec, zh->m_group ? zh->m_group : "none");
281 for (i = 0; i<32; i++)
282 if (matchFlag[i] && w[i])
294 yaz_log (YLOG_WARN, "Record didn't contain match"
295 " fields in (%s,%s)", attset_str, attname_str);
303 const char *spec_src = NULL;
304 const char *s1 = ++s;
305 while (*s1 && *s1 != ' ' && *s1 != '\t')
311 memcpy (special, s, spec_len);
312 special[spec_len] = '\0';
315 if (!strcmp (special, "group"))
316 spec_src = zh->m_group;
317 else if (!strcmp (special, "database"))
318 spec_src = zh->basenames[0];
319 else if (!strcmp (special, "filename")) {
322 else if (!strcmp (special, "type"))
323 spec_src = zh->m_record_type;
328 strcpy (dst, spec_src);
329 dst += strlen (spec_src);
332 else if (*s == '\"' || *s == '\'')
334 int stopMarker = *s++;
338 while (*s && *s != stopMarker)
341 tmpString[i++] = *s++;
346 strcpy (dst, tmpString);
347 dst += strlen (tmpString);
351 yaz_log (YLOG_WARN, "Syntax error in match criteria %s in group %s",
352 spec, zh->m_group ? zh->m_group : "none");
359 yaz_log (YLOG_WARN, "No match criteria for record %s in group %s",
360 fname, zh->m_group ? zh->m_group : "none");
367 struct recordLogInfo {
370 struct recordGroup *rGroup;
373 void create_rec_keys_codec(struct recKeys *keys)
376 iscz1_reset(keys->codec_handle);
379 static int file_extract_record(ZebraHandle zh,
380 SYSNO *sysno, const char *fname,
382 struct file_read_info *fi,
385 void *recTypeClientData)
387 RecordAttr *recordAttr;
389 const char *matchStr = 0;
392 off_t recordOffset = 0;
394 /* announce database */
395 if (zebraExplain_curDatabase (zh->reg->zei, zh->basenames[0]))
397 if (zebraExplain_newDatabase (zh->reg->zei, zh->basenames[0],
398 zh->m_explain_database))
404 struct recExtractCtrl extractCtrl;
406 /* we are going to read from a file, so prepare the extraction */
409 create_rec_keys_codec(&zh->reg->keys);
411 zh->reg->sortKeys.buf_used = 0;
414 recordOffset = fi->file_moffset;
415 extractCtrl.handle = zh;
416 extractCtrl.offset = fi->file_moffset;
417 extractCtrl.readf = file_read;
418 extractCtrl.seekf = file_seek;
419 extractCtrl.tellf = file_tell;
420 extractCtrl.endf = file_end;
422 extractCtrl.init = extract_init;
423 extractCtrl.tokenAdd = extract_token_add;
424 extractCtrl.schemaAdd = extract_schema_add;
425 extractCtrl.dh = zh->reg->dh;
426 extractCtrl.match_criteria[0] = '\0';
427 extractCtrl.first_record = fi->file_offset ? 0 : 1;
429 extract_set_store_data_prepare(&extractCtrl);
431 for (i = 0; i<256; i++)
433 if (zebra_maps_is_positioned(zh->reg->zebra_maps, i))
434 extractCtrl.seqno[i] = 1;
436 extractCtrl.seqno[i] = 0;
438 extractCtrl.zebra_maps = zh->reg->zebra_maps;
439 extractCtrl.flagShowRecords = !zh->m_flag_rw;
442 printf ("File: %s " PRINTF_OFF_T "\n", fname, recordOffset);
446 sprintf (msg, "%s:" PRINTF_OFF_T , fname, recordOffset);
447 yaz_log_init_prefix2 (msg);
450 r = (*recType->extract)(recTypeClientData, &extractCtrl);
452 yaz_log_init_prefix2 (0);
453 if (r == RECCTRL_EXTRACT_EOF)
455 else if (r == RECCTRL_EXTRACT_ERROR_GENERIC)
457 /* error occured during extraction ... */
459 zh->records_processed < zh->m_file_verbose_limit)
461 yaz_log (YLOG_WARN, "fail %s %s " PRINTF_OFF_T, zh->m_record_type,
462 fname, recordOffset);
466 else if (r == RECCTRL_EXTRACT_ERROR_NO_SUCH_FILTER)
468 /* error occured during extraction ... */
470 zh->records_processed < zh->m_file_verbose_limit)
472 yaz_log (YLOG_WARN, "no filter for %s %s "
473 PRINTF_OFF_T, zh->m_record_type,
474 fname, recordOffset);
478 if (extractCtrl.match_criteria[0])
479 matchStr = extractCtrl.match_criteria;
482 /* perform match if sysno not known and if match criteria is specified */
488 if (matchStr == 0 && zh->m_record_id && *zh->m_record_id)
491 matchStr = fileMatchStr (zh, &zh->reg->keys, fname,
495 yaz_log(YLOG_WARN, "Bad match criteria");
501 char *rinfo = dict_lookup (zh->reg->matchDict, matchStr);
504 assert(*rinfo == sizeof(*sysno));
505 memcpy (sysno, rinfo+1, sizeof(*sysno));
509 if (! *sysno && zh->reg->keys.buf_used == 0)
511 /* the extraction process returned no information - the record
512 is probably empty - unless flagShowRecords is in use */
516 if (zh->records_processed < zh->m_file_verbose_limit)
517 yaz_log (YLOG_WARN, "empty %s %s " PRINTF_OFF_T, zh->m_record_type,
518 fname, recordOffset);
527 yaz_log (YLOG_LOG, "delete %s %s " PRINTF_OFF_T, zh->m_record_type,
528 fname, recordOffset);
529 yaz_log (YLOG_WARN, "cannot delete record above (seems new)");
532 if (zh->records_processed < zh->m_file_verbose_limit)
533 yaz_log (YLOG_LOG, "add %s %s " PRINTF_OFF_T, zh->m_record_type,
534 fname, recordOffset);
535 rec = rec_new (zh->reg->records);
539 recordAttr = rec_init_attr (zh->reg->zei, rec);
543 dict_insert (zh->reg->matchDict, matchStr, sizeof(*sysno), sysno);
545 extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys);
546 extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys);
548 zh->records_inserted++;
552 /* record already exists */
553 struct recKeys delkeys;
554 struct sortKeys sortKeys;
556 rec = rec_get (zh->reg->records, *sysno);
559 recordAttr = rec_init_attr (zh->reg->zei, rec);
561 if (!force_update && recordAttr->runNumber ==
562 zebraExplain_runNumberIncrement (zh->reg->zei, 0))
564 yaz_log (YLOG_LOG, "run number = " ZINT_FORMAT,
565 recordAttr->runNumber);
566 yaz_log (YLOG_LOG, "skipped %s %s " PRINTF_OFF_T,
567 zh->m_record_type, fname, recordOffset);
568 extract_flushSortKeys (zh, *sysno, -1, &zh->reg->sortKeys);
573 delkeys.buf_used = rec->size[recInfo_delKeys];
574 delkeys.buf = rec->info[recInfo_delKeys];
576 sortKeys.buf_used = rec->size[recInfo_sortKeys];
577 sortKeys.buf = rec->info[recInfo_sortKeys];
579 extract_flushSortKeys (zh, *sysno, 0, &sortKeys);
580 extract_flushRecordKeys (zh, *sysno, 0, &delkeys);
583 /* record going to be deleted */
584 if (!delkeys.buf_used)
586 yaz_log (YLOG_LOG, "delete %s %s " PRINTF_OFF_T,
587 zh->m_record_type, fname, recordOffset);
588 yaz_log (YLOG_WARN, "cannot delete file above, storeKeys false");
592 if (zh->records_processed < zh->m_file_verbose_limit)
593 yaz_log (YLOG_LOG, "delete %s %s " PRINTF_OFF_T,
594 zh->m_record_type, fname, recordOffset);
595 zh->records_deleted++;
597 dict_delete (zh->reg->matchDict, matchStr);
598 rec_del (zh->reg->records, &rec);
606 /* record going to be updated */
607 if (!delkeys.buf_used)
609 yaz_log (YLOG_LOG, "update %s %s " PRINTF_OFF_T,
610 zh->m_record_type, fname, recordOffset);
611 yaz_log (YLOG_WARN, "cannot update file above, storeKeys false");
615 if (zh->records_processed < zh->m_file_verbose_limit)
616 yaz_log (YLOG_LOG, "update %s %s " PRINTF_OFF_T,
617 zh->m_record_type, fname, recordOffset);
618 extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys);
619 extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys);
620 zh->records_updated++;
624 /* update file type */
625 xfree (rec->info[recInfo_fileType]);
626 rec->info[recInfo_fileType] =
627 rec_strdup (zh->m_record_type, &rec->size[recInfo_fileType]);
629 /* update filename */
630 xfree (rec->info[recInfo_filename]);
631 rec->info[recInfo_filename] =
632 rec_strdup (fname, &rec->size[recInfo_filename]);
634 /* update delete keys */
635 xfree (rec->info[recInfo_delKeys]);
636 if (zh->reg->keys.buf_used > 0 && zh->m_store_keys == 1)
638 rec->size[recInfo_delKeys] = zh->reg->keys.buf_used;
639 rec->info[recInfo_delKeys] = zh->reg->keys.buf;
640 zh->reg->keys.buf = NULL;
641 zh->reg->keys.buf_max = 0;
645 rec->info[recInfo_delKeys] = NULL;
646 rec->size[recInfo_delKeys] = 0;
649 /* update sort keys */
650 xfree (rec->info[recInfo_sortKeys]);
652 rec->size[recInfo_sortKeys] = zh->reg->sortKeys.buf_used;
653 rec->info[recInfo_sortKeys] = zh->reg->sortKeys.buf;
654 zh->reg->sortKeys.buf = NULL;
655 zh->reg->sortKeys.buf_max = 0;
657 /* save file size of original record */
658 zebraExplain_recordBytesIncrement (zh->reg->zei,
659 - recordAttr->recordSize);
660 recordAttr->recordSize = fi->file_moffset - recordOffset;
661 if (!recordAttr->recordSize)
662 recordAttr->recordSize = fi->file_max - recordOffset;
663 zebraExplain_recordBytesIncrement (zh->reg->zei,
664 recordAttr->recordSize);
666 /* set run-number for this record */
667 recordAttr->runNumber = zebraExplain_runNumberIncrement (zh->reg->zei,
670 /* update store data */
671 xfree (rec->info[recInfo_storeData]);
672 if (zh->store_data_buf)
674 rec->size[recInfo_storeData] = zh->store_data_size;
675 rec->info[recInfo_storeData] = zh->store_data_buf;
676 zh->store_data_buf = 0;
677 file_end(fi, fi->file_offset);
679 else if (zh->m_store_data)
681 rec->size[recInfo_storeData] = recordAttr->recordSize;
682 rec->info[recInfo_storeData] = (char *)
683 xmalloc (recordAttr->recordSize);
684 if (lseek (fi->fd, recordOffset, SEEK_SET) < 0)
686 yaz_log (YLOG_ERRNO|YLOG_FATAL, "seek to " PRINTF_OFF_T " in %s",
687 recordOffset, fname);
690 if (read (fi->fd, rec->info[recInfo_storeData], recordAttr->recordSize)
691 < recordAttr->recordSize)
693 yaz_log (YLOG_ERRNO|YLOG_FATAL, "read %d bytes of %s",
694 recordAttr->recordSize, fname);
700 rec->info[recInfo_storeData] = NULL;
701 rec->size[recInfo_storeData] = 0;
703 /* update database name */
704 xfree (rec->info[recInfo_databaseName]);
705 rec->info[recInfo_databaseName] =
706 rec_strdup (zh->basenames[0], &rec->size[recInfo_databaseName]);
709 recordAttr->recordOffset = recordOffset;
711 /* commit this record */
712 rec_put (zh->reg->records, &rec);
717 int fileExtract (ZebraHandle zh, SYSNO *sysno, const char *fname,
724 struct file_read_info *fi;
725 const char *original_record_type = 0;
727 void *recTypeClientData;
729 if (!zh->m_group || !*zh->m_group)
732 sprintf (gprefix, "%s.", zh->m_group);
734 yaz_log (YLOG_DEBUG, "fileExtract %s", fname);
736 /* determine file extension */
738 for (i = strlen(fname); --i >= 0; )
741 else if (fname[i] == '.')
743 strcpy (ext, fname+i+1);
746 /* determine file type - depending on extension */
747 original_record_type = zh->m_record_type;
748 if (!zh->m_record_type)
750 sprintf (ext_res, "%srecordType.%s", gprefix, ext);
751 zh->m_record_type = res_get (zh->res, ext_res);
753 if (!zh->m_record_type)
755 if (zh->records_processed < zh->m_file_verbose_limit)
756 yaz_log (YLOG_LOG, "? %s", fname);
759 /* determine match criteria */
760 if (!zh->m_record_id)
762 sprintf (ext_res, "%srecordId.%s", gprefix, ext);
763 zh->m_record_id = res_get (zh->res, ext_res);
767 recType_byName (zh->reg->recTypes, zh->res, zh->m_record_type,
768 &recTypeClientData)))
770 yaz_log(YLOG_WARN, "No such record type: %s", zh->m_record_type);
774 switch(recType->version)
779 yaz_log(YLOG_WARN, "Bad filter version: %s", zh->m_record_type);
781 if (sysno && deleteFlag)
787 if (zh->path_reg && !yaz_is_abspath (fname))
789 strcpy (full_rep, zh->path_reg);
790 strcat (full_rep, "/");
791 strcat (full_rep, fname);
794 strcpy (full_rep, fname);
797 if ((fd = open (full_rep, O_BINARY|O_RDONLY)) == -1)
799 yaz_log (YLOG_WARN|YLOG_ERRNO, "open %s", full_rep);
800 zh->m_record_type = original_record_type;
804 fi = file_read_start (fd);
808 r = file_extract_record (zh, sysno, fname, deleteFlag, fi, 1,
809 recType, recTypeClientData);
811 while (r && !sysno && fi->file_more);
815 zh->m_record_type = original_record_type;
820 If sysno is provided, then it's used to identify the reocord.
821 If not, and match_criteria is provided, then sysno is guessed
822 If not, and a record is provided, then sysno is got from there
825 ZEBRA_RES buffer_extract_record (ZebraHandle zh,
826 const char *buf, size_t buf_size,
829 const char *recordType,
831 const char *match_criteria,
836 RecordAttr *recordAttr;
837 struct recExtractCtrl extractCtrl;
839 const char *matchStr = 0;
840 RecType recType = NULL;
843 long recordOffset = 0;
844 struct zebra_fetch_control fc;
845 const char *pr_fname = fname; /* filename to print .. */
846 int show_progress = zh->records_processed < zh->m_file_verbose_limit ? 1:0;
849 pr_fname = "<no file>"; /* make it printable if file is omitted */
852 fc.record_int_buf = buf;
853 fc.record_int_len = buf_size;
854 fc.record_int_pos = 0;
856 fc.record_offset = 0;
858 extractCtrl.offset = 0;
859 extractCtrl.readf = zebra_record_int_read;
860 extractCtrl.seekf = zebra_record_int_seek;
861 extractCtrl.tellf = zebra_record_int_tell;
862 extractCtrl.endf = zebra_record_int_end;
863 extractCtrl.first_record = 1;
864 extractCtrl.fh = &fc;
866 create_rec_keys_codec(&zh->reg->keys);
868 zh->reg->sortKeys.buf_used = 0;
870 if (zebraExplain_curDatabase (zh->reg->zei, zh->basenames[0]))
872 if (zebraExplain_newDatabase (zh->reg->zei, zh->basenames[0],
873 zh->m_explain_database))
877 if (recordType && *recordType)
879 yaz_log (YLOG_DEBUG, "Record type explicitly specified: %s", recordType);
880 recType = recType_byName (zh->reg->recTypes, zh->res, recordType,
885 if (!(zh->m_record_type))
887 yaz_log (YLOG_WARN, "No such record type defined");
890 yaz_log (YLOG_DEBUG, "Get record type from rgroup: %s",zh->m_record_type);
891 recType = recType_byName (zh->reg->recTypes, zh->res,
892 zh->m_record_type, &clientData);
893 recordType = zh->m_record_type;
898 yaz_log (YLOG_WARN, "No such record type: %s", zh->m_record_type);
902 extractCtrl.init = extract_init;
903 extractCtrl.tokenAdd = extract_token_add;
904 extractCtrl.schemaAdd = extract_schema_add;
905 extractCtrl.dh = zh->reg->dh;
906 extractCtrl.handle = zh;
907 extractCtrl.zebra_maps = zh->reg->zebra_maps;
908 extractCtrl.flagShowRecords = 0;
909 extractCtrl.match_criteria[0] = '\0';
910 for (i = 0; i<256; i++)
912 if (zebra_maps_is_positioned(zh->reg->zebra_maps, i))
913 extractCtrl.seqno[i] = 1;
915 extractCtrl.seqno[i] = 0;
917 extract_set_store_data_prepare(&extractCtrl);
919 r = (*recType->extract)(clientData, &extractCtrl);
921 if (r == RECCTRL_EXTRACT_EOF)
923 else if (r == RECCTRL_EXTRACT_ERROR_GENERIC)
925 /* error occured during extraction ... */
926 yaz_log (YLOG_WARN, "extract error: generic");
929 else if (r == RECCTRL_EXTRACT_ERROR_NO_SUCH_FILTER)
931 /* error occured during extraction ... */
932 yaz_log (YLOG_WARN, "extract error: no such filter");
938 if (extractCtrl.match_criteria[0])
939 match_criteria = extractCtrl.match_criteria;
943 if (match_criteria && *match_criteria) {
944 matchStr = match_criteria;
946 if (zh->m_record_id && *zh->m_record_id) {
947 matchStr = fileMatchStr (zh, &zh->reg->keys, pr_fname,
951 yaz_log (YLOG_WARN, "Bad match criteria (recordID)");
957 rinfo = dict_lookup (zh->reg->matchDict, matchStr);
960 assert(*rinfo == sizeof(*sysno));
961 memcpy (sysno, rinfo+1, sizeof(*sysno));
965 if (zh->reg->keys.buf_used == 0)
967 /* the extraction process returned no information - the record
968 is probably empty - unless flagShowRecords is in use */
979 yaz_log (YLOG_LOG, "delete %s %s %ld", recordType,
980 pr_fname, (long) recordOffset);
981 yaz_log (YLOG_WARN, "cannot delete record above (seems new)");
985 yaz_log (YLOG_LOG, "add %s %s %ld", recordType, pr_fname,
986 (long) recordOffset);
987 rec = rec_new (zh->reg->records);
991 recordAttr = rec_init_attr (zh->reg->zei, rec);
995 dict_insert (zh->reg->matchDict, matchStr,
996 sizeof(*sysno), sysno);
998 extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys);
999 extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys);
1001 zh->records_inserted++;
1005 /* record already exists */
1006 struct recKeys delkeys;
1007 struct sortKeys sortKeys;
1012 yaz_log (YLOG_LOG, "skipped %s %s %ld",
1013 recordType, pr_fname, (long) recordOffset);
1018 rec = rec_get (zh->reg->records, *sysno);
1021 recordAttr = rec_init_attr (zh->reg->zei, rec);
1023 if (!force_update) {
1024 if (recordAttr->runNumber ==
1025 zebraExplain_runNumberIncrement (zh->reg->zei, 0))
1028 yaz_log (YLOG_LOG, "skipped %s %s %ld", recordType,
1029 pr_fname, (long) recordOffset);
1030 extract_flushSortKeys (zh, *sysno, -1, &zh->reg->sortKeys);
1037 delkeys.buf_used = rec->size[recInfo_delKeys];
1038 delkeys.buf = rec->info[recInfo_delKeys];
1040 sortKeys.buf_used = rec->size[recInfo_sortKeys];
1041 sortKeys.buf = rec->info[recInfo_sortKeys];
1043 extract_flushSortKeys (zh, *sysno, 0, &sortKeys);
1044 extract_flushRecordKeys (zh, *sysno, 0, &delkeys);
1047 /* record going to be deleted */
1048 if (!delkeys.buf_used)
1052 yaz_log (YLOG_LOG, "delete %s %s %ld", recordType,
1053 pr_fname, (long) recordOffset);
1054 yaz_log (YLOG_WARN, "cannot delete file above, "
1061 yaz_log (YLOG_LOG, "delete %s %s %ld", recordType,
1062 pr_fname, (long) recordOffset);
1063 zh->records_deleted++;
1065 dict_delete (zh->reg->matchDict, matchStr);
1066 rec_del (zh->reg->records, &rec);
1074 /* record going to be updated */
1075 if (!delkeys.buf_used)
1079 yaz_log (YLOG_LOG, "update %s %s %ld", recordType,
1080 pr_fname, (long) recordOffset);
1081 yaz_log (YLOG_WARN, "cannot update file above, storeKeys false");
1087 yaz_log (YLOG_LOG, "update %s %s %ld", recordType,
1088 pr_fname, (long) recordOffset);
1089 extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys);
1090 extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys);
1091 zh->records_updated++;
1095 /* update file type */
1096 xfree (rec->info[recInfo_fileType]);
1097 rec->info[recInfo_fileType] =
1098 rec_strdup (recordType, &rec->size[recInfo_fileType]);
1100 /* update filename */
1101 xfree (rec->info[recInfo_filename]);
1102 rec->info[recInfo_filename] =
1103 rec_strdup (fname, &rec->size[recInfo_filename]);
1105 /* update delete keys */
1106 xfree (rec->info[recInfo_delKeys]);
1107 if (zh->reg->keys.buf_used > 0 && zh->m_store_keys == 1)
1109 rec->size[recInfo_delKeys] = zh->reg->keys.buf_used;
1110 rec->info[recInfo_delKeys] = zh->reg->keys.buf;
1111 zh->reg->keys.buf = NULL;
1112 zh->reg->keys.buf_max = 0;
1116 rec->info[recInfo_delKeys] = NULL;
1117 rec->size[recInfo_delKeys] = 0;
1120 /* update sort keys */
1121 xfree (rec->info[recInfo_sortKeys]);
1123 rec->size[recInfo_sortKeys] = zh->reg->sortKeys.buf_used;
1124 rec->info[recInfo_sortKeys] = zh->reg->sortKeys.buf;
1125 zh->reg->sortKeys.buf = NULL;
1126 zh->reg->sortKeys.buf_max = 0;
1128 /* save file size of original record */
1129 zebraExplain_recordBytesIncrement (zh->reg->zei,
1130 - recordAttr->recordSize);
1132 recordAttr->recordSize = fi->file_moffset - recordOffset;
1133 if (!recordAttr->recordSize)
1134 recordAttr->recordSize = fi->file_max - recordOffset;
1136 recordAttr->recordSize = buf_size;
1138 zebraExplain_recordBytesIncrement (zh->reg->zei,
1139 recordAttr->recordSize);
1141 /* set run-number for this record */
1142 recordAttr->runNumber =
1143 zebraExplain_runNumberIncrement (zh->reg->zei, 0);
1145 /* update store data */
1146 xfree (rec->info[recInfo_storeData]);
1147 if (zh->m_store_data)
1149 rec->size[recInfo_storeData] = recordAttr->recordSize;
1150 rec->info[recInfo_storeData] = (char *)
1151 xmalloc (recordAttr->recordSize);
1152 memcpy (rec->info[recInfo_storeData], buf, recordAttr->recordSize);
1156 rec->info[recInfo_storeData] = NULL;
1157 rec->size[recInfo_storeData] = 0;
1159 /* update database name */
1160 xfree (rec->info[recInfo_databaseName]);
1161 rec->info[recInfo_databaseName] =
1162 rec_strdup (zh->basenames[0], &rec->size[recInfo_databaseName]);
1165 recordAttr->recordOffset = recordOffset;
1167 /* commit this record */
1168 rec_put (zh->reg->records, &rec);
1173 int explain_extract (void *handle, Record rec, data1_node *n)
1175 ZebraHandle zh = (ZebraHandle) handle;
1176 struct recExtractCtrl extractCtrl;
1179 if (zebraExplain_curDatabase (zh->reg->zei,
1180 rec->info[recInfo_databaseName]))
1183 if (zebraExplain_newDatabase (zh->reg->zei,
1184 rec->info[recInfo_databaseName], 0))
1188 create_rec_keys_codec(&zh->reg->keys);
1190 zh->reg->sortKeys.buf_used = 0;
1192 extractCtrl.init = extract_init;
1193 extractCtrl.tokenAdd = extract_token_add;
1194 extractCtrl.schemaAdd = extract_schema_add;
1195 extractCtrl.dh = zh->reg->dh;
1196 for (i = 0; i<256; i++)
1197 extractCtrl.seqno[i] = 0;
1198 extractCtrl.zebra_maps = zh->reg->zebra_maps;
1199 extractCtrl.flagShowRecords = 0;
1200 extractCtrl.match_criteria[0] = '\0';
1201 extractCtrl.handle = handle;
1202 extractCtrl.first_record = 1;
1204 extract_set_store_data_prepare(&extractCtrl);
1207 grs_extract_tree(&extractCtrl, n);
1209 if (rec->size[recInfo_delKeys])
1211 struct recKeys delkeys;
1212 struct sortKeys sortkeys;
1214 delkeys.buf_used = rec->size[recInfo_delKeys];
1215 delkeys.buf = rec->info[recInfo_delKeys];
1217 sortkeys.buf_used = rec->size[recInfo_sortKeys];
1218 sortkeys.buf = rec->info[recInfo_sortKeys];
1220 extract_flushSortKeys (zh, rec->sysno, 0, &sortkeys);
1221 extract_flushRecordKeys (zh, rec->sysno, 0, &delkeys);
1223 extract_flushRecordKeys (zh, rec->sysno, 1, &zh->reg->keys);
1224 extract_flushSortKeys (zh, rec->sysno, 1, &zh->reg->sortKeys);
1226 xfree (rec->info[recInfo_delKeys]);
1227 rec->size[recInfo_delKeys] = zh->reg->keys.buf_used;
1228 rec->info[recInfo_delKeys] = zh->reg->keys.buf;
1229 zh->reg->keys.buf = NULL;
1230 zh->reg->keys.buf_max = 0;
1232 xfree (rec->info[recInfo_sortKeys]);
1233 rec->size[recInfo_sortKeys] = zh->reg->sortKeys.buf_used;
1234 rec->info[recInfo_sortKeys] = zh->reg->sortKeys.buf;
1235 zh->reg->sortKeys.buf = NULL;
1236 zh->reg->sortKeys.buf_max = 0;
1241 void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno,
1242 int cmd, struct recKeys *reckeys)
1244 void *decode_handle = iscz1_start();
1247 ZebraExplainInfo zei = zh->reg->zei;
1249 if (!zh->reg->key_buf)
1251 int mem= 1024*1024* atoi( res_get_def( zh->res, "memmax", "8"));
1254 yaz_log(YLOG_WARN, "Invalid memory setting, using default 8 MB");
1257 /* FIXME: That "8" should be in a default settings include */
1258 /* not hard-coded here! -H */
1259 zh->reg->key_buf = (char**) xmalloc (mem);
1260 zh->reg->ptr_top = mem/sizeof(char*);
1262 zh->reg->key_buf_used = 0;
1263 zh->reg->key_file_no = 0;
1265 zebraExplain_recordCountIncrement (zei, cmd ? 1 : -1);
1267 while (off < reckeys->buf_used)
1269 const char *src = reckeys->buf + off;
1271 char *dst = (char*) &key;
1273 iscz1_decode(decode_handle, &dst, &src);
1274 assert(key.len == 4);
1276 if (zh->reg->key_buf_used + 1024 >
1277 (zh->reg->ptr_top -zh->reg->ptr_i)*sizeof(char*))
1278 extract_flushWriteKeys (zh,0);
1280 assert(zh->reg->ptr_i > 0);
1281 (zh->reg->key_buf)[zh->reg->ptr_top - zh->reg->ptr_i] =
1282 (char*)zh->reg->key_buf + zh->reg->key_buf_used;
1284 ch = (int) key.mem[0]; /* ordinal for field/use/attribute */
1286 zh->reg->key_buf_used +=
1287 key_SU_encode (ch,((char*)zh->reg->key_buf) +
1288 zh->reg->key_buf_used);
1290 ((char*)zh->reg->key_buf) [(zh->reg->key_buf_used)++] = *src++;
1292 ((char*)(zh->reg->key_buf))[(zh->reg->key_buf_used)++] = '\0';
1293 ((char*)(zh->reg->key_buf))[(zh->reg->key_buf_used)++] = cmd;
1296 if (key.mem[1]) /* filter specified record ID */
1297 key.mem[0] = key.mem[1];
1300 key.mem[1] = key.mem[2]; /* section_id */
1301 key.mem[2] = key.mem[3]; /* sequence .. */
1303 memcpy ((char*)zh->reg->key_buf + zh->reg->key_buf_used,
1305 (zh->reg->key_buf_used) += sizeof(key);
1306 off = src - reckeys->buf;
1308 assert (off == reckeys->buf_used);
1309 iscz1_stop(decode_handle);
1312 void extract_flushWriteKeys (ZebraHandle zh, int final)
1313 /* optimizing: if final=1, and no files written yet */
1314 /* push the keys directly to merge, sidestepping the */
1315 /* temp file altogether. Speeds small updates */
1318 char out_fname[200];
1320 struct encode_info encode_info;
1321 int ptr_i = zh->reg->ptr_i;
1326 if (!zh->reg->key_buf || ptr_i <= 0)
1328 yaz_log (YLOG_DEBUG, " nothing to flush section=%d buf=%p i=%d",
1329 zh->reg->key_file_no, zh->reg->key_buf, ptr_i);
1330 yaz_log (YLOG_DEBUG, " buf=%p ",
1332 yaz_log (YLOG_DEBUG, " ptr=%d ",zh->reg->ptr_i);
1333 yaz_log (YLOG_DEBUG, " reg=%p ",zh->reg);
1338 (zh->reg->key_file_no)++;
1339 yaz_log (YLOG_LOG, "sorting section %d", (zh->reg->key_file_no));
1340 yaz_log (YLOG_DEBUG, " sort_buff at %p n=%d",
1341 zh->reg->key_buf + zh->reg->ptr_top - ptr_i,ptr_i);
1343 qsort (zh->reg->key_buf + zh->reg->ptr_top - ptr_i, ptr_i,
1344 sizeof(char*), key_qsort_compare);
1346 /* zebra.cfg: tempfiles:
1347 Y: always use temp files (old way)
1348 A: use temp files, if more than one (auto)
1349 = if this is both the last and the first
1350 N: never bother with temp files (new) */
1352 temp_policy=toupper(res_get_def(zh->res,"tempfiles","auto")[0]);
1353 if (temp_policy != 'Y' && temp_policy != 'N' && temp_policy != 'A') {
1354 yaz_log (YLOG_WARN, "Illegal tempfiles setting '%c'. using 'Auto' ",
1359 if ( ( temp_policy =='N' ) || /* always from memory */
1360 ( ( temp_policy =='A' ) && /* automatic */
1361 (zh->reg->key_file_no == 1) && /* this is first time */
1362 (final) ) ) /* and last (=only) time */
1363 { /* go directly from memory */
1364 zh->reg->key_file_no =0; /* signal not to read files */
1365 zebra_index_merge(zh);
1367 zh->reg->key_buf_used = 0;
1371 /* Not doing directly from memory, write into a temp file */
1372 extract_get_fname_tmp (zh, out_fname, zh->reg->key_file_no);
1374 if (!(outf = fopen (out_fname, "wb")))
1376 yaz_log (YLOG_FATAL|YLOG_ERRNO, "fopen %s", out_fname);
1379 yaz_log (YLOG_LOG, "writing section %d", zh->reg->key_file_no);
1380 prevcp = cp = (zh->reg->key_buf)[zh->reg->ptr_top - ptr_i];
1382 encode_key_init (&encode_info);
1383 encode_key_write (cp, &encode_info, outf);
1387 cp = (zh->reg->key_buf)[zh->reg->ptr_top - ptr_i];
1388 if (strcmp (cp, prevcp))
1390 encode_key_flush ( &encode_info, outf);
1391 encode_key_init (&encode_info);
1392 encode_key_write (cp, &encode_info, outf);
1396 encode_key_write (cp + strlen(cp), &encode_info, outf);
1398 encode_key_flush ( &encode_info, outf);
1400 qsort (key_buf + ptr_top-ptr_i, ptr_i, sizeof(char*), key_x_compare);
1401 extract_get_fname_tmp (out_fname, key_file_no);
1403 if (!(outf = fopen (out_fname, "wb")))
1405 yaz_log (YLOG_FATAL|YLOG_ERRNO, "fopen %s", out_fname);
1408 yaz_log (YLOG_LOG, "writing section %d", key_file_no);
1410 prevcp = key_buf[ptr_top-i];
1412 if (!--i || strcmp (prevcp, key_buf[ptr_top-i]))
1414 key_y_len = strlen(prevcp)+1;
1416 yaz_log (YLOG_LOG, "key_y_len: %2d %02x %02x %s",
1417 key_y_len, prevcp[0], prevcp[1], 2+prevcp);
1419 qsort (key_buf + ptr_top-ptr_i, ptr_i - i,
1420 sizeof(char*), key_y_compare);
1421 cp = key_buf[ptr_top-ptr_i];
1423 encode_key_init (&encode_info);
1424 encode_key_write (cp, &encode_info, outf);
1427 cp = key_buf[ptr_top-ptr_i];
1428 encode_key_write (cp+key_y_len, &encode_info, outf);
1430 encode_key_flush ( &encode_info, outf);
1433 prevcp = key_buf[ptr_top-ptr_i];
1438 yaz_log (YLOG_FATAL|YLOG_ERRNO, "fclose %s", out_fname);
1441 yaz_log (YLOG_LOG, "finished section %d", zh->reg->key_file_no);
1443 zh->reg->key_buf_used = 0;
1446 void extract_add_it_key (ZebraHandle zh,
1448 const char *str, int slen, struct it_key *key)
1451 struct recKeys *keys = &zh->reg->keys;
1452 const char *src = (char*) key;
1454 if (keys->buf_used+1024 > keys->buf_max)
1456 char *b = (char *) xmalloc (keys->buf_max += 128000);
1457 if (keys->buf_used > 0)
1458 memcpy (b, keys->buf, keys->buf_used);
1462 dst = keys->buf + keys->buf_used;
1464 iscz1_encode(keys->codec_handle, &dst, &src);
1467 memcpy (dst, str, slen);
1470 keys->buf_used = dst - keys->buf;
1473 void extract_add_index_string (RecWord *p, const char *str, int length)
1477 ZebraHandle zh = p->extractCtrl->handle;
1478 ZebraExplainInfo zei = zh->reg->zei;
1483 ch = zebraExplain_lookup_attr_str(zei, p->attrStr);
1485 ch = zebraExplain_add_attr_str(zei, p->attrStr);
1489 ch = zebraExplain_lookup_attr_su(zei, p->attrSet, p->attrUse);
1491 ch = zebraExplain_add_attr_su(zei, p->attrSet, p->attrUse);
1495 key.mem[1] = p->record_id;
1496 key.mem[2] = p->section_id;
1497 key.mem[3] = p->seqno;
1500 /* just for debugging .. */
1501 yaz_log(YLOG_LOG, "add: set=%d use=%d "
1502 "record_id=%lld section_id=%lld seqno=%lld",
1503 p->attrSet, p->attrUse, p->record_id, p->section_id, p->seqno);
1506 extract_add_it_key(p->extractCtrl->handle, p->reg_type, str,
1510 static void extract_add_sort_string (RecWord *p, const char *str,
1513 ZebraHandle zh = p->extractCtrl->handle;
1514 struct sortKeys *sk = &zh->reg->sortKeys;
1517 while (off < sk->buf_used)
1521 off += key_SU_decode(&set, sk->buf + off);
1522 off += key_SU_decode(&use, sk->buf + off);
1523 off += key_SU_decode(&slen, sk->buf + off);
1525 if (p->attrSet == set && p->attrUse == use)
1528 assert (off == sk->buf_used);
1530 if (sk->buf_used + IT_MAX_WORD > sk->buf_max)
1534 b = (char *) xmalloc (sk->buf_max += 128000);
1535 if (sk->buf_used > 0)
1536 memcpy (b, sk->buf, sk->buf_used);
1540 off += key_SU_encode(p->attrSet, sk->buf + off);
1541 off += key_SU_encode(p->attrUse, sk->buf + off);
1542 off += key_SU_encode(length, sk->buf + off);
1543 memcpy (sk->buf + off, str, length);
1544 sk->buf_used = off + length;
1547 void extract_add_string (RecWord *p, const char *string, int length)
1549 assert (length > 0);
1550 if (zebra_maps_is_sort (p->zebra_maps, p->reg_type))
1551 extract_add_sort_string (p, string, length);
1553 extract_add_index_string (p, string, length);
1556 static void extract_add_incomplete_field (RecWord *p)
1558 const char *b = p->term_buf;
1559 int remain = p->term_len;
1560 const char **map = 0;
1562 yaz_log(YLOG_DEBUG, "Incomplete field, w='%.*s'", p->term_len, p->term_buf);
1565 map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain, 0);
1569 char buf[IT_MAX_WORD+1];
1573 while (map && *map && **map == *CHR_SPACE)
1575 remain = p->term_len - (b - p->term_buf);
1577 map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain, 0);
1584 while (map && *map && **map != *CHR_SPACE)
1586 const char *cp = *map;
1588 while (i < IT_MAX_WORD && *cp)
1590 remain = p->term_len - (b - p->term_buf);
1592 map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain, 0);
1598 extract_add_string (p, buf, i);
1603 static void extract_add_complete_field (RecWord *p)
1605 const char *b = p->term_buf;
1606 char buf[IT_MAX_WORD+1];
1607 const char **map = 0;
1608 int i = 0, remain = p->term_len;
1610 yaz_log(YLOG_DEBUG, "Complete field, w='%.*s'",
1611 p->term_len, p->term_buf);
1614 map = zebra_maps_input (p->zebra_maps, p->reg_type, &b, remain, 1);
1616 while (remain > 0 && i < IT_MAX_WORD)
1618 while (map && *map && **map == *CHR_SPACE)
1620 remain = p->term_len - (b - p->term_buf);
1624 int first = i ? 0 : 1; /* first position */
1625 map = zebra_maps_input(p->zebra_maps, p->reg_type, &b, remain, first);
1633 if (i && i < IT_MAX_WORD)
1634 buf[i++] = *CHR_SPACE;
1635 while (map && *map && **map != *CHR_SPACE)
1637 const char *cp = *map;
1639 if (**map == *CHR_CUT)
1645 if (i >= IT_MAX_WORD)
1647 yaz_log(YLOG_DEBUG, "Adding string to index '%d'", **map);
1648 while (i < IT_MAX_WORD && *cp)
1651 remain = p->term_len - (b - p->term_buf);
1654 map = zebra_maps_input (p->zebra_maps, p->reg_type, &b,
1663 extract_add_string (p, buf, i);
1666 void extract_token_add (RecWord *p)
1670 yaz_log (YLOG_LOG, "token_add "
1671 "reg_type=%c attrSet=%d attrUse=%d seqno=%d s=%.*s",
1672 p->reg_type, p->attrSet, p->attrUse, p->seqno, p->length,
1675 if ((wrbuf = zebra_replace(p->zebra_maps, p->reg_type, 0,
1676 p->term_buf, p->term_len)))
1678 p->term_buf = wrbuf_buf(wrbuf);
1679 p->term_len = wrbuf_len(wrbuf);
1681 if (zebra_maps_is_complete (p->zebra_maps, p->reg_type))
1682 extract_add_complete_field (p);
1684 extract_add_incomplete_field(p);
1687 static void extract_set_store_data_cb(struct recExtractCtrl *p,
1688 void *buf, size_t sz)
1690 ZebraHandle zh = (ZebraHandle) p->handle;
1692 xfree(zh->store_data_buf);
1693 zh->store_data_buf = 0;
1694 zh->store_data_size = 0;
1697 zh->store_data_buf = xmalloc(sz);
1698 zh->store_data_size = sz;
1699 memcpy(zh->store_data_buf, buf, sz);
1703 static void extract_set_store_data_prepare(struct recExtractCtrl *p)
1705 ZebraHandle zh = (ZebraHandle) p->handle;
1706 xfree(zh->store_data_buf);
1707 zh->store_data_buf = 0;
1708 zh->store_data_size = 0;
1709 p->setStoreData = extract_set_store_data_cb;
1712 void extract_schema_add (struct recExtractCtrl *p, Odr_oid *oid)
1714 ZebraHandle zh = (ZebraHandle) p->handle;
1715 zebraExplain_addSchema (zh->reg->zei, oid);
1718 void extract_flushSortKeys (ZebraHandle zh, SYSNO sysno,
1719 int cmd, struct sortKeys *sk)
1721 SortIdx sortIdx = zh->reg->sortIdx;
1724 sortIdx_sysno (sortIdx, sysno);
1726 while (off < sk->buf_used)
1730 off += key_SU_decode(&set, sk->buf + off);
1731 off += key_SU_decode(&use, sk->buf + off);
1732 off += key_SU_decode(&slen, sk->buf + off);
1734 sortIdx_type(sortIdx, use);
1736 sortIdx_add(sortIdx, sk->buf + off, slen);
1738 sortIdx_add(sortIdx, "", 1);
1743 void encode_key_init (struct encode_info *i)
1752 i->encode_handle = iscz1_start();
1758 /* this is the old encode_key_write
1759 * may be deleted once we are confident that the new works
1762 void encode_key_write (char *k, struct encode_info *i, FILE *outf)
1765 char *bp = i->buf, *bp0;
1766 const char *src = (char *) &key;
1768 /* copy term to output buf */
1769 while ((*bp++ = *k++))
1771 /* and copy & align key so we can mangle */
1772 memcpy (&key, k+1, sizeof(struct it_key)); /* *k is insert/delete */
1775 iscz1_encode(i->encode_handle, &bp, &src);
1776 *bp0 = (*k * 128) + bp - bp0 - 1; /* length and insert/delete combined */
1777 if (fwrite (i->buf, bp - i->buf, 1, outf) != 1)
1779 yaz_log (YLOG_FATAL|YLOG_ERRNO, "fwrite");
1784 void encode_key_flush (struct encode_info *i, FILE *outf)
1785 { /* dummy routine */
1786 iscz1_stop(i->encode_handle);
1791 /* new encode_key_write
1792 * The idea is to buffer one more key, and compare them
1793 * If we are going to delete and insert the same key,
1794 * we may as well not bother. Should make a difference in
1795 * updates with small modifications (appending to a mbox)
1797 void encode_key_write (char *k, struct encode_info *i, FILE *outf)
1802 if (*k) /* first time for new key */
1805 while ((*bp++ = *k++))
1807 i->keylen= bp - i->buf -1;
1808 assert(i->keylen+1+sizeof(struct it_key) < ENCODE_BUFLEN);
1812 bp=i->buf + i->keylen;
1817 memcpy (&key, k+1, sizeof(struct it_key));
1818 if (0==i->prevsys) /* no previous filter, fill up */
1820 i->prevsys=key.sysno;
1821 i->prevseq=key.seqno;
1824 else if ( (i->prevsys==key.sysno) &&
1825 (i->prevseq==key.seqno) &&
1827 { /* same numbers, diff cmd, they cancel out */
1831 { /* different stuff, write previous, move buf */
1832 bp = encode_key_int ( (i->prevsys - i->sysno) * 2 + i->prevcmd, bp);
1833 if (i->sysno != i->prevsys)
1835 i->sysno = i->prevsys;
1838 else if (!i->seqno && !i->prevseq && i->cmd == i->prevcmd)
1840 return; /* ??? Filters some sort of duplicates away */
1841 /* ??? Can this ever happen -H 15oct02 */
1843 bp = encode_key_int (i->prevseq - i->seqno, bp);
1844 i->seqno = i->prevseq;
1845 i->cmd = i->prevcmd;
1846 if (fwrite (i->buf, bp - i->buf, 1, outf) != 1)
1848 yaz_log (YLOG_FATAL|YLOG_ERRNO, "fwrite");
1851 i->keylen=0; /* ok, it's written, forget it */
1852 i->prevsys=key.sysno;
1853 i->prevseq=key.seqno;
1858 void encode_key_flush (struct encode_info *i, FILE *outf)
1859 { /* flush the last key from i */
1860 char *bp =i->buf + i->keylen;
1863 return; /* nothing to flush */
1866 bp = encode_key_int ( (i->prevsys - i->sysno) * 2 + i->prevcmd, bp);
1867 if (i->sysno != i->prevsys)
1869 i->sysno = i->prevsys;
1872 else if (!i->seqno && !i->prevseq && i->cmd == i->prevcmd)
1874 return; /* ??? Filters some sort of duplicates away */
1875 /* ??? Can this ever happen -H 15oct02 */
1877 bp = encode_key_int (i->prevseq - i->seqno, bp);
1878 i->seqno = i->prevseq;
1879 i->cmd = i->prevcmd;
1880 if (fwrite (i->buf, bp - i->buf, 1, outf) != 1)
1882 yaz_log (YLOG_FATAL|YLOG_ERRNO, "fwrite");
1885 i->keylen=0; /* ok, it's written, forget it */
1886 i->prevsys=0; /* forget the values too */