1 /* This file is part of the Zebra server.
2 Copyright (C) 1994-2011 Index Data
4 Zebra is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
9 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
21 \brief indexes records and extract tokens for indexing and sorting
43 #include <yaz/snprintf.h>
45 static int log_level_extract = 0;
46 static int log_level_details = 0;
47 static int log_level_initialized = 0;
49 /* 1 if we use eliminitate identical delete/insert keys */
50 /* eventually this the 0-case code will be removed */
54 static void extract_flush_record_keys2(ZebraHandle zh, zint sysno,
55 zebra_rec_keys_t ins_keys,
57 zebra_rec_keys_t del_keys,
60 static void extract_flush_record_keys(ZebraHandle zh, zint sysno,
62 zebra_rec_keys_t reckeys,
66 static void zebra_init_log_level(void)
68 if (!log_level_initialized)
70 log_level_initialized = 1;
72 log_level_extract = yaz_log_module_level("extract");
73 log_level_details = yaz_log_module_level("indexdetails");
77 static WRBUF wrbuf_hex_str(const char *cstr)
80 WRBUF w = wrbuf_alloc();
81 for (i = 0; cstr[i]; i++)
83 if (cstr[i] < ' ' || cstr[i] > 126)
84 wrbuf_printf(w, "\\%02X", cstr[i] & 0xff);
86 wrbuf_putc(w, cstr[i]);
92 static void extract_flush_sort_keys(ZebraHandle zh, zint sysno,
93 int cmd, zebra_rec_keys_t skp);
94 static void extract_schema_add(struct recExtractCtrl *p, Odr_oid *oid);
95 static void extract_token_add(RecWord *p);
97 static void check_log_limit(ZebraHandle zh)
99 if (zh->records_processed + zh->records_skipped == zh->m_file_verbose_limit)
101 yaz_log(YLOG_LOG, "More than %d file log entries. Omitting rest",
102 zh->m_file_verbose_limit);
106 static void logRecord(ZebraHandle zh)
109 ++zh->records_processed;
110 if (!(zh->records_processed % 1000))
112 yaz_log(YLOG_LOG, "Records: "ZINT_FORMAT" i/u/d "
113 ZINT_FORMAT"/"ZINT_FORMAT"/"ZINT_FORMAT,
114 zh->records_processed, zh->records_inserted,
115 zh->records_updated, zh->records_deleted);
119 static void init_extractCtrl(ZebraHandle zh, struct recExtractCtrl *ctrl)
121 ctrl->flagShowRecords = !zh->m_flag_rw;
125 static void extract_add_index_string(RecWord *p,
126 zinfo_index_category_t cat,
127 const char *str, int length);
129 static void extract_set_store_data_prepare(struct recExtractCtrl *p);
131 static void extract_init(struct recExtractCtrl *p, RecWord *w)
134 w->index_name = "any";
142 struct snip_rec_info {
144 zebra_snippets *snippets;
148 static void snippet_add_complete_field(RecWord *p, int ord,
151 struct snip_rec_info *h = p->extractCtrl->handle;
153 const char *b = p->term_buf;
154 char buf[IT_MAX_WORD+1];
155 const char **map = 0;
156 int i = 0, remain = p->term_len;
157 const char *start = b;
158 const char *last = 0;
161 map = zebra_maps_input(zm, &b, remain, 1);
163 while (remain > 0 && i < IT_MAX_WORD)
165 while (map && *map && **map == *CHR_SPACE)
167 remain = p->term_len - (b - p->term_buf);
170 start = b; /* set to first non-ws area */
173 int first = i ? 0 : 1; /* first position */
175 map = zebra_maps_input(zm, &b, remain, first);
183 if (i && i < IT_MAX_WORD)
184 buf[i++] = *CHR_SPACE;
185 while (map && *map && **map != *CHR_SPACE)
187 const char *cp = *map;
189 if (**map == *CHR_CUT)
195 if (i >= IT_MAX_WORD)
197 while (i < IT_MAX_WORD && *cp)
201 remain = p->term_len - (b - p->term_buf);
204 map = zebra_maps_input(zm, &b, remain, 0);
212 if (last && start != last && zebra_maps_is_index(zm))
213 zebra_snippets_appendn(h->snippets, p->seqno, 0, ord,
214 start, last - start);
217 static void snippet_add_incomplete_field(RecWord *p, int ord, zebra_map_t zm)
219 struct snip_rec_info *h = p->extractCtrl->handle;
220 const char *b = p->term_buf;
221 int remain = p->term_len;
223 const char **map = 0;
224 const char *start = b;
225 const char *last = b;
228 map = zebra_maps_input(zm, &b, remain, 0);
232 char buf[IT_MAX_WORD+1];
236 while (map && *map && **map == *CHR_SPACE)
238 remain = p->term_len - (b - p->term_buf);
241 map = zebra_maps_input(zm, &b, remain, 0);
247 if (start != last && zebra_maps_is_index(zm))
249 zebra_snippets_appendn(h->snippets, p->seqno, 1, ord,
250 start, last - start);
256 while (map && *map && **map != *CHR_SPACE)
258 const char *cp = *map;
260 while (i < IT_MAX_WORD && *cp)
262 remain = p->term_len - (b - p->term_buf);
265 map = zebra_maps_input(zm, &b, remain, 0);
275 if (zebra_maps_is_first_in_field(zm))
277 /* first in field marker */
281 if (start != last && zebra_maps_is_index(zm))
282 zebra_snippets_appendn(h->snippets, p->seqno, 0, ord,
283 start, last - start);
290 static void snippet_add_icu(RecWord *p, int ord, zebra_map_t zm)
292 struct snip_rec_info *h = p->extractCtrl->handle;
294 const char *res_buf = 0;
297 const char *display_buf = 0;
298 size_t display_len = 0;
300 zebra_map_tokenize_start(zm, p->term_buf, p->term_len);
301 while (zebra_map_tokenize_next(zm, &res_buf, &res_len,
302 &display_buf, &display_len))
304 if (zebra_maps_is_index(zm))
305 zebra_snippets_appendn(h->snippets, p->seqno, 0, ord,
306 display_buf, display_len);
311 static void snippet_token_add(RecWord *p)
313 struct snip_rec_info *h = p->extractCtrl->handle;
314 ZebraHandle zh = h->zh;
315 zebra_map_t zm = zebra_map_get_or_add(zh->reg->zebra_maps, p->index_type);
319 ZebraExplainInfo zei = zh->reg->zei;
320 int ch = zebraExplain_lookup_attr_str(
321 zei, zinfo_index_category_index, p->index_type, p->index_name);
323 if (zebra_maps_is_icu(zm))
324 snippet_add_icu(p, ch, zm);
327 if (zebra_maps_is_complete(zm))
328 snippet_add_complete_field(p, ch, zm);
330 snippet_add_incomplete_field(p, ch, zm);
335 static void snippet_schema_add(
336 struct recExtractCtrl *p, Odr_oid *oid)
341 void extract_snippet(ZebraHandle zh, zebra_snippets *sn,
342 struct ZebraRecStream *stream,
343 RecType rt, void *recTypeClientData)
345 struct recExtractCtrl extractCtrl;
346 struct snip_rec_info info;
349 extractCtrl.stream = stream;
350 extractCtrl.first_record = 1;
351 extractCtrl.init = extract_init;
352 extractCtrl.tokenAdd = snippet_token_add;
353 extractCtrl.schemaAdd = snippet_schema_add;
357 extractCtrl.dh = zh->reg->dh;
361 extractCtrl.handle = &info;
362 extractCtrl.match_criteria[0] = '\0';
363 extractCtrl.staticrank = 0;
364 extractCtrl.action = action_insert;
366 init_extractCtrl(zh, &extractCtrl);
368 extractCtrl.setStoreData = 0;
370 r = (*rt->extract)(recTypeClientData, &extractCtrl);
374 static void searchRecordKey(ZebraHandle zh,
375 zebra_rec_keys_t reckeys,
376 const char *index_name,
377 const char **ws, int ws_length)
381 zinfo_index_category_t cat = zinfo_index_category_index;
383 for (i = 0; i<ws_length; i++)
387 ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, "0", index_name);
389 ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, "p", index_name);
391 ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, "w", index_name);
396 if (zebra_rec_keys_rewind(reckeys))
403 while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
405 assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
407 seqno = key.mem[key.len-1];
409 if (key.mem[0] == ch)
415 woff = seqno - startSeq;
416 if (woff >= 0 && woff < ws_length)
423 #define FILE_MATCH_BLANK "\t "
425 static char *get_match_from_spec(ZebraHandle zh,
426 zebra_rec_keys_t reckeys,
427 const char *fname, const char *spec)
429 static char dstBuf[2048]; /* static here ??? */
431 const char *s = spec;
435 for (; *s && strchr(FILE_MATCH_BLANK, *s); s++)
442 char attset_str[64], attname_str[64];
446 for (s++; strchr(FILE_MATCH_BLANK, *s); s++)
448 for (i = 0; *s && *s != ',' && *s != ')' &&
449 !strchr(FILE_MATCH_BLANK, *s); s++)
450 if (i+1 < sizeof(attset_str))
451 attset_str[i++] = *s;
452 attset_str[i] = '\0';
454 for (; strchr(FILE_MATCH_BLANK, *s); s++)
457 strcpy(attname_str, attset_str);
460 for (s++; strchr(FILE_MATCH_BLANK, *s); s++)
462 for (i = 0; *s && *s != ')' &&
463 !strchr(FILE_MATCH_BLANK, *s); s++)
464 if (i+1 < sizeof(attname_str))
465 attname_str[i++] = *s;
466 attname_str[i] = '\0';
470 yaz_log(YLOG_WARN, "Missing ) in match criteria %s in group %s",
471 spec, zh->m_group ? zh->m_group : "none");
476 searchRecordKey(zh, reckeys, attname_str, ws, 32);
477 if (0) /* for debugging */
479 for (i = 0; i<32; i++)
483 WRBUF w = wrbuf_hex_str(ws[i]);
484 yaz_log(YLOG_LOG, "ws[%d] = %s", i, wrbuf_cstr(w));
490 for (i = 0; i<32; i++)
499 dst += strlen(ws[i]);
503 yaz_log(YLOG_WARN, "Record didn't contain match"
504 " fields in (%s,%s)", attset_str, attname_str);
512 const char *spec_src = NULL;
513 const char *s1 = ++s;
514 while (*s1 && !strchr(FILE_MATCH_BLANK, *s1))
518 if (spec_len > sizeof(special)-1)
519 spec_len = sizeof(special)-1;
520 memcpy(special, s, spec_len);
521 special[spec_len] = '\0';
524 if (!strcmp(special, "group"))
525 spec_src = zh->m_group;
526 else if (!strcmp(special, "database"))
527 spec_src = zh->basenames[0];
528 else if (!strcmp(special, "filename")) {
531 else if (!strcmp(special, "type"))
532 spec_src = zh->m_record_type;
537 strcpy(dst, spec_src);
538 dst += strlen(spec_src);
541 else if (*s == '\"' || *s == '\'')
543 int stopMarker = *s++;
547 while (*s && *s != stopMarker)
549 if (i+1 < sizeof(tmpString))
550 tmpString[i++] = *s++;
555 strcpy(dst, tmpString);
556 dst += strlen(tmpString);
560 yaz_log(YLOG_WARN, "Syntax error in match criteria %s in group %s",
561 spec, zh->m_group ? zh->m_group : "none");
568 yaz_log(YLOG_WARN, "No match criteria for record %s in group %s",
569 fname, zh->m_group ? zh->m_group : "none");
574 if (0) /* for debugging */
576 WRBUF w = wrbuf_hex_str(dstBuf);
577 yaz_log(YLOG_LOG, "get_match_from_spec %s", wrbuf_cstr(w));
584 struct recordLogInfo {
587 struct recordGroup *rGroup;
590 /** \brief add the always-matches index entry and map to real record ID
591 \param ctrl record control
592 \param record_id custom record ID
593 \param sysno system record ID
595 This function serves two purposes.. It adds the always matches
596 entry and makes a pointer from the custom record ID (if defined)
597 back to the system record ID (sysno)
598 See zebra_recid_to_sysno .
600 static void all_matches_add(struct recExtractCtrl *ctrl, zint record_id,
604 extract_init(ctrl, &word);
605 word.record_id = record_id;
606 /* we use the seqno as placeholder for a way to get back to
607 record database from _ALLRECORDS.. This is used if a custom
608 RECORD was defined */
610 word.index_name = "_ALLRECORDS";
611 word.index_type = "w";
613 extract_add_index_string(&word, zinfo_index_category_alwaysmatches,
617 /* forward declaration */
618 ZEBRA_RES zebra_extract_records_stream(ZebraHandle zh,
619 struct ZebraRecStream *stream,
620 enum zebra_recctrl_action_t action,
621 const char *recordType,
623 const char *match_criteria,
626 void *recTypeClientData);
629 ZEBRA_RES zebra_extract_file(ZebraHandle zh, zint *sysno, const char *fname,
630 enum zebra_recctrl_action_t action)
632 ZEBRA_RES r = ZEBRA_OK;
637 struct file_read_info *fi = 0;
638 const char *original_record_type = 0;
640 void *recTypeClientData;
641 struct ZebraRecStream stream, *streamp;
643 zebra_init_log_level();
645 if (!zh->m_group || !*zh->m_group)
648 sprintf(gprefix, "%s.", zh->m_group);
650 yaz_log(log_level_extract, "zebra_extract_file %s", fname);
652 /* determine file extension */
654 for (i = strlen(fname); --i >= 0; )
657 else if (fname[i] == '.')
659 strcpy(ext, fname+i+1);
662 /* determine file type - depending on extension */
663 original_record_type = zh->m_record_type;
664 if (!zh->m_record_type)
666 sprintf(ext_res, "%srecordType.%s", gprefix, ext);
667 zh->m_record_type = res_get(zh->res, ext_res);
669 if (!zh->m_record_type)
672 if (zh->records_processed + zh->records_skipped
673 < zh->m_file_verbose_limit)
674 yaz_log(YLOG_LOG, "? %s", fname);
675 zh->records_skipped++;
678 /* determine match criteria */
679 if (!zh->m_record_id)
681 sprintf(ext_res, "%srecordId.%s", gprefix, ext);
682 zh->m_record_id = res_get(zh->res, ext_res);
686 recType_byName(zh->reg->recTypes, zh->res, zh->m_record_type,
687 &recTypeClientData)))
689 yaz_log(YLOG_WARN, "No such record type: %s", zh->m_record_type);
693 switch(recType->version)
698 yaz_log(YLOG_WARN, "Bad filter version: %s", zh->m_record_type);
700 if (sysno && (action == action_delete || action == action_a_delete))
709 if (zh->path_reg && !yaz_is_abspath(fname))
711 strcpy(full_rep, zh->path_reg);
712 strcat(full_rep, "/");
713 strcat(full_rep, fname);
716 strcpy(full_rep, fname);
718 if ((fd = open(full_rep, O_BINARY|O_RDONLY)) == -1)
720 yaz_log(YLOG_WARN|YLOG_ERRNO, "open %s", full_rep);
721 zh->m_record_type = original_record_type;
725 zebra_create_stream_fd(streamp, fd, 0);
727 r = zebra_extract_records_stream(zh, streamp,
731 0, /*match_criteria */
733 recType, recTypeClientData);
735 stream.destroy(streamp);
736 zh->m_record_type = original_record_type;
741 If sysno is provided, then it's used to identify the reocord.
742 If not, and match_criteria is provided, then sysno is guessed
743 If not, and a record is provided, then sysno is got from there
747 ZEBRA_RES zebra_buffer_extract_record(ZebraHandle zh,
748 const char *buf, size_t buf_size,
749 enum zebra_recctrl_action_t action,
750 const char *recordType,
752 const char *match_criteria,
755 struct ZebraRecStream stream;
760 if (recordType && *recordType)
762 yaz_log(log_level_extract,
763 "Record type explicitly specified: %s", recordType);
764 recType = recType_byName(zh->reg->recTypes, zh->res, recordType,
769 if (!(zh->m_record_type))
771 yaz_log(YLOG_WARN, "No such record type defined");
774 yaz_log(log_level_extract, "Get record type from rgroup: %s",
776 recType = recType_byName(zh->reg->recTypes, zh->res,
777 zh->m_record_type, &clientData);
778 recordType = zh->m_record_type;
783 yaz_log(YLOG_WARN, "No such record type: %s", recordType);
787 zebra_create_stream_mem(&stream, buf, buf_size);
789 res = zebra_extract_records_stream(zh, &stream,
795 recType, clientData);
796 stream.destroy(&stream);
800 static ZEBRA_RES zebra_extract_record_stream(ZebraHandle zh,
801 struct ZebraRecStream *stream,
802 enum zebra_recctrl_action_t action,
803 const char *recordType,
805 const char *match_criteria,
808 void *recTypeClientData,
813 RecordAttr *recordAttr;
814 struct recExtractCtrl extractCtrl;
816 const char *matchStr = 0;
818 off_t start_offset = 0, end_offset = 0;
819 const char *pr_fname = fname; /* filename to print .. */
820 int show_progress = zh->records_processed + zh->records_skipped
821 < zh->m_file_verbose_limit ? 1:0;
823 zebra_init_log_level();
826 pr_fname = "<no file>"; /* make it printable if file is omitted */
828 zebra_rec_keys_reset(zh->reg->keys);
829 zebra_rec_keys_reset(zh->reg->sortKeys);
831 if (zebraExplain_curDatabase(zh->reg->zei, zh->basenames[0]))
833 if (zebraExplain_newDatabase(zh->reg->zei, zh->basenames[0],
834 zh->m_explain_database))
840 off_t null_offset = 0;
841 extractCtrl.stream = stream;
843 start_offset = stream->tellf(stream);
845 extractCtrl.first_record = start_offset ? 0 : 1;
847 stream->endf(stream, &null_offset);;
849 extractCtrl.init = extract_init;
850 extractCtrl.tokenAdd = extract_token_add;
851 extractCtrl.schemaAdd = extract_schema_add;
852 extractCtrl.dh = zh->reg->dh;
853 extractCtrl.handle = zh;
854 extractCtrl.match_criteria[0] = '\0';
855 extractCtrl.staticrank = 0;
856 extractCtrl.action = action;
858 init_extractCtrl(zh, &extractCtrl);
860 extract_set_store_data_prepare(&extractCtrl);
862 r = (*recType->extract)(recTypeClientData, &extractCtrl);
864 if (action == action_update)
866 action = extractCtrl.action;
871 case RECCTRL_EXTRACT_EOF:
873 case RECCTRL_EXTRACT_ERROR_GENERIC:
874 /* error occured during extraction ... */
875 yaz_log(YLOG_WARN, "extract error: generic");
877 case RECCTRL_EXTRACT_ERROR_NO_SUCH_FILTER:
878 /* error occured during extraction ... */
879 yaz_log(YLOG_WARN, "extract error: no such filter");
881 case RECCTRL_EXTRACT_SKIP:
883 yaz_log(YLOG_LOG, "skip %s %s " ZINT_FORMAT,
884 recordType, pr_fname, (zint) start_offset);
887 end_offset = stream->endf(stream, 0);
889 stream->seekf(stream, end_offset);
892 case RECCTRL_EXTRACT_OK:
895 yaz_log(YLOG_WARN, "extract error: unknown error: %d", r);
898 end_offset = stream->endf(stream, 0);
900 stream->seekf(stream, end_offset);
902 end_offset = stream->tellf(stream);
904 if (extractCtrl.match_criteria[0])
905 match_criteria = extractCtrl.match_criteria;
910 if (zh->m_flag_rw == 0)
912 yaz_log(YLOG_LOG, "test %s %s " ZINT_FORMAT, recordType,
913 pr_fname, (zint) start_offset);
914 /* test mode .. Do not perform match */
922 if (match_criteria && *match_criteria)
923 matchStr = match_criteria;
926 if (zh->m_record_id && *zh->m_record_id)
928 matchStr = get_match_from_spec(zh, zh->reg->keys, pr_fname,
932 yaz_log(YLOG_LOG, "error %s %s " ZINT_FORMAT, recordType,
933 pr_fname, (zint) start_offset);
938 WRBUF w = wrbuf_alloc();
940 for (i = 0; i < strlen(matchStr); i++)
942 wrbuf_printf(w, "%02X", matchStr[i] & 0xff);
944 yaz_log(YLOG_LOG, "Got match %s", wrbuf_cstr(w));
951 int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
952 char *rinfo = dict_lookup_ord(zh->reg->matchDict, db_ord,
956 if (log_level_extract)
958 WRBUF w = wrbuf_hex_str(matchStr);
959 yaz_log(log_level_extract, "matchStr: %s", wrbuf_cstr(w));
964 assert(*rinfo == sizeof(*sysno));
965 memcpy(sysno, rinfo+1, sizeof(*sysno));
972 /* new record AKA does not exist already */
973 if (action == action_delete)
975 yaz_log(YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
976 pr_fname, (zint) start_offset);
977 yaz_log(YLOG_WARN, "cannot delete record above (seems new)");
980 else if (action == action_a_delete)
983 yaz_log(YLOG_LOG, "adelete %s %s " ZINT_FORMAT, recordType,
984 pr_fname, (zint) start_offset);
987 else if (action == action_replace)
989 yaz_log(YLOG_LOG, "update %s %s " ZINT_FORMAT, recordType,
990 pr_fname, (zint) start_offset);
991 yaz_log(YLOG_WARN, "cannot update record above (seems new)");
995 yaz_log(YLOG_LOG, "add %s %s " ZINT_FORMAT, recordType, pr_fname,
996 (zint) start_offset);
997 rec = rec_new(zh->reg->records);
1004 all_matches_add(&extractCtrl,
1005 zebra_rec_keys_get_custom_record_id(zh->reg->keys),
1010 recordAttr = rec_init_attr(zh->reg->zei, rec);
1011 if (extractCtrl.staticrank < 0)
1013 yaz_log(YLOG_WARN, "Negative staticrank for record. Set to 0");
1014 extractCtrl.staticrank = 0;
1019 int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
1020 dict_insert_ord(zh->reg->matchDict, db_ord, matchStr,
1021 sizeof(*sysno), sysno);
1024 extract_flush_sort_keys(zh, *sysno, 1, zh->reg->sortKeys);
1026 extract_flush_record_keys2(zh, *sysno,
1027 zh->reg->keys, extractCtrl.staticrank,
1028 0, recordAttr->staticrank);
1030 extract_flush_record_keys(zh, *sysno, 1, zh->reg->keys,
1031 extractCtrl.staticrank);
1033 recordAttr->staticrank = extractCtrl.staticrank;
1034 zh->records_inserted++;
1038 /* record already exists */
1039 zebra_rec_keys_t delkeys = zebra_rec_keys_open();
1040 zebra_rec_keys_t sortKeys = zebra_rec_keys_open();
1041 if (action == action_insert)
1043 yaz_log(YLOG_LOG, "skipped %s %s " ZINT_FORMAT,
1044 recordType, pr_fname, (zint) start_offset);
1049 rec = rec_get(zh->reg->records, *sysno);
1054 all_matches_add(&extractCtrl,
1055 zebra_rec_keys_get_custom_record_id(zh->reg->keys),
1059 recordAttr = rec_init_attr(zh->reg->zei, rec);
1061 /* decrease total size */
1062 zebraExplain_recordBytesIncrement(zh->reg->zei,
1063 - recordAttr->recordSize);
1065 zebra_rec_keys_set_buf(delkeys,
1066 rec->info[recInfo_delKeys],
1067 rec->size[recInfo_delKeys],
1069 zebra_rec_keys_set_buf(sortKeys,
1070 rec->info[recInfo_sortKeys],
1071 rec->size[recInfo_sortKeys],
1074 extract_flush_sort_keys(zh, *sysno, 0, sortKeys);
1076 extract_flush_record_keys(zh, *sysno, 0, delkeys,
1077 recordAttr->staticrank);
1079 if (action == action_delete || action == action_a_delete)
1081 /* record going to be deleted */
1083 extract_flush_record_keys2(zh, *sysno, 0, recordAttr->staticrank,
1084 delkeys, recordAttr->staticrank);
1086 if (zebra_rec_keys_empty(delkeys))
1088 yaz_log(YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
1089 pr_fname, (zint) start_offset);
1090 yaz_log(YLOG_WARN, "cannot delete file above, "
1091 "storeKeys false (3)");
1096 yaz_log(YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
1097 pr_fname, (zint) start_offset);
1098 zh->records_deleted++;
1101 int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
1102 dict_delete_ord(zh->reg->matchDict, db_ord, matchStr);
1104 rec_del(zh->reg->records, &rec);
1106 zebra_rec_keys_close(delkeys);
1107 zebra_rec_keys_close(sortKeys);
1113 { /* update or special_update */
1115 yaz_log(YLOG_LOG, "update %s %s " ZINT_FORMAT, recordType,
1116 pr_fname, (zint) start_offset);
1117 extract_flush_sort_keys(zh, *sysno, 1, zh->reg->sortKeys);
1120 extract_flush_record_keys2(zh, *sysno,
1121 zh->reg->keys, extractCtrl.staticrank,
1122 delkeys, recordAttr->staticrank);
1124 extract_flush_record_keys(zh, *sysno, 1,
1125 zh->reg->keys, extractCtrl.staticrank);
1127 recordAttr->staticrank = extractCtrl.staticrank;
1128 zh->records_updated++;
1130 zebra_rec_keys_close(delkeys);
1131 zebra_rec_keys_close(sortKeys);
1133 /* update file type */
1134 xfree(rec->info[recInfo_fileType]);
1135 rec->info[recInfo_fileType] =
1136 rec_strdup(recordType, &rec->size[recInfo_fileType]);
1138 /* update filename */
1139 xfree(rec->info[recInfo_filename]);
1140 rec->info[recInfo_filename] =
1141 rec_strdup(fname, &rec->size[recInfo_filename]);
1143 /* update delete keys */
1144 xfree(rec->info[recInfo_delKeys]);
1145 if (!zebra_rec_keys_empty(zh->reg->keys) && zh->m_store_keys == 1)
1147 zebra_rec_keys_get_buf(zh->reg->keys,
1148 &rec->info[recInfo_delKeys],
1149 &rec->size[recInfo_delKeys]);
1153 rec->info[recInfo_delKeys] = NULL;
1154 rec->size[recInfo_delKeys] = 0;
1156 /* update sort keys */
1157 xfree(rec->info[recInfo_sortKeys]);
1159 zebra_rec_keys_get_buf(zh->reg->sortKeys,
1160 &rec->info[recInfo_sortKeys],
1161 &rec->size[recInfo_sortKeys]);
1165 recordAttr->recordSize = end_offset - start_offset;
1166 zebraExplain_recordBytesIncrement(zh->reg->zei,
1167 recordAttr->recordSize);
1170 /* set run-number for this record */
1171 recordAttr->runNumber =
1172 zebraExplain_runNumberIncrement(zh->reg->zei, 0);
1174 /* update store data */
1175 xfree(rec->info[recInfo_storeData]);
1177 /* update store data */
1178 if (zh->store_data_buf)
1180 rec->size[recInfo_storeData] = zh->store_data_size;
1181 rec->info[recInfo_storeData] = zh->store_data_buf;
1182 zh->store_data_buf = 0;
1183 recordAttr->recordSize = zh->store_data_size;
1185 else if (zh->m_store_data)
1187 off_t cur_offset = stream->tellf(stream);
1189 rec->size[recInfo_storeData] = recordAttr->recordSize;
1190 rec->info[recInfo_storeData] = (char *)
1191 xmalloc(recordAttr->recordSize);
1192 stream->seekf(stream, start_offset);
1193 stream->readf(stream, rec->info[recInfo_storeData],
1194 recordAttr->recordSize);
1195 stream->seekf(stream, cur_offset);
1199 rec->info[recInfo_storeData] = NULL;
1200 rec->size[recInfo_storeData] = 0;
1202 /* update database name */
1203 xfree(rec->info[recInfo_databaseName]);
1204 rec->info[recInfo_databaseName] =
1205 rec_strdup(zh->basenames[0], &rec->size[recInfo_databaseName]);
1208 recordAttr->recordOffset = start_offset;
1210 /* commit this record */
1211 rec_put(zh->reg->records, &rec);
1216 /** \brief extracts records from stream
1217 \param zh Zebra Handle
1218 \param stream stream that we read from
1219 \param action (action_insert, action_replace, action_delete, ..)
1220 \param recordType Record filter type "grs.xml", etc.
1221 \param sysno pointer to sysno if already known; NULL otherwise
1222 \param match_criteria (NULL if not already given)
1223 \param fname filename that we read from (for logging purposes only)
1224 \param recType record type
1225 \param recTypeClientData client data for record type
1226 \returns ZEBRA_OK for success; ZEBRA_FAIL for failure
1228 ZEBRA_RES zebra_extract_records_stream(ZebraHandle zh,
1229 struct ZebraRecStream *stream,
1230 enum zebra_recctrl_action_t action,
1231 const char *recordType,
1233 const char *match_criteria,
1236 void *recTypeClientData)
1238 ZEBRA_RES res = ZEBRA_OK;
1242 res = zebra_extract_record_stream(zh, stream,
1248 recType, recTypeClientData, &more);
1254 if (res != ZEBRA_OK)
1262 ZEBRA_RES zebra_extract_explain(void *handle, Record rec, data1_node *n)
1264 ZebraHandle zh = (ZebraHandle) handle;
1265 struct recExtractCtrl extractCtrl;
1267 if (zebraExplain_curDatabase(zh->reg->zei,
1268 rec->info[recInfo_databaseName]))
1271 if (zebraExplain_newDatabase(zh->reg->zei,
1272 rec->info[recInfo_databaseName], 0))
1276 zebra_rec_keys_reset(zh->reg->keys);
1277 zebra_rec_keys_reset(zh->reg->sortKeys);
1279 extractCtrl.init = extract_init;
1280 extractCtrl.tokenAdd = extract_token_add;
1281 extractCtrl.schemaAdd = extract_schema_add;
1282 extractCtrl.dh = zh->reg->dh;
1284 init_extractCtrl(zh, &extractCtrl);
1286 extractCtrl.flagShowRecords = 0;
1287 extractCtrl.match_criteria[0] = '\0';
1288 extractCtrl.staticrank = 0;
1289 extractCtrl.action = action_update;
1291 extractCtrl.handle = handle;
1292 extractCtrl.first_record = 1;
1294 extract_set_store_data_prepare(&extractCtrl);
1297 grs_extract_tree(&extractCtrl, n);
1299 if (rec->size[recInfo_delKeys])
1301 zebra_rec_keys_t delkeys = zebra_rec_keys_open();
1303 zebra_rec_keys_t sortkeys = zebra_rec_keys_open();
1305 zebra_rec_keys_set_buf(delkeys, rec->info[recInfo_delKeys],
1306 rec->size[recInfo_delKeys],
1309 extract_flush_record_keys2(zh, rec->sysno,
1310 zh->reg->keys, 0, delkeys, 0);
1312 extract_flush_record_keys(zh, rec->sysno, 0, delkeys, 0);
1313 extract_flush_record_keys(zh, rec->sysno, 1, zh->reg->keys, 0);
1315 zebra_rec_keys_close(delkeys);
1317 zebra_rec_keys_set_buf(sortkeys, rec->info[recInfo_sortKeys],
1318 rec->size[recInfo_sortKeys],
1321 extract_flush_sort_keys(zh, rec->sysno, 0, sortkeys);
1322 zebra_rec_keys_close(sortkeys);
1327 extract_flush_record_keys2(zh, rec->sysno, zh->reg->keys, 0, 0, 0);
1329 extract_flush_record_keys(zh, rec->sysno, 1, zh->reg->keys, 0);
1332 extract_flush_sort_keys(zh, rec->sysno, 1, zh->reg->sortKeys);
1334 xfree(rec->info[recInfo_delKeys]);
1335 zebra_rec_keys_get_buf(zh->reg->keys,
1336 &rec->info[recInfo_delKeys],
1337 &rec->size[recInfo_delKeys]);
1339 xfree(rec->info[recInfo_sortKeys]);
1340 zebra_rec_keys_get_buf(zh->reg->sortKeys,
1341 &rec->info[recInfo_sortKeys],
1342 &rec->size[recInfo_sortKeys]);
1346 void zebra_it_key_str_dump(ZebraHandle zh, struct it_key *key,
1347 const char *str, size_t slen, NMEM nmem, int level)
1349 char keystr[200]; /* room for zints to print */
1351 int ord = CAST_ZINT_TO_INT(key->mem[0]);
1352 const char *index_type;
1354 const char *string_index;
1356 zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type,
1357 0/* db */, &string_index);
1359 zebra_term_untrans_iconv(zh, nmem, index_type,
1362 for (i = 0; i < key->len; i++)
1364 sprintf(keystr + strlen(keystr), ZINT_FORMAT " ", key->mem[i]);
1367 if (*str < CHR_BASE_CHAR)
1370 char dst_buf[200]; /* room for special chars */
1372 strcpy(dst_buf , "?");
1374 if (!strcmp(str, ""))
1375 strcpy(dst_buf, "alwaysmatches");
1376 if (!strcmp(str, FIRST_IN_FIELD_STR))
1377 strcpy(dst_buf, "firstinfield");
1378 else if (!strcmp(str, CHR_UNKNOWN))
1379 strcpy(dst_buf, "unknown");
1380 else if (!strcmp(str, CHR_SPACE))
1381 strcpy(dst_buf, "space");
1383 for (i = 0; i<slen; i++)
1385 sprintf(dst_buf + strlen(dst_buf), " %d", str[i] & 0xff);
1387 yaz_log(level, "%s%s %s %s", keystr, index_type,
1388 string_index, dst_buf);
1392 yaz_log(level, "%s%s %s \"%s\"", keystr, index_type,
1393 string_index, dst_term);
1396 void extract_rec_keys_log(ZebraHandle zh, int is_insert,
1397 zebra_rec_keys_t reckeys,
1400 if (zebra_rec_keys_rewind(reckeys))
1405 NMEM nmem = nmem_create();
1407 while(zebra_rec_keys_read(reckeys, &str, &slen, &key))
1409 zebra_it_key_str_dump(zh, &key, str, slen, nmem, level);
1416 void extract_rec_keys_adjust(ZebraHandle zh, int is_insert,
1417 zebra_rec_keys_t reckeys)
1419 ZebraExplainInfo zei = zh->reg->zei;
1423 struct ord_stat *next;
1426 if (zebra_rec_keys_rewind(reckeys))
1428 struct ord_stat *ord_list = 0;
1432 struct it_key key_in;
1433 while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1435 int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
1437 for (p = ord_list; p ; p = p->next)
1445 p = xmalloc(sizeof(*p));
1456 struct ord_stat *p1 = p;
1459 zebraExplain_ord_adjust_occurrences(zei, p->ord, p->no, 1);
1461 zebraExplain_ord_adjust_occurrences(zei, p->ord, - p->no, -1);
1469 static void extract_flush_record_keys2(
1470 ZebraHandle zh, zint sysno,
1471 zebra_rec_keys_t ins_keys, zint ins_rank,
1472 zebra_rec_keys_t del_keys, zint del_rank)
1474 ZebraExplainInfo zei = zh->reg->zei;
1478 if (!zh->reg->key_block)
1480 int mem = 1024*1024 * atoi( res_get_def( zh->res, "memmax", "8"));
1481 const char *key_tmp_dir = res_get_def(zh->res, "keyTmpDir", ".");
1482 int use_threads = atoi(res_get_def(zh->res, "threads", "1"));
1483 zh->reg->key_block = key_block_create(mem, key_tmp_dir, use_threads);
1488 extract_rec_keys_adjust(zh, 1, ins_keys);
1490 zebraExplain_recordCountIncrement(zei, 1);
1491 zebra_rec_keys_rewind(ins_keys);
1495 extract_rec_keys_adjust(zh, 0, del_keys);
1497 zebraExplain_recordCountIncrement(zei, -1);
1498 zebra_rec_keys_rewind(del_keys);
1504 const char *del_str;
1505 struct it_key del_key_in;
1509 const char *ins_str;
1510 struct it_key ins_key_in;
1514 del = zebra_rec_keys_read(del_keys, &del_str, &del_slen,
1517 ins = zebra_rec_keys_read(ins_keys, &ins_str, &ins_slen,
1520 if (del && ins && ins_rank == del_rank
1521 && !key_compare(&del_key_in, &ins_key_in)
1522 && ins_slen == del_slen && !memcmp(del_str, ins_str, del_slen))
1532 key_block_write(zh->reg->key_block, sysno,
1533 &del_key_in, 0, del_str, del_slen,
1534 del_rank, zh->m_staticrank);
1536 key_block_write(zh->reg->key_block, sysno,
1537 &ins_key_in, 1, ins_str, ins_slen,
1538 ins_rank, zh->m_staticrank);
1540 yaz_log(log_level_extract, "normal=%d optimized=%d", normal, optimized);
1543 static void extract_flush_record_keys(
1544 ZebraHandle zh, zint sysno, int cmd,
1545 zebra_rec_keys_t reckeys,
1548 ZebraExplainInfo zei = zh->reg->zei;
1550 extract_rec_keys_adjust(zh, cmd, reckeys);
1552 if (log_level_details)
1554 yaz_log(log_level_details, "Keys for record " ZINT_FORMAT " %s",
1555 sysno, cmd ? "insert" : "delete");
1556 extract_rec_keys_log(zh, cmd, reckeys, log_level_details);
1559 if (!zh->reg->key_block)
1561 int mem = 1024*1024 * atoi( res_get_def( zh->res, "memmax", "8"));
1562 const char *key_tmp_dir = res_get_def(zh->res, "keyTmpDir", ".");
1563 int use_threads = atoi(res_get_def(zh->res, "threads", "1"));
1564 zh->reg->key_block = key_block_create(mem, key_tmp_dir, use_threads);
1566 zebraExplain_recordCountIncrement(zei, cmd ? 1 : -1);
1569 yaz_log(YLOG_LOG, "sysno=" ZINT_FORMAT " cmd=%d", sysno, cmd);
1570 print_rec_keys(zh, reckeys);
1572 if (zebra_rec_keys_rewind(reckeys))
1576 struct it_key key_in;
1577 while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1579 key_block_write(zh->reg->key_block, sysno,
1580 &key_in, cmd, str, slen,
1581 staticrank, zh->m_staticrank);
1587 ZEBRA_RES zebra_rec_keys_to_snippets1(ZebraHandle zh,
1588 zebra_rec_keys_t reckeys,
1589 zebra_snippets *snippets)
1591 NMEM nmem = nmem_create();
1592 if (zebra_rec_keys_rewind(reckeys))
1597 while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
1602 const char *index_type;
1604 assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
1605 seqno = key.mem[key.len-1];
1606 ord = CAST_ZINT_TO_INT(key.mem[0]);
1608 zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type,
1609 0/* db */, 0 /* string_index */);
1611 zebra_term_untrans_iconv(zh, nmem, index_type,
1613 zebra_snippets_append(snippets, seqno, 0, ord, dst_term);
1621 void print_rec_keys(ZebraHandle zh, zebra_rec_keys_t reckeys)
1623 yaz_log(YLOG_LOG, "print_rec_keys");
1624 if (zebra_rec_keys_rewind(reckeys))
1629 while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
1631 char dst_buf[IT_MAX_WORD];
1633 const char *index_type;
1634 int ord = CAST_ZINT_TO_INT(key.mem[0]);
1636 assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
1638 zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, &db, 0);
1640 seqno = key.mem[key.len-1];
1642 zebra_term_untrans(zh, index_type, dst_buf, str);
1644 yaz_log(YLOG_LOG, "ord=%d seqno=" ZINT_FORMAT
1645 " term=%s", ord, seqno, dst_buf);
1650 static void extract_add_index_string(RecWord *p, zinfo_index_category_t cat,
1651 const char *str, int length)
1654 ZebraHandle zh = p->extractCtrl->handle;
1655 ZebraExplainInfo zei = zh->reg->zei;
1658 ch = zebraExplain_lookup_attr_str(zei, cat, p->index_type, p->index_name);
1660 ch = zebraExplain_add_attr_str(zei, cat, p->index_type, p->index_name);
1664 key.mem[i++] = p->record_id;
1665 key.mem[i++] = p->section_id;
1667 if (zh->m_segment_indexing)
1668 key.mem[i++] = p->segment;
1669 key.mem[i++] = p->seqno;
1672 zebra_rec_keys_write(zh->reg->keys, str, length, &key);
1675 static void extract_add_sort_string(RecWord *p, const char *str, int length)
1678 ZebraHandle zh = p->extractCtrl->handle;
1679 ZebraExplainInfo zei = zh->reg->zei;
1681 zinfo_index_category_t cat = zinfo_index_category_sort;
1683 ch = zebraExplain_lookup_attr_str(zei, cat, p->index_type, p->index_name);
1685 ch = zebraExplain_add_attr_str(zei, cat, p->index_type, p->index_name);
1688 key.mem[1] = p->record_id;
1689 key.mem[2] = p->section_id;
1691 zebra_rec_keys_write(zh->reg->sortKeys, str, length, &key);
1694 static void extract_add_staticrank_string(RecWord *p,
1695 const char *str, int length)
1698 struct recExtractCtrl *ctrl = p->extractCtrl;
1700 if (length > sizeof(valz)-1)
1701 length = sizeof(valz)-1;
1703 memcpy(valz, str, length);
1704 valz[length] = '\0';
1705 ctrl->staticrank = atozint(valz);
1708 static void extract_add_string(RecWord *p, zebra_map_t zm,
1709 const char *string, int length)
1715 if (log_level_details)
1718 WRBUF w = wrbuf_alloc();
1720 wrbuf_write_escaped(w, string, length);
1721 yaz_log(log_level_details, "extract_add_string: %s", wrbuf_cstr(w));
1724 if (zebra_maps_is_index(zm))
1726 extract_add_index_string(p, zinfo_index_category_index,
1728 if (zebra_maps_is_alwaysmatches(zm))
1731 memcpy(&word, p, sizeof(word));
1734 extract_add_index_string(
1735 &word, zinfo_index_category_alwaysmatches, "", 0);
1738 else if (zebra_maps_is_sort(zm))
1740 extract_add_sort_string(p, string, length);
1742 else if (zebra_maps_is_staticrank(zm))
1744 extract_add_staticrank_string(p, string, length);
1748 static void extract_add_incomplete_field(RecWord *p, zebra_map_t zm)
1750 const char *b = p->term_buf;
1751 int remain = p->term_len;
1753 const char **map = 0;
1756 map = zebra_maps_input(zm, &b, remain, 0);
1760 char buf[IT_MAX_WORD+1];
1764 while (map && *map && **map == *CHR_SPACE)
1766 remain = p->term_len - (b - p->term_buf);
1768 map = zebra_maps_input(zm, &b, remain, 0);
1775 while (map && *map && **map != *CHR_SPACE)
1777 const char *cp = *map;
1779 while (i < IT_MAX_WORD && *cp)
1781 remain = p->term_len - (b - p->term_buf);
1783 map = zebra_maps_input(zm, &b, remain, 0);
1793 if (zebra_maps_is_first_in_field(zm))
1795 /* first in field marker */
1796 extract_add_string(p, zm, FIRST_IN_FIELD_STR, FIRST_IN_FIELD_LEN);
1800 extract_add_string(p, zm, buf, i);
1805 static void extract_add_complete_field(RecWord *p, zebra_map_t zm)
1807 const char *b = p->term_buf;
1808 char buf[IT_MAX_WORD+1];
1809 const char **map = 0;
1810 int i = 0, remain = p->term_len;
1813 map = zebra_maps_input(zm, &b, remain, 1);
1815 while (remain > 0 && i < IT_MAX_WORD)
1817 while (map && *map && **map == *CHR_SPACE)
1819 remain = p->term_len - (b - p->term_buf);
1823 int first = i ? 0 : 1; /* first position */
1824 map = zebra_maps_input(zm, &b, remain, first);
1832 if (i && i < IT_MAX_WORD)
1833 buf[i++] = *CHR_SPACE;
1834 while (map && *map && **map != *CHR_SPACE)
1836 const char *cp = *map;
1838 if (**map == *CHR_CUT)
1844 if (i >= IT_MAX_WORD)
1846 while (i < IT_MAX_WORD && *cp)
1849 remain = p->term_len - (b - p->term_buf);
1852 map = zebra_maps_input(zm, &b, remain, 0);
1860 extract_add_string(p, zm, buf, i);
1863 static void extract_add_icu(RecWord *p, zebra_map_t zm)
1865 const char *res_buf = 0;
1868 zebra_map_tokenize_start(zm, p->term_buf, p->term_len);
1869 while (zebra_map_tokenize_next(zm, &res_buf, &res_len, 0, 0))
1871 extract_add_string(p, zm, res_buf, res_len);
1877 /** \brief top-level indexing handler for recctrl system
1878 \param p token data to be indexed
1882 extract_add_{in}_complete / extract_add_icu
1885 extract_add_index_string
1887 extract_add_sort_string
1889 extract_add_staticrank_string
1892 static void extract_token_add(RecWord *p)
1894 ZebraHandle zh = p->extractCtrl->handle;
1895 zebra_map_t zm = zebra_map_get_or_add(zh->reg->zebra_maps, p->index_type);
1898 if (log_level_details)
1900 yaz_log(log_level_details, "extract_token_add "
1901 "type=%s index=%s seqno=" ZINT_FORMAT " s=%.*s",
1902 p->index_type, p->index_name,
1903 p->seqno, p->term_len, p->term_buf);
1905 if ((wrbuf = zebra_replace(zm, 0, p->term_buf, p->term_len)))
1907 p->term_buf = wrbuf_buf(wrbuf);
1908 p->term_len = wrbuf_len(wrbuf);
1910 if (zebra_maps_is_icu(zm))
1912 extract_add_icu(p, zm);
1916 if (zebra_maps_is_complete(zm))
1917 extract_add_complete_field(p, zm);
1919 extract_add_incomplete_field(p, zm);
1923 static void extract_set_store_data_cb(struct recExtractCtrl *p,
1924 void *buf, size_t sz)
1926 ZebraHandle zh = (ZebraHandle) p->handle;
1928 xfree(zh->store_data_buf);
1929 zh->store_data_buf = 0;
1930 zh->store_data_size = 0;
1933 zh->store_data_buf = xmalloc(sz);
1934 zh->store_data_size = sz;
1935 memcpy(zh->store_data_buf, buf, sz);
1939 static void extract_set_store_data_prepare(struct recExtractCtrl *p)
1941 ZebraHandle zh = (ZebraHandle) p->handle;
1942 xfree(zh->store_data_buf);
1943 zh->store_data_buf = 0;
1944 zh->store_data_size = 0;
1945 p->setStoreData = extract_set_store_data_cb;
1948 static void extract_schema_add(struct recExtractCtrl *p, Odr_oid *oid)
1950 ZebraHandle zh = (ZebraHandle) p->handle;
1951 zebraExplain_addSchema(zh->reg->zei, oid);
1954 void extract_flush_sort_keys(ZebraHandle zh, zint sysno,
1955 int cmd, zebra_rec_keys_t reckeys)
1958 yaz_log(YLOG_LOG, "extract_flush_sort_keys cmd=%d sysno=" ZINT_FORMAT,
1960 extract_rec_keys_log(zh, cmd, reckeys, YLOG_LOG);
1963 if (zebra_rec_keys_rewind(reckeys))
1965 zebra_sort_index_t si = zh->reg->sort_index;
1968 struct it_key key_in;
1970 NMEM nmem = nmem_create();
1971 struct sort_add_ent {
1974 struct sort_add_ent *next;
1979 struct sort_add_ent *sort_ent_list = 0;
1981 while (zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1983 int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
1984 zint filter_sysno = key_in.mem[1];
1985 zint section_id = key_in.mem[2];
1987 struct sort_add_ent **e = &sort_ent_list;
1988 for (; *e; e = &(*e)->next)
1989 if ((*e)->ord == ord && section_id == (*e)->section_id)
1993 *e = nmem_malloc(nmem, sizeof(**e));
1995 (*e)->wrbuf = wrbuf_alloc();
1998 (*e)->sysno = filter_sysno ? filter_sysno : sysno;
1999 (*e)->section_id = section_id;
2002 wrbuf_write((*e)->wrbuf, str, slen);
2003 wrbuf_putc((*e)->wrbuf, '\0');
2007 zint last_sysno = 0;
2008 struct sort_add_ent *e = sort_ent_list;
2009 for (; e; e = e->next)
2011 if (last_sysno != e->sysno)
2013 zebra_sort_sysno(si, e->sysno);
2014 last_sysno = e->sysno;
2016 zebra_sort_type(si, e->ord);
2018 zebra_sort_add(si, e->section_id, e->wrbuf);
2020 zebra_sort_delete(si, e->section_id);
2021 wrbuf_destroy(e->wrbuf);
2031 * c-file-style: "Stroustrup"
2032 * indent-tabs-mode: nil
2034 * vim: shiftwidth=4 tabstop=8 expandtab