1 /* $Id: icu_I18N.c,v 1.22 2007-05-25 13:27:21 marc Exp $
2 Copyright (c) 2006-2007, Index Data.
4 This file is part of Pazpar2.
6 Pazpar2 is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 2, or (at your option) any later
11 Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with Pazpar2; see the file LICENSE. If not, write to the
18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
28 #include <yaz/timing.h>
41 #include <unicode/ustring.h> /* some more string fcns*/
42 #include <unicode/uchar.h> /* char names */
45 //#include <unicode/ustdio.h>
46 //#include <unicode/utypes.h> /* Basic ICU data types */
47 #include <unicode/ucol.h>
48 //#include <unicode/ucnv.h> /* C Converter API */
49 //#include <unicode/uloc.h>
50 //#include <unicode/ubrk.h>
51 /* #include <unicode/unistr.h> */
56 int icu_check_status (UErrorCode status)
58 if(U_FAILURE(status)){
60 "ICU: %d %s\n", status, u_errorName(status));
69 struct icu_buf_utf16 * icu_buf_utf16_create(size_t capacity)
71 struct icu_buf_utf16 * buf16
72 = (struct icu_buf_utf16 *) malloc(sizeof(struct icu_buf_utf16));
79 buf16->utf16 = (UChar *) malloc(sizeof(UChar) * capacity);
80 buf16->utf16[0] = (UChar) 0;
81 buf16->utf16_cap = capacity;
87 struct icu_buf_utf16 * icu_buf_utf16_resize(struct icu_buf_utf16 * buf16,
92 if (0 == buf16->utf16)
93 buf16->utf16 = (UChar *) malloc(sizeof(UChar) * capacity);
96 = (UChar *) realloc(buf16->utf16, sizeof(UChar) * capacity);
97 buf16->utf16[0] = (UChar) 0;
99 buf16->utf16_cap = capacity;
105 buf16->utf16_len = 0;
106 buf16->utf16_cap = 0;
114 struct icu_buf_utf16 * icu_buf_utf16_copy(struct icu_buf_utf16 * dest16,
115 struct icu_buf_utf16 * src16)
121 if (dest16->utf16_cap < src16->utf16_len)
122 icu_buf_utf16_resize(dest16, src16->utf16_len * 2);
124 u_strncpy(dest16->utf16, src16->utf16, src16->utf16_len);
125 dest16->utf16_len = src16->utf16_len;
131 void icu_buf_utf16_destroy(struct icu_buf_utf16 * buf16)
145 struct icu_buf_utf8 * icu_buf_utf8_create(size_t capacity)
147 struct icu_buf_utf8 * buf8
148 = (struct icu_buf_utf8 *) malloc(sizeof(struct icu_buf_utf8));
155 buf8->utf8 = (uint8_t *) malloc(sizeof(uint8_t) * capacity);
156 buf8->utf8[0] = (uint8_t) 0;
157 buf8->utf8_cap = capacity;
164 struct icu_buf_utf8 * icu_buf_utf8_resize(struct icu_buf_utf8 * buf8,
170 buf8->utf8 = (uint8_t *) malloc(sizeof(uint8_t) * capacity);
173 = (uint8_t *) realloc(buf8->utf8,
174 sizeof(uint8_t) * capacity);
175 buf8->utf8[0] = (uint8_t) 0;
177 buf8->utf8_cap = capacity;
192 struct icu_buf_utf8 * icu_buf_utf8_copy(struct icu_buf_utf8 * dest8,
193 struct icu_buf_utf8 * src8)
200 if (dest8->utf8_cap < src8->utf8_len)
201 icu_buf_utf8_resize(dest8, src8->utf8_len * 2);
203 strncpy((char*) dest8->utf8, (char*) src8->utf8, src8->utf8_len);
210 void icu_buf_utf8_destroy(struct icu_buf_utf8 * buf8)
221 UErrorCode icu_utf16_from_utf8(struct icu_buf_utf16 * dest16,
222 struct icu_buf_utf8 * src8,
225 int32_t utf16_len = 0;
227 u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
229 (const char *) src8->utf8, src8->utf8_len, status);
231 // check for buffer overflow, resize and retry
232 if (*status == U_BUFFER_OVERFLOW_ERROR
233 //|| dest16->utf16_len > dest16->utf16_cap
235 icu_buf_utf16_resize(dest16, utf16_len * 2);
236 *status = U_ZERO_ERROR;
237 u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
239 (const char *) src8->utf8, src8->utf8_len, status);
242 //if (*status != U_BUFFER_OVERFLOW_ERROR
243 if (U_SUCCESS(*status)
244 && utf16_len < dest16->utf16_cap)
245 dest16->utf16_len = utf16_len;
247 dest16->utf16[0] = (UChar) 0;
248 dest16->utf16_len = 0;
256 UErrorCode icu_utf16_from_utf8_cstr(struct icu_buf_utf16 * dest16,
257 const char * src8cstr,
260 size_t src8cstr_len = 0;
261 int32_t utf16_len = 0;
263 src8cstr_len = strlen(src8cstr);
265 u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
267 src8cstr, src8cstr_len, status);
269 // check for buffer overflow, resize and retry
270 if (*status == U_BUFFER_OVERFLOW_ERROR
271 //|| dest16->utf16_len > dest16->utf16_cap
273 icu_buf_utf16_resize(dest16, utf16_len * 2);
274 *status = U_ZERO_ERROR;
275 u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
277 src8cstr, src8cstr_len, status);
280 // if (*status != U_BUFFER_OVERFLOW_ERROR
281 if (U_SUCCESS(*status)
282 && utf16_len < dest16->utf16_cap)
283 dest16->utf16_len = utf16_len;
285 dest16->utf16[0] = (UChar) 0;
286 dest16->utf16_len = 0;
295 UErrorCode icu_utf16_to_utf8(struct icu_buf_utf8 * dest8,
296 struct icu_buf_utf16 * src16,
299 int32_t utf8_len = 0;
301 u_strToUTF8((char *) dest8->utf8, dest8->utf8_cap,
303 src16->utf16, src16->utf16_len, status);
305 // check for buffer overflow, resize and retry
306 if (*status == U_BUFFER_OVERFLOW_ERROR
307 //|| dest8->utf8_len > dest8->utf8_cap
309 icu_buf_utf8_resize(dest8, utf8_len * 2);
310 *status = U_ZERO_ERROR;
311 u_strToUTF8((char *) dest8->utf8, dest8->utf8_cap,
313 src16->utf16, src16->utf16_len, status);
317 //if (*status != U_BUFFER_OVERFLOW_ERROR
318 if (U_SUCCESS(*status)
319 && utf8_len < dest8->utf8_cap)
320 dest8->utf8_len = utf8_len;
322 dest8->utf8[0] = (uint8_t) 0;
331 struct icu_casemap * icu_casemap_create(const char *locale, char action,
334 struct icu_casemap * casemap
335 = (struct icu_casemap *) malloc(sizeof(struct icu_casemap));
336 strcpy(casemap->locale, locale);
337 casemap->action = action;
339 switch(casemap->action) {
349 icu_casemap_destroy(casemap);
356 void icu_casemap_destroy(struct icu_casemap * casemap)
363 int icu_casemap_casemap(struct icu_casemap * casemap,
364 struct icu_buf_utf16 * dest16,
365 struct icu_buf_utf16 * src16,
371 return icu_utf16_casemap(dest16, src16,
372 casemap->locale, casemap->action, status);
376 int icu_utf16_casemap(struct icu_buf_utf16 * dest16,
377 struct icu_buf_utf16 * src16,
378 const char *locale, char action,
381 int32_t dest16_len = 0;
385 dest16_len = u_strToLower(dest16->utf16, dest16->utf16_cap,
386 src16->utf16, src16->utf16_len,
390 dest16_len = u_strToUpper(dest16->utf16, dest16->utf16_cap,
391 src16->utf16, src16->utf16_len,
395 dest16_len = u_strToTitle(dest16->utf16, dest16->utf16_cap,
396 src16->utf16, src16->utf16_len,
400 dest16_len = u_strFoldCase(dest16->utf16, dest16->utf16_cap,
401 src16->utf16, src16->utf16_len,
402 U_FOLD_CASE_DEFAULT, status);
406 return U_UNSUPPORTED_ERROR;
410 // check for buffer overflow, resize and retry
411 if (*status == U_BUFFER_OVERFLOW_ERROR
412 && dest16 != src16 // do not resize if in-place conversion
413 //|| dest16_len > dest16->utf16_cap
415 icu_buf_utf16_resize(dest16, dest16_len * 2);
416 *status = U_ZERO_ERROR;
421 dest16_len = u_strToLower(dest16->utf16, dest16->utf16_cap,
422 src16->utf16, src16->utf16_len,
426 dest16_len = u_strToUpper(dest16->utf16, dest16->utf16_cap,
427 src16->utf16, src16->utf16_len,
431 dest16_len = u_strToTitle(dest16->utf16, dest16->utf16_cap,
432 src16->utf16, src16->utf16_len,
436 dest16_len = u_strFoldCase(dest16->utf16, dest16->utf16_cap,
437 src16->utf16, src16->utf16_len,
438 U_FOLD_CASE_DEFAULT, status);
442 return U_UNSUPPORTED_ERROR;
447 if (U_SUCCESS(*status)
448 && dest16_len < dest16->utf16_cap)
449 dest16->utf16_len = dest16_len;
451 dest16->utf16[0] = (UChar) 0;
452 dest16->utf16_len = 0;
460 UErrorCode icu_sortkey8_from_utf16(UCollator *coll,
461 struct icu_buf_utf8 * dest8,
462 struct icu_buf_utf16 * src16,
466 int32_t sortkey_len = 0;
468 sortkey_len = ucol_getSortKey(coll, src16->utf16, src16->utf16_len,
469 dest8->utf8, dest8->utf8_cap);
471 // check for buffer overflow, resize and retry
472 if (sortkey_len > dest8->utf8_cap) {
473 icu_buf_utf8_resize(dest8, sortkey_len * 2);
474 sortkey_len = ucol_getSortKey(coll, src16->utf16, src16->utf16_len,
475 dest8->utf8, dest8->utf8_cap);
478 if (U_SUCCESS(*status)
480 dest8->utf8_len = sortkey_len;
482 dest8->utf8[0] = (UChar) 0;
491 struct icu_tokenizer * icu_tokenizer_create(const char *locale, char action,
494 struct icu_tokenizer * tokenizer
495 = (struct icu_tokenizer *) malloc(sizeof(struct icu_tokenizer));
497 strcpy(tokenizer->locale, locale);
498 tokenizer->action = action;
500 tokenizer->buf16 = 0;
501 tokenizer->token_count = 0;
502 tokenizer->token_id = 0;
503 tokenizer->token_start = 0;
504 tokenizer->token_end = 0;
507 switch(tokenizer->action) {
510 = ubrk_open(UBRK_LINE, tokenizer->locale,
515 = ubrk_open(UBRK_SENTENCE, tokenizer->locale,
520 = ubrk_open(UBRK_WORD, tokenizer->locale,
525 = ubrk_open(UBRK_CHARACTER, tokenizer->locale,
530 = ubrk_open(UBRK_TITLE, tokenizer->locale,
534 *status = U_UNSUPPORTED_ERROR;
539 // ICU error stuff is a very funny business
540 if (U_SUCCESS(*status))
544 icu_tokenizer_destroy(tokenizer);
548 void icu_tokenizer_destroy(struct icu_tokenizer * tokenizer)
552 ubrk_close(tokenizer->bi);
557 int icu_tokenizer_attach(struct icu_tokenizer * tokenizer,
558 struct icu_buf_utf16 * src16,
561 if (!tokenizer || !tokenizer->bi || !src16)
565 tokenizer->buf16 = src16;
566 tokenizer->token_count = 0;
567 tokenizer->token_id = 0;
568 tokenizer->token_start = 0;
569 tokenizer->token_end = 0;
571 ubrk_setText(tokenizer->bi, src16->utf16, src16->utf16_len, status);
574 if (U_FAILURE(*status))
580 int32_t icu_tokenizer_next_token(struct icu_tokenizer * tokenizer,
581 struct icu_buf_utf16 * tkn16,
584 int32_t tkn_start = 0;
589 if (!tokenizer || !tokenizer->bi
590 || !tokenizer->buf16 || !tokenizer->buf16->utf16_len)
593 // never change tokenizer->buf16 and keep always invariant
594 // 0 <= tokenizer->token_start
595 // <= tokenizer->token_end
596 // <= tokenizer->buf16->utf16_len
597 // returns length of token
599 if (0 == tokenizer->token_end) // first call
600 tkn_start = ubrk_first(tokenizer->bi);
601 else //successive calls
602 tkn_start = tokenizer->token_end;
605 tkn_end = ubrk_next(tokenizer->bi);
607 // repairing invariant at end of ubrk, which is UBRK_DONE = -1
608 if (UBRK_DONE == tkn_end)
609 tkn_end = tokenizer->buf16->utf16_len;
611 // copy out if everything is well
612 if(U_FAILURE(*status))
615 // everything OK, now update internal state
616 tkn_len = tkn_end - tkn_start;
619 tokenizer->token_count++;
620 tokenizer->token_id++;
622 tokenizer->token_id = 0;
624 tokenizer->token_start = tkn_start;
625 tokenizer->token_end = tkn_end;
628 // copying into token buffer if it exists
630 if (tkn16->utf16_cap < tkn_len)
631 icu_buf_utf16_resize(tkn16, (size_t) tkn_len * 2);
633 u_strncpy(tkn16->utf16, &(tokenizer->buf16->utf16)[tkn_start],
636 tkn16->utf16_len = tkn_len;
643 int32_t icu_tokenizer_token_id(struct icu_tokenizer * tokenizer)
645 return tokenizer->token_id;
648 int32_t icu_tokenizer_token_start(struct icu_tokenizer * tokenizer)
650 return tokenizer->token_start;
653 int32_t icu_tokenizer_token_end(struct icu_tokenizer * tokenizer)
655 return tokenizer->token_end;
658 int32_t icu_tokenizer_token_length(struct icu_tokenizer * tokenizer)
660 return (tokenizer->token_end - tokenizer->token_start);
663 int32_t icu_tokenizer_token_count(struct icu_tokenizer * tokenizer)
665 return tokenizer->token_count;
670 struct icu_normalizer * icu_normalizer_create(const char *rules, char action,
674 struct icu_normalizer * normalizer
675 = (struct icu_normalizer *) malloc(sizeof(struct icu_normalizer));
677 normalizer->action = action;
678 normalizer->trans = 0;
679 normalizer->rules16 = icu_buf_utf16_create(0);
680 icu_utf16_from_utf8_cstr(normalizer->rules16, rules, status);
682 switch(normalizer->action) {
685 = utrans_openU(normalizer->rules16->utf16,
686 normalizer->rules16->utf16_len,
689 normalizer->parse_error, status);
690 // yaz_log(YLOG_LOG, "utrans_open %p", normalizer->trans);
694 = utrans_openU(normalizer->rules16->utf16,
695 normalizer->rules16->utf16_len,
698 normalizer->parse_error, status);
699 // yaz_log(YLOG_LOG, "utrans_open %p", normalizer->trans);
702 *status = U_UNSUPPORTED_ERROR;
707 if (U_SUCCESS(*status))
711 icu_normalizer_destroy(normalizer);
716 void icu_normalizer_destroy(struct icu_normalizer * normalizer){
718 if (normalizer->rules16)
719 icu_buf_utf16_destroy(normalizer->rules16);
720 if (normalizer->trans)
722 // yaz_log(YLOG_LOG, "utrans_close %p", normalizer->trans);
723 utrans_close(normalizer->trans);
731 int icu_normalizer_normalize(struct icu_normalizer * normalizer,
732 struct icu_buf_utf16 * dest16,
733 struct icu_buf_utf16 * src16,
736 if (!normalizer || !normalizer->trans || !src16 || !dest16)
739 if (!icu_buf_utf16_copy(dest16, src16))
742 utrans_transUChars (normalizer->trans,
743 dest16->utf16, &(dest16->utf16_len),
745 0, &(src16->utf16_len), status);
747 if (U_FAILURE(*status)){
748 dest16->utf16[0] = (UChar) 0;
749 dest16->utf16_len = 0;
752 return dest16->utf16_len;
758 struct icu_chain_step * icu_chain_step_create(struct icu_chain * chain,
759 enum icu_chain_step_type type,
760 const uint8_t * rule,
761 struct icu_buf_utf16 * buf16,
764 struct icu_chain_step * step = 0;
766 if(!chain || !type || !rule)
769 step = (struct icu_chain_step *) malloc(sizeof(struct icu_chain_step));
775 // create auxilary objects
777 case ICU_chain_step_type_display:
779 case ICU_chain_step_type_index:
781 case ICU_chain_step_type_sortkey:
783 case ICU_chain_step_type_casemap:
784 step->u.casemap = icu_casemap_create((char *) chain->locale,
785 (char) rule[0], status);
787 case ICU_chain_step_type_normalize:
788 step->u.normalizer = icu_normalizer_create((char *) rule, 'f', status);
790 case ICU_chain_step_type_tokenize:
791 step->u.tokenizer = icu_tokenizer_create((char *) chain->locale,
792 (char) rule[0], status);
802 void icu_chain_step_destroy(struct icu_chain_step * step){
807 icu_chain_step_destroy(step->previous);
810 case ICU_chain_step_type_display:
812 case ICU_chain_step_type_index:
814 case ICU_chain_step_type_sortkey:
816 case ICU_chain_step_type_casemap:
817 icu_casemap_destroy(step->u.casemap);
818 icu_buf_utf16_destroy(step->buf16);
820 case ICU_chain_step_type_normalize:
821 icu_normalizer_destroy(step->u.normalizer);
822 icu_buf_utf16_destroy(step->buf16);
824 case ICU_chain_step_type_tokenize:
825 icu_tokenizer_destroy(step->u.tokenizer);
826 icu_buf_utf16_destroy(step->buf16);
836 struct icu_chain * icu_chain_create(const uint8_t * identifier,
837 const uint8_t * locale)
840 struct icu_chain * chain
841 = (struct icu_chain *) malloc(sizeof(struct icu_chain));
843 strncpy((char *) chain->identifier, (const char *) identifier, 128);
844 chain->identifier[128 - 1] = '\0';
845 strncpy((char *) chain->locale, (const char *) locale, 16);
846 chain->locale[16 - 1] = '\0';
848 chain->token_count = 0;
850 chain->display8 = icu_buf_utf8_create(0);
851 chain->norm8 = icu_buf_utf8_create(0);
852 chain->sort8 = icu_buf_utf8_create(0);
854 chain->src16 = icu_buf_utf16_create(0);
862 void icu_chain_destroy(struct icu_chain * chain)
865 icu_buf_utf8_destroy(chain->display8);
866 icu_buf_utf8_destroy(chain->norm8);
867 icu_buf_utf8_destroy(chain->sort8);
869 icu_buf_utf16_destroy(chain->src16);
871 icu_chain_step_destroy(chain->steps);
878 struct icu_chain * icu_chain_xml_config(xmlNode *xml_node,
879 UErrorCode * status){
882 struct icu_chain * chain = 0;
885 ||xml_node->type != XML_ELEMENT_NODE
886 || strcmp((const char *) xml_node->name, "icu_chain"))
890 xmlChar *xml_id = xmlGetProp(xml_node, (xmlChar *) "id");
891 xmlChar *xml_locale = xmlGetProp(xml_node, (xmlChar *) "locale");
893 if (!xml_id || !strlen((const char *) xml_id)
894 || !xml_locale || !strlen((const char *) xml_locale))
897 chain = icu_chain_create((const uint8_t *) xml_id,
898 (const uint8_t *) xml_locale);
905 for (node = xml_node->children; node; node = node->next)
907 if (node->type != XML_ELEMENT_NODE)
910 xmlChar *xml_rule = xmlGetProp(node, (xmlChar *) "rule");
911 struct icu_chain_step * step = 0;
913 if (!strcmp((const char *) node->name,
914 (const char *) "casemap")){
915 step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap,
916 (const uint8_t *) xml_rule, status);
918 else if (!strcmp((const char *) node->name,
919 (const char *) "normalize")){
920 step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize,
921 (const uint8_t *) xml_rule, status);
923 else if (!strcmp((const char *) node->name,
924 (const char *) "tokenize")){
925 step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,
926 (const uint8_t *) xml_rule, status);
928 else if (!strcmp((const char *) node->name,
929 (const char *) "display")){
930 step = icu_chain_insert_step(chain, ICU_chain_step_type_display,
931 (const uint8_t *) "", status);
933 else if (!strcmp((const char *) node->name,
934 (const char *) "index")){
935 step = icu_chain_insert_step(chain, ICU_chain_step_type_index,
936 (const uint8_t *) "", status);
938 else if (!strcmp((const char *) node->name,
939 (const char *) "sortkey")){
940 step = icu_chain_insert_step(chain, ICU_chain_step_type_sortkey,
941 (const uint8_t *) "", status);
945 if (!step || U_FAILURE(*status)){
946 icu_chain_destroy(chain);
958 struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain,
959 enum icu_chain_step_type type,
960 const uint8_t * rule,
963 struct icu_chain_step * step = 0;
964 struct icu_buf_utf16 * src16 = 0;
965 struct icu_buf_utf16 * buf16 = 0;
967 if (!chain || !type || !rule)
970 // assign utf16 src buffers as needed
971 if (chain->steps && chain->steps->buf16)
972 src16 = chain->steps->buf16;
973 else if (chain->src16)
974 src16 = chain->src16;
979 // create utf16 destination buffers as needed, or
981 case ICU_chain_step_type_display:
984 case ICU_chain_step_type_index:
987 case ICU_chain_step_type_sortkey:
990 case ICU_chain_step_type_casemap:
991 buf16 = icu_buf_utf16_create(0);
993 case ICU_chain_step_type_normalize:
994 buf16 = icu_buf_utf16_create(0);
996 case ICU_chain_step_type_tokenize:
997 buf16 = icu_buf_utf16_create(0);
1003 // create actual chain step with this buffer
1004 step = icu_chain_step_create(chain, type, rule, buf16, status);
1006 step->previous = chain->steps;
1007 chain->steps = step;
1013 int icu_chain_step_next_token(struct icu_chain * chain,
1014 struct icu_chain_step * step,
1017 struct icu_buf_utf16 * src16 = 0;
1019 //printf("icu_chain_step_next_token %d\n", (int) step);
1021 if (!chain || !chain->src16 || !step || !step->more_tokens)
1024 // assign utf16 src buffers as neeed, advance in previous steps
1025 // tokens until non-zero token met, and setting stop condition
1026 if (step->previous){
1027 src16 = step->previous->buf16;
1028 if (step->need_new_token)
1029 //while (step->more_tokens && !src16->utf16_len)
1031 = icu_chain_step_next_token(chain, step->previous, status);
1033 else { // first step can only work once on chain->src16 input buffer
1034 src16 = chain->src16;
1035 step->more_tokens = 1;
1038 // stop if nothing to process
1039 // i.e new token source was not properly assigned
1040 if (!step->more_tokens || !src16) // || !src16->utf16_len
1043 //printf("icu_chain_step_next_token %d working\n", (int) step);
1046 // perform the work, eventually put this steps output in
1047 // step->buf16 or the chains UTF8 output buffers
1048 switch(step->type) {
1049 case ICU_chain_step_type_display:
1050 icu_utf16_to_utf8(chain->display8, src16, status);
1052 case ICU_chain_step_type_index:
1053 icu_utf16_to_utf8(chain->norm8, src16, status);
1055 case ICU_chain_step_type_sortkey:
1056 icu_utf16_to_utf8(chain->sort8, src16, status);
1058 case ICU_chain_step_type_casemap:
1059 icu_casemap_casemap(step->u.casemap,
1060 step->buf16, src16, status);
1062 case ICU_chain_step_type_normalize:
1063 icu_normalizer_normalize(step->u.normalizer,
1064 step->buf16, src16, status);
1066 case ICU_chain_step_type_tokenize:
1067 // attach to new src16 token only first time during splitting
1068 if (step->need_new_token){
1069 icu_tokenizer_attach(step->u.tokenizer, src16, status);
1070 step->need_new_token = 0;
1072 // splitting one src16 token into multiple buf16 tokens
1074 = icu_tokenizer_next_token(step->u.tokenizer,
1075 step->buf16, status);
1076 // make sure to get new previous token if this one had been used up
1077 if (step->previous && !step->more_tokens){
1078 if (icu_chain_step_next_token(chain, step->previous, status)){
1079 icu_tokenizer_attach(step->u.tokenizer, src16, status);
1080 step->need_new_token = 0;
1082 = icu_tokenizer_next_token(step->u.tokenizer,
1083 step->buf16, status);
1086 if (0 == step->more_tokens)
1096 // stop further token processing if last step and
1097 // new tokens are needed from previous (non-existing) step
1098 if (!step->previous && step->need_new_token)
1099 step->more_tokens = 0;
1101 //printf("%d %d %d\n",
1102 // step->more_tokens, src16->utf16_len, step->buf16->utf16_len);
1105 if (U_FAILURE(*status))
1113 int icu_chain_assign_cstr(struct icu_chain * chain,
1114 const char * src8cstr,
1117 struct icu_chain_step * stp = 0;
1119 if (!chain || !src8cstr)
1124 // clear token count
1125 chain->token_count = 0;
1127 // clear all steps stop states
1130 stp->more_tokens = 1;
1131 stp->need_new_token = 1;
1132 stp = stp->previous;
1135 // finally convert UTF8 to UTF16 string
1136 icu_utf16_from_utf8_cstr(chain->src16, src8cstr, status);
1138 if (U_FAILURE(*status))
1146 int icu_chain_next_token(struct icu_chain * chain,
1151 if (!chain || !chain->steps)
1154 success = icu_chain_step_next_token(chain, chain->steps, status);
1157 chain->token_count++;
1158 return chain->token_count;
1164 int icu_chain_get_token_count(struct icu_chain * chain)
1169 return chain->token_count;
1174 const char * icu_chain_get_display(struct icu_chain * chain)
1176 if (chain->display8)
1177 return (const char *) chain->display8->utf8;
1182 const char * icu_chain_get_norm(struct icu_chain * chain)
1185 return (const char *) chain->norm8->utf8;
1190 const char * icu_chain_get_sort(struct icu_chain * chain)
1193 return (const char *) chain->sort8->utf8;
1209 * indent-tabs-mode: nil
1211 * vim: shiftwidth=4 tabstop=8 expandtab