From: Marc Cromme Date: Sun, 20 May 2007 19:00:17 +0000 (+0000) Subject: ICU chain working correctly with tokenizer, normalizer and casemap operations, and... X-Git-Tag: PAZPAR2.1.0.0~121 X-Git-Url: http://lists.indexdata.dk/?a=commitdiff_plain;h=968c591d366c50df78a3dda223fdb3491cbcb6cd;p=pazpar2-moved-to-github.git ICU chain working correctly with tokenizer, normalizer and casemap operations, and data extraction in normalform, sortkey form and display form all possible. New unit test added --- diff --git a/src/icu_I18N.c b/src/icu_I18N.c index d791aba..29b9edd 100644 --- a/src/icu_I18N.c +++ b/src/icu_I18N.c @@ -1,4 +1,4 @@ -/* $Id: icu_I18N.c,v 1.16 2007-05-16 19:50:01 marc Exp $ +/* $Id: icu_I18N.c,v 1.17 2007-05-20 19:00:17 marc Exp $ Copyright (c) 2006-2007, Index Data. This file is part of Pazpar2. @@ -328,6 +328,51 @@ UErrorCode icu_utf16_to_utf8(struct icu_buf_utf8 * dest8, +struct icu_casemap * icu_casemap_create(const char *locale, char action, + UErrorCode *status) +{ + struct icu_casemap * casemap + = (struct icu_casemap *) malloc(sizeof(struct icu_casemap)); + strcpy(casemap->locale, locale); + casemap->action = action; + + switch(casemap->action) { + case 'l': + break; + case 'u': + break; + case 't': + break; + case 'f': + break; + default: + icu_casemap_destroy(casemap); + return 0; + } + + return casemap; +}; + +void icu_casemap_destroy(struct icu_casemap * casemap) +{ + if (casemap) + free(casemap); +}; + + +int icu_casemap_casemap(struct icu_casemap * casemap, + struct icu_buf_utf16 * dest16, + struct icu_buf_utf16 * src16, + UErrorCode *status) +{ + if(!casemap) + return 0; + + return icu_utf16_casemap(dest16, src16, + casemap->locale, casemap->action, status); +}; + + int icu_utf16_casemap(struct icu_buf_utf16 * dest16, struct icu_buf_utf16 * src16, const char *locale, char action, @@ -622,15 +667,6 @@ int32_t icu_tokenizer_token_count(struct icu_tokenizer * tokenizer) -//struct icu_normalizer -//{ -// char action; -// struct icu_buf_utf16 * rules16; -// UParseError parse_error[256]; -// UTransliterator * trans; -//}; - - struct icu_normalizer * icu_normalizer_create(const char *rules, char action, UErrorCode *status) { @@ -729,6 +765,7 @@ struct icu_chain_step * icu_chain_step_create(struct icu_chain * chain, step->type = type; step->more_tokens = 0; + step->need_new_token = 1; if (buf16) step->buf16 = buf16; @@ -743,7 +780,9 @@ struct icu_chain_step * icu_chain_step_create(struct icu_chain * chain, break; case ICU_chain_step_type_sort: break; - case ICU_chain_step_type_charmap: + case ICU_chain_step_type_casemap: + step->u.casemap = icu_casemap_create((char *) chain->locale, + (char) rule[0], status); break; case ICU_chain_step_type_normalize: step->u.normalizer = icu_normalizer_create((char *) rule, 'f', status); @@ -774,7 +813,8 @@ void icu_chain_step_destroy(struct icu_chain_step * step){ break; case ICU_chain_step_type_sort: break; - case ICU_chain_step_type_charmap: + case ICU_chain_step_type_casemap: + icu_casemap_destroy(step->u.casemap); icu_buf_utf16_destroy(step->buf16); break; case ICU_chain_step_type_normalize: @@ -853,8 +893,7 @@ struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain, return 0; - // assign utf16 destination buffers as needed, or - // re-use previous uft18 buffer if this step does not touch it + // create utf16 destination buffers as needed, or switch(type) { case ICU_chain_step_type_display: buf16 = src16; @@ -865,7 +904,7 @@ struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain, case ICU_chain_step_type_sort: buf16 = src16; break; - case ICU_chain_step_type_charmap: + case ICU_chain_step_type_casemap: buf16 = icu_buf_utf16_create(0); break; case ICU_chain_step_type_normalize: @@ -894,17 +933,19 @@ int icu_chain_step_next_token(struct icu_chain * chain, { struct icu_buf_utf16 * src16 = 0; - printf("icu_chain_step_next_token %d\n", (int) step); + //printf("icu_chain_step_next_token %d\n", (int) step); if (!chain || !chain->src16 || !step || !step->more_tokens) return 0; // assign utf16 src buffers as neeed, advance in previous steps - // tokens, and setting stop condition + // tokens until non-zero token met, and setting stop condition if (step->previous){ src16 = step->previous->buf16; - step->more_tokens - = icu_chain_step_next_token(chain, step->previous, status); + if (step->need_new_token) + //while (step->more_tokens && !src16->utf16_len) + step->more_tokens + = icu_chain_step_next_token(chain, step->previous, status); } else { // first step can only work once on chain->src16 input buffer src16 = chain->src16; @@ -913,10 +954,10 @@ int icu_chain_step_next_token(struct icu_chain * chain, // stop if nothing to process // i.e new token source was not properly assigned - if (!step->more_tokens || !src16 || !src16->utf16_len) // + if (!step->more_tokens || !src16) // || !src16->utf16_len return 0; - printf("icu_chain_step_next_token %d working\n", (int) step); + //printf("icu_chain_step_next_token %d working\n", (int) step); // perform the work, eventually put this steps output in @@ -931,28 +972,52 @@ int icu_chain_step_next_token(struct icu_chain * chain, case ICU_chain_step_type_sort: icu_utf16_to_utf8(chain->sort8, src16, status); break; - case ICU_chain_step_type_charmap: + case ICU_chain_step_type_casemap: + icu_casemap_casemap(step->u.casemap, + step->buf16, src16, status); break; case ICU_chain_step_type_normalize: icu_normalizer_normalize(step->u.normalizer, step->buf16, src16, status); break; case ICU_chain_step_type_tokenize: - icu_tokenizer_attach(step->u.tokenizer, src16, status); + // attach to new src16 token only first time during splitting + if (step->need_new_token){ + icu_tokenizer_attach(step->u.tokenizer, src16, status); + step->need_new_token = 0; + } + // splitting one src16 token into multiple buf16 tokens step->more_tokens = icu_tokenizer_next_token(step->u.tokenizer, step->buf16, status); + // make sure to get new previous token if this one had been used up + if (step->previous && !step->more_tokens){ + if (icu_chain_step_next_token(chain, step->previous, status)){ + icu_tokenizer_attach(step->u.tokenizer, src16, status); + step->need_new_token = 0; + step->more_tokens + = icu_tokenizer_next_token(step->u.tokenizer, + step->buf16, status); + } + } + if (0 == step->more_tokens) + return 0; break; default: return 0; break; } - - // stop further token processing if last step - if (!step->previous) + + + // stop further token processing if last step and + // new tokens are needed from previous (non-existing) step + if (!step->previous && step->need_new_token) step->more_tokens = 0; + //printf("%d %d %d\n", + // step->more_tokens, src16->utf16_len, step->buf16->utf16_len); + if (U_FAILURE(*status)) return 0; diff --git a/src/icu_I18N.h b/src/icu_I18N.h index 91c356a..299eb7b 100644 --- a/src/icu_I18N.h +++ b/src/icu_I18N.h @@ -1,4 +1,4 @@ -/* $Id: icu_I18N.h,v 1.14 2007-05-16 12:39:49 marc Exp $ +/* $Id: icu_I18N.h,v 1.15 2007-05-20 19:00:17 marc Exp $ Copyright (c) 2006-2007, Index Data. This file is part of Pazpar2. @@ -87,6 +87,22 @@ UErrorCode icu_utf16_to_utf8(struct icu_buf_utf8 * dest8, struct icu_buf_utf16 * src16, UErrorCode * status); +struct icu_casemap +{ + char locale[16]; + char action; +}; + +struct icu_casemap * icu_casemap_create(const char *locale, char action, + UErrorCode *status); + +void icu_casemap_destroy(struct icu_casemap * casemap); + +int icu_casemap_casemap(struct icu_casemap * casemap, + struct icu_buf_utf16 * dest16, + struct icu_buf_utf16 * src16, + UErrorCode *status); + int icu_utf16_casemap(struct icu_buf_utf16 * dest16, struct icu_buf_utf16 * src16, const char *locale, char action, @@ -165,12 +181,13 @@ struct icu_token } #endif + enum icu_chain_step_type { ICU_chain_step_type_none, // ICU_chain_step_type_display, // convert to utf8 display format ICU_chain_step_type_norm, // convert to utf8 norm format ICU_chain_step_type_sort, // convert to utf8 sort format - ICU_chain_step_type_charmap, // apply utf16 charmap + ICU_chain_step_type_casemap, // apply utf16 charmap ICU_chain_step_type_normalize, // apply utf16 normalization ICU_chain_step_type_tokenize // apply utf16 tokenization }; @@ -182,6 +199,7 @@ struct icu_chain_step // type and action object enum icu_chain_step_type type; union { + struct icu_casemap * casemap; struct icu_normalizer * normalizer; struct icu_tokenizer * tokenizer; } u; @@ -189,6 +207,7 @@ struct icu_chain_step struct icu_buf_utf16 * buf16; struct icu_chain_step * previous; int more_tokens; + int need_new_token; }; diff --git a/src/test_icu_I18N.c b/src/test_icu_I18N.c index 00671f3..8861765 100644 --- a/src/test_icu_I18N.c +++ b/src/test_icu_I18N.c @@ -1,4 +1,4 @@ -/* $Id: test_icu_I18N.c,v 1.21 2007-05-16 19:50:01 marc Exp $ +/* $Id: test_icu_I18N.c,v 1.22 2007-05-20 19:00:17 marc Exp $ Copyright (c) 2006-2007, Index Data. This file is part of Pazpar2. @@ -498,31 +498,39 @@ void test_icu_I18N_tokenizer(int argc, char **argv) void test_icu_I18N_chain(int argc, char **argv) { const char * en_str - = "O Romeo, Romeo! wherefore art thou Romeo?"; + = "O Romeo, Romeo! wherefore art\nthou\tRomeo?"; + + printf("ICU chain:\ninput: '%s'\n", en_str); UErrorCode status = U_ZERO_ERROR; struct icu_chain_step * step = 0; struct icu_chain * chain - = icu_chain_create((uint8_t *) "en:sentence", (uint8_t *) "en"); -/* step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, */ -/* (const uint8_t *) "[:Control:] Any-Remove", */ -/* &status); */ + = icu_chain_create((uint8_t *) "en:word", (uint8_t *) "en"); + step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, + (const uint8_t *) "[:Control:] Any-Remove", + &status); + step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize, + (const uint8_t *) "s", + &status); step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize, (const uint8_t *) "l", &status); -/* step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, */ -/* (const uint8_t *) */ -/* "[[:WhiteSpace:][:Punctuation:]] Any-Remove", */ -/* &status); */ + step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, + (const uint8_t *) + "[[:WhiteSpace:][:Punctuation:]] Any-Remove", + &status); step = icu_chain_insert_step(chain, ICU_chain_step_type_display, (const uint8_t *)"", &status); /* step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, */ /* (const uint8_t *) "Lower", */ /* &status); */ -/* step = icu_chain_insert_step(chain, ICU_chain_step_type_norm, */ -/* (const uint8_t *)"", */ -/* &status); */ + step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap, + (const uint8_t *) "l", + &status); + step = icu_chain_insert_step(chain, ICU_chain_step_type_norm, + (const uint8_t *)"", + &status); /* step = icu_chain_insert_step(chain, ICU_chain_step_type_sort, */ /* (const uint8_t *)"", */ /* &status); */ @@ -533,7 +541,7 @@ void test_icu_I18N_chain(int argc, char **argv) YAZ_CHECK(icu_chain_assign_cstr(chain, en_str, &status)); while (icu_chain_next_token(chain, &status)){ - printf("token %d norm: '%s' display: '%s'\n", + printf("%d '%s' '%s'\n", icu_chain_get_token_count(chain), icu_chain_get_norm(chain), icu_chain_get_display(chain)); @@ -561,7 +569,7 @@ int main(int argc, char **argv) test_icu_I18N_sortmap(argc, argv); test_icu_I18N_normalizer(argc, argv); test_icu_I18N_tokenizer(argc, argv); - //test_icu_I18N_chain(argc, argv); + test_icu_I18N_chain(argc, argv); #else // HAVE_ICU