From ee2314e5828aac6055031f9ed4441eb9852ec27c Mon Sep 17 00:00:00 2001 From: Marc Cromme Date: Thu, 25 Oct 2007 08:32:50 +0000 Subject: [PATCH] added sortkey flag to icu_chain constructor --- include/yaz/icu_I18N.h | 20 +++++++++++----- src/icu_I18N.c | 61 +++++++++++++++++++++++++++++++++++++++++------- test/tst_icu_I18N.c | 46 ++++-------------------------------- util/yaz-icu.c | 5 ++-- 4 files changed, 73 insertions(+), 59 deletions(-) diff --git a/include/yaz/icu_I18N.h b/include/yaz/icu_I18N.h index e4f5c8c..4e80ad1 100644 --- a/include/yaz/icu_I18N.h +++ b/include/yaz/icu_I18N.h @@ -41,6 +41,8 @@ #include +// #define ICU_CHAIN_SORTKEY +#undef ICU_CHAIN_SORTKEY /* declared structs and functions */ @@ -185,8 +187,10 @@ int icu_normalizer_normalize(struct icu_normalizer * normalizer, enum icu_chain_step_type { ICU_chain_step_type_none, ICU_chain_step_type_display, /* convert to utf8 display format */ +#ifdef ICU_CHAIN_SORTKEY ICU_chain_step_type_index, /* convert to utf8 index format */ ICU_chain_step_type_sortkey, /* convert to utf8 sortkey format */ +#endif ICU_chain_step_type_casemap, /* apply utf16 charmap */ ICU_chain_step_type_normalize, /* apply utf16 normalization */ ICU_chain_step_type_tokenize /* apply utf16 tokenization */ @@ -225,8 +229,10 @@ void icu_chain_step_destroy(struct icu_chain_step * step); struct icu_chain { - uint8_t identifier[128]; uint8_t locale[16]; + int sort; + + UCollator * coll; /* number of tokens returned so far */ int32_t token_count; @@ -243,22 +249,22 @@ struct icu_chain struct icu_chain_step * steps; }; -struct icu_chain * icu_chain_create( // const uint8_t * identifier, - const uint8_t * locale); +struct icu_chain * icu_chain_create(const uint8_t * locale, + int sort, + UErrorCode * status); void icu_chain_destroy(struct icu_chain * chain); struct icu_chain * icu_chain_xml_config(xmlNode *xml_node, - const uint8_t * locale, + const uint8_t * locale, + int sort, UErrorCode * status); - struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain, enum icu_chain_step_type type, const uint8_t * rule, UErrorCode *status); - int icu_chain_step_next_token(struct icu_chain * chain, struct icu_chain_step * step, UErrorCode *status); @@ -278,6 +284,8 @@ const char * icu_chain_get_norm(struct icu_chain * chain); const char * icu_chain_get_sort(struct icu_chain * chain); +const UCollator * icu_chain_get_coll(struct icu_chain * chain); + #endif /* ICU_I18NL_H */ /* diff --git a/src/icu_I18N.c b/src/icu_I18N.c index 230d59f..305316e 100644 --- a/src/icu_I18N.c +++ b/src/icu_I18N.c @@ -2,7 +2,7 @@ * Copyright (C) 1995-2007, Index Data ApS * See the file LICENSE for details. * - * $Id: icu_I18N.c,v 1.5 2007-10-24 14:48:17 marc Exp $ + * $Id: icu_I18N.c,v 1.6 2007-10-25 08:32:50 marc Exp $ */ #if HAVE_CONFIG_H @@ -801,10 +801,12 @@ struct icu_chain_step * icu_chain_step_create(struct icu_chain * chain, switch(step->type) { case ICU_chain_step_type_display: break; +#ifdef ICU_CHAIN_SORTKEY case ICU_chain_step_type_index: break; case ICU_chain_step_type_sortkey: break; +#endif case ICU_chain_step_type_casemap: step->u.casemap = icu_casemap_create((char *) chain->locale, (char) rule[0], status); @@ -834,10 +836,12 @@ void icu_chain_step_destroy(struct icu_chain_step * step){ switch(step->type) { case ICU_chain_step_type_display: break; +#ifdef ICU_CHAIN_SORTKEY case ICU_chain_step_type_index: break; case ICU_chain_step_type_sortkey: break; +#endif case ICU_chain_step_type_casemap: icu_casemap_destroy(step->u.casemap); icu_buf_utf16_destroy(step->buf16); @@ -858,18 +862,25 @@ void icu_chain_step_destroy(struct icu_chain_step * step){ -struct icu_chain * icu_chain_create( //const uint8_t * identifier, - const uint8_t * locale) +struct icu_chain * icu_chain_create(const uint8_t * locale, + int sort, + UErrorCode * status) { struct icu_chain * chain = (struct icu_chain *) malloc(sizeof(struct icu_chain)); - //strncpy((char *) chain->identifier, (const char *) identifier, 128); - //chain->identifier[128 - 1] = '\0'; strncpy((char *) chain->locale, (const char *) locale, 16); chain->locale[16 - 1] = '\0'; + chain->sort = sort; + + chain->coll = ucol_open((const char *) chain->locale, status); + + if (U_FAILURE(*status)) + return 0; + + chain->token_count = 0; chain->display8 = icu_buf_utf8_create(0); @@ -880,6 +891,7 @@ struct icu_chain * icu_chain_create( //const uint8_t * identifier, chain->steps = 0; + return chain; } @@ -887,6 +899,10 @@ struct icu_chain * icu_chain_create( //const uint8_t * identifier, void icu_chain_destroy(struct icu_chain * chain) { if (chain){ + + if (chain->coll) + ucol_close(chain->coll); + icu_buf_utf8_destroy(chain->display8); icu_buf_utf8_destroy(chain->norm8); icu_buf_utf8_destroy(chain->sort8); @@ -902,6 +918,7 @@ void icu_chain_destroy(struct icu_chain * chain) struct icu_chain * icu_chain_xml_config(xmlNode *xml_node, const uint8_t * locale, + int sort, UErrorCode * status){ xmlNode *node = 0; @@ -914,8 +931,7 @@ struct icu_chain * icu_chain_xml_config(xmlNode *xml_node, return 0; - chain = icu_chain_create( // (const uint8_t *) xml_id, - locale); + chain = icu_chain_create(locale, sort, status); if (!chain) return 0; @@ -951,6 +967,7 @@ struct icu_chain * icu_chain_xml_config(xmlNode *xml_node, step = icu_chain_insert_step(chain, ICU_chain_step_type_display, (const uint8_t *) "", status); } +#ifdef ICU_CHAIN_SORTKEY else if (!strcmp((const char *) node->name, (const char *) "index")){ step = icu_chain_insert_step(chain, ICU_chain_step_type_index, @@ -961,7 +978,7 @@ struct icu_chain * icu_chain_xml_config(xmlNode *xml_node, step = icu_chain_insert_step(chain, ICU_chain_step_type_sortkey, (const uint8_t *) "", status); } - +#endif xmlFree(xml_rule); if (!step || U_FAILURE(*status)){ icu_chain_destroy(chain); @@ -1002,12 +1019,14 @@ struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain, case ICU_chain_step_type_display: buf16 = src16; break; +#ifdef ICU_CHAIN_SORTKEY case ICU_chain_step_type_index: buf16 = src16; break; case ICU_chain_step_type_sortkey: buf16 = src16; break; +#endif case ICU_chain_step_type_casemap: buf16 = icu_buf_utf16_create(0); break; @@ -1076,12 +1095,19 @@ int icu_chain_step_next_token(struct icu_chain * chain, case ICU_chain_step_type_display: icu_utf16_to_utf8(chain->display8, src16, status); break; +#ifdef ICU_CHAIN_SORTKEY + // TODO case ICU_chain_step_type_index: icu_utf16_to_utf8(chain->norm8, src16, status); break; case ICU_chain_step_type_sortkey: icu_utf16_to_utf8(chain->sort8, src16, status); + //UErrorCode icu_sortkey8_from_utf16(UCollator *coll, + // struct icu_buf_utf8 * dest8, + // struct icu_buf_utf16 * src16, + // UErrorCode * status); break; +#endif case ICU_chain_step_type_casemap: icu_casemap_casemap(step->u.casemap, step->buf16, src16, status); @@ -1123,6 +1149,9 @@ int icu_chain_step_next_token(struct icu_chain * chain, /* if token disappered into thin air, tell caller */ if (!step->buf16->utf16_len) return 0; + + if (U_FAILURE(*status)) + return 0; return 1; } @@ -1266,10 +1295,17 @@ int icu_chain_next_token(struct icu_chain * chain, if (!chain || !chain->steps) return 0; - got_token = icu_chain_step_next_token(chain, chain->steps, status); + while(!got_token && chain->steps->more_tokens) + got_token = icu_chain_step_next_token(chain, chain->steps, status); if (got_token){ chain->token_count++; + + icu_utf16_to_utf8(chain->norm8, chain->steps->buf16, status); + + icu_sortkey8_from_utf16(chain->coll, + chain->sort8, chain->steps->buf16, status); + return chain->token_count; } @@ -1310,6 +1346,13 @@ const char * icu_chain_get_sort(struct icu_chain * chain) return 0; } +const UCollator * icu_chain_get_coll(struct icu_chain * chain) +{ + return chain->coll; +} + + + #endif /* HAVE_ICU */ diff --git a/test/tst_icu_I18N.c b/test/tst_icu_I18N.c index 35b4c56..08099e9 100644 --- a/test/tst_icu_I18N.c +++ b/test/tst_icu_I18N.c @@ -1,4 +1,4 @@ -/* $Id: tst_icu_I18N.c,v 1.6 2007-10-24 14:48:17 marc Exp $ +/* $Id: tst_icu_I18N.c,v 1.7 2007-10-25 08:32:51 marc Exp $ Copyright (c) 2006-2007, Index Data. This file is part of Pazpar2. @@ -499,8 +499,6 @@ void test_icu_I18N_chain(int argc, char **argv) "" "" "" - "" - "" ""; @@ -511,40 +509,7 @@ void test_icu_I18N_chain(int argc, char **argv) // printf("ICU chain:\ninput: '%s'\n", en_str); - chain = icu_chain_xml_config(xml_node, (uint8_t *) "en", &status); - -#if 0 - chain = icu_chain_create((uint8_t *) "en:word", (uint8_t *) "en"); - step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, - (const uint8_t *) "[:Control:] Any-Remove", - &status); - step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize, - (const uint8_t *) "s", - &status); - step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize, - (const uint8_t *) "l", - &status); - step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, - (const uint8_t *) - "[[:WhiteSpace:][:Punctuation:]] Any-Remove", - &status); - step = icu_chain_insert_step(chain, ICU_chain_step_type_display, - (const uint8_t *)"", - &status); -/* step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, */ -/* (const uint8_t *) "Lower", */ -/* &status); */ - step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap, - (const uint8_t *) "l", - &status); - step = icu_chain_insert_step(chain, ICU_chain_step_type_index, - (const uint8_t *)"", - &status); -/* step = icu_chain_insert_step(chain, ICU_chain_step_type_sortkey, */ -/* (const uint8_t *)"", */ -/* &status); */ - -#endif + chain = icu_chain_xml_config(xml_node, (uint8_t *) "en", 0, &status); xmlFreeDoc(doc); YAZ_CHECK(chain); @@ -594,8 +559,6 @@ void test_bug_1140(void) "" "" "" - "" - "" ""; @@ -603,7 +566,7 @@ void test_bug_1140(void) xmlNode *xml_node = xmlDocGetRootElement(doc); YAZ_CHECK(xml_node); - chain = icu_chain_xml_config(xml_node, (uint8_t *) "en", &status); + chain = icu_chain_xml_config(xml_node, (uint8_t *) "en", 0, &status); xmlFreeDoc(doc); YAZ_CHECK(chain); @@ -650,14 +613,13 @@ void test_chain_empty_token(void) const char * xml_str = "" "" "" - "" ""; xmlDoc *doc = xmlParseMemory(xml_str, strlen(xml_str)); xmlNode *xml_node = xmlDocGetRootElement(doc); YAZ_CHECK(xml_node); - chain = icu_chain_xml_config(xml_node, (uint8_t *) "en", &status); + chain = icu_chain_xml_config(xml_node, (uint8_t *) "en", 0, &status); xmlFreeDoc(doc); YAZ_CHECK(chain); diff --git a/util/yaz-icu.c b/util/yaz-icu.c index a876186..df78e91 100644 --- a/util/yaz-icu.c +++ b/util/yaz-icu.c @@ -2,7 +2,7 @@ * Copyright (C) 1995-2007, Index Data ApS * See the file LICENSE for details. * - * $Id: yaz-icu.c,v 1.3 2007-10-24 14:48:18 marc Exp $ + * $Id: yaz-icu.c,v 1.4 2007-10-25 08:32:51 marc Exp $ */ #if HAVE_CONFIG_H @@ -445,7 +445,8 @@ static void process_text_file(const struct config_t *p_config) if (!xml_locale || !strlen((const char *) xml_locale)) return; - config.chain = icu_chain_xml_config(xml_node, (uint8_t *) "en", &status); + config.chain = icu_chain_xml_config(xml_node, (uint8_t *) xml_locale, 0, + &status); xmlFree(xml_locale); -- 1.7.10.4