added sortkey flag to icu_chain constructor

diff --git a/include/yaz/icu_I18N.h b/include/yaz/icu_I18N.h
index e4f5c8c..4e80ad1 100644 (file)
--- a/include/yaz/icu_I18N.h
+++ b/include/yaz/icu_I18N.h
@@ -41,6 +41,8 @@
 #include <unicode/utrans.h>
 
 
+// #define ICU_CHAIN_SORTKEY
+#undef ICU_CHAIN_SORTKEY
 
 /* declared structs and functions */
 
@@ -185,8 +187,10 @@ int icu_normalizer_normalize(struct icu_normalizer * normalizer,
 enum icu_chain_step_type {
     ICU_chain_step_type_none,
     ICU_chain_step_type_display,   /* convert to utf8 display format */
+#ifdef ICU_CHAIN_SORTKEY
     ICU_chain_step_type_index,     /* convert to utf8 index format  */
     ICU_chain_step_type_sortkey,   /* convert to utf8 sortkey format */
+#endif
     ICU_chain_step_type_casemap,   /* apply utf16 charmap */
     ICU_chain_step_type_normalize, /* apply utf16 normalization */
     ICU_chain_step_type_tokenize   /* apply utf16 tokenization */
@@ -225,8 +229,10 @@ void icu_chain_step_destroy(struct icu_chain_step * step);
 
 struct icu_chain
 {
-    uint8_t identifier[128];
     uint8_t locale[16];
+    int sort;
+
+    UCollator * coll;
     
     /* number of tokens returned so far */
     int32_t token_count;
@@ -243,22 +249,22 @@ struct icu_chain
     struct icu_chain_step * steps;
 };
 
-struct icu_chain * icu_chain_create( // const uint8_t * identifier, 
-                                    const uint8_t * locale);
+struct icu_chain * icu_chain_create(const uint8_t * locale,
+                                    int sort,
+                                    UErrorCode * status);
 
 void icu_chain_destroy(struct icu_chain * chain);
 
 struct icu_chain * icu_chain_xml_config(xmlNode *xml_node,
-                                        const uint8_t * locale, 
+                                        const uint8_t * locale,
+                                        int sort,
                                         UErrorCode * status);
 
-
 struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain,
                                               enum icu_chain_step_type type,
                                               const uint8_t * rule,
                                               UErrorCode *status);
 
-
 int icu_chain_step_next_token(struct icu_chain * chain,
                               struct icu_chain_step * step,
                               UErrorCode *status);
@@ -278,6 +284,8 @@ const char * icu_chain_get_norm(struct icu_chain * chain);
 
 const char * icu_chain_get_sort(struct icu_chain * chain);
 
+const UCollator * icu_chain_get_coll(struct icu_chain * chain);
+
 #endif /* ICU_I18NL_H */
 
 /*

diff --git a/src/icu_I18N.c b/src/icu_I18N.c
index 230d59f..305316e 100644 (file)
--- a/src/icu_I18N.c
+++ b/src/icu_I18N.c
@@ -2,7 +2,7 @@
  * Copyright (C) 1995-2007, Index Data ApS
  * See the file LICENSE for details.
  *
- * $Id: icu_I18N.c,v 1.5 2007-10-24 14:48:17 marc Exp $
+ * $Id: icu_I18N.c,v 1.6 2007-10-25 08:32:50 marc Exp $
  */
 
 #if HAVE_CONFIG_H
@@ -801,10 +801,12 @@ struct icu_chain_step * icu_chain_step_create(struct icu_chain * chain,
     switch(step->type) {
     case ICU_chain_step_type_display:
         break;
+#ifdef ICU_CHAIN_SORTKEY
     case ICU_chain_step_type_index:
         break;
     case ICU_chain_step_type_sortkey:
         break;
+#endif
     case ICU_chain_step_type_casemap:
         step->u.casemap = icu_casemap_create((char *) chain->locale, 
                                              (char) rule[0], status);
@@ -834,10 +836,12 @@ void icu_chain_step_destroy(struct icu_chain_step * step){
     switch(step->type) {
     case ICU_chain_step_type_display:
         break;
+#ifdef ICU_CHAIN_SORTKEY
     case ICU_chain_step_type_index:
         break;
     case ICU_chain_step_type_sortkey:
         break;
+#endif
     case ICU_chain_step_type_casemap:
         icu_casemap_destroy(step->u.casemap);
         icu_buf_utf16_destroy(step->buf16);
@@ -858,18 +862,25 @@ void icu_chain_step_destroy(struct icu_chain_step * step){
 
 
 
-struct icu_chain * icu_chain_create( //const uint8_t * identifier,
-                                    const uint8_t * locale)
+struct icu_chain * icu_chain_create(const uint8_t * locale, 
+                                    int sort,
+                                    UErrorCode * status)
 {
 
     struct icu_chain * chain 
         = (struct icu_chain *) malloc(sizeof(struct icu_chain));
 
-    //strncpy((char *) chain->identifier, (const char *) identifier, 128);
-    //chain->identifier[128 - 1] = '\0';
     strncpy((char *) chain->locale, (const char *) locale, 16);    
     chain->locale[16 - 1] = '\0';
 
+    chain->sort = sort;
+
+    chain->coll = ucol_open((const char *) chain->locale, status);
+
+    if (U_FAILURE(*status))
+        return 0;
+
+
     chain->token_count = 0;
 
     chain->display8 = icu_buf_utf8_create(0);
@@ -880,6 +891,7 @@ struct icu_chain * icu_chain_create( //const uint8_t * identifier,
 
     chain->steps = 0;
 
+
     return chain;
 }
 
@@ -887,6 +899,10 @@ struct icu_chain * icu_chain_create( //const uint8_t * identifier,
 void icu_chain_destroy(struct icu_chain * chain)
 {
     if (chain){
+
+        if (chain->coll)
+            ucol_close(chain->coll);
+
         icu_buf_utf8_destroy(chain->display8);
         icu_buf_utf8_destroy(chain->norm8);
         icu_buf_utf8_destroy(chain->sort8);
@@ -902,6 +918,7 @@ void icu_chain_destroy(struct icu_chain * chain)
 
 struct icu_chain * icu_chain_xml_config(xmlNode *xml_node, 
                                         const uint8_t * locale, 
+                                        int sort,
                                         UErrorCode * status){
 
     xmlNode *node = 0;
@@ -914,8 +931,7 @@ struct icu_chain * icu_chain_xml_config(xmlNode *xml_node,
 
         return 0;
 
-        chain = icu_chain_create( // (const uint8_t *) xml_id, 
-                                 locale);
+        chain = icu_chain_create(locale, sort, status);
 
     if (!chain)
         return 0;
@@ -951,6 +967,7 @@ struct icu_chain * icu_chain_xml_config(xmlNode *xml_node,
             step = icu_chain_insert_step(chain, ICU_chain_step_type_display, 
                                          (const uint8_t *) "", status);
         }
+#ifdef ICU_CHAIN_SORTKEY
         else if (!strcmp((const char *) node->name,
                          (const char *) "index")){
             step = icu_chain_insert_step(chain, ICU_chain_step_type_index, 
@@ -961,7 +978,7 @@ struct icu_chain * icu_chain_xml_config(xmlNode *xml_node,
             step = icu_chain_insert_step(chain, ICU_chain_step_type_sortkey, 
                                          (const uint8_t *) "", status);
         }
-
+#endif
         xmlFree(xml_rule);
         if (!step || U_FAILURE(*status)){
             icu_chain_destroy(chain);
@@ -1002,12 +1019,14 @@ struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain,
     case ICU_chain_step_type_display:
         buf16 = src16;
         break;
+#ifdef ICU_CHAIN_SORTKEY
     case ICU_chain_step_type_index:
         buf16 = src16;
         break;
     case ICU_chain_step_type_sortkey:
         buf16 = src16;
         break;
+#endif
     case ICU_chain_step_type_casemap:
         buf16 = icu_buf_utf16_create(0);
         break;
@@ -1076,12 +1095,19 @@ int icu_chain_step_next_token(struct icu_chain * chain,
     case ICU_chain_step_type_display:
         icu_utf16_to_utf8(chain->display8, src16, status);
         break;
+#ifdef ICU_CHAIN_SORTKEY
+        // TODO
     case ICU_chain_step_type_index:
         icu_utf16_to_utf8(chain->norm8, src16, status);
         break;
     case ICU_chain_step_type_sortkey:
         icu_utf16_to_utf8(chain->sort8, src16, status);
+        //UErrorCode icu_sortkey8_from_utf16(UCollator *coll,
+        //                           struct icu_buf_utf8 * dest8, 
+        //                           struct icu_buf_utf16 * src16,
+        //                           UErrorCode * status);
         break;
+#endif
     case ICU_chain_step_type_casemap:
         icu_casemap_casemap(step->u.casemap,
                             step->buf16, src16, status);
@@ -1123,6 +1149,9 @@ int icu_chain_step_next_token(struct icu_chain * chain,
     /* if token disappered into thin air, tell caller */
     if (!step->buf16->utf16_len)
         return 0;
+    
+    if (U_FAILURE(*status))
+        return 0;
 
     return 1;
 }
@@ -1266,10 +1295,17 @@ int icu_chain_next_token(struct icu_chain * chain,
     if (!chain || !chain->steps)
         return 0;
 
-    got_token = icu_chain_step_next_token(chain, chain->steps, status);
+    while(!got_token && chain->steps->more_tokens)
+        got_token = icu_chain_step_next_token(chain, chain->steps, status);
     
     if (got_token){
         chain->token_count++;
+
+        icu_utf16_to_utf8(chain->norm8, chain->steps->buf16, status);
+
+        icu_sortkey8_from_utf16(chain->coll,
+                                chain->sort8, chain->steps->buf16, status);
+
         return chain->token_count;
     }
 
@@ -1310,6 +1346,13 @@ const char * icu_chain_get_sort(struct icu_chain * chain)
     return 0;
 }
 
+const UCollator * icu_chain_get_coll(struct icu_chain * chain)
+{
+    return chain->coll;
+}
+
+
+
 
 #endif /* HAVE_ICU */
 

diff --git a/test/tst_icu_I18N.c b/test/tst_icu_I18N.c
index 35b4c56..08099e9 100644 (file)
--- a/test/tst_icu_I18N.c
+++ b/test/tst_icu_I18N.c
@@ -1,4 +1,4 @@
-/* $Id: tst_icu_I18N.c,v 1.6 2007-10-24 14:48:17 marc Exp $
+/* $Id: tst_icu_I18N.c,v 1.7 2007-10-25 08:32:51 marc Exp $
    Copyright (c) 2006-2007, Index Data.
 
    This file is part of Pazpar2.
@@ -499,8 +499,6 @@ void test_icu_I18N_chain(int argc, char **argv)
         "<normalize rule=\"[[:WhiteSpace:][:Punctuation:]] Remove\"/>"
         "<display/>"
         "<casemap rule=\"l\"/>"
-        "<index/>"
-        "<sortkey/>"
         "</icu_chain>";
 
     
@@ -511,40 +509,7 @@ void test_icu_I18N_chain(int argc, char **argv)
     // printf("ICU chain:\ninput: '%s'\n", en_str);
 
 
-    chain = icu_chain_xml_config(xml_node, (uint8_t *) "en", &status);
-
-#if 0
-    chain  = icu_chain_create((uint8_t *) "en:word", (uint8_t *) "en");
-    step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize,
-                                 (const uint8_t *) "[:Control:] Any-Remove",
-                                 &status);
-    step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,
-                                 (const uint8_t *) "s",
-                                 &status);
-    step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,
-                                 (const uint8_t *) "l",
-                                 &status);
-    step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize,
-                                 (const uint8_t *)
-                                 "[[:WhiteSpace:][:Punctuation:]] Any-Remove",
-                                 &status);
-    step = icu_chain_insert_step(chain, ICU_chain_step_type_display,
-                                 (const uint8_t *)"",
-                                 &status);
-/*     step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, */
-/*                                  (const uint8_t *) "Lower", */
-/*                                  &status); */
-    step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap,
-                                 (const uint8_t *) "l",
-                                 &status);
-    step = icu_chain_insert_step(chain, ICU_chain_step_type_index,
-                                 (const uint8_t *)"",
-                                 &status);
-/*     step = icu_chain_insert_step(chain, ICU_chain_step_type_sortkey, */
-/*                                  (const uint8_t *)"", */
-/*                                  &status); */
-    
-#endif
+    chain = icu_chain_xml_config(xml_node, (uint8_t *) "en", 0, &status);
 
     xmlFreeDoc(doc);
     YAZ_CHECK(chain);
@@ -594,8 +559,6 @@ void test_bug_1140(void)
         "<normalize rule=\"[[:WhiteSpace:][:Punctuation:]] Remove\"/>"
         "<display/>"
         "<casemap rule=\"l\"/>"
-        "<index/>"
-        "<sortkey/>"
         "</icu_chain>";
 
     
@@ -603,7 +566,7 @@ void test_bug_1140(void)
     xmlNode *xml_node = xmlDocGetRootElement(doc);
     YAZ_CHECK(xml_node);
 
-    chain = icu_chain_xml_config(xml_node, (uint8_t *) "en", &status);
+    chain = icu_chain_xml_config(xml_node, (uint8_t *) "en", 0, &status);
 
     xmlFreeDoc(doc);
     YAZ_CHECK(chain);
@@ -650,14 +613,13 @@ void test_chain_empty_token(void)
     const char * xml_str = "<icu_chain>"
         "<tokenize rule=\"w\"/>"
         "<normalize rule=\"[[:WhiteSpace:][:Punctuation:]] Remove\"/>"
-        "<index/>"
         "</icu_chain>";
     
     xmlDoc *doc = xmlParseMemory(xml_str, strlen(xml_str));
     xmlNode *xml_node = xmlDocGetRootElement(doc);
     YAZ_CHECK(xml_node);
 
-    chain = icu_chain_xml_config(xml_node, (uint8_t *) "en", &status);
+    chain = icu_chain_xml_config(xml_node, (uint8_t *) "en", 0, &status);
 
     xmlFreeDoc(doc);
     YAZ_CHECK(chain);

diff --git a/util/yaz-icu.c b/util/yaz-icu.c
index a876186..df78e91 100644 (file)
--- a/util/yaz-icu.c
+++ b/util/yaz-icu.c
@@ -2,7 +2,7 @@
  * Copyright (C) 1995-2007, Index Data ApS
  * See the file LICENSE for details.
  *
- * $Id: yaz-icu.c,v 1.3 2007-10-24 14:48:18 marc Exp $
+ * $Id: yaz-icu.c,v 1.4 2007-10-25 08:32:51 marc Exp $
  */
 
 #if HAVE_CONFIG_H
@@ -445,7 +445,8 @@ static void process_text_file(const struct config_t *p_config)
     if (!xml_locale || !strlen((const char *) xml_locale))
         return;        
     
-    config.chain = icu_chain_xml_config(xml_node, (uint8_t *) "en", &status);
+    config.chain = icu_chain_xml_config(xml_node, (uint8_t *) xml_locale, 0,
+                                        &status);
 
     xmlFree(xml_locale);