+
+ return success;
+}
+
+
+// DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8
+
+void test_icu_I18N_tokenizer(int argc, char **argv)
+{
+
+
+ const char * en_str
+ = "O Romeo, Romeo! wherefore art thou Romeo?";
+
+ YAZ_CHECK(test_icu_tokenizer("en", 's', en_str, 2));
+ YAZ_CHECK(test_icu_tokenizer("en", 'l', en_str, 7));
+ YAZ_CHECK(test_icu_tokenizer("en", 'w', en_str, 16));
+ YAZ_CHECK(test_icu_tokenizer("en", 'c', en_str, 41));
+
+
+
+ const char * fr_str
+ = "O Romeo, Romeo! wherefore art thou Romeo?";
+
+ YAZ_CHECK(test_icu_tokenizer("fr", 's', fr_str, 2));
+ YAZ_CHECK(test_icu_tokenizer("fr", 'l', fr_str, 7));
+ YAZ_CHECK(test_icu_tokenizer("fr", 'w', fr_str, 16));
+ YAZ_CHECK(test_icu_tokenizer("fr", 'c', fr_str, 41));
+