/* This file is part of the YAZ toolkit.
- * Copyright (C) 1995-2012 Index Data.
+ * Copyright (C) 1995-2013 Index Data.
* All rights reserved.
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
struct icu_buf_utf16 *icu_buf_utf16_copy(struct icu_buf_utf16 * dest16,
const struct icu_buf_utf16 * src16);
+struct icu_buf_utf16 *icu_buf_utf16_append(struct icu_buf_utf16 *dest16,
+ const struct icu_buf_utf16 * src16);
+
+void icu_buf_utf16_log(const char *lead, struct icu_buf_utf16 *src16);
+
void icu_buf_utf16_destroy(struct icu_buf_utf16 * buf16);
struct icu_buf_utf8;
int32_t icu_tokenizer_next_token(struct icu_tokenizer * tokenizer,
struct icu_buf_utf16 * tkn16,
- UErrorCode *status);
+ UErrorCode *status,
+ size_t *start, size_t *len);
int32_t icu_tokenizer_token_count(struct icu_tokenizer * tokenizer);