mjohnson/espeak-ng
1
0
Fork 0
mirror of https://github.com/rhasspy/espeak-ng.git synced 2026-10-31 05:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

tokenizer.c: Tokenize line separator codepoints as newline tokens.

Browse Source
This commit is contained in:
Reece H. Dunn
2017-04-28 07:16:38 +01:00
parent bf45e7ce36
commit d2d718d700
2 changed files with 13 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libespeak-ng/tokenizer.c
+2 -1
View File
@@ -93,6 +93,7 @@ typedef enum {
#define ESPEAKNG_CTYPE_PROPERTY_MASK 0x0000000000000001ull
// Reference: http://www.unicode.org/reports/tr14/tr14-32.html -- Unicode Line Breaking Algorithm
static espeakng_CTYPE codepoint_type(uint32_t c)
{
// 1. Detect and classify specific codepoints.
@@ -110,7 +111,7 @@ static espeakng_CTYPE codepoint_type(uint32_t c)
ucd_category cat = ucd_lookup_category(c);
switch (cat)
{
case UCD_CATEGORY_Zl: return ESPEAKNG_CTYPE_WHITESPACE;
case UCD_CATEGORY_Zl: return ESPEAKNG_CTYPE_NEWLINE;
case UCD_CATEGORY_Zp: return ESPEAKNG_CTYPE_WHITESPACE;
case UCD_CATEGORY_Zs: return ESPEAKNG_CTYPE_WHITESPACE;
}
tests/tokenizer.c
+11 -11
View File
@@ -268,7 +268,7 @@ test_unicode_newline_tokens()
espeak_ng_TOKENIZER *tokenizer = create_tokenizer();
espeak_ng_TEXT_DECODER *decoder = create_text_decoder();
assert(text_decoder_decode_string(decoder, "\xC2\x85\xC2\x85", -1, ESPEAKNG_ENCODING_UTF_8) == ENS_OK);
assert(text_decoder_decode_string(decoder, "\xC2\x85\xC2\x85\xE2\x80\xA8\xE2\x80\xA8", -1, ESPEAKNG_ENCODING_UTF_8) == ENS_OK);
assert(tokenizer_reset(tokenizer, decoder) == 1);
// U+0085 : NEXT LINE (NEL) -- Used in EBCDIC systems as a combined CR+LF character.
@@ -280,6 +280,15 @@ test_unicode_newline_tokens()
assert(tokenizer_get_token_text(tokenizer) != NULL);
assert(strcmp(tokenizer_get_token_text(tokenizer), "\xC2\x85") == 0);
// General Category: Zl -- LINE SEPARATOR
assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_NEWLINE);
assert(tokenizer_get_token_text(tokenizer) != NULL);
assert(strcmp(tokenizer_get_token_text(tokenizer), "\xE2\x80\xA8") == 0);
assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_NEWLINE);
assert(tokenizer_get_token_text(tokenizer) != NULL);
assert(strcmp(tokenizer_get_token_text(tokenizer), "\xE2\x80\xA8") == 0);
assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_END_OF_BUFFER);
assert(tokenizer_get_token_text(tokenizer) != NULL);
assert(*tokenizer_get_token_text(tokenizer) == '\0');
@@ -296,7 +305,7 @@ test_whitespace_tokens()
espeak_ng_TOKENIZER *tokenizer = create_tokenizer();
espeak_ng_TEXT_DECODER *decoder = create_text_decoder();
assert(text_decoder_decode_string(decoder, "\t\t\n \xE3\x80\x80 \n\xC2\xA0\xC2\xA0\n\xE2\x80\xA8\n\xE2\x80\xA9", -1, ESPEAKNG_ENCODING_UTF_8) == ENS_OK);
assert(text_decoder_decode_string(decoder, "\t\t\n \xE3\x80\x80 \n\xC2\xA0\xC2\xA0\n\xE2\x80\xA9", -1, ESPEAKNG_ENCODING_UTF_8) == ENS_OK);
assert(tokenizer_reset(tokenizer, decoder) == 1);
// General Category: Cc, Property: White_Space
@@ -326,15 +335,6 @@ test_whitespace_tokens()
assert(tokenizer_get_token_text(tokenizer) != NULL);
assert(strcmp(tokenizer_get_token_text(tokenizer), "\n") == 0);
// General Category: Zl -- LINE SEPARATOR
assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_WHITESPACE);
assert(tokenizer_get_token_text(tokenizer) != NULL);
assert(strcmp(tokenizer_get_token_text(tokenizer), "\xE2\x80\xA8") == 0);
assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_NEWLINE);
assert(tokenizer_get_token_text(tokenizer) != NULL);
assert(strcmp(tokenizer_get_token_text(tokenizer), "\n") == 0);
// General Category: Zp -- PARAGRAPH SEPARATOR
assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_WHITESPACE);
assert(tokenizer_get_token_text(tokenizer) != NULL);