From d2d718d70000c3f7914039740f84428e2944fdef Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Fri, 28 Apr 2017 07:16:38 +0100 Subject: [PATCH] tokenizer.c: Tokenize line separator codepoints as newline tokens. --- src/libespeak-ng/tokenizer.c | 3 ++- tests/tokenizer.c | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/libespeak-ng/tokenizer.c b/src/libespeak-ng/tokenizer.c index 0abf0bb8..61193c0e 100644 --- a/src/libespeak-ng/tokenizer.c +++ b/src/libespeak-ng/tokenizer.c @@ -93,6 +93,7 @@ typedef enum { #define ESPEAKNG_CTYPE_PROPERTY_MASK 0x0000000000000001ull +// Reference: http://www.unicode.org/reports/tr14/tr14-32.html -- Unicode Line Breaking Algorithm static espeakng_CTYPE codepoint_type(uint32_t c) { // 1. Detect and classify specific codepoints. @@ -110,7 +111,7 @@ static espeakng_CTYPE codepoint_type(uint32_t c) ucd_category cat = ucd_lookup_category(c); switch (cat) { - case UCD_CATEGORY_Zl: return ESPEAKNG_CTYPE_WHITESPACE; + case UCD_CATEGORY_Zl: return ESPEAKNG_CTYPE_NEWLINE; case UCD_CATEGORY_Zp: return ESPEAKNG_CTYPE_WHITESPACE; case UCD_CATEGORY_Zs: return ESPEAKNG_CTYPE_WHITESPACE; } diff --git a/tests/tokenizer.c b/tests/tokenizer.c index 2294fbe6..51c7fc10 100644 --- a/tests/tokenizer.c +++ b/tests/tokenizer.c @@ -268,7 +268,7 @@ test_unicode_newline_tokens() espeak_ng_TOKENIZER *tokenizer = create_tokenizer(); espeak_ng_TEXT_DECODER *decoder = create_text_decoder(); - assert(text_decoder_decode_string(decoder, "\xC2\x85\xC2\x85", -1, ESPEAKNG_ENCODING_UTF_8) == ENS_OK); + assert(text_decoder_decode_string(decoder, "\xC2\x85\xC2\x85\xE2\x80\xA8\xE2\x80\xA8", -1, ESPEAKNG_ENCODING_UTF_8) == ENS_OK); assert(tokenizer_reset(tokenizer, decoder) == 1); // U+0085 : NEXT LINE (NEL) -- Used in EBCDIC systems as a combined CR+LF character. @@ -280,6 +280,15 @@ test_unicode_newline_tokens() assert(tokenizer_get_token_text(tokenizer) != NULL); assert(strcmp(tokenizer_get_token_text(tokenizer), "\xC2\x85") == 0); + // General Category: Zl -- LINE SEPARATOR + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_NEWLINE); + assert(tokenizer_get_token_text(tokenizer) != NULL); + assert(strcmp(tokenizer_get_token_text(tokenizer), "\xE2\x80\xA8") == 0); + + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_NEWLINE); + assert(tokenizer_get_token_text(tokenizer) != NULL); + assert(strcmp(tokenizer_get_token_text(tokenizer), "\xE2\x80\xA8") == 0); + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_END_OF_BUFFER); assert(tokenizer_get_token_text(tokenizer) != NULL); assert(*tokenizer_get_token_text(tokenizer) == '\0'); @@ -296,7 +305,7 @@ test_whitespace_tokens() espeak_ng_TOKENIZER *tokenizer = create_tokenizer(); espeak_ng_TEXT_DECODER *decoder = create_text_decoder(); - assert(text_decoder_decode_string(decoder, "\t\t\n \xE3\x80\x80 \n\xC2\xA0\xC2\xA0\n\xE2\x80\xA8\n\xE2\x80\xA9", -1, ESPEAKNG_ENCODING_UTF_8) == ENS_OK); + assert(text_decoder_decode_string(decoder, "\t\t\n \xE3\x80\x80 \n\xC2\xA0\xC2\xA0\n\xE2\x80\xA9", -1, ESPEAKNG_ENCODING_UTF_8) == ENS_OK); assert(tokenizer_reset(tokenizer, decoder) == 1); // General Category: Cc, Property: White_Space @@ -326,15 +335,6 @@ test_whitespace_tokens() assert(tokenizer_get_token_text(tokenizer) != NULL); assert(strcmp(tokenizer_get_token_text(tokenizer), "\n") == 0); - // General Category: Zl -- LINE SEPARATOR - assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_WHITESPACE); - assert(tokenizer_get_token_text(tokenizer) != NULL); - assert(strcmp(tokenizer_get_token_text(tokenizer), "\xE2\x80\xA8") == 0); - - assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_NEWLINE); - assert(tokenizer_get_token_text(tokenizer) != NULL); - assert(strcmp(tokenizer_get_token_text(tokenizer), "\n") == 0); - // General Category: Zp -- PARAGRAPH SEPARATOR assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_WHITESPACE); assert(tokenizer_get_token_text(tokenizer) != NULL);