From 786575c6ed7c73b5b160710bd2ad012fcf117331 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Wed, 3 May 2017 07:56:53 +0100 Subject: [PATCH] tokenizer.c: Support general punctuation tokens. --- src/include/espeak-ng/tokenizer.h | 1 + src/libespeak-ng/tokenizer.c | 12 +++++ tests/tokenizer.c | 82 +++++++++++++++++++++++++++++-- 3 files changed, 90 insertions(+), 5 deletions(-) diff --git a/src/include/espeak-ng/tokenizer.h b/src/include/espeak-ng/tokenizer.h index d0a3e1d3..ebfe9f77 100644 --- a/src/include/espeak-ng/tokenizer.h +++ b/src/include/espeak-ng/tokenizer.h @@ -58,6 +58,7 @@ typedef enum ESPEAKNG_TOKEN_COLON, ESPEAKNG_TOKEN_SEMICOLON, ESPEAKNG_TOKEN_ELLIPSIS, + ESPEAKNG_TOKEN_PUNCTUATION, } espeak_ng_TOKEN_TYPE; ESPEAK_NG_API espeak_ng_TOKEN_TYPE diff --git a/src/libespeak-ng/tokenizer.c b/src/libespeak-ng/tokenizer.c index b4a3ddea..101fbe4c 100644 --- a/src/libespeak-ng/tokenizer.c +++ b/src/libespeak-ng/tokenizer.c @@ -99,6 +99,7 @@ typedef enum { ESPEAKNG_CTYPE_COLON, ESPEAKNG_CTYPE_SEMICOLON, ESPEAKNG_CTYPE_ELLIPSIS, + ESPEAKNG_CTYPE_PUNCTUATION, } espeakng_CTYPE; #define ESPEAKNG_CTYPE_PROPERTY_MASK 0xFE0000000000C001ull @@ -159,6 +160,13 @@ static espeakng_CTYPE codepoint_type(uint32_t c) { case UCD_CATEGORY_Lu: return ESPEAKNG_CTYPE_UPPERCASE; case UCD_CATEGORY_Ll: return ESPEAKNG_CTYPE_LOWERCASE; + case UCD_CATEGORY_Pc: return ESPEAKNG_CTYPE_PUNCTUATION; + case UCD_CATEGORY_Pd: return ESPEAKNG_CTYPE_PUNCTUATION; + case UCD_CATEGORY_Pe: return ESPEAKNG_CTYPE_PUNCTUATION; + case UCD_CATEGORY_Pf: return ESPEAKNG_CTYPE_PUNCTUATION; + case UCD_CATEGORY_Pi: return ESPEAKNG_CTYPE_PUNCTUATION; + case UCD_CATEGORY_Po: return ESPEAKNG_CTYPE_PUNCTUATION; + case UCD_CATEGORY_Ps: return ESPEAKNG_CTYPE_PUNCTUATION; } // 5. Classify the remaining codepoints. @@ -330,6 +338,10 @@ tokenizer_state_default(espeak_ng_TOKENIZER *tokenizer) current += utf8_out(c, current); *current = '\0'; return ESPEAKNG_TOKEN_ELLIPSIS; + case ESPEAKNG_CTYPE_PUNCTUATION: + current += utf8_out(c, current); + *current = '\0'; + return ESPEAKNG_TOKEN_PUNCTUATION; default: current += utf8_out(c, current); *current = '\0'; diff --git a/tests/tokenizer.c b/tests/tokenizer.c index 3cf1cdfa..cfc9e493 100644 --- a/tests/tokenizer.c +++ b/tests/tokenizer.c @@ -288,7 +288,7 @@ test_unicode_newline_tokens() assert(text_decoder_decode_string(decoder, "\x0C\x0C\xC2\x85\xC2\x85\xE2\x80\xA8\xE2\x80\xA8", -1, ESPEAKNG_ENCODING_UTF_8) == ENS_OK); assert(tokenizer_reset(tokenizer, decoder, ESPEAKNG_TOKENIZER_OPTION_TEXT) == 1); - // U+000C : FORM FEED (FF) -- Used as a page (not paragraph) break. + // FORM FEED (FF) -- Used as a page (not paragraph) break. assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_NEWLINE); assert(tokenizer_get_token_text(tokenizer) != NULL); assert(strcmp(tokenizer_get_token_text(tokenizer), "\x0C") == 0); @@ -297,7 +297,7 @@ test_unicode_newline_tokens() assert(tokenizer_get_token_text(tokenizer) != NULL); assert(strcmp(tokenizer_get_token_text(tokenizer), "\x0C") == 0); - // U+0085 : NEXT LINE (NEL) -- Used in EBCDIC systems as a combined CR+LF character. + // NEXT LINE (NEL) [U+0085] -- Used in EBCDIC systems as a combined CR+LF character. assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_NEWLINE); assert(tokenizer_get_token_text(tokenizer) != NULL); assert(strcmp(tokenizer_get_token_text(tokenizer), "\xC2\x85") == 0); @@ -306,7 +306,7 @@ test_unicode_newline_tokens() assert(tokenizer_get_token_text(tokenizer) != NULL); assert(strcmp(tokenizer_get_token_text(tokenizer), "\xC2\x85") == 0); - // General Category: Zl -- LINE SEPARATOR + // General Category: Zl -- LINE SEPARATOR [U+2028] assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_NEWLINE); assert(tokenizer_get_token_text(tokenizer) != NULL); assert(strcmp(tokenizer_get_token_text(tokenizer), "\xE2\x80\xA8") == 0); @@ -334,7 +334,7 @@ test_paragraph_tokens() assert(text_decoder_decode_string(decoder, "\xE2\x80\xA9\xE2\x80\xA9", -1, ESPEAKNG_ENCODING_UTF_8) == ENS_OK); assert(tokenizer_reset(tokenizer, decoder, ESPEAKNG_TOKENIZER_OPTION_TEXT) == 1); - // General Category: Zp -- PARAGRAPH SEPARATOR + // General Category: Zp, PARAGRAPH SEPARATOR [U+2029] assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_PARAGRAPH); assert(tokenizer_get_token_text(tokenizer) != NULL); assert(strcmp(tokenizer_get_token_text(tokenizer), "\xE2\x80\xA9") == 0); @@ -389,7 +389,7 @@ test_whitespace_tokens() assert(tokenizer_get_token_text(tokenizer) != NULL); assert(strcmp(tokenizer_get_token_text(tokenizer), "\n") == 0); - // General Category: Zs, Property: White_Space, Decomposition: + // General Category: Zs, Property: White_Space, Decomposition: , NO-BREAK SPACE [U+00A0] assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_WHITESPACE); assert(tokenizer_get_token_text(tokenizer) != NULL); assert(strcmp(tokenizer_get_token_text(tokenizer), "\xC2\xA0\xC2\xA0") == 0); @@ -548,6 +548,7 @@ test_Latn_punctuation_tokens() assert(tokenizer_get_token_text(tokenizer) != NULL); assert(strcmp(tokenizer_get_token_text(tokenizer), " ") == 0); + // HORIZONTAL ELLIPSIS [U+2026] assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_ELLIPSIS); assert(tokenizer_get_token_text(tokenizer) != NULL); assert(strcmp(tokenizer_get_token_text(tokenizer), "\xE2\x80\xA6") == 0); @@ -560,6 +561,76 @@ test_Latn_punctuation_tokens() destroy_tokenizer(tokenizer); } +void +test_Latn_general_punctuation_tokens() +{ + printf("testing Latin (Latn) script general punctuation tokens\n"); + + espeak_ng_TOKENIZER *tokenizer = create_tokenizer(); + espeak_ng_TEXT_DECODER *decoder = create_text_decoder(); + + assert(text_decoder_decode_string(decoder, "\" () - _ \xC2\xAB\xC2\xBB", -1, ESPEAKNG_ENCODING_UTF_8) == ENS_OK); + assert(tokenizer_reset(tokenizer, decoder, ESPEAKNG_TOKENIZER_OPTION_TEXT) == 1); + + // General Category: Po + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_PUNCTUATION); + assert(tokenizer_get_token_text(tokenizer) != NULL); + assert(strcmp(tokenizer_get_token_text(tokenizer), "\"") == 0); + + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_WHITESPACE); + assert(tokenizer_get_token_text(tokenizer) != NULL); + assert(strcmp(tokenizer_get_token_text(tokenizer), " ") == 0); + + // General Category: Ps + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_PUNCTUATION); + assert(tokenizer_get_token_text(tokenizer) != NULL); + assert(strcmp(tokenizer_get_token_text(tokenizer), "(") == 0); + + // General Category: Pe + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_PUNCTUATION); + assert(tokenizer_get_token_text(tokenizer) != NULL); + assert(strcmp(tokenizer_get_token_text(tokenizer), ")") == 0); + + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_WHITESPACE); + assert(tokenizer_get_token_text(tokenizer) != NULL); + assert(strcmp(tokenizer_get_token_text(tokenizer), " ") == 0); + + // General Category: Pd + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_PUNCTUATION); + assert(tokenizer_get_token_text(tokenizer) != NULL); + assert(strcmp(tokenizer_get_token_text(tokenizer), "-") == 0); + + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_WHITESPACE); + assert(tokenizer_get_token_text(tokenizer) != NULL); + assert(strcmp(tokenizer_get_token_text(tokenizer), " ") == 0); + + // General Category: Pc + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_PUNCTUATION); + assert(tokenizer_get_token_text(tokenizer) != NULL); + assert(strcmp(tokenizer_get_token_text(tokenizer), "_") == 0); + + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_WHITESPACE); + assert(tokenizer_get_token_text(tokenizer) != NULL); + assert(strcmp(tokenizer_get_token_text(tokenizer), " ") == 0); + + // General Category: Pi, LEFT-POINTING DOUBLE ANGLE QUOTATION MARK [U+00AB] + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_PUNCTUATION); + assert(tokenizer_get_token_text(tokenizer) != NULL); + assert(strcmp(tokenizer_get_token_text(tokenizer), "\xC2\xAB") == 0); + + // General Category: Pf, RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK [U+00BB] + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_PUNCTUATION); + assert(tokenizer_get_token_text(tokenizer) != NULL); + assert(strcmp(tokenizer_get_token_text(tokenizer), "\xC2\xBB") == 0); + + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_END_OF_BUFFER); + assert(tokenizer_get_token_text(tokenizer) != NULL); + assert(*tokenizer_get_token_text(tokenizer) == '\0'); + + destroy_text_decoder(decoder); + destroy_tokenizer(tokenizer); +} + void run_tests() { @@ -586,6 +657,7 @@ run_tests() test_Latn_word_tokens(); test_Latn_punctuation_tokens(); + test_Latn_general_punctuation_tokens(); printf("done\n"); }