From c9199cfacb73c414b928154cd06ba84060c10512 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Mon, 1 May 2017 19:00:40 +0100 Subject: [PATCH] tokenizer.c: Support exclamation mark tokens. --- src/libespeak-ng/tokenizer.c | 9 ++++++++- src/libespeak-ng/tokenizer.h | 1 + tests/tokenizer.c | 13 ++++++++++++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/libespeak-ng/tokenizer.c b/src/libespeak-ng/tokenizer.c index 9c1030c8..78f3b8d7 100644 --- a/src/libespeak-ng/tokenizer.c +++ b/src/libespeak-ng/tokenizer.c @@ -94,9 +94,10 @@ typedef enum { ESPEAKNG_CTYPE_UPPERCASE, ESPEAKNG_CTYPE_FULL_STOP, ESPEAKNG_CTYPE_QUESTION_MARK, + ESPEAKNG_CTYPE_EXCLAMATION_MARK, } espeakng_CTYPE; -#define ESPEAKNG_CTYPE_PROPERTY_MASK 0xC00000000000C001ull +#define ESPEAKNG_CTYPE_PROPERTY_MASK 0xE00000000000C001ull // Reference: http://www.unicode.org/reports/tr14/tr14-32.html -- Unicode Line Breaking Algorithm static espeakng_CTYPE codepoint_type(uint32_t c) @@ -139,6 +140,8 @@ static espeakng_CTYPE codepoint_type(uint32_t c) return ESPEAKNG_CTYPE_FULL_STOP; case ESPEAKNG_PROPERTY_QUESTION_MARK: return ESPEAKNG_CTYPE_QUESTION_MARK; + case ESPEAKNG_PROPERTY_EXCLAMATION_MARK: + return ESPEAKNG_CTYPE_EXCLAMATION_MARK; } // 4. Classify the remaining codepoints. @@ -278,6 +281,10 @@ tokenizer_state_default(espeak_ng_TOKENIZER *tokenizer) current += utf8_out(c, current); *current = '\0'; return ESPEAKNG_TOKEN_QUESTION_MARK; + case ESPEAKNG_CTYPE_EXCLAMATION_MARK: + current += utf8_out(c, current); + *current = '\0'; + return ESPEAKNG_TOKEN_EXCLAMATION_MARK; default: current += utf8_out(c, current); *current = '\0'; diff --git a/src/libespeak-ng/tokenizer.h b/src/libespeak-ng/tokenizer.h index 61a59300..4c061f6d 100644 --- a/src/libespeak-ng/tokenizer.h +++ b/src/libespeak-ng/tokenizer.h @@ -53,6 +53,7 @@ typedef enum ESPEAKNG_TOKEN_WORD_CAPITALIZED, ESPEAKNG_TOKEN_FULL_STOP, ESPEAKNG_TOKEN_QUESTION_MARK, + ESPEAKNG_TOKEN_EXCLAMATION_MARK, } espeak_ng_TOKEN_TYPE; espeak_ng_TOKEN_TYPE diff --git a/tests/tokenizer.c b/tests/tokenizer.c index 9fffd220..688c8bae 100644 --- a/tests/tokenizer.c +++ b/tests/tokenizer.c @@ -465,7 +465,7 @@ test_Latn_punctuation_tokens() espeak_ng_TOKENIZER *tokenizer = create_tokenizer(); espeak_ng_TEXT_DECODER *decoder = create_text_decoder(); - assert(text_decoder_decode_string(decoder, ". ?", -1, ESPEAKNG_ENCODING_US_ASCII) == ENS_OK); + assert(text_decoder_decode_string(decoder, ". ? !", -1, ESPEAKNG_ENCODING_US_ASCII) == ENS_OK); assert(tokenizer_reset(tokenizer, decoder, ESPEAKNG_TOKENIZER_OPTION_TEXT) == 1); assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_FULL_STOP); @@ -480,6 +480,14 @@ test_Latn_punctuation_tokens() assert(tokenizer_get_token_text(tokenizer) != NULL); assert(strcmp(tokenizer_get_token_text(tokenizer), "?") == 0); + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_WHITESPACE); + assert(tokenizer_get_token_text(tokenizer) != NULL); + assert(strcmp(tokenizer_get_token_text(tokenizer), " ") == 0); + + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_EXCLAMATION_MARK); + assert(tokenizer_get_token_text(tokenizer) != NULL); + assert(strcmp(tokenizer_get_token_text(tokenizer), "!") == 0); + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_END_OF_BUFFER); assert(tokenizer_get_token_text(tokenizer) != NULL); assert(*tokenizer_get_token_text(tokenizer) == '\0'); @@ -575,6 +583,9 @@ print_tokens(espeak_ng_TEXT_DECODER *decoder) case ESPEAKNG_TOKEN_QUESTION_MARK: printf("question mark : %s\n", tokenizer_get_token_text(tokenizer)); break; + case ESPEAKNG_TOKEN_EXCLAMATION_MARK: + printf("exclamation mark : %s\n", tokenizer_get_token_text(tokenizer)); + break; } }