diff --git a/src/libespeak-ng/tokenizer.c b/src/libespeak-ng/tokenizer.c index 451555a9..274d6b41 100644 --- a/src/libespeak-ng/tokenizer.c +++ b/src/libespeak-ng/tokenizer.c @@ -88,6 +88,7 @@ typedef enum { ESPEAKNG_CTYPE_CARRIAGE_RETURN, ESPEAKNG_CTYPE_NEWLINE, ESPEAKNG_CTYPE_END_OF_STRING, + ESPEAKNG_CTYPE_PARAGRAPH, ESPEAKNG_CTYPE_WHITESPACE, } espeakng_CTYPE; @@ -113,7 +114,7 @@ static espeakng_CTYPE codepoint_type(uint32_t c) switch (cat) { case UCD_CATEGORY_Zl: return ESPEAKNG_CTYPE_NEWLINE; - case UCD_CATEGORY_Zp: return ESPEAKNG_CTYPE_WHITESPACE; + case UCD_CATEGORY_Zp: return ESPEAKNG_CTYPE_PARAGRAPH; case UCD_CATEGORY_Zs: return ESPEAKNG_CTYPE_WHITESPACE; } @@ -170,6 +171,9 @@ tokenizer_state_default(espeak_ng_TOKENIZER *tokenizer) switch (codepoint_type(c)) { + case ESPEAKNG_CTYPE_END_OF_STRING: // '\0' + tokenizer->read = tokenizer_state_end_of_buffer; + return tokenizer_state_end_of_buffer(tokenizer); case ESPEAKNG_CTYPE_CARRIAGE_RETURN: // '\r' if (text_decoder_peekc(tokenizer->decoder) == '\n') { current += utf8_out(c, current); @@ -180,9 +184,10 @@ tokenizer_state_default(espeak_ng_TOKENIZER *tokenizer) current += utf8_out(c, current); *current = '\0'; return ESPEAKNG_TOKEN_NEWLINE; - case ESPEAKNG_CTYPE_END_OF_STRING: // '\0' - tokenizer->read = tokenizer_state_end_of_buffer; - return tokenizer_state_end_of_buffer(tokenizer); + case ESPEAKNG_CTYPE_PARAGRAPH: + current += utf8_out(c, current); + *current = '\0'; + return ESPEAKNG_TOKEN_PARAGRAPH; case ESPEAKNG_CTYPE_WHITESPACE: current += utf8_out(c, current); while (!text_decoder_eof(tokenizer->decoder) && diff --git a/src/libespeak-ng/tokenizer.h b/src/libespeak-ng/tokenizer.h index c1ba7fb9..74f0c9bb 100644 --- a/src/libespeak-ng/tokenizer.h +++ b/src/libespeak-ng/tokenizer.h @@ -39,6 +39,7 @@ typedef enum ESPEAKNG_TOKEN_END_OF_BUFFER, ESPEAKNG_TOKEN_UNKNOWN, ESPEAKNG_TOKEN_NEWLINE, + ESPEAKNG_TOKEN_PARAGRAPH, ESPEAKNG_TOKEN_WHITESPACE, } espeak_ng_TOKEN_TYPE; diff --git a/tests/tokenizer.c b/tests/tokenizer.c index e8d4a3da..990f0df5 100644 --- a/tests/tokenizer.c +++ b/tests/tokenizer.c @@ -306,6 +306,34 @@ test_unicode_newline_tokens() destroy_tokenizer(tokenizer); } +void +test_paragraph_tokens() +{ + printf("testing paragraph tokens\n"); + + espeak_ng_TOKENIZER *tokenizer = create_tokenizer(); + espeak_ng_TEXT_DECODER *decoder = create_text_decoder(); + + assert(text_decoder_decode_string(decoder, "\xE2\x80\xA9\xE2\x80\xA9", -1, ESPEAKNG_ENCODING_UTF_8) == ENS_OK); + assert(tokenizer_reset(tokenizer, decoder) == 1); + + // General Category: Zp -- PARAGRAPH SEPARATOR + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_PARAGRAPH); + assert(tokenizer_get_token_text(tokenizer) != NULL); + assert(strcmp(tokenizer_get_token_text(tokenizer), "\xE2\x80\xA9") == 0); + + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_PARAGRAPH); + assert(tokenizer_get_token_text(tokenizer) != NULL); + assert(strcmp(tokenizer_get_token_text(tokenizer), "\xE2\x80\xA9") == 0); + + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_END_OF_BUFFER); + assert(tokenizer_get_token_text(tokenizer) != NULL); + assert(*tokenizer_get_token_text(tokenizer) == '\0'); + + destroy_text_decoder(decoder); + destroy_tokenizer(tokenizer); +} + void test_whitespace_tokens() { @@ -314,7 +342,7 @@ test_whitespace_tokens() espeak_ng_TOKENIZER *tokenizer = create_tokenizer(); espeak_ng_TEXT_DECODER *decoder = create_text_decoder(); - assert(text_decoder_decode_string(decoder, "\t\t\n\x0B\x0B\n \xE3\x80\x80 \n\xC2\xA0\xC2\xA0\n\xE2\x80\xA9", -1, ESPEAKNG_ENCODING_UTF_8) == ENS_OK); + assert(text_decoder_decode_string(decoder, "\t\t\n\x0B\x0B\n \xE3\x80\x80 \n\xC2\xA0\xC2\xA0", -1, ESPEAKNG_ENCODING_UTF_8) == ENS_OK); assert(tokenizer_reset(tokenizer, decoder) == 1); // General Category: Cc, Property: White_Space @@ -349,15 +377,6 @@ test_whitespace_tokens() assert(tokenizer_get_token_text(tokenizer) != NULL); assert(strcmp(tokenizer_get_token_text(tokenizer), "\xC2\xA0\xC2\xA0") == 0); - assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_NEWLINE); - assert(tokenizer_get_token_text(tokenizer) != NULL); - assert(strcmp(tokenizer_get_token_text(tokenizer), "\n") == 0); - - // General Category: Zp -- PARAGRAPH SEPARATOR - assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_WHITESPACE); - assert(tokenizer_get_token_text(tokenizer) != NULL); - assert(strcmp(tokenizer_get_token_text(tokenizer), "\xE2\x80\xA9") == 0); - assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_END_OF_BUFFER); assert(tokenizer_get_token_text(tokenizer) != NULL); assert(*tokenizer_get_token_text(tokenizer) == '\0'); @@ -387,6 +406,7 @@ main(int argc, char **argv) test_mac_newline_tokens(); test_windows_newline_tokens(); test_unicode_newline_tokens(); + test_paragraph_tokens(); test_whitespace_tokens(); printf("done\n");