tokenizer.c: Tokenise Zp codepoints as paragraphs.

src/libespeak-ng/tokenizer.c

@@ -88,6 +88,7 @@ typedef enum {
 	ESPEAKNG_CTYPE_CARRIAGE_RETURN,
 	ESPEAKNG_CTYPE_NEWLINE,
 	ESPEAKNG_CTYPE_END_OF_STRING,
+	ESPEAKNG_CTYPE_PARAGRAPH,
 	ESPEAKNG_CTYPE_WHITESPACE,
 } espeakng_CTYPE;
 
@@ -113,7 +114,7 @@ static espeakng_CTYPE codepoint_type(uint32_t c)
 	switch (cat)
 	{
 	case UCD_CATEGORY_Zl: return ESPEAKNG_CTYPE_NEWLINE;
-	case UCD_CATEGORY_Zp: return ESPEAKNG_CTYPE_WHITESPACE;
+	case UCD_CATEGORY_Zp: return ESPEAKNG_CTYPE_PARAGRAPH;
 	case UCD_CATEGORY_Zs: return ESPEAKNG_CTYPE_WHITESPACE;
 	}
 
@@ -170,6 +171,9 @@ tokenizer_state_default(espeak_ng_TOKENIZER *tokenizer)
 
 	switch (codepoint_type(c))
 	{
+	case ESPEAKNG_CTYPE_END_OF_STRING: // '\0'
+		tokenizer->read = tokenizer_state_end_of_buffer;
+		return tokenizer_state_end_of_buffer(tokenizer);
 	case ESPEAKNG_CTYPE_CARRIAGE_RETURN: // '\r'
 		if (text_decoder_peekc(tokenizer->decoder) == '\n') {
 			current += utf8_out(c, current);
@@ -180,9 +184,10 @@ tokenizer_state_default(espeak_ng_TOKENIZER *tokenizer)
 		current += utf8_out(c, current);
 		*current = '\0';
 		return ESPEAKNG_TOKEN_NEWLINE;
-	case ESPEAKNG_CTYPE_END_OF_STRING: // '\0'
-		tokenizer->read = tokenizer_state_end_of_buffer;
-		return tokenizer_state_end_of_buffer(tokenizer);
+	case ESPEAKNG_CTYPE_PARAGRAPH:
+		current += utf8_out(c, current);
+		*current = '\0';
+		return ESPEAKNG_TOKEN_PARAGRAPH;
 	case ESPEAKNG_CTYPE_WHITESPACE:
 		current += utf8_out(c, current);
 		while (!text_decoder_eof(tokenizer->decoder) &&

src/libespeak-ng/tokenizer.h

@@ -39,6 +39,7 @@ typedef enum
 	ESPEAKNG_TOKEN_END_OF_BUFFER,
 	ESPEAKNG_TOKEN_UNKNOWN,
 	ESPEAKNG_TOKEN_NEWLINE,
+	ESPEAKNG_TOKEN_PARAGRAPH,
 	ESPEAKNG_TOKEN_WHITESPACE,
 } espeak_ng_TOKEN_TYPE;
 

tests/tokenizer.c

@@ -306,6 +306,34 @@ test_unicode_newline_tokens()
 	destroy_tokenizer(tokenizer);
 }
 
+void
+test_paragraph_tokens()
+{
+	printf("testing paragraph tokens\n");
+
+	espeak_ng_TOKENIZER *tokenizer = create_tokenizer();
+	espeak_ng_TEXT_DECODER *decoder = create_text_decoder();
+
+	assert(text_decoder_decode_string(decoder, "\xE2\x80\xA9\xE2\x80\xA9", -1, ESPEAKNG_ENCODING_UTF_8) == ENS_OK);
+	assert(tokenizer_reset(tokenizer, decoder) == 1);
+
+	// General Category: Zp -- PARAGRAPH SEPARATOR
+	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_PARAGRAPH);
+	assert(tokenizer_get_token_text(tokenizer) != NULL);
+	assert(strcmp(tokenizer_get_token_text(tokenizer), "\xE2\x80\xA9") == 0);
+
+	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_PARAGRAPH);
+	assert(tokenizer_get_token_text(tokenizer) != NULL);
+	assert(strcmp(tokenizer_get_token_text(tokenizer), "\xE2\x80\xA9") == 0);
+
+	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_END_OF_BUFFER);
+	assert(tokenizer_get_token_text(tokenizer) != NULL);
+	assert(*tokenizer_get_token_text(tokenizer) == '\0');
+
+	destroy_text_decoder(decoder);
+	destroy_tokenizer(tokenizer);
+}
+
 void
 test_whitespace_tokens()
 {
@@ -314,7 +342,7 @@ test_whitespace_tokens()
 	espeak_ng_TOKENIZER *tokenizer = create_tokenizer();
 	espeak_ng_TEXT_DECODER *decoder = create_text_decoder();
 
-	assert(text_decoder_decode_string(decoder, "\t\t\n\x0B\x0B\n \xE3\x80\x80 \n\xC2\xA0\xC2\xA0\n\xE2\x80\xA9", -1, ESPEAKNG_ENCODING_UTF_8) == ENS_OK);
+	assert(text_decoder_decode_string(decoder, "\t\t\n\x0B\x0B\n \xE3\x80\x80 \n\xC2\xA0\xC2\xA0", -1, ESPEAKNG_ENCODING_UTF_8) == ENS_OK);
 	assert(tokenizer_reset(tokenizer, decoder) == 1);
 
 	// General Category: Cc, Property: White_Space
@@ -349,15 +377,6 @@ test_whitespace_tokens()
 	assert(tokenizer_get_token_text(tokenizer) != NULL);
 	assert(strcmp(tokenizer_get_token_text(tokenizer), "\xC2\xA0\xC2\xA0") == 0);
 
-	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_NEWLINE);
-	assert(tokenizer_get_token_text(tokenizer) != NULL);
-	assert(strcmp(tokenizer_get_token_text(tokenizer), "\n") == 0);
-
-	// General Category: Zp -- PARAGRAPH SEPARATOR
-	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_WHITESPACE);
-	assert(tokenizer_get_token_text(tokenizer) != NULL);
-	assert(strcmp(tokenizer_get_token_text(tokenizer), "\xE2\x80\xA9") == 0);
-
 	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_END_OF_BUFFER);
 	assert(tokenizer_get_token_text(tokenizer) != NULL);
 	assert(*tokenizer_get_token_text(tokenizer) == '\0');
@@ -387,6 +406,7 @@ main(int argc, char **argv)
 	test_mac_newline_tokens();
 	test_windows_newline_tokens();
 	test_unicode_newline_tokens();
+	test_paragraph_tokens();
 	test_whitespace_tokens();
 
 	printf("done\n");