From dd90d3812d1f4b9dadfa2cdc9462d4a59ecc472f Mon Sep 17 00:00:00 2001
From: "Reece H. Dunn" <msclrhd@gmail.com>
Date: Wed, 3 May 2017 08:08:40 +0100
Subject: [PATCH] tokenizer.c: Support general symbol tokens.

---
 src/include/espeak-ng/tokenizer.h |  1 +
 src/libespeak-ng/tokenizer.c      |  9 ++++++
 tests/tokenizer.c                 | 52 +++++++++++++++++++++++++++++++
 3 files changed, 62 insertions(+)

diff --git a/src/include/espeak-ng/tokenizer.h b/src/include/espeak-ng/tokenizer.h
index ebfe9f77..fc35257b 100644
--- a/src/include/espeak-ng/tokenizer.h
+++ b/src/include/espeak-ng/tokenizer.h
@@ -59,6 +59,7 @@ typedef enum
 	ESPEAKNG_TOKEN_SEMICOLON,
 	ESPEAKNG_TOKEN_ELLIPSIS,
 	ESPEAKNG_TOKEN_PUNCTUATION,
+	ESPEAKNG_TOKEN_SYMBOL,
 } espeak_ng_TOKEN_TYPE;
 
 ESPEAK_NG_API espeak_ng_TOKEN_TYPE
diff --git a/src/libespeak-ng/tokenizer.c b/src/libespeak-ng/tokenizer.c
index 101fbe4c..7b7c8a64 100644
--- a/src/libespeak-ng/tokenizer.c
+++ b/src/libespeak-ng/tokenizer.c
@@ -100,6 +100,7 @@ typedef enum {
 	ESPEAKNG_CTYPE_SEMICOLON,
 	ESPEAKNG_CTYPE_ELLIPSIS,
 	ESPEAKNG_CTYPE_PUNCTUATION,
+	ESPEAKNG_CTYPE_SYMBOL,
 } espeakng_CTYPE;
 
 #define ESPEAKNG_CTYPE_PROPERTY_MASK 0xFE0000000000C001ull
@@ -167,6 +168,10 @@ static espeakng_CTYPE codepoint_type(uint32_t c)
 	case UCD_CATEGORY_Pi: return ESPEAKNG_CTYPE_PUNCTUATION;
 	case UCD_CATEGORY_Po: return ESPEAKNG_CTYPE_PUNCTUATION;
 	case UCD_CATEGORY_Ps: return ESPEAKNG_CTYPE_PUNCTUATION;
+	case UCD_CATEGORY_Sc: return ESPEAKNG_CTYPE_SYMBOL;
+	case UCD_CATEGORY_Sk: return ESPEAKNG_CTYPE_SYMBOL;
+	case UCD_CATEGORY_Sm: return ESPEAKNG_CTYPE_SYMBOL;
+	case UCD_CATEGORY_So: return ESPEAKNG_CTYPE_SYMBOL;
 	}
 
 	// 5. Classify the remaining codepoints.
@@ -342,6 +347,10 @@ tokenizer_state_default(espeak_ng_TOKENIZER *tokenizer)
 		current += utf8_out(c, current);
 		*current = '\0';
 		return ESPEAKNG_TOKEN_PUNCTUATION;
+	case ESPEAKNG_CTYPE_SYMBOL:
+		current += utf8_out(c, current);
+		*current = '\0';
+		return ESPEAKNG_TOKEN_SYMBOL;
 	default:
 		current += utf8_out(c, current);
 		*current = '\0';
diff --git a/tests/tokenizer.c b/tests/tokenizer.c
index cfc9e493..d450c36a 100644
--- a/tests/tokenizer.c
+++ b/tests/tokenizer.c
@@ -631,6 +631,57 @@ test_Latn_general_punctuation_tokens()
 	destroy_tokenizer(tokenizer);
 }
 
+void
+test_Latn_symbol_tokens()
+{
+	printf("testing Latin (Latn) script symbol tokens\n");
+
+	espeak_ng_TOKENIZER *tokenizer = create_tokenizer();
+	espeak_ng_TEXT_DECODER *decoder = create_text_decoder();
+
+	assert(text_decoder_decode_string(decoder, "$ ^ + \xC2\xA9", -1, ESPEAKNG_ENCODING_UTF_8) == ENS_OK);
+	assert(tokenizer_reset(tokenizer, decoder, ESPEAKNG_TOKENIZER_OPTION_TEXT) == 1);
+
+	// General Category: Sc
+	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_SYMBOL);
+	assert(tokenizer_get_token_text(tokenizer) != NULL);
+	assert(strcmp(tokenizer_get_token_text(tokenizer), "$") == 0);
+
+	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_WHITESPACE);
+	assert(tokenizer_get_token_text(tokenizer) != NULL);
+	assert(strcmp(tokenizer_get_token_text(tokenizer), " ") == 0);
+
+	// General Category: Sk
+	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_SYMBOL);
+	assert(tokenizer_get_token_text(tokenizer) != NULL);
+	assert(strcmp(tokenizer_get_token_text(tokenizer), "^") == 0);
+
+	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_WHITESPACE);
+	assert(tokenizer_get_token_text(tokenizer) != NULL);
+	assert(strcmp(tokenizer_get_token_text(tokenizer), " ") == 0);
+
+	// General Category: Sm
+	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_SYMBOL);
+	assert(tokenizer_get_token_text(tokenizer) != NULL);
+	assert(strcmp(tokenizer_get_token_text(tokenizer), "+") == 0);
+
+	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_WHITESPACE);
+	assert(tokenizer_get_token_text(tokenizer) != NULL);
+	assert(strcmp(tokenizer_get_token_text(tokenizer), " ") == 0);
+
+	// General Category: So, COPYRIGHT SIGN [U+00A9]
+	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_SYMBOL);
+	assert(tokenizer_get_token_text(tokenizer) != NULL);
+	assert(strcmp(tokenizer_get_token_text(tokenizer), "\xC2\xA9") == 0);
+
+	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_END_OF_BUFFER);
+	assert(tokenizer_get_token_text(tokenizer) != NULL);
+	assert(*tokenizer_get_token_text(tokenizer) == '\0');
+
+	destroy_text_decoder(decoder);
+	destroy_tokenizer(tokenizer);
+}
+
 void
 run_tests()
 {
@@ -658,6 +709,7 @@ run_tests()
 	test_Latn_word_tokens();
 	test_Latn_punctuation_tokens();
 	test_Latn_general_punctuation_tokens();
+	test_Latn_symbol_tokens();
 
 	printf("done\n");
 }