tokenizer.c: Support comma tokens.

This commit is contained in:
Reece H. Dunn
2017-05-01 19:14:47 +01:00
parent c9199cfacb
commit 7560070dcd
3 changed files with 21 additions and 2 deletions
+8 -1
View File
@@ -95,9 +95,10 @@ typedef enum {
ESPEAKNG_CTYPE_FULL_STOP,
ESPEAKNG_CTYPE_QUESTION_MARK,
ESPEAKNG_CTYPE_EXCLAMATION_MARK,
ESPEAKNG_CTYPE_COMMA,
} espeakng_CTYPE;
#define ESPEAKNG_CTYPE_PROPERTY_MASK 0xE00000000000C001ull
#define ESPEAKNG_CTYPE_PROPERTY_MASK 0xF00000000000C001ull
// Reference: http://www.unicode.org/reports/tr14/tr14-32.html -- Unicode Line Breaking Algorithm
static espeakng_CTYPE codepoint_type(uint32_t c)
@@ -142,6 +143,8 @@ static espeakng_CTYPE codepoint_type(uint32_t c)
return ESPEAKNG_CTYPE_QUESTION_MARK;
case ESPEAKNG_PROPERTY_EXCLAMATION_MARK:
return ESPEAKNG_CTYPE_EXCLAMATION_MARK;
case ESPEAKNG_PROPERTY_COMMA:
return ESPEAKNG_CTYPE_COMMA;
}
// 4. Classify the remaining codepoints.
@@ -285,6 +288,10 @@ tokenizer_state_default(espeak_ng_TOKENIZER *tokenizer)
current += utf8_out(c, current);
*current = '\0';
return ESPEAKNG_TOKEN_EXCLAMATION_MARK;
case ESPEAKNG_CTYPE_COMMA:
current += utf8_out(c, current);
*current = '\0';
return ESPEAKNG_TOKEN_COMMA;
default:
current += utf8_out(c, current);
*current = '\0';
+1
View File
@@ -54,6 +54,7 @@ typedef enum
ESPEAKNG_TOKEN_FULL_STOP,
ESPEAKNG_TOKEN_QUESTION_MARK,
ESPEAKNG_TOKEN_EXCLAMATION_MARK,
ESPEAKNG_TOKEN_COMMA,
} espeak_ng_TOKEN_TYPE;
espeak_ng_TOKEN_TYPE