mirror of
https://github.com/rhasspy/espeak-ng.git
synced 2026-11-18 18:50:25 +00:00
tokenizer.c: Support comma tokens.
This commit is contained in:
@@ -95,9 +95,10 @@ typedef enum {
|
||||
ESPEAKNG_CTYPE_FULL_STOP,
|
||||
ESPEAKNG_CTYPE_QUESTION_MARK,
|
||||
ESPEAKNG_CTYPE_EXCLAMATION_MARK,
|
||||
ESPEAKNG_CTYPE_COMMA,
|
||||
} espeakng_CTYPE;
|
||||
|
||||
#define ESPEAKNG_CTYPE_PROPERTY_MASK 0xE00000000000C001ull
|
||||
#define ESPEAKNG_CTYPE_PROPERTY_MASK 0xF00000000000C001ull
|
||||
|
||||
// Reference: http://www.unicode.org/reports/tr14/tr14-32.html -- Unicode Line Breaking Algorithm
|
||||
static espeakng_CTYPE codepoint_type(uint32_t c)
|
||||
@@ -142,6 +143,8 @@ static espeakng_CTYPE codepoint_type(uint32_t c)
|
||||
return ESPEAKNG_CTYPE_QUESTION_MARK;
|
||||
case ESPEAKNG_PROPERTY_EXCLAMATION_MARK:
|
||||
return ESPEAKNG_CTYPE_EXCLAMATION_MARK;
|
||||
case ESPEAKNG_PROPERTY_COMMA:
|
||||
return ESPEAKNG_CTYPE_COMMA;
|
||||
}
|
||||
|
||||
// 4. Classify the remaining codepoints.
|
||||
@@ -285,6 +288,10 @@ tokenizer_state_default(espeak_ng_TOKENIZER *tokenizer)
|
||||
current += utf8_out(c, current);
|
||||
*current = '\0';
|
||||
return ESPEAKNG_TOKEN_EXCLAMATION_MARK;
|
||||
case ESPEAKNG_CTYPE_COMMA:
|
||||
current += utf8_out(c, current);
|
||||
*current = '\0';
|
||||
return ESPEAKNG_TOKEN_COMMA;
|
||||
default:
|
||||
current += utf8_out(c, current);
|
||||
*current = '\0';
|
||||
|
||||
@@ -54,6 +54,7 @@ typedef enum
|
||||
ESPEAKNG_TOKEN_FULL_STOP,
|
||||
ESPEAKNG_TOKEN_QUESTION_MARK,
|
||||
ESPEAKNG_TOKEN_EXCLAMATION_MARK,
|
||||
ESPEAKNG_TOKEN_COMMA,
|
||||
} espeak_ng_TOKEN_TYPE;
|
||||
|
||||
espeak_ng_TOKEN_TYPE
|
||||
|
||||
Reference in New Issue
Block a user