mjohnson/espeak-ng
1
0
Fork 0
mirror of https://github.com/rhasspy/espeak-ng.git synced 2026-11-08 07:32:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

tokenizer.c: Support general symbol tokens.

Browse Source
This commit is contained in:
Reece H. Dunn
2017-05-03 08:08:40 +01:00
parent 786575c6ed
commit dd90d3812d
3 changed files with 62 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/include/espeak-ng/tokenizer.h
+1
View File
@@ -59,6 +59,7 @@ typedef enum
ESPEAKNG_TOKEN_SEMICOLON,
ESPEAKNG_TOKEN_ELLIPSIS,
ESPEAKNG_TOKEN_PUNCTUATION,
ESPEAKNG_TOKEN_SYMBOL,
} espeak_ng_TOKEN_TYPE;
ESPEAK_NG_API espeak_ng_TOKEN_TYPE
src/libespeak-ng/tokenizer.c
+9
View File
@@ -100,6 +100,7 @@ typedef enum {
ESPEAKNG_CTYPE_SEMICOLON,
ESPEAKNG_CTYPE_ELLIPSIS,
ESPEAKNG_CTYPE_PUNCTUATION,
ESPEAKNG_CTYPE_SYMBOL,
} espeakng_CTYPE;
#define ESPEAKNG_CTYPE_PROPERTY_MASK 0xFE0000000000C001ull
@@ -167,6 +168,10 @@ static espeakng_CTYPE codepoint_type(uint32_t c)
case UCD_CATEGORY_Pi: return ESPEAKNG_CTYPE_PUNCTUATION;
case UCD_CATEGORY_Po: return ESPEAKNG_CTYPE_PUNCTUATION;
case UCD_CATEGORY_Ps: return ESPEAKNG_CTYPE_PUNCTUATION;
case UCD_CATEGORY_Sc: return ESPEAKNG_CTYPE_SYMBOL;
case UCD_CATEGORY_Sk: return ESPEAKNG_CTYPE_SYMBOL;
case UCD_CATEGORY_Sm: return ESPEAKNG_CTYPE_SYMBOL;
case UCD_CATEGORY_So: return ESPEAKNG_CTYPE_SYMBOL;
}
// 5. Classify the remaining codepoints.
@@ -342,6 +347,10 @@ tokenizer_state_default(espeak_ng_TOKENIZER *tokenizer)
current += utf8_out(c, current);
*current = '\0';
return ESPEAKNG_TOKEN_PUNCTUATION;
case ESPEAKNG_CTYPE_SYMBOL:
current += utf8_out(c, current);
*current = '\0';
return ESPEAKNG_TOKEN_SYMBOL;
default:
current += utf8_out(c, current);
*current = '\0';