mjohnson/espeak-ng
1
0
Fork 0
mirror of https://github.com/rhasspy/espeak-ng.git synced 2026-11-11 16:39:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

tokenizer.c: Support general punctuation tokens.

Browse Source
This commit is contained in:
Reece H. Dunn
2017-05-03 07:56:53 +01:00
parent 0705844bf8
commit 786575c6ed
3 changed files with 90 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/include/espeak-ng/tokenizer.h
+1
View File
@@ -58,6 +58,7 @@ typedef enum
ESPEAKNG_TOKEN_COLON,
ESPEAKNG_TOKEN_SEMICOLON,
ESPEAKNG_TOKEN_ELLIPSIS,
ESPEAKNG_TOKEN_PUNCTUATION,
} espeak_ng_TOKEN_TYPE;
ESPEAK_NG_API espeak_ng_TOKEN_TYPE
src/libespeak-ng/tokenizer.c
+12
View File
@@ -99,6 +99,7 @@ typedef enum {
ESPEAKNG_CTYPE_COLON,
ESPEAKNG_CTYPE_SEMICOLON,
ESPEAKNG_CTYPE_ELLIPSIS,
ESPEAKNG_CTYPE_PUNCTUATION,
} espeakng_CTYPE;
#define ESPEAKNG_CTYPE_PROPERTY_MASK 0xFE0000000000C001ull
@@ -159,6 +160,13 @@ static espeakng_CTYPE codepoint_type(uint32_t c)
{
case UCD_CATEGORY_Lu: return ESPEAKNG_CTYPE_UPPERCASE;
case UCD_CATEGORY_Ll: return ESPEAKNG_CTYPE_LOWERCASE;
case UCD_CATEGORY_Pc: return ESPEAKNG_CTYPE_PUNCTUATION;
case UCD_CATEGORY_Pd: return ESPEAKNG_CTYPE_PUNCTUATION;
case UCD_CATEGORY_Pe: return ESPEAKNG_CTYPE_PUNCTUATION;
case UCD_CATEGORY_Pf: return ESPEAKNG_CTYPE_PUNCTUATION;
case UCD_CATEGORY_Pi: return ESPEAKNG_CTYPE_PUNCTUATION;
case UCD_CATEGORY_Po: return ESPEAKNG_CTYPE_PUNCTUATION;
case UCD_CATEGORY_Ps: return ESPEAKNG_CTYPE_PUNCTUATION;
}
// 5. Classify the remaining codepoints.
@@ -330,6 +338,10 @@ tokenizer_state_default(espeak_ng_TOKENIZER *tokenizer)
current += utf8_out(c, current);
*current = '\0';
return ESPEAKNG_TOKEN_ELLIPSIS;
case ESPEAKNG_CTYPE_PUNCTUATION:
current += utf8_out(c, current);
*current = '\0';
return ESPEAKNG_TOKEN_PUNCTUATION;
default:
current += utf8_out(c, current);
*current = '\0';