mjohnson/espeak-ng
1
0
Fork 0
mirror of https://github.com/rhasspy/espeak-ng.git synced 2026-11-12 16:58:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

tokenizer.c: Recognise U+0085 [NEW LINE (NEL)] as a newline codepoint.

Browse Source
This commit is contained in:
Reece H. Dunn
2017-04-28 07:08:02 +01:00
parent df6ca7a22c
commit bf45e7ce36
2 changed files with 34 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libespeak-ng/tokenizer.c
+5 -4
View File
@@ -99,9 +99,10 @@ static espeakng_CTYPE codepoint_type(uint32_t c)
switch (c)
{
case '\r': return ESPEAKNG_CTYPE_CARRIAGE_RETURN;
case '\n': return ESPEAKNG_CTYPE_NEWLINE;
case '\0': return ESPEAKNG_CTYPE_END_OF_STRING;
case 0x0000: return ESPEAKNG_CTYPE_END_OF_STRING; // NULL
case 0x000A: return ESPEAKNG_CTYPE_NEWLINE; // LINE FEED (LF)
case 0x000D: return ESPEAKNG_CTYPE_CARRIAGE_RETURN; // CARRIAGE RETURN (CR)
case 0x0085: return ESPEAKNG_CTYPE_NEWLINE; // NEW LINE (NEL)
}
// 2. Classify codepoints by their Unicode General Category.
@@ -173,7 +174,7 @@ tokenizer_state_default(espeak_ng_TOKENIZER *tokenizer)
c = text_decoder_getc(tokenizer->decoder);
}
// fallthrough
case ESPEAKNG_CTYPE_NEWLINE: // '\n'
case ESPEAKNG_CTYPE_NEWLINE:
current += utf8_out(c, current);
*current = '\0';
return ESPEAKNG_TOKEN_NEWLINE;