From fc7a4e6701271fcb8089b877132898c79f2c7dbe Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Fri, 28 Apr 2017 07:31:05 +0100 Subject: [PATCH] tokenizer.c: Recognise U+000C [FORM FEED (FF)] as a newline codepoint. --- src/libespeak-ng/tokenizer.c | 1 + tests/tokenizer.c | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/libespeak-ng/tokenizer.c b/src/libespeak-ng/tokenizer.c index 61193c0e..451555a9 100644 --- a/src/libespeak-ng/tokenizer.c +++ b/src/libespeak-ng/tokenizer.c @@ -102,6 +102,7 @@ static espeakng_CTYPE codepoint_type(uint32_t c) { case 0x0000: return ESPEAKNG_CTYPE_END_OF_STRING; // NULL case 0x000A: return ESPEAKNG_CTYPE_NEWLINE; // LINE FEED (LF) + case 0x000C: return ESPEAKNG_CTYPE_NEWLINE; // FORM FEED (FF) case 0x000D: return ESPEAKNG_CTYPE_CARRIAGE_RETURN; // CARRIAGE RETURN (CR) case 0x0085: return ESPEAKNG_CTYPE_NEWLINE; // NEW LINE (NEL) } diff --git a/tests/tokenizer.c b/tests/tokenizer.c index 51c7fc10..2019111c 100644 --- a/tests/tokenizer.c +++ b/tests/tokenizer.c @@ -268,9 +268,18 @@ test_unicode_newline_tokens() espeak_ng_TOKENIZER *tokenizer = create_tokenizer(); espeak_ng_TEXT_DECODER *decoder = create_text_decoder(); - assert(text_decoder_decode_string(decoder, "\xC2\x85\xC2\x85\xE2\x80\xA8\xE2\x80\xA8", -1, ESPEAKNG_ENCODING_UTF_8) == ENS_OK); + assert(text_decoder_decode_string(decoder, "\x0C\x0C\xC2\x85\xC2\x85\xE2\x80\xA8\xE2\x80\xA8", -1, ESPEAKNG_ENCODING_UTF_8) == ENS_OK); assert(tokenizer_reset(tokenizer, decoder) == 1); + // U+000C : FORM FEED (FF) -- Used as a page (not paragraph) break. + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_NEWLINE); + assert(tokenizer_get_token_text(tokenizer) != NULL); + assert(strcmp(tokenizer_get_token_text(tokenizer), "\x0C") == 0); + + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_NEWLINE); + assert(tokenizer_get_token_text(tokenizer) != NULL); + assert(strcmp(tokenizer_get_token_text(tokenizer), "\x0C") == 0); + // U+0085 : NEXT LINE (NEL) -- Used in EBCDIC systems as a combined CR+LF character. assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_NEWLINE); assert(tokenizer_get_token_text(tokenizer) != NULL);