diff --git a/src/libespeak-ng/tokenizer.c b/src/libespeak-ng/tokenizer.c index 5f8db45b..3b4039dc 100644 --- a/src/libespeak-ng/tokenizer.c +++ b/src/libespeak-ng/tokenizer.c @@ -112,6 +112,11 @@ tokenizer_state_default(espeak_ng_TOKENIZER *tokenizer) switch (c = text_decoder_getc(tokenizer->decoder)) { case '\r': + if (text_decoder_peekc(tokenizer->decoder) == '\n') { + current += utf8_out(c, current); + c = text_decoder_getc(tokenizer->decoder); + } + // fallthrough case '\n': current += utf8_out(c, current); *current = '\0'; diff --git a/tests/tokenizer.c b/tests/tokenizer.c index 1df2d8c5..dbc6b518 100644 --- a/tests/tokenizer.c +++ b/tests/tokenizer.c @@ -233,6 +233,33 @@ test_mac_newline_tokens() destroy_tokenizer(tokenizer); } +void +test_windows_newline_tokens() +{ + printf("testing windows newline tokens\n"); + + espeak_ng_TOKENIZER *tokenizer = create_tokenizer(); + espeak_ng_TEXT_DECODER *decoder = create_text_decoder(); + + assert(text_decoder_decode_string(decoder, "\r\n\r\n", -1, ESPEAKNG_ENCODING_US_ASCII) == ENS_OK); + assert(tokenizer_reset(tokenizer, decoder) == 1); + + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_NEWLINE); + assert(tokenizer_get_token_text(tokenizer) != NULL); + assert(strcmp(tokenizer_get_token_text(tokenizer), "\r\n") == 0); + + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_NEWLINE); + assert(tokenizer_get_token_text(tokenizer) != NULL); + assert(strcmp(tokenizer_get_token_text(tokenizer), "\r\n") == 0); + + assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_END_OF_BUFFER); + assert(tokenizer_get_token_text(tokenizer) != NULL); + assert(*tokenizer_get_token_text(tokenizer) == '\0'); + + destroy_text_decoder(decoder); + destroy_tokenizer(tokenizer); +} + int main(int argc, char **argv) { @@ -251,6 +278,7 @@ main(int argc, char **argv) test_unbound_tokenizer(); test_linux_newline_tokens(); test_mac_newline_tokens(); + test_windows_newline_tokens(); printf("done\n");