tokenizer.c: Support mac newlines.

src/libespeak-ng/tokenizer.c

@@ -111,6 +111,7 @@ tokenizer_state_default(espeak_ng_TOKENIZER *tokenizer)
 
 	switch (c = text_decoder_getc(tokenizer->decoder))
 	{
+	case '\r':
 	case '\n':
 		current += utf8_out(c, current);
 		*current = '\0';

tests/tokenizer.c

@@ -206,6 +206,33 @@ test_linux_newline_tokens()
 	destroy_tokenizer(tokenizer);
 }
 
+void
+test_mac_newline_tokens()
+{
+	printf("testing mac newline tokens\n");
+
+	espeak_ng_TOKENIZER *tokenizer = create_tokenizer();
+	espeak_ng_TEXT_DECODER *decoder = create_text_decoder();
+
+	assert(text_decoder_decode_string(decoder, "\r\r", -1, ESPEAKNG_ENCODING_US_ASCII) == ENS_OK);
+	assert(tokenizer_reset(tokenizer, decoder) == 1);
+
+	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_NEWLINE);
+	assert(tokenizer_get_token_text(tokenizer) != NULL);
+	assert(strcmp(tokenizer_get_token_text(tokenizer), "\r") == 0);
+
+	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_NEWLINE);
+	assert(tokenizer_get_token_text(tokenizer) != NULL);
+	assert(strcmp(tokenizer_get_token_text(tokenizer), "\r") == 0);
+
+	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_END_OF_BUFFER);
+	assert(tokenizer_get_token_text(tokenizer) != NULL);
+	assert(*tokenizer_get_token_text(tokenizer) == '\0');
+
+	destroy_text_decoder(decoder);
+	destroy_tokenizer(tokenizer);
+}
+
 int
 main(int argc, char **argv)
 {
@@ -223,6 +250,7 @@ main(int argc, char **argv)
 
 	test_unbound_tokenizer();
 	test_linux_newline_tokens();
+	test_mac_newline_tokens();
 
 	printf("done\n");