From 1c8ed9c1909c279ce716127490278df18926faed Mon Sep 17 00:00:00 2001
From: "Reece H. Dunn" <msclrhd@gmail.com>
Date: Wed, 26 Apr 2017 07:39:06 +0100
Subject: [PATCH] tokenizer.c: Support mac newlines.

---
 src/libespeak-ng/tokenizer.c |  1 +
 tests/tokenizer.c            | 28 ++++++++++++++++++++++++++++
 2 files changed, 29 insertions(+)

diff --git a/src/libespeak-ng/tokenizer.c b/src/libespeak-ng/tokenizer.c
index ff39727a..5f8db45b 100644
--- a/src/libespeak-ng/tokenizer.c
+++ b/src/libespeak-ng/tokenizer.c
@@ -111,6 +111,7 @@ tokenizer_state_default(espeak_ng_TOKENIZER *tokenizer)
 
 	switch (c = text_decoder_getc(tokenizer->decoder))
 	{
+	case '\r':
 	case '\n':
 		current += utf8_out(c, current);
 		*current = '\0';
diff --git a/tests/tokenizer.c b/tests/tokenizer.c
index 7ff07e0c..1df2d8c5 100644
--- a/tests/tokenizer.c
+++ b/tests/tokenizer.c
@@ -206,6 +206,33 @@ test_linux_newline_tokens()
 	destroy_tokenizer(tokenizer);
 }
 
+void
+test_mac_newline_tokens()
+{
+	printf("testing mac newline tokens\n");
+
+	espeak_ng_TOKENIZER *tokenizer = create_tokenizer();
+	espeak_ng_TEXT_DECODER *decoder = create_text_decoder();
+
+	assert(text_decoder_decode_string(decoder, "\r\r", -1, ESPEAKNG_ENCODING_US_ASCII) == ENS_OK);
+	assert(tokenizer_reset(tokenizer, decoder) == 1);
+
+	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_NEWLINE);
+	assert(tokenizer_get_token_text(tokenizer) != NULL);
+	assert(strcmp(tokenizer_get_token_text(tokenizer), "\r") == 0);
+
+	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_NEWLINE);
+	assert(tokenizer_get_token_text(tokenizer) != NULL);
+	assert(strcmp(tokenizer_get_token_text(tokenizer), "\r") == 0);
+
+	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_END_OF_BUFFER);
+	assert(tokenizer_get_token_text(tokenizer) != NULL);
+	assert(*tokenizer_get_token_text(tokenizer) == '\0');
+
+	destroy_text_decoder(decoder);
+	destroy_tokenizer(tokenizer);
+}
+
 int
 main(int argc, char **argv)
 {
@@ -223,6 +250,7 @@ main(int argc, char **argv)
 
 	test_unbound_tokenizer();
 	test_linux_newline_tokens();
+	test_mac_newline_tokens();
 
 	printf("done\n");