From bce44316bb3bd7207c7b1abe6e5a2711dc87db30 Mon Sep 17 00:00:00 2001
From: "Reece H. Dunn" <msclrhd@gmail.com>
Date: Tue, 25 Apr 2017 07:55:10 +0100
Subject: [PATCH] Create a basic tokenizer API using a structure that mirrors
 the TtsTokenizer interface in the tts-dev-studio project.

---
 src/libespeak-ng/tokenizer.c | 44 ++++++++++++++++++++++++++++++
 src/libespeak-ng/tokenizer.h | 53 ++++++++++++++++++++++++++++++++++++
 tests/tokenizer.c            | 27 ++++++++++++++++++
 3 files changed, 124 insertions(+)
 create mode 100644 src/libespeak-ng/tokenizer.h

diff --git a/src/libespeak-ng/tokenizer.c b/src/libespeak-ng/tokenizer.c
index 5cc6da30..7acd2e3f 100644
--- a/src/libespeak-ng/tokenizer.c
+++ b/src/libespeak-ng/tokenizer.c
@@ -28,6 +28,7 @@
 #include <ucd/ucd.h>
 
 #include "encoding.h"
+#include "tokenizer.h"
 #include "speech.h"
 #include "phoneme.h"
 #include "synthesize.h"
@@ -81,3 +82,46 @@ int clause_type_from_codepoint(uint32_t c)
 
 	return CLAUSE_NONE;
 }
+
+struct espeak_ng_TOKENIZER_
+{
+	espeak_ng_TEXT_DECODER *decoder;
+};
+
+espeak_ng_TOKENIZER *
+create_tokenizer(void)
+{
+	espeak_ng_TOKENIZER *tokenizer = malloc(sizeof(espeak_ng_TOKENIZER));
+	if (!tokenizer) return NULL;
+
+	tokenizer->decoder = NULL;
+	return tokenizer;
+}
+
+void
+destroy_tokenizer(espeak_ng_TOKENIZER *tokenizer)
+{
+	if (tokenizer) free(tokenizer);
+}
+
+int
+tokenizer_reset(espeak_ng_TOKENIZER *tokenizer,
+                espeak_ng_TEXT_DECODER *decoder)
+{
+	if (!tokenizer || !decoder) return 0;
+
+	tokenizer->decoder = decoder;
+	return 1;
+}
+
+espeak_ng_TOKEN_TYPE
+tokenizer_read_next_token(espeak_ng_TOKENIZER *tokenizer)
+{
+	return ESPEAKNG_TOKEN_END_OF_BUFFER;
+}
+
+const char *
+tokenizer_get_token_text(espeak_ng_TOKENIZER *tokenizer)
+{
+	return "";
+}
diff --git a/src/libespeak-ng/tokenizer.h b/src/libespeak-ng/tokenizer.h
new file mode 100644
index 00000000..260bc481
--- /dev/null
+++ b/src/libespeak-ng/tokenizer.h
@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2017 Reece H. Dunn
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see: <http://www.gnu.org/licenses/>.
+ */
+#ifndef ESPEAK_NG_TOKENIZER_H
+#define ESPEAK_NG_TOKENIZER_H
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+typedef struct espeak_ng_TOKENIZER_ espeak_ng_TOKENIZER;
+
+espeak_ng_TOKENIZER *
+create_tokenizer(void);
+
+void
+destroy_tokenizer(espeak_ng_TOKENIZER *tokenizer);
+
+int
+tokenizer_reset(espeak_ng_TOKENIZER *tokenizer,
+                espeak_ng_TEXT_DECODER *decoder);
+
+typedef enum
+{
+	ESPEAKNG_TOKEN_END_OF_BUFFER,
+	ESPEAKNG_TOKEN_UNKNOWN,
+} espeak_ng_TOKEN_TYPE;
+
+espeak_ng_TOKEN_TYPE
+tokenizer_read_next_token(espeak_ng_TOKENIZER *tokenizer);
+
+const char *
+tokenizer_get_token_text(espeak_ng_TOKENIZER *tokenizer);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/tests/tokenizer.c b/tests/tokenizer.c
index 4719aa84..61c02ff2 100644
--- a/tests/tokenizer.c
+++ b/tests/tokenizer.c
@@ -26,6 +26,7 @@
 #include <espeak-ng/espeak_ng.h>
 
 #include "encoding.h"
+#include "tokenizer.h"
 #include "speech.h"
 #include "phoneme.h"
 #include "synthesize.h"
@@ -153,6 +154,30 @@ test_fullwidth()
 	assert(clause_type_from_codepoint(0xFF1F) == (CLAUSE_QUESTION | CLAUSE_OPTIONAL_SPACE_AFTER));
 }
 
+void
+test_unbound_tokenizer()
+{
+	printf("testing unbound tokenizer\n");
+
+	espeak_ng_TOKENIZER *tokenizer = create_tokenizer();
+	assert(tokenizer != NULL);
+
+	assert(tokenizer_get_token_text(tokenizer) != NULL);
+	assert(*tokenizer_get_token_text(tokenizer) == '\0');
+
+	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_END_OF_BUFFER);
+	assert(tokenizer_get_token_text(tokenizer) != NULL);
+	assert(*tokenizer_get_token_text(tokenizer) == '\0');
+
+	assert(tokenizer_reset(tokenizer, NULL) == 0);
+
+	assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_END_OF_BUFFER);
+	assert(tokenizer_get_token_text(tokenizer) != NULL);
+	assert(*tokenizer_get_token_text(tokenizer) == '\0');
+
+	destroy_tokenizer(tokenizer);
+}
+
 int
 main(int argc, char **argv)
 {
@@ -168,6 +193,8 @@ main(int argc, char **argv)
 	test_ideographic();
 	test_fullwidth();
 
+	test_unbound_tokenizer();
+
 	printf("done\n");
 
 	return EXIT_SUCCESS;