mjohnson/espeak-ng
1
0
Fork 0
mirror of https://github.com/rhasspy/espeak-ng.git synced 2026-11-15 01:54:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

tokenizer.c: Support exclamation mark tokens.

Browse Source
This commit is contained in:
Reece H. Dunn
2017-05-01 19:00:40 +01:00
parent 128ceaff6a
commit c9199cfacb
3 changed files with 21 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libespeak-ng/tokenizer.c
+8 -1
View File
@@ -94,9 +94,10 @@ typedef enum {
ESPEAKNG_CTYPE_UPPERCASE,
ESPEAKNG_CTYPE_FULL_STOP,
ESPEAKNG_CTYPE_QUESTION_MARK,
ESPEAKNG_CTYPE_EXCLAMATION_MARK,
} espeakng_CTYPE;
#define ESPEAKNG_CTYPE_PROPERTY_MASK 0xC00000000000C001ull
#define ESPEAKNG_CTYPE_PROPERTY_MASK 0xE00000000000C001ull
// Reference: http://www.unicode.org/reports/tr14/tr14-32.html -- Unicode Line Breaking Algorithm
static espeakng_CTYPE codepoint_type(uint32_t c)
@@ -139,6 +140,8 @@ static espeakng_CTYPE codepoint_type(uint32_t c)
return ESPEAKNG_CTYPE_FULL_STOP;
case ESPEAKNG_PROPERTY_QUESTION_MARK:
return ESPEAKNG_CTYPE_QUESTION_MARK;
case ESPEAKNG_PROPERTY_EXCLAMATION_MARK:
return ESPEAKNG_CTYPE_EXCLAMATION_MARK;
}
// 4. Classify the remaining codepoints.
@@ -278,6 +281,10 @@ tokenizer_state_default(espeak_ng_TOKENIZER *tokenizer)
current += utf8_out(c, current);
*current = '\0';
return ESPEAKNG_TOKEN_QUESTION_MARK;
case ESPEAKNG_CTYPE_EXCLAMATION_MARK:
current += utf8_out(c, current);
*current = '\0';
return ESPEAKNG_TOKEN_EXCLAMATION_MARK;
default:
current += utf8_out(c, current);
*current = '\0';
src/libespeak-ng/tokenizer.h
+1
View File
@@ -53,6 +53,7 @@ typedef enum
ESPEAKNG_TOKEN_WORD_CAPITALIZED,
ESPEAKNG_TOKEN_FULL_STOP,
ESPEAKNG_TOKEN_QUESTION_MARK,
ESPEAKNG_TOKEN_EXCLAMATION_MARK,
} espeak_ng_TOKEN_TYPE;
espeak_ng_TOKEN_TYPE
tests/tokenizer.c
+12 -1
View File
@@ -465,7 +465,7 @@ test_Latn_punctuation_tokens()
espeak_ng_TOKENIZER *tokenizer = create_tokenizer();
espeak_ng_TEXT_DECODER *decoder = create_text_decoder();
assert(text_decoder_decode_string(decoder, ". ?", -1, ESPEAKNG_ENCODING_US_ASCII) == ENS_OK);
assert(text_decoder_decode_string(decoder, ". ? !", -1, ESPEAKNG_ENCODING_US_ASCII) == ENS_OK);
assert(tokenizer_reset(tokenizer, decoder, ESPEAKNG_TOKENIZER_OPTION_TEXT) == 1);
assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_FULL_STOP);
@@ -480,6 +480,14 @@ test_Latn_punctuation_tokens()
assert(tokenizer_get_token_text(tokenizer) != NULL);
assert(strcmp(tokenizer_get_token_text(tokenizer), "?") == 0);
assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_WHITESPACE);
assert(tokenizer_get_token_text(tokenizer) != NULL);
assert(strcmp(tokenizer_get_token_text(tokenizer), " ") == 0);
assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_EXCLAMATION_MARK);
assert(tokenizer_get_token_text(tokenizer) != NULL);
assert(strcmp(tokenizer_get_token_text(tokenizer), "!") == 0);
assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_END_OF_BUFFER);
assert(tokenizer_get_token_text(tokenizer) != NULL);
assert(*tokenizer_get_token_text(tokenizer) == '\0');
@@ -575,6 +583,9 @@ print_tokens(espeak_ng_TEXT_DECODER *decoder)
case ESPEAKNG_TOKEN_QUESTION_MARK:
printf("question mark : %s\n", tokenizer_get_token_text(tokenizer));
break;
case ESPEAKNG_TOKEN_EXCLAMATION_MARK:
printf("exclamation mark : %s\n", tokenizer_get_token_text(tokenizer));
break;
}
}