mirror of
https://github.com/rhasspy/espeak-ng.git
synced 2026-11-19 19:03:53 +00:00
tokenizer.c: Support semicolon tokens.
This commit is contained in:
+12
-1
@@ -465,7 +465,7 @@ test_Latn_punctuation_tokens()
|
||||
espeak_ng_TOKENIZER *tokenizer = create_tokenizer();
|
||||
espeak_ng_TEXT_DECODER *decoder = create_text_decoder();
|
||||
|
||||
assert(text_decoder_decode_string(decoder, ". ? ! , :", -1, ESPEAKNG_ENCODING_US_ASCII) == ENS_OK);
|
||||
assert(text_decoder_decode_string(decoder, ". ? ! , : ;", -1, ESPEAKNG_ENCODING_US_ASCII) == ENS_OK);
|
||||
assert(tokenizer_reset(tokenizer, decoder, ESPEAKNG_TOKENIZER_OPTION_TEXT) == 1);
|
||||
|
||||
assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_FULL_STOP);
|
||||
@@ -504,6 +504,14 @@ test_Latn_punctuation_tokens()
|
||||
assert(tokenizer_get_token_text(tokenizer) != NULL);
|
||||
assert(strcmp(tokenizer_get_token_text(tokenizer), ":") == 0);
|
||||
|
||||
assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_WHITESPACE);
|
||||
assert(tokenizer_get_token_text(tokenizer) != NULL);
|
||||
assert(strcmp(tokenizer_get_token_text(tokenizer), " ") == 0);
|
||||
|
||||
assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_SEMICOLON);
|
||||
assert(tokenizer_get_token_text(tokenizer) != NULL);
|
||||
assert(strcmp(tokenizer_get_token_text(tokenizer), ";") == 0);
|
||||
|
||||
assert(tokenizer_read_next_token(tokenizer) == ESPEAKNG_TOKEN_END_OF_BUFFER);
|
||||
assert(tokenizer_get_token_text(tokenizer) != NULL);
|
||||
assert(*tokenizer_get_token_text(tokenizer) == '\0');
|
||||
@@ -608,6 +616,9 @@ print_tokens(espeak_ng_TEXT_DECODER *decoder)
|
||||
case ESPEAKNG_TOKEN_COLON:
|
||||
printf("colon : %s\n", tokenizer_get_token_text(tokenizer));
|
||||
break;
|
||||
case ESPEAKNG_TOKEN_SEMICOLON:
|
||||
printf("semicolon : %s\n", tokenizer_get_token_text(tokenizer));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user