From 119c200e00a46248bbaa149e8d0c3b1f19f86135 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 1 Oct 2017 08:41:22 +0100 Subject: [PATCH 1/4] Use strcpy instead of memcpy+strlen. This replaces uses of: memcpy(dst, src, strlen(src)) with: strcpy(dst, src) This fixes issues with reading past the end of the copied buffer (e.g. when processing word-based replacements for emoji characters) by ensuring that the destination buffer is null terminated. Reported by Michael Curran --- .gitignore | 1 + Makefile.am | 1 + src/libespeak-ng/dictionary.c | 4 ++-- src/libespeak-ng/translate.c | 12 ++++++------ tests/phoneme-output.test | 21 +++++++++++++++++++++ 5 files changed, 31 insertions(+), 8 deletions(-) create mode 100755 tests/phoneme-output.test diff --git a/.gitignore b/.gitignore index 60f7758e..47826973 100644 --- a/.gitignore +++ b/.gitignore @@ -96,6 +96,7 @@ src/speak-ng tests/*.test !tests/languages.test +!tests/phoneme-output.test espeak-ng.pc diff --git a/Makefile.am b/Makefile.am index 0ae0c326..427083de 100644 --- a/Makefile.am +++ b/Makefile.am @@ -250,6 +250,7 @@ tests_api_test_SOURCES = tests/api.c check: tests/encoding.check \ tests/readclause.check \ tests/api.check \ + tests/phoneme-output.check \ tests/languages.check ##### phoneme data: diff --git a/src/libespeak-ng/dictionary.c b/src/libespeak-ng/dictionary.c index c82e4272..4ce6d8df 100644 --- a/src/libespeak-ng/dictionary.c +++ b/src/libespeak-ng/dictionary.c @@ -2394,7 +2394,7 @@ int TranslateRules(Translator *tr, char *p_start, char *phonemes, int ph_size, c match1.end_type |= p - p_start; } strcpy(end_phonemes, match1.phonemes); - memcpy(p_start, word_copy, strlen(word_copy)); + strcpy(p_start, word_copy); return match1.end_type; } } @@ -2404,7 +2404,7 @@ int TranslateRules(Translator *tr, char *p_start, char *phonemes, int ph_size, c } } - memcpy(p_start, word_copy, strlen(word_copy)); + strcpy(p_start, word_copy); return 0; } diff --git a/src/libespeak-ng/translate.c b/src/libespeak-ng/translate.c index 780d4486..73cd9948 100644 --- a/src/libespeak-ng/translate.c +++ b/src/libespeak-ng/translate.c @@ -795,7 +795,7 @@ static int TranslateWord3(Translator *tr, char *word_start, WORD_TAB *wtab, char if (end2) { RemoveEnding(tr, wordx, end2, word_copy); end_type = TranslateRules(tr, wordx, phonemes, N_WORD_PHONEMES, end_phonemes, wflags|FLAG_NO_TRACE, dictionary_flags); - memcpy(wordx, word_copy, strlen(word_copy)); + strcpy(wordx, word_copy); if ((end_type & SUFX_P) == 0) { // after removing the suffix, the prefix is no longer recognised. // Keep the suffix, but don't use the prefix @@ -902,7 +902,7 @@ static int TranslateWord3(Translator *tr, char *word_start, WORD_TAB *wtab, char wordx[-1] = ' '; if (phonemes[0] == phonSWITCH) { // change to another language in order to translate this word - memcpy(wordx, word_copy, strlen(word_copy)); + strcpy(wordx, word_copy); strcpy(word_phonemes, phonemes); return 0; } @@ -920,7 +920,7 @@ static int TranslateWord3(Translator *tr, char *word_start, WORD_TAB *wtab, char found = LookupDictList(tr, &wordx, phonemes, dictionary_flags2, end_flags, wtab); // without prefix and suffix if (phonemes[0] == phonSWITCH) { // change to another language in order to translate this word - memcpy(wordx, word_copy, strlen(word_copy)); + strcpy(wordx, word_copy); strcpy(word_phonemes, phonemes); return 0; } @@ -960,7 +960,7 @@ static int TranslateWord3(Translator *tr, char *word_start, WORD_TAB *wtab, char if (phonemes[0] == phonSWITCH) { // change to another language in order to translate this word strcpy(word_phonemes, phonemes); - memcpy(wordx, word_copy, strlen(word_copy)); + strcpy(wordx, word_copy); wordx[-1] = c_temp; return 0; } @@ -974,7 +974,7 @@ static int TranslateWord3(Translator *tr, char *word_start, WORD_TAB *wtab, char AppendPhonemes(tr, phonemes, N_WORD_PHONEMES, end_phonemes); end_phonemes[0] = 0; } - memcpy(wordx, word_copy, strlen(word_copy)); + strcpy(wordx, word_copy); } wordx[-1] = c_temp; @@ -1139,7 +1139,7 @@ int TranslateWord(Translator *tr, char *word_start, WORD_TAB *wtab, char *word_o char word[N_WORD_BYTES+1]; word[0] = 0; word[1] = ' '; - memcpy(word+2, word_out, strlen(word_out)); + strcpy(word+2, word_out); word_out = word+2; while (*word_out && available > 1) { diff --git a/tests/phoneme-output.test b/tests/phoneme-output.test new file mode 100755 index 00000000..1468bec3 --- /dev/null +++ b/tests/phoneme-output.test @@ -0,0 +1,21 @@ +#!/bin/sh + +test_phonemes() { + TEST_LANG=$1 + EXPECTED=$2 + TEST_TEXT=$3 + + echo "testing ${TEST_LANG} \"${TEST_TEXT}\"" + ESPEAK_DATA_PATH=`pwd` LD_LIBRARY_PATH=src:${LD_LIBRARY_PATH} \ + src/espeak-ng -xq -v ${TEST_LANG} "${TEST_TEXT}" > actual.txt + echo "${EXPECTED}" > expected.txt + diff expected.txt actual.txt || exit 1 +} + +test_phonemes en " h@l'oU" "hello" + +# Emoji + +# ED-3 - emoji_character [http://www.unicode.org/reports/tr51/tr51-12.html#def_emoji_character] +test_phonemes en " Ekskla#m'eIS@N kw'EstS@n m'A@k" "โ‰" +test_phonemes en " Ekskla#m'eIS@N kw'EstS@n m'A@k r'eInboU" "โ‰ ๐ŸŒˆ" From 516786aecb74f2057305f4e8b1f4079725ca6f37 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 1 Oct 2017 11:28:33 +0100 Subject: [PATCH 2/4] ka: Comment out the emoji translations that were breaking the pronunciation of '3' and '7'. --- dictsource/ka_emoji | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dictsource/ka_emoji b/dictsource/ka_emoji index d4f55111..92b78ee9 100644 --- a/dictsource/ka_emoji +++ b/dictsource/ka_emoji @@ -926,7 +926,8 @@ $textmode ๐Ÿ˜– แƒจแƒ”แƒชแƒ‘แƒฃแƒœแƒ”แƒ‘แƒฃแƒšแƒ˜ แƒกแƒแƒฎแƒ” // [1F616] ๐Ÿ˜— แƒ™แƒแƒชแƒœแƒ˜แƒก แƒ’แƒแƒ›แƒแƒ›แƒฎแƒแƒขแƒ•แƒ”แƒšแƒ˜ แƒกแƒแƒฎแƒ” // [1F617] ๐Ÿ˜˜ แƒกแƒแƒฎแƒ”, แƒ แƒแƒ›แƒ”แƒšแƒ˜แƒช แƒ™แƒแƒชแƒœแƒแƒก แƒแƒ’แƒ–แƒแƒ•แƒœแƒ˜แƒก // [1F618] -๐Ÿ˜™ แƒ™แƒแƒชแƒœแƒ˜แƒก แƒ’แƒแƒ›แƒแƒ›แƒฎแƒแƒขแƒ•แƒ”แƒšแƒ˜ แƒกแƒแƒฎแƒ” แƒ›แƒแƒ›แƒฆแƒ˜แƒ›แƒแƒ แƒ” แƒ—แƒ•แƒแƒšแƒ”แƒ‘แƒ˜แƒ— // [1F619] +๐Ÿ˜— kissing face // [1F617] +//๐Ÿ˜™ แƒ™แƒแƒชแƒœแƒ˜แƒก แƒ’แƒแƒ›แƒแƒ›แƒฎแƒแƒขแƒ•แƒ”แƒšแƒ˜ แƒกแƒแƒฎแƒ” แƒ›แƒแƒ›แƒฆแƒ˜แƒ›แƒแƒ แƒ” แƒ—แƒ•แƒแƒšแƒ”แƒ‘แƒ˜แƒ— // [1F619] TODO: This breaks speaking "7". ๐Ÿ˜š แƒ™แƒแƒชแƒœแƒ˜แƒก แƒ’แƒแƒ›แƒแƒ›แƒฎแƒแƒขแƒ•แƒ”แƒšแƒ˜ แƒกแƒแƒฎแƒ” แƒ“แƒแƒฎแƒฃแƒญแƒฃแƒšแƒ˜ แƒ—แƒ•แƒแƒšแƒ”แƒ‘แƒ˜แƒ— // [1F61A] ๐Ÿ˜› แƒกแƒแƒฎแƒ” แƒ’แƒแƒ›แƒแƒงแƒแƒคแƒ˜แƒšแƒ˜ แƒ”แƒœแƒ˜แƒ— // [1F61B] ๐Ÿ˜œ แƒกแƒแƒฎแƒ” แƒ’แƒแƒ›แƒแƒงแƒแƒคแƒ˜แƒšแƒ˜ แƒ”แƒœแƒ˜แƒ—แƒ แƒ“แƒ แƒฉแƒแƒ™แƒ แƒฃแƒšแƒ˜ แƒ—แƒ•แƒแƒšแƒ˜แƒ— // [1F61C] @@ -938,7 +939,8 @@ $textmode ๐Ÿ˜ข แƒ›แƒขแƒ˜แƒ แƒแƒšแƒ แƒกแƒแƒฎแƒ” // [1F622] ๐Ÿ˜ฃ แƒจแƒ”แƒฃแƒžแƒแƒ•แƒแƒ แƒ˜ แƒกแƒแƒฎแƒ” // [1F623] ๐Ÿ˜ค แƒกแƒแƒฎแƒ” แƒชแƒฎแƒ•แƒ˜แƒ แƒ˜แƒ“แƒแƒœ แƒ’แƒแƒ›แƒแƒ›แƒแƒ•แƒแƒšแƒ˜ แƒแƒ แƒ—แƒฅแƒšแƒ˜แƒ— // [1F624] -๐Ÿ˜ฅ แƒ˜แƒ›แƒ”แƒ“แƒ’แƒแƒชแƒ แƒฃแƒ”แƒ‘แƒฃแƒšแƒ˜, แƒ›แƒแƒ’แƒ แƒแƒ› แƒจแƒ•แƒ”แƒ‘แƒ˜แƒก แƒ’แƒแƒ›แƒแƒ›แƒฎแƒแƒขแƒ•แƒ”แƒšแƒ˜ แƒกแƒแƒฎแƒ” // [1F625] +๐Ÿ˜ฅ disappointed but relieved face // [1F625] +//๐Ÿ˜ฅ แƒ˜แƒ›แƒ”แƒ“แƒ’แƒแƒชแƒ แƒฃแƒ”แƒ‘แƒฃแƒšแƒ˜, แƒ›แƒแƒ’แƒ แƒแƒ› แƒจแƒ•แƒ”แƒ‘แƒ˜แƒก แƒ’แƒแƒ›แƒแƒ›แƒฎแƒแƒขแƒ•แƒ”แƒšแƒ˜ แƒกแƒแƒฎแƒ” // [1F625] TODO: This breaks speaking "3". ๐Ÿ˜ฆ แƒจแƒ”แƒญแƒ›แƒฃแƒฎแƒœแƒ˜แƒšแƒ˜ แƒกแƒแƒฎแƒ” แƒฆแƒ˜แƒ แƒžแƒ˜แƒ แƒ˜แƒ— // [1F626] ๐Ÿ˜ง แƒขแƒแƒœแƒฏแƒฃแƒšแƒ˜ แƒกแƒแƒฎแƒ” // [1F627] ๐Ÿ˜จ แƒจแƒ”แƒจแƒ˜แƒœแƒ”แƒ‘แƒฃแƒšแƒ˜ แƒกแƒแƒฎแƒ” // [1F628] From 2442dd05aefea74d7c486439394664a14a007d3c Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 1 Oct 2017 12:23:12 +0100 Subject: [PATCH 3/4] Document the ignored characters in tr_languages.c. --- src/libespeak-ng/tr_languages.c | 43 ++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/src/libespeak-ng/tr_languages.c b/src/libespeak-ng/tr_languages.c index 38034312..083ed186 100644 --- a/src/libespeak-ng/tr_languages.c +++ b/src/libespeak-ng/tr_languages.c @@ -158,19 +158,44 @@ static void SetLetterBitsRange(Translator *tr, int group, int first, int last) // ignore these characters static const unsigned short chars_ignore_default[] = { - 0xad, 1, // soft hyphen - 0x200c, 1, // zero width non-joiner - 0x200d, 1, // zero width joiner - 0, 0 + // U+00AD SOFT HYPHEN + // Used to mark hyphenation points in words for where to split a + // word at the end of a line to provide readable justified text. + 0xad, 1, + // U+200C ZERO WIDTH NON-JOINER + // Used to prevent combined ligatures being displayed in their + // combined form. + 0x200c, 1, + // U+200D ZERO WIDTH JOINER + // Used to indicate an alternative connected form made up of the + // characters surrounding the ZWJ in Devanagari, Kannada, Malayalam + // and Emoji. + 0x200d, 1, + // End of the ignored character list. + 0, 0 }; // alternatively, ignore characters but allow zero-width-non-joiner (lang-fa) static const unsigned short chars_ignore_zwnj_hyphen[] = { - 0xad, 1, // soft hyphen - 0x640, 1, // igniore Arabic Tatweel (lang=FA) - 0x200c, '-', // zero width non-joiner, replace with hyphen - 0x200d, 1, // zero width joiner - 0, 0 + // U+00AD SOFT HYPHEN + // Used to mark hyphenation points in words for where to split a + // word at the end of a line to provide readable justified text. + 0xad, 1, + // U+0640 TATWEEL (KASHIDA) + // Used in Arabic scripts to stretch characters for justifying + // the text. + 0x640, 1, + // U+200C ZERO WIDTH NON-JOINER + // Used to prevent combined ligatures being displayed in their + // combined form. + 0x200c, '-', + // U+200D ZERO WIDTH JOINER + // Used to indicate an alternative connected form made up of the + // characters surrounding the ZWJ in Devanagari, Kannada, Malayalam + // and Emoji. + 0x200d, 1, + // End of the ignored character list. + 0, 0 }; const unsigned char utf8_ordinal[] = { 0xc2, 0xba, 0 }; // masculine ordinal character, UTF-8 From 7f5914c0ded36f20556c984cecb94074c2f7ea41 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 1 Oct 2017 12:24:30 +0100 Subject: [PATCH 4/4] Don't ignore ZWJ characters. --- src/libespeak-ng/tr_languages.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libespeak-ng/tr_languages.c b/src/libespeak-ng/tr_languages.c index 083ed186..6556d224 100644 --- a/src/libespeak-ng/tr_languages.c +++ b/src/libespeak-ng/tr_languages.c @@ -170,7 +170,7 @@ static const unsigned short chars_ignore_default[] = { // Used to indicate an alternative connected form made up of the // characters surrounding the ZWJ in Devanagari, Kannada, Malayalam // and Emoji. - 0x200d, 1, +// 0x200d, 1, // Not ignored. // End of the ignored character list. 0, 0 }; @@ -193,7 +193,7 @@ static const unsigned short chars_ignore_zwnj_hyphen[] = { // Used to indicate an alternative connected form made up of the // characters surrounding the ZWJ in Devanagari, Kannada, Malayalam // and Emoji. - 0x200d, 1, +// 0x200d, 1, // Not ignored. // End of the ignored character list. 0, 0 };