mjohnson/espeak-ng
1
0
Fork 0
mirror of https://github.com/rhasspy/espeak-ng.git synced 2026-11-13 17:23:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert the new (broken) replacement rule logic.

Browse Source 
The replacement tests for bs, hr, and sr are no longer marked as
broken as they work using the old code. The mk tests keep the
broken annotation, as they don't work in the old code either.

This reverts commit 801a8d197c.
This reverts commit 64d5701e5e.
This reverts commit 3b51ebf617.
This reverts commit 1fd235d2c0.
This reverts commit 9f0667de86.
This commit is contained in:
Reece H. Dunn
2018-07-23 19:38:46 +01:00
parent bae92dab38
commit 424f705525
7 changed files with 96 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libespeak-ng/compiledict.c
+17 -40
View File
@@ -1448,53 +1448,30 @@ static espeak_ng_STATUS compile_dictrules(FILE *f_in, FILE *f_out, char *fname_t
break;
case 2: // .replace
{
int from[N_WORD_BYTES + 1] = { 0 }; // Size of N_WORD_BYTES + null byte
int to[N_WORD_BYTES + 1] = { 0 };
int replace1;
int replace2;
char *p;
p = buf;
while (isspace2(*p)) // skip spaces in xx_rules file
p++;
replace1 = 0;
replace2 = 0;
while (isspace2(*p)) p++;
ix = 0;
while ((unsigned char) (*p) > ' ') { // prepare 'from' characters
p += utf8_in(&from[ix], p);
ix++;
while ((unsigned char)(*p) > 0x20) { // not space or zero-byte
p += utf8_in(&c, p);
replace1 += (c << ix);
ix += 16;
}
while (isspace2(*p)) // skip spaces again
p++;
while (isspace2(*p)) p++;
ix = 0;
while ((unsigned char) (*p) > ' ') { // prepare 'to' characters
p += utf8_in(&to[ix], p);
ix++;
while ((unsigned char)(*p) > 0x20) {
p += utf8_in(&c, p);
replace2 += (c << ix);
ix += 16;
}
// save into file
if (from[0] != 0) {
ix = 0;
while (from[ix] != 0) {
Write4Bytes(f_out, from[ix]);
ix++;
if (ix > N_WORD_BYTES / 4) {
fprintf(stderr,
"Too long .replace 'from' part (limit is: %d characters)\n",
N_WORD_BYTES / 4);
break;
}
}
Write4Bytes(f_out, ' '); // end of 'from'
ix = 0;
while (to[ix] != 0) {
Write4Bytes(f_out, to[ix]);
ix++;
if (ix > N_WORD_BYTES / 4) {
fprintf(stderr,
"Too long .replace 'to' part (limit is: %d characters)\n",
N_WORD_BYTES / 4);
break;
}
}
Write4Bytes(f_out, ' '); // end of 'to'
if (replace1 != 0) {
Write4Bytes(f_out, replace1); // write as little-endian
Write4Bytes(f_out, replace2); // if big-endian, reverse the bytes in LoadDictionary()
}
}
break;
src/libespeak-ng/dictionary.c
+1 -1
View File
@@ -152,7 +152,7 @@ static void InitGroups(Translator *tr)
pw = (unsigned int *)(((intptr_t)p+4) & ~3); // advance to next word boundary
tr->langopts.replace_chars = pw;
while (pw[0] != 0)
pw ++; // find the end of the replacement list
pw += 2; // find the end of the replacement list, each entry is 2 words.
p = (char *)(pw+1);
#ifdef ARCH_BIG
src/libespeak-ng/translate.c
+66 -145
View File
@@ -1790,158 +1790,81 @@ static int EmbeddedCommand(unsigned int *source_index_out)
return 1;
}
static bool GetNextTranslateChars(const int **from, const int **to, bool next_step)
// handle .replace rule in xx_rules file
static int SubstituteChar(Translator *tr, unsigned int c, unsigned int next_in, int *insert, int *wordflags)
{
// Set pointers to next 'from' and 'to' part for replacement
// next_step indicates second and succeeding replacement group
// Return true if got data, false, if reached end
if (next_step) { // first 'from' is pointed already
int spaces = 0;
while (**from != 0 && spaces < 2) { // Get next 'from' position skipping 'to' part
(*from)++;
if (**from == ' ')
spaces++;
}
while (**from == ' ') // skip remaining spaces, if exists
(*from)++;
int ix;
unsigned int word;
unsigned int new_c, c2 = ' ', c_lower;
int upper_case = 0;
static bool ignore_next = false;
const unsigned int *replace_chars;
if (ignore_next) {
ignore_next = false;
return 8;
}
if (**from == 0)
return false;
if (c == 0) return 0;
*to = *from; // start seeking 'to' from 'from' position
while (**to != 0 && **to != ' ') // get next 'to' position
(*to)++;
if ((replace_chars = tr->langopts.replace_chars) == NULL)
return c;
while (**to == ' ') // skip remaining spaces, if exists
(*to)++;
// there is a list of character codes to be substituted with alternative codes
return **to != 0;
}
static void ReplaceMatchingChars(char **matchstart, const int *from, const int *to, int upper_case)
{
// Replace characters from 'from' to 'to' group in source clause, starting
// with place indicated in matchstart.
// Rewrites remaining part of source, if length of 'from' and 'to' groups differs.
char tmpsource[N_TR_SOURCE + 40]; // Prepare working copy of source
memcpy(&tmpsource, *matchstart, N_TR_SOURCE + 40);
// prepare from part
char charfrom[N_WORD_BYTES + 1] = { 0 };
int usedfrom = 0;
int const *pintfrom = from;
char *pcharfrom = charfrom;
while (*pintfrom != ' ') {
usedfrom += utf8_out(*pintfrom, pcharfrom + usedfrom);
pintfrom++;
if (iswupper(c_lower = c)) {
c_lower = towlower2(c, tr);
upper_case = 1;
}
// prepare to part
char charto[N_WORD_BYTES + 1] = { 0 };
int usedto = 0;
int const *pintto = to;
char *pcharto = charto;
while (*pintto != 0 && *pintto != ' ') {
usedto += utf8_out(*pintto, pcharto + usedto);
pintto++;
}
// Set up replacement in temporary buffer
char *psource = *matchstart;
char *ptmpsource = tmpsource;
pcharto = charto;
// Do replacement writing 'to' part in buffer
while (*pcharto != 0 ) {
if(upper_case) { // TODO: should check for capital letter from source (maybe using word_flags)
*ptmpsource = toupper(*pcharto);
upper_case = 0; // capitalize only first character if necessary (will work in most cases)
}
else
*ptmpsource = *pcharto;
pcharto++;
ptmpsource++;
}
// If 'to' part is different than 'from' part, fix remaining part of the buffer.
// (Read after 'from' in source, write after 'to' in buffer.
// Don't touch last 40 bytes of source, which could contain control data.
if (usedfrom != usedto) {
psource = *matchstart + usedfrom;
while (*psource != 0 && psource < &source[N_TR_SOURCE]
&& ptmpsource < &tmpsource[N_TR_SOURCE]) {
*ptmpsource = *psource;
ptmpsource++;
psource++;
}
}
// Write temporary buffer back to source
psource = *matchstart;
ptmpsource = tmpsource;
while (*ptmpsource != 0 && ptmpsource < &tmpsource[N_TR_SOURCE]
&& psource < &source[N_TR_SOURCE]) {
*psource = *ptmpsource;
ptmpsource++;
psource++;
}
// if requested, print trace
if (option_phonemes & espeakPHONEMES_TRACE)
fprintf(f_trans, "Replace %s > %s\n", charfrom, charto);
}
static void TranslateChars(Translator *tr, char *source)
{
// Replace characters using Translator.replace_chars in passed source buffer
char *startmatch;
const int *from;
const int *to;
bool upper_case = false;
if ((from = (const int *)tr->langopts.replace_chars) == NULL)
return;
char *curchar = source; // pointer to the source clause
int curint; // current UTF-16 character
int used; // bytes used in buffer
const int *curfrom; // pointer to current 'from'
bool next_step = false; // signal to jumping to next 'from' part in dictionary
while (GetNextTranslateChars(&from, &to, next_step)) {
next_step = true;
curfrom = from;
curchar = source;
startmatch = 0;
curint = 0;
do {
used = utf8_in(&curint, curchar);
if (curint < ' ') { // skip control characters
curchar += used;
continue;
new_c = 0;
for (ix = 0; (word = replace_chars[ix]) != 0; ix += 2) {
if (c_lower == (word & 0xffff)) {
if ((word >> 16) == 0) {
new_c = replace_chars[ix+1];
break;
}
if ((upper_case = iswupper(curint)) == true) // remember, if uppercase
curint = ucd_tolower(curint);
if (curint == *curfrom) { // if matches
if (startmatch == 0)
startmatch = curchar;
curfrom++; // move check to next character
if (*curfrom == ' ') { // if full match, replace
ReplaceMatchingChars(&startmatch, from, to, upper_case);
startmatch = 0; // reset match to start of 'from'
curfrom = from;
}
} else { // reset match to start of 'from'
startmatch = 0;
curfrom = from;
if ((word >> 16) == (unsigned int)towlower2(next_in, tr)) {
new_c = replace_chars[ix+1];
ignore_next = true;
break;
}
curchar += used;
} while (curint > 0);
}
}
if (new_c == 0)
return c; // no substitution
if (new_c & 0xffe00000) {
// there is a second character to be inserted
// don't convert the case of the second character unless the next letter is also upper case
c2 = new_c >> 16;
if (upper_case && iswupper(next_in))
c2 = ucd_toupper(c2);
*insert = c2;
new_c &= 0xffff;
}
if (upper_case)
new_c = ucd_toupper(new_c);
*wordflags |= FLAG_CHAR_REPLACED;
if (option_phonemes & espeakPHONEMES_TRACE) {
char msg[21] = {'R','e','p','l','a','c','e',':',' '};
char *index = msg;
index += 9;
index += utf8_out(c, index);
*index++ = ' ';
*index++ = '>';
*index++ = ' ';
index += utf8_out(new_c, index);
index += utf8_out(c2, index);
*index = 0;
fprintf(f_trans, "%s\n", msg);
}
return new_c;
}
static int TranslateChar(Translator *tr, char *ptr, int prev_in, unsigned int c, unsigned int next_in, int *insert)
static int TranslateChar(Translator *tr, char *ptr, int prev_in, unsigned int c, unsigned int next_in, int *insert, int *wordflags)
{
// To allow language specific examination and replacement of characters
@@ -2007,7 +1930,8 @@ static int TranslateChar(Translator *tr, char *ptr, int prev_in, unsigned int c,
}
break;
}
return c;
// handle .replace rule in xx_rules file
return SubstituteChar(tr, c, next_in, insert, wordflags);
}
static const char *UCase_ga[] = { "bp", "bhf", "dt", "gc", "hA", "mb", "nd", "ng", "ts", "tA", "nA", NULL };
@@ -2180,9 +2104,6 @@ void TranslateClause(Translator *tr, int *tone_out, char **voice_change)
}
words[0].length = k;
// handle .replace rule of xx_rules file
TranslateChars(tr, source);
while (!finished && (ix < (int)sizeof(sbuf) - 1) && (n_ph_list2 < N_PHONEME_LIST-4)) {
prev_out2 = prev_out;
utf8_in2(&prev_out, &sbuf[ix-1], 1);
@@ -2285,7 +2206,7 @@ void TranslateClause(Translator *tr, int *tone_out, char **voice_change)
word_flags |= FLAG_COMMA_AFTER;
}
// language specific character translations
c = TranslateChar(tr, &source[source_index], prev_in, c, next_in, &char_inserted);
c = TranslateChar(tr, &source[source_index], prev_in, c, next_in, &char_inserted, &word_flags);
if (c == 8)
continue; // ignore this character