From 1443c970cdffe258276e1a6442316268c1d380be Mon Sep 17 00:00:00 2001
From: Juho Hiltunen <jaacoppi@kapsi.fi>
Date: Wed, 31 Aug 2022 21:06:45 +0300
Subject: [PATCH] cmn: search for dictionary matches instead of translating
 characters.

cmn (Mandarin chinese) has been broken since 4825905.

This fix makes mandarin behave more like Cantonese. Instead of
translating characters, we search for dictionary matches.

The functionality of normal vs Chao tones should be investigated more.
Looks like latin characters as pinyin still uses Chao tones whereas
the characters in cmn_list and cmn_listx do not.

See #1044 for discussion. See also #1028 and #1163.
---
 dictsource/cmn_list             |  1 -
 dictsource/extra/cmn_listx      |  1 -
 phsource/ph_cmn                 | 46 ++++++++++++++++++++++++++++-----
 src/libespeak-ng/tr_languages.c |  2 +-
 4 files changed, 41 insertions(+), 9 deletions(-)

diff --git a/dictsource/cmn_list b/dictsource/cmn_list
index ee5fbb3b..d32b1171 100644
--- a/dictsource/cmn_list
+++ b/dictsource/cmn_list
@@ -93,7 +93,6 @@ z zi51
 ㄨ wu55
 ㄩ y55
 
-$textmode
 // Most frequent pronunciations of the 3799 most common characters (from Unihan database ftp://ftp.unicode.org/Public/UNIDATA/Unihan.zip, kHanyuPinlu field with some corrections)
 涉	she4
 礦	kuang4
diff --git a/dictsource/extra/cmn_listx b/dictsource/extra/cmn_listx
index 86bc6942..3cbc29be 100644
--- a/dictsource/extra/cmn_listx
+++ b/dictsource/extra/cmn_listx
@@ -1,4 +1,3 @@
-$textmode
 //From Unihan database ftp://ftp.unicode.org/Public/UNIDATA/Unihan.zip kMandarin entries (except the ones that have kHanyuPinlu, which are in zh_list)
 //with compounds from CC-CEDICT http://www.mdbg.net/chindict/chindict.php?page=cedict and some corrections
 //21611 single characters plus 36500 compound exceptions (includes 320 added 'yi' and 10721 added 'bu' exceptions, and 9700 extra 2-syllable words for 3rd-tone sandhi blocking)
diff --git a/phsource/ph_cmn b/phsource/ph_cmn
index 49c6d622..b62d466e 100644
--- a/phsource/ph_cmn
+++ b/phsource/ph_cmn
@@ -1,23 +1,39 @@
-
 //====================================================
 //  Tone Numbers
+//  Note: 
+//  For tones 1-5, both normal tones and Chao tones
+// are defined
 //====================================================
 
-phoneme 11   //  tone: low level
+//  tone: low level
+phoneme 11
   stress
   Tone(12, 9, envelope/i_risefall, NULL)
 endphoneme
 
+phoneme 5
+  stress
+  Tone(12, 9, envelope/i_risefall, NULL)
+endphoneme
+
+
 phoneme 21   //  tone: low fall
   stress
   Tone(20, 10, envelope/p_fall, NULL)
 endphoneme
 
-phoneme 214   //  tone: fall rise
+//  tone: fall rise
+phoneme 214
   stress
   Tone(18, 42, envelope/p_214, NULL)
 endphoneme
 
+phoneme 3
+  stress
+  Tone(18, 42, envelope/p_214, NULL)
+endphoneme
+
+
 phoneme 22   //  tone: mid-low level
   stress
   Tone(22, 20, envelope/p_fall, NULL)
@@ -28,7 +44,13 @@ phoneme 33   //  tone: mid level
   Tone(32, 30, envelope/p_fall, NULL)
 endphoneme
 
-phoneme 35   //  tone: mid rise
+//  tone: mid rise
+phoneme 35   
+  stress
+  Tone(30, 50, envelope/p_rise, NULL)
+endphoneme
+
+phoneme 2
   stress
   Tone(30, 50, envelope/p_rise, NULL)
 endphoneme
@@ -38,7 +60,13 @@ phoneme 44   //  tone: mid-high level
   Tone(38, 41, envelope/p_rise, NULL)
 endphoneme
 
-phoneme 51   //  tone: high fall
+//  tone: high fall
+phoneme 51
+  stress
+  Tone(50, 10, envelope/p_fall, NULL)
+endphoneme
+
+phoneme 4
   stress
   Tone(50, 10, envelope/p_fall, NULL)
 endphoneme
@@ -48,7 +76,13 @@ phoneme 53   //  tone: high fall
   Tone(50, 30, envelope/p_fall, NULL)
 endphoneme
 
-phoneme 55   //  tone: high level
+//  tone: high level
+phoneme 55   
+  stress
+  Tone(55, 50, envelope/p_level, NULL)
+endphoneme
+
+phoneme 1
   stress
   Tone(55, 50, envelope/p_level, NULL)
 endphoneme
diff --git a/src/libespeak-ng/tr_languages.c b/src/libespeak-ng/tr_languages.c
index c12aad37..91ea3842 100644
--- a/src/libespeak-ng/tr_languages.c
+++ b/src/libespeak-ng/tr_languages.c
@@ -1588,8 +1588,8 @@ Translator *SelectTranslator(const char *name)
 		tr->langopts.ideographs = 1;
 		tr->langopts.our_alphabet = 0x3100;
 		tr->langopts.word_gap = 0x21; // length of a final vowel is less dependent on the next consonant, don't merge consonant with next word
+		tr->langopts.textmode = true;
 		if (name2 == L3('y', 'u', 'e')) {
-			tr->langopts.textmode = true;
 			tr->langopts.listx = 1; // compile zh_listx after zh_list
 			tr->langopts.numbers = NUM_DEFAULT;
 			tr->langopts.numbers2 = NUM2_ZERO_TENS;