diff --git a/src/libespeak-ng/tokenizer.c b/src/libespeak-ng/tokenizer.c index d9858b75..9291b229 100644 --- a/src/libespeak-ng/tokenizer.c +++ b/src/libespeak-ng/tokenizer.c @@ -1,7 +1,5 @@ /* Tokenizer APIs. * - * Copyright (C) 2005 to 2015 by Jonathan Duddington - * email: jonsd@users.sourceforge.net * Copyright (C) 2017 Reece H. Dunn * * This program is free software; you can redistribute it and/or modify @@ -34,20 +32,6 @@ #include "synthesize.h" #include "translate.h" -// punctuations symbols that can end a clause -static const unsigned short punct_chars[] = { - 0x00a1, // inverted exclamation - 0x00bf, // inverted question - 0 -}; - -// indexed by entry num. in punct_chars -static const unsigned int punct_attributes[] = { - CLAUSE_SEMICOLON | CLAUSE_OPTIONAL_SPACE_AFTER, // inverted exclamation - CLAUSE_SEMICOLON | CLAUSE_OPTIONAL_SPACE_AFTER, // inverted question - 0 -}; - #define ESPEAKNG_CLAUSE_TYPE_PROPERTY_MASK 0xFFF0000000000000ull int clause_type_from_codepoint(uint32_t c) @@ -55,11 +39,6 @@ int clause_type_from_codepoint(uint32_t c) ucd_category cat = ucd_lookup_category(c); ucd_property props = ucd_properties(c, cat); - for (int ix = 0; punct_chars[ix] != 0; ++ix) { - if (punct_chars[ix] == c) - return punct_attributes[ix]; - } - switch (props & ESPEAKNG_CLAUSE_TYPE_PROPERTY_MASK) { case ESPEAKNG_PROPERTY_FULL_STOP: @@ -90,6 +69,8 @@ int clause_type_from_codepoint(uint32_t c) case ESPEAKNG_PROPERTY_EXTENDED_DASH: return CLAUSE_SEMICOLON; case ESPEAKNG_PROPERTY_SEMI_COLON | ESPEAKNG_PROPERTY_OPTIONAL_SPACE_AFTER: + case ESPEAKNG_PROPERTY_QUESTION_MARK | ESPEAKNG_PROPERTY_OPTIONAL_SPACE_AFTER | ESPEAKNG_PROPERTY_INVERTED_TERMINAL_PUNCTUATION: + case ESPEAKNG_PROPERTY_EXCLAMATION_MARK | ESPEAKNG_PROPERTY_OPTIONAL_SPACE_AFTER | ESPEAKNG_PROPERTY_INVERTED_TERMINAL_PUNCTUATION: return CLAUSE_SEMICOLON | CLAUSE_OPTIONAL_SPACE_AFTER; case ESPEAKNG_PROPERTY_ELLIPSIS: return CLAUSE_SEMICOLON | CLAUSE_SPEAK_PUNCTUATION_NAME | CLAUSE_OPTIONAL_SPACE_AFTER; diff --git a/src/ucd-tools/data/espeak-ng/PropList.txt b/src/ucd-tools/data/espeak-ng/PropList.txt index f27deb9e..f091fdd7 100644 --- a/src/ucd-tools/data/espeak-ng/PropList.txt +++ b/src/ucd-tools/data/espeak-ng/PropList.txt @@ -213,3 +213,10 @@ FF1F ; Optional_Space_After # Po FULLWIDTH QUESTION MARK 055E ; Punctuation_In_Word # Po ARMENIAN QUESTION MARK # Total code points: 3 + +# ================================================ + +00A1 ; Inverted_Terminal_Punctuation # Po INVERTED EXCLAMATION MARK +00BF ; Inverted_Terminal_Punctuation # Po INVERTED QUESTION MARK + +# Total code points: 2 diff --git a/src/ucd-tools/src/include/ucd/ucd.h b/src/ucd-tools/src/include/ucd/ucd.h index 609f1a51..3d96b22c 100644 --- a/src/ucd-tools/src/include/ucd/ucd.h +++ b/src/ucd-tools/src/include/ucd/ucd.h @@ -363,6 +363,7 @@ typedef uint64_t ucd_property; #define UCD_PROPERTY_PREPENDED_CONCATENATION_MARK 0x0000000100000000ull /**< @brief Prepended_Concatenation_Mark */ // eSpeak NG extended properties: +#define ESPEAKNG_PROPERTY_INVERTED_TERMINAL_PUNCTUATION 0x0010000000000000ull /**< @brief Inverted_Terminal_Punctuation */ #define ESPEAKNG_PROPERTY_PUNCTUATION_IN_WORD 0x0020000000000000ull /**< @brief Punctuation_In_Word */ #define ESPEAKNG_PROPERTY_OPTIONAL_SPACE_AFTER 0x0040000000000000ull /**< @brief Optional_Space_After */ #define ESPEAKNG_PROPERTY_EXTENDED_DASH 0x0080000000000000ull /**< @brief Extended_Dash */ diff --git a/src/ucd-tools/src/proplist.c b/src/ucd-tools/src/proplist.c index 5784c49b..f48910b0 100644 --- a/src/ucd-tools/src/proplist.c +++ b/src/ucd-tools/src/proplist.c @@ -1342,9 +1342,9 @@ static ucd_property properties_Po(codepoint_t c) if (c == 0x003A) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_PATTERN_SYNTAX | ESPEAKNG_PROPERTY_COLON; if (c == 0x003B) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_PATTERN_SYNTAX | ESPEAKNG_PROPERTY_SEMI_COLON; if (c == 0x003F) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL | UCD_PROPERTY_PATTERN_SYNTAX | ESPEAKNG_PROPERTY_QUESTION_MARK; - if (c == 0x00A1) return UCD_PROPERTY_PATTERN_SYNTAX | ESPEAKNG_PROPERTY_EXCLAMATION_MARK | ESPEAKNG_PROPERTY_OPTIONAL_SPACE_AFTER; + if (c == 0x00A1) return UCD_PROPERTY_PATTERN_SYNTAX | ESPEAKNG_PROPERTY_EXCLAMATION_MARK | ESPEAKNG_PROPERTY_OPTIONAL_SPACE_AFTER | ESPEAKNG_PROPERTY_INVERTED_TERMINAL_PUNCTUATION; if (c == 0x00B7) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_EXTENDER | UCD_PROPERTY_OTHER_ID_CONTINUE; - if (c == 0x00BF) return UCD_PROPERTY_PATTERN_SYNTAX | ESPEAKNG_PROPERTY_QUESTION_MARK | ESPEAKNG_PROPERTY_OPTIONAL_SPACE_AFTER; + if (c == 0x00BF) return UCD_PROPERTY_PATTERN_SYNTAX | ESPEAKNG_PROPERTY_QUESTION_MARK | ESPEAKNG_PROPERTY_OPTIONAL_SPACE_AFTER | ESPEAKNG_PROPERTY_INVERTED_TERMINAL_PUNCTUATION; return UCD_PROPERTY_PATTERN_SYNTAX; case 0x0300: if (c == 0x037E) return UCD_PROPERTY_TERMINAL_PUNCTUATION | ESPEAKNG_PROPERTY_QUESTION_MARK; diff --git a/src/ucd-tools/tools/printdata.py b/src/ucd-tools/tools/printdata.py index cbfc7801..e4ea971a 100755 --- a/src/ucd-tools/tools/printdata.py +++ b/src/ucd-tools/tools/printdata.py @@ -161,6 +161,7 @@ def properties(data): props += (2 ** 31) * data.get('Pattern_Syntax', 0) props += (2 ** 32) * data.get('Prepended_Concatenation_Mark', 0) # eSpeak NG extended properties: + props += (2 ** 52) * data.get('Inverted_Terminal_Punctuation', 0) props += (2 ** 53) * data.get('Punctuation_In_Word', 0) props += (2 ** 54) * data.get('Optional_Space_After', 0) props += (2 ** 55) * data.get('Extended_Dash', 0)