From b716b95febd098d4e7f07bf4b7543984480556a7 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Tue, 28 Mar 2017 07:47:20 +0100 Subject: [PATCH 01/44] mkencodingtable: support mapping ISO 8859 encoding tables --- tools/mkencodingtable | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100755 tools/mkencodingtable diff --git a/tools/mkencodingtable b/tools/mkencodingtable new file mode 100755 index 00000000..9e0a0584 --- /dev/null +++ b/tools/mkencodingtable @@ -0,0 +1,39 @@ +#!/usr/bin/python3 + +# Copyright (C) 2017 Reece H. Dunn +# +# This file is part of ucd-tools. +# +# ucd-tools is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ucd-tools is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with ucd-tools. If not, see . + +import os +import sys + +def readtable(filename): + with open(filename) as f: + for line in f: + if line == '\n' or line.startswith('#'): + continue + yield line.split()[0:2] + +table = {} +for code, mapped in readtable(sys.argv[1]): + table[int(code, 0)] = mapped.lower() + +for code in range(0x80, 0x100): + if (code % 8 == 0): + print('\t', end='') + print('%s, ' % table.get(code, '0xfffd'), end='') + if (code % 8 == 7): + print('// %02x\n' % (code - 7), end='') From 54a8ea2a707190170e046259968ed757aa58fd87 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sat, 8 Apr 2017 18:51:22 +0100 Subject: [PATCH 02/44] Add White_Space support from PropList.txt. --- Makefile.am | 1 + src/include/ucd/ucd.h | 33 +++++++++++++++++++++++++++++++ src/proplist.c | 46 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+) create mode 100644 src/proplist.c diff --git a/Makefile.am b/Makefile.am index 6ffe4d11..2b208ac8 100644 --- a/Makefile.am +++ b/Makefile.am @@ -122,6 +122,7 @@ src_libucd_la_SOURCES = \ src/case.c \ src/categories.c \ src/ctype.c \ + src/proplist.c \ src/scripts.c \ src/tostring.c diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 840b2995..d834db60 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -325,6 +325,21 @@ const char *ucd_get_script_string(ucd_script s); */ ucd_script ucd_lookup_script(codepoint_t c); +/** @brief Properties + */ +typedef enum ucd_property_ +{ + UCD_PROPERTY_WHITE_SPACE = 0x0001, /**< @brief White_Space */ +} ucd_property; + +/** @brief Return the properties of the specified codepoint. + * + * @param c The Unicode codepoint to lookup. + * @param category The General Category of the codepoint. + * @return The properties associated with the codepoint. + */ +ucd_property ucd_properties(codepoint_t c, ucd_category category); + /** @brief Is the codepoint in the 'alnum' class? * * @param c The Unicode codepoint to check. @@ -770,6 +785,24 @@ namespace ucd return (script)ucd_lookup_script(c); } + /** @brief Properties + */ + enum property + { + White_Space = UCD_PROPERTY_WHITE_SPACE, /**< @brief White_Space */ + }; + + /** @brief Return the properties of the specified codepoint. + * + * @param c The Unicode codepoint to lookup. + * @param cat The General Category of the codepoint. + * @return The properties associated with the codepoint. + */ + inline property properties(codepoint_t c, category cat) + { + return (property)ucd_properties(c, (ucd_category)cat); + } + /** @brief Is the codepoint in the 'alnum' class? * * @param c The Unicode codepoint to check. diff --git a/src/proplist.c b/src/proplist.c new file mode 100644 index 00000000..a9a15a6e --- /dev/null +++ b/src/proplist.c @@ -0,0 +1,46 @@ +/* PropList APIs. + * + * Copyright (C) 2017 Reece H. Dunn + * + * This file is part of ucd-tools. + * + * ucd-tools is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * ucd-tools is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with ucd-tools. If not, see . + */ + +#include "ucd/ucd.h" + +ucd_property ucd_properties(codepoint_t c, ucd_category category) +{ + switch (category) + { + case UCD_CATEGORY_Cc: + switch (c) + { + case 0x09: // CHARACTER TABULATION + case 0x0A: // LINE FEED + case 0x0B: // LINE TABULATION + case 0x0C: // FORM FEED + case 0x0D: // CARRIAGE RETURN + case 0x85: // NEXT LINE + return UCD_PROPERTY_WHITE_SPACE; + } + return 0; + case UCD_CATEGORY_Zl: + case UCD_CATEGORY_Zp: + case UCD_CATEGORY_Zs: + return UCD_PROPERTY_WHITE_SPACE; + default: + return 0; + }; +} From 8cc80d671d7beb92a9ea2d39579bac6890001bb9 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sat, 8 Apr 2017 19:08:50 +0100 Subject: [PATCH 03/44] Add support from UnicodeData.txt. --- src/include/ucd/ucd.h | 2 ++ src/proplist.c | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index d834db60..65658fa1 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -330,6 +330,7 @@ ucd_script ucd_lookup_script(codepoint_t c); typedef enum ucd_property_ { UCD_PROPERTY_WHITE_SPACE = 0x0001, /**< @brief White_Space */ + UCD_PROPERTY_NO_BREAK = 0x0002, /**< @brief */ } ucd_property; /** @brief Return the properties of the specified codepoint. @@ -790,6 +791,7 @@ namespace ucd enum property { White_Space = UCD_PROPERTY_WHITE_SPACE, /**< @brief White_Space */ + noBreak = UCD_PROPERTY_NO_BREAK, /**< @brief */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index a9a15a6e..c5b60b7b 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -36,9 +36,25 @@ ucd_property ucd_properties(codepoint_t c, ucd_category category) return UCD_PROPERTY_WHITE_SPACE; } return 0; + case UCD_CATEGORY_Pd: + if (c == 0x2011) // NON-BREAKING HYPHEN + return UCD_PROPERTY_NO_BREAK; + return 0; + case UCD_CATEGORY_Po: + if (c == 0x0F0C) // TIBETAN MARK DELIMITER TSHEG BSTAR + return UCD_PROPERTY_NO_BREAK; + return 0; case UCD_CATEGORY_Zl: case UCD_CATEGORY_Zp: + return UCD_PROPERTY_WHITE_SPACE; case UCD_CATEGORY_Zs: + switch (c) + { + case 0x00A0: // NO-BREAK SPACE + case 0x2007: // FIGURE SPACE + case 0x202F: // NARROW NO-BREAK SPACE + return UCD_PROPERTY_WHITE_SPACE | UCD_PROPERTY_NO_BREAK; + } return UCD_PROPERTY_WHITE_SPACE; default: return 0; From bc16887a7932123b594f2d4647629039804399a5 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sat, 8 Apr 2017 19:11:45 +0100 Subject: [PATCH 04/44] Use ucd_properties to implement ucd_isspace. --- src/ctype.c | 30 ++---------------------------- 1 file changed, 2 insertions(+), 28 deletions(-) diff --git a/src/ctype.c b/src/ctype.c index 4632b55e..373d15e3 100644 --- a/src/ctype.c +++ b/src/ctype.c @@ -437,34 +437,8 @@ int ucd_ispunct(codepoint_t c) int ucd_isspace(codepoint_t c) { - switch (ucd_lookup_category(c)) - { - case UCD_CATEGORY_Zl: - case UCD_CATEGORY_Zp: - return 1; - case UCD_CATEGORY_Zs: - switch (c) // Exclude characters with the DispositionType - { - case 0x00A0: // U+00A0 : NO-BREAK SPACE - case 0x2007: // U+2007 : FIGURE SPACE - case 0x202F: // U+202F : NARROW NO-BREAK SPACE - return 0; - } - return 1; - case UCD_CATEGORY_Cc: - switch (c) // Include control characters marked as White_Space - { - case 0x09: // U+0009 : CHARACTER TABULATION - case 0x0A: // U+000A : LINE FEED - case 0x0B: // U+000B : LINE TABULATION - case 0x0C: // U+000C : FORM FEED - case 0x0D: // U+000D : CARRIAGE RETURN - case 0x85: // U+0085 : NEXT LINE - return 1; - } - default: - return 0; - } + ucd_property props = ucd_properties(c, ucd_lookup_category(c)); + return (props & (UCD_PROPERTY_WHITE_SPACE | UCD_PROPERTY_NO_BREAK)) == UCD_PROPERTY_WHITE_SPACE; } int ucd_isupper(codepoint_t c) From eae8740117dc7555ffed192fb829e1acefdeb4d2 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sat, 8 Apr 2017 19:21:56 +0100 Subject: [PATCH 05/44] Add Bidi_Control support from PropList.txt. --- src/include/ucd/ucd.h | 6 ++++-- src/proplist.c | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 65658fa1..99b33960 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -329,8 +329,9 @@ ucd_script ucd_lookup_script(codepoint_t c); */ typedef enum ucd_property_ { - UCD_PROPERTY_WHITE_SPACE = 0x0001, /**< @brief White_Space */ - UCD_PROPERTY_NO_BREAK = 0x0002, /**< @brief */ + UCD_PROPERTY_WHITE_SPACE = 0x0001, /**< @brief White_Space */ + UCD_PROPERTY_NO_BREAK = 0x0002, /**< @brief */ + UCD_PROPERTY_BIDI_CONTROL = 0x0004, /**< @brief Bidi_Control */ } ucd_property; /** @brief Return the properties of the specified codepoint. @@ -792,6 +793,7 @@ namespace ucd { White_Space = UCD_PROPERTY_WHITE_SPACE, /**< @brief White_Space */ noBreak = UCD_PROPERTY_NO_BREAK, /**< @brief */ + Bidi_Control = UCD_PROPERTY_BIDI_CONTROL, /**< @brief Bidi_Control */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index c5b60b7b..a3d421b8 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -36,6 +36,12 @@ ucd_property ucd_properties(codepoint_t c, ucd_category category) return UCD_PROPERTY_WHITE_SPACE; } return 0; + case UCD_CATEGORY_Cf: + if (c == 0x061C) return UCD_PROPERTY_BIDI_CONTROL; + if (c >= 0x200E && c <= 0x200F) return UCD_PROPERTY_BIDI_CONTROL; + if (c >= 0x202A && c <= 0x202E) return UCD_PROPERTY_BIDI_CONTROL; + if (c >= 0x2066 && c <= 0x2069) return UCD_PROPERTY_BIDI_CONTROL; + return 0; case UCD_CATEGORY_Pd: if (c == 0x2011) // NON-BREAKING HYPHEN return UCD_PROPERTY_NO_BREAK; From fd8a8d17b048a9ea7bb470e9f663a8b7f659ec95 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sat, 8 Apr 2017 19:25:17 +0100 Subject: [PATCH 06/44] Add Join_Control support from PropList.txt. --- src/include/ucd/ucd.h | 2 ++ src/proplist.c | 1 + 2 files changed, 3 insertions(+) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 99b33960..2ad132de 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -332,6 +332,7 @@ typedef enum ucd_property_ UCD_PROPERTY_WHITE_SPACE = 0x0001, /**< @brief White_Space */ UCD_PROPERTY_NO_BREAK = 0x0002, /**< @brief */ UCD_PROPERTY_BIDI_CONTROL = 0x0004, /**< @brief Bidi_Control */ + UCD_PROPERTY_JOIN_CONTROL = 0x0008, /**< @brief Join_Control */ } ucd_property; /** @brief Return the properties of the specified codepoint. @@ -794,6 +795,7 @@ namespace ucd White_Space = UCD_PROPERTY_WHITE_SPACE, /**< @brief White_Space */ noBreak = UCD_PROPERTY_NO_BREAK, /**< @brief */ Bidi_Control = UCD_PROPERTY_BIDI_CONTROL, /**< @brief Bidi_Control */ + Join_Control = UCD_PROPERTY_JOIN_CONTROL, /**< @brief Join_Control */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index a3d421b8..325da2a6 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -38,6 +38,7 @@ ucd_property ucd_properties(codepoint_t c, ucd_category category) return 0; case UCD_CATEGORY_Cf: if (c == 0x061C) return UCD_PROPERTY_BIDI_CONTROL; + if (c >= 0x200C && c <= 0x200D) return UCD_PROPERTY_JOIN_CONTROL; if (c >= 0x200E && c <= 0x200F) return UCD_PROPERTY_BIDI_CONTROL; if (c >= 0x202A && c <= 0x202E) return UCD_PROPERTY_BIDI_CONTROL; if (c >= 0x2066 && c <= 0x2069) return UCD_PROPERTY_BIDI_CONTROL; From 249960bae4811b3b18e46490ac63f55d3cb65645 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sat, 8 Apr 2017 19:49:41 +0100 Subject: [PATCH 07/44] Add Dash support from PropList.txt. --- src/include/ucd/ucd.h | 2 + src/proplist.c | 85 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 79 insertions(+), 8 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 2ad132de..5d1e59e6 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -333,6 +333,7 @@ typedef enum ucd_property_ UCD_PROPERTY_NO_BREAK = 0x0002, /**< @brief */ UCD_PROPERTY_BIDI_CONTROL = 0x0004, /**< @brief Bidi_Control */ UCD_PROPERTY_JOIN_CONTROL = 0x0008, /**< @brief Join_Control */ + UCD_PROPERTY_DASH = 0x0010, /**< @brief Dash */ } ucd_property; /** @brief Return the properties of the specified codepoint. @@ -796,6 +797,7 @@ namespace ucd noBreak = UCD_PROPERTY_NO_BREAK, /**< @brief */ Bidi_Control = UCD_PROPERTY_BIDI_CONTROL, /**< @brief Bidi_Control */ Join_Control = UCD_PROPERTY_JOIN_CONTROL, /**< @brief Join_Control */ + Dash = UCD_PROPERTY_DASH, /**< @brief Dash */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index 325da2a6..4ce96ee6 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -20,6 +20,80 @@ #include "ucd/ucd.h" +static int properties_Pd(codepoint_t c) +{ + switch (c & 0xFFFFFF00) + { + case 0x0000: + if (c == 0x002D) return UCD_PROPERTY_DASH; + break; + case 0x0500: + if (c == 0x058A) return UCD_PROPERTY_DASH; + if (c == 0x05BE) return UCD_PROPERTY_DASH; + break; + case 0x1400: + if (c == 0x1400) return UCD_PROPERTY_DASH; + break; + case 0x1800: + if (c == 0x1806) return UCD_PROPERTY_DASH; + break; + case 0x2000: + if (c == 0x2010) return UCD_PROPERTY_DASH; + if (c == 0x2011) return UCD_PROPERTY_DASH | UCD_PROPERTY_NO_BREAK; + if (c >= 0x2012 && c <= 0x2015) return UCD_PROPERTY_DASH; + break; + case 0x2E00: + if (c == 0x2E17) return UCD_PROPERTY_DASH; + if (c == 0x2E1A) return UCD_PROPERTY_DASH; + if (c >= 0x2E3A && c <= 0x2E3B) return UCD_PROPERTY_DASH; + if (c == 0x2E40) return UCD_PROPERTY_DASH; + break; + case 0x3000: + if (c == 0x301C) return UCD_PROPERTY_DASH; + if (c == 0x3030) return UCD_PROPERTY_DASH; + if (c == 0x30A0) return UCD_PROPERTY_DASH; + break; + case 0xFE00: + if (c >= 0xFE31 && c <= 0xFE32) return UCD_PROPERTY_DASH; + if (c == 0xFE58) return UCD_PROPERTY_DASH; + if (c == 0xFE63) return UCD_PROPERTY_DASH; + break; + case 0xFF00: + if (c == 0xFF0D) return UCD_PROPERTY_DASH; + break; + } + return 0; +} + +static int properties_Po(codepoint_t c) +{ + switch (c & 0xFFFFFF00) + { + case 0x0F00: + if (c == 0x0F0C) return UCD_PROPERTY_NO_BREAK; + break; + case 0x2000: + if (c == 0x2053) return UCD_PROPERTY_DASH; + break; + } + return 0; +} + +static int properties_Sm(codepoint_t c) +{ + switch (c & 0xFFFFFF00) + { + case 0x2000: + if (c == 0x207B) return UCD_PROPERTY_DASH; + if (c == 0x208B) return UCD_PROPERTY_DASH; + break; + case 0x2200: + if (c == 0x2212) return UCD_PROPERTY_DASH; + break; + } + return 0; +} + ucd_property ucd_properties(codepoint_t c, ucd_category category) { switch (category) @@ -43,14 +117,9 @@ ucd_property ucd_properties(codepoint_t c, ucd_category category) if (c >= 0x202A && c <= 0x202E) return UCD_PROPERTY_BIDI_CONTROL; if (c >= 0x2066 && c <= 0x2069) return UCD_PROPERTY_BIDI_CONTROL; return 0; - case UCD_CATEGORY_Pd: - if (c == 0x2011) // NON-BREAKING HYPHEN - return UCD_PROPERTY_NO_BREAK; - return 0; - case UCD_CATEGORY_Po: - if (c == 0x0F0C) // TIBETAN MARK DELIMITER TSHEG BSTAR - return UCD_PROPERTY_NO_BREAK; - return 0; + case UCD_CATEGORY_Pd: return properties_Pd(c); + case UCD_CATEGORY_Po: return properties_Po(c); + case UCD_CATEGORY_Sm: return properties_Sm(c); case UCD_CATEGORY_Zl: case UCD_CATEGORY_Zp: return UCD_PROPERTY_WHITE_SPACE; From 1e02d05c612c63aae0134870a08bd19eb716e3e8 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sat, 8 Apr 2017 19:56:28 +0100 Subject: [PATCH 08/44] proplist.c: Create helper functions for all matched categories. --- src/proplist.c | 75 ++++++++++++++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 33 deletions(-) diff --git a/src/proplist.c b/src/proplist.c index 4ce96ee6..d971a8ab 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -20,6 +20,30 @@ #include "ucd/ucd.h" +static int properties_Cc(codepoint_t c) +{ + if (c >= 0x0009 && c <= 0x000D) return UCD_PROPERTY_WHITE_SPACE; + if (c == 0x0085) return UCD_PROPERTY_WHITE_SPACE; + return 0; +} + +static int properties_Cf(codepoint_t c) +{ + switch (c & 0xFFFFFF00) + { + case 0x0600: + if (c == 0x061C) return UCD_PROPERTY_BIDI_CONTROL; + break; + case 0x2000: + if (c >= 0x200C && c <= 0x200D) return UCD_PROPERTY_JOIN_CONTROL; + if (c >= 0x200E && c <= 0x200F) return UCD_PROPERTY_BIDI_CONTROL; + if (c >= 0x202A && c <= 0x202E) return UCD_PROPERTY_BIDI_CONTROL; + if (c >= 0x2066 && c <= 0x2069) return UCD_PROPERTY_BIDI_CONTROL; + break; + } + return 0; +} + static int properties_Pd(codepoint_t c) { switch (c & 0xFFFFFF00) @@ -94,45 +118,30 @@ static int properties_Sm(codepoint_t c) return 0; } +static int properties_Zs(codepoint_t c) +{ + switch (c) + { + case 0x00A0: // NO-BREAK SPACE + case 0x2007: // FIGURE SPACE + case 0x202F: // NARROW NO-BREAK SPACE + return UCD_PROPERTY_WHITE_SPACE | UCD_PROPERTY_NO_BREAK; + } + return UCD_PROPERTY_WHITE_SPACE; +} + ucd_property ucd_properties(codepoint_t c, ucd_category category) { switch (category) { - case UCD_CATEGORY_Cc: - switch (c) - { - case 0x09: // CHARACTER TABULATION - case 0x0A: // LINE FEED - case 0x0B: // LINE TABULATION - case 0x0C: // FORM FEED - case 0x0D: // CARRIAGE RETURN - case 0x85: // NEXT LINE - return UCD_PROPERTY_WHITE_SPACE; - } - return 0; - case UCD_CATEGORY_Cf: - if (c == 0x061C) return UCD_PROPERTY_BIDI_CONTROL; - if (c >= 0x200C && c <= 0x200D) return UCD_PROPERTY_JOIN_CONTROL; - if (c >= 0x200E && c <= 0x200F) return UCD_PROPERTY_BIDI_CONTROL; - if (c >= 0x202A && c <= 0x202E) return UCD_PROPERTY_BIDI_CONTROL; - if (c >= 0x2066 && c <= 0x2069) return UCD_PROPERTY_BIDI_CONTROL; - return 0; + case UCD_CATEGORY_Cc: return properties_Cc(c); + case UCD_CATEGORY_Cf: return properties_Cf(c); case UCD_CATEGORY_Pd: return properties_Pd(c); case UCD_CATEGORY_Po: return properties_Po(c); case UCD_CATEGORY_Sm: return properties_Sm(c); - case UCD_CATEGORY_Zl: - case UCD_CATEGORY_Zp: - return UCD_PROPERTY_WHITE_SPACE; - case UCD_CATEGORY_Zs: - switch (c) - { - case 0x00A0: // NO-BREAK SPACE - case 0x2007: // FIGURE SPACE - case 0x202F: // NARROW NO-BREAK SPACE - return UCD_PROPERTY_WHITE_SPACE | UCD_PROPERTY_NO_BREAK; - } - return UCD_PROPERTY_WHITE_SPACE; - default: - return 0; + case UCD_CATEGORY_Zl: return UCD_PROPERTY_WHITE_SPACE; + case UCD_CATEGORY_Zp: return UCD_PROPERTY_WHITE_SPACE; + case UCD_CATEGORY_Zs: return properties_Zs(c); + default: return 0; }; } From d1eb23a741ae0df9d8e1c08a657c2c75b6acbb7e Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sat, 8 Apr 2017 20:05:33 +0100 Subject: [PATCH 09/44] Add Hyphen support from PropList.txt. --- src/include/ucd/ucd.h | 2 ++ src/proplist.c | 25 +++++++++++++++++-------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 5d1e59e6..8356a31c 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -334,6 +334,7 @@ typedef enum ucd_property_ UCD_PROPERTY_BIDI_CONTROL = 0x0004, /**< @brief Bidi_Control */ UCD_PROPERTY_JOIN_CONTROL = 0x0008, /**< @brief Join_Control */ UCD_PROPERTY_DASH = 0x0010, /**< @brief Dash */ + UCD_PROPERTY_HYPHEN = 0x0020, /**< @brief Hyphen */ } ucd_property; /** @brief Return the properties of the specified codepoint. @@ -798,6 +799,7 @@ namespace ucd Bidi_Control = UCD_PROPERTY_BIDI_CONTROL, /**< @brief Bidi_Control */ Join_Control = UCD_PROPERTY_JOIN_CONTROL, /**< @brief Join_Control */ Dash = UCD_PROPERTY_DASH, /**< @brief Dash */ + Hyphen = UCD_PROPERTY_HYPHEN, /**< @brief Hyphen */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index d971a8ab..69bdafe7 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -31,6 +31,9 @@ static int properties_Cf(codepoint_t c) { switch (c & 0xFFFFFF00) { + case 0x0000: + if (c == 0x00AD) return UCD_PROPERTY_HYPHEN; + break; case 0x0600: if (c == 0x061C) return UCD_PROPERTY_BIDI_CONTROL; break; @@ -49,25 +52,25 @@ static int properties_Pd(codepoint_t c) switch (c & 0xFFFFFF00) { case 0x0000: - if (c == 0x002D) return UCD_PROPERTY_DASH; + if (c == 0x002D) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; break; case 0x0500: - if (c == 0x058A) return UCD_PROPERTY_DASH; + if (c == 0x058A) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; if (c == 0x05BE) return UCD_PROPERTY_DASH; break; case 0x1400: if (c == 0x1400) return UCD_PROPERTY_DASH; break; case 0x1800: - if (c == 0x1806) return UCD_PROPERTY_DASH; + if (c == 0x1806) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; break; case 0x2000: - if (c == 0x2010) return UCD_PROPERTY_DASH; - if (c == 0x2011) return UCD_PROPERTY_DASH | UCD_PROPERTY_NO_BREAK; + if (c == 0x2010) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; + if (c == 0x2011) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN | UCD_PROPERTY_NO_BREAK; if (c >= 0x2012 && c <= 0x2015) return UCD_PROPERTY_DASH; break; case 0x2E00: - if (c == 0x2E17) return UCD_PROPERTY_DASH; + if (c == 0x2E17) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; if (c == 0x2E1A) return UCD_PROPERTY_DASH; if (c >= 0x2E3A && c <= 0x2E3B) return UCD_PROPERTY_DASH; if (c == 0x2E40) return UCD_PROPERTY_DASH; @@ -80,10 +83,10 @@ static int properties_Pd(codepoint_t c) case 0xFE00: if (c >= 0xFE31 && c <= 0xFE32) return UCD_PROPERTY_DASH; if (c == 0xFE58) return UCD_PROPERTY_DASH; - if (c == 0xFE63) return UCD_PROPERTY_DASH; + if (c == 0xFE63) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; break; case 0xFF00: - if (c == 0xFF0D) return UCD_PROPERTY_DASH; + if (c == 0xFF0D) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; break; } return 0; @@ -99,6 +102,12 @@ static int properties_Po(codepoint_t c) case 0x2000: if (c == 0x2053) return UCD_PROPERTY_DASH; break; + case 0x3000: + if (c == 0x30FB) return UCD_PROPERTY_HYPHEN; + break; + case 0xFF00: + if (c == 0xFF65) return UCD_PROPERTY_HYPHEN; + break; } return 0; } From ec1a17b229eb3c3e3335ef65237dc5943f0ef391 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 9 Apr 2017 13:30:51 +0100 Subject: [PATCH 10/44] Add Quotation_Mark support from PropList.txt. --- src/include/ucd/ucd.h | 14 ++++--- src/proplist.c | 90 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 6 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 8356a31c..40103581 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -329,12 +329,13 @@ ucd_script ucd_lookup_script(codepoint_t c); */ typedef enum ucd_property_ { - UCD_PROPERTY_WHITE_SPACE = 0x0001, /**< @brief White_Space */ - UCD_PROPERTY_NO_BREAK = 0x0002, /**< @brief */ - UCD_PROPERTY_BIDI_CONTROL = 0x0004, /**< @brief Bidi_Control */ - UCD_PROPERTY_JOIN_CONTROL = 0x0008, /**< @brief Join_Control */ - UCD_PROPERTY_DASH = 0x0010, /**< @brief Dash */ - UCD_PROPERTY_HYPHEN = 0x0020, /**< @brief Hyphen */ + UCD_PROPERTY_WHITE_SPACE = 0x00000001, /**< @brief White_Space */ + UCD_PROPERTY_NO_BREAK = 0x00000002, /**< @brief */ + UCD_PROPERTY_BIDI_CONTROL = 0x00000004, /**< @brief Bidi_Control */ + UCD_PROPERTY_JOIN_CONTROL = 0x00000008, /**< @brief Join_Control */ + UCD_PROPERTY_DASH = 0x00000010, /**< @brief Dash */ + UCD_PROPERTY_HYPHEN = 0x00000020, /**< @brief Hyphen */ + UCD_PROPERTY_QUOTATION_MARK = 0x00000020, /**< @brief Quotation_Mark */ } ucd_property; /** @brief Return the properties of the specified codepoint. @@ -800,6 +801,7 @@ namespace ucd Join_Control = UCD_PROPERTY_JOIN_CONTROL, /**< @brief Join_Control */ Dash = UCD_PROPERTY_DASH, /**< @brief Dash */ Hyphen = UCD_PROPERTY_HYPHEN, /**< @brief Hyphen */ + Quotation_Mark = UCD_PROPERTY_QUOTATION_MARK, /**< @brief Quotation_Mark */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index 69bdafe7..636c609f 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -92,10 +92,67 @@ static int properties_Pd(codepoint_t c) return 0; } +static int properties_Pe(codepoint_t c) +{ + switch (c & 0xFFFFFF00) + { + case 0x3000: + if (c == 0x300D) return UCD_PROPERTY_QUOTATION_MARK; + if (c == 0x300F) return UCD_PROPERTY_QUOTATION_MARK; + if (c >= 0x301E && c <= 0x301F) return UCD_PROPERTY_QUOTATION_MARK; + break; + case 0xFE00: + if (c == 0xFE42) return UCD_PROPERTY_QUOTATION_MARK; + if (c == 0xFE44) return UCD_PROPERTY_QUOTATION_MARK; + break; + case 0xFF00: + if (c == 0xFF63) return UCD_PROPERTY_QUOTATION_MARK; + break; + } + return 0; +} + +static int properties_Pf(codepoint_t c) +{ + switch (c & 0xFFFFFF00) + { + case 0x0000: + if (c == 0x00BB) return UCD_PROPERTY_QUOTATION_MARK; + break; + case 0x2000: + if (c == 0x2019) return UCD_PROPERTY_QUOTATION_MARK; + if (c == 0x201D) return UCD_PROPERTY_QUOTATION_MARK; + if (c == 0x203A) return UCD_PROPERTY_QUOTATION_MARK; + break; + } + return 0; +} + +static int properties_Pi(codepoint_t c) +{ + switch (c & 0xFFFFFF00) + { + case 0x0000: + if (c == 0x00AB) return UCD_PROPERTY_QUOTATION_MARK; + break; + case 0x2000: + if (c == 0x2018) return UCD_PROPERTY_QUOTATION_MARK; + if (c >= 0x201B && c <= 0x201C) return UCD_PROPERTY_QUOTATION_MARK; + if (c == 0x201F) return UCD_PROPERTY_QUOTATION_MARK; + if (c == 0x2039) return UCD_PROPERTY_QUOTATION_MARK; + break; + } + return 0; +} + static int properties_Po(codepoint_t c) { switch (c & 0xFFFFFF00) { + case 0x0000: + if (c == 0x0022) return UCD_PROPERTY_QUOTATION_MARK; + if (c == 0x0027) return UCD_PROPERTY_QUOTATION_MARK; + break; case 0x0F00: if (c == 0x0F0C) return UCD_PROPERTY_NO_BREAK; break; @@ -106,12 +163,41 @@ static int properties_Po(codepoint_t c) if (c == 0x30FB) return UCD_PROPERTY_HYPHEN; break; case 0xFF00: + if (c == 0xFF02) return UCD_PROPERTY_QUOTATION_MARK; + if (c == 0xFF07) return UCD_PROPERTY_QUOTATION_MARK; if (c == 0xFF65) return UCD_PROPERTY_HYPHEN; break; } return 0; } +static int properties_Ps(codepoint_t c) +{ + switch (c & 0xFFFFFF00) + { + case 0x2000: + if (c == 0x201A) return UCD_PROPERTY_QUOTATION_MARK; + if (c == 0x201E) return UCD_PROPERTY_QUOTATION_MARK; + break; + case 0x2E00: + if (c == 0x2E42) return UCD_PROPERTY_QUOTATION_MARK; + break; + case 0x3000: + if (c == 0x300C) return UCD_PROPERTY_QUOTATION_MARK; + if (c == 0x300E) return UCD_PROPERTY_QUOTATION_MARK; + if (c == 0x301D) return UCD_PROPERTY_QUOTATION_MARK; + break; + case 0xFE00: + if (c == 0xFE41) return UCD_PROPERTY_QUOTATION_MARK; + if (c == 0xFE43) return UCD_PROPERTY_QUOTATION_MARK; + break; + case 0xFF00: + if (c == 0xFF62) return UCD_PROPERTY_QUOTATION_MARK; + break; + } + return 0; +} + static int properties_Sm(codepoint_t c) { switch (c & 0xFFFFFF00) @@ -146,7 +232,11 @@ ucd_property ucd_properties(codepoint_t c, ucd_category category) case UCD_CATEGORY_Cc: return properties_Cc(c); case UCD_CATEGORY_Cf: return properties_Cf(c); case UCD_CATEGORY_Pd: return properties_Pd(c); + case UCD_CATEGORY_Pe: return properties_Pe(c); + case UCD_CATEGORY_Pf: return properties_Pf(c); + case UCD_CATEGORY_Pi: return properties_Pi(c); case UCD_CATEGORY_Po: return properties_Po(c); + case UCD_CATEGORY_Ps: return properties_Ps(c); case UCD_CATEGORY_Sm: return properties_Sm(c); case UCD_CATEGORY_Zl: return UCD_PROPERTY_WHITE_SPACE; case UCD_CATEGORY_Zp: return UCD_PROPERTY_WHITE_SPACE; From a9aabc6242d88a678140f1a32c87fc8a5cb2f41b Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 9 Apr 2017 14:01:59 +0100 Subject: [PATCH 11/44] Add tests for the PropList API. --- src/include/ucd/ucd.h | 2 +- tests/printcdata.c | 5 ++++- tests/printucddata.c | 5 ++++- tests/printucddata_cpp.cpp | 5 ++++- tools/printdata.py | 22 ++++++++++++++++++++-- 5 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 40103581..294c3386 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -335,7 +335,7 @@ typedef enum ucd_property_ UCD_PROPERTY_JOIN_CONTROL = 0x00000008, /**< @brief Join_Control */ UCD_PROPERTY_DASH = 0x00000010, /**< @brief Dash */ UCD_PROPERTY_HYPHEN = 0x00000020, /**< @brief Hyphen */ - UCD_PROPERTY_QUOTATION_MARK = 0x00000020, /**< @brief Quotation_Mark */ + UCD_PROPERTY_QUOTATION_MARK = 0x00000040, /**< @brief Quotation_Mark */ } ucd_property; /** @brief Return the properties of the specified codepoint. diff --git a/tests/printcdata.c b/tests/printcdata.c index f9af2d08..49e29a42 100644 --- a/tests/printcdata.c +++ b/tests/printcdata.c @@ -163,6 +163,9 @@ void uprintf(FILE *out, codepoint_t c, const char *format) case 'p': // codepoint uprintf_codepoint(out, c, *++format); break; + case 'P': // properties + fprintf(out, "%08x", ucd_properties(c, ucd_lookup_category(c))); + break; case 'i': // is* uprintf_is(out, c, *++format); break; @@ -249,7 +252,7 @@ int main(int argc, char **argv) { for (codepoint_t c = 0; c <= 0x10FFFF; ++c) uprintf(stdout, c, format ? format : - "%pH %s %C %c %UH %LH %TH %id %ix %ic %is %ib %ip %iP %ig %iA %ia %iu %il\n"); + "%pH %s %C %c %UH %LH %TH %id %ix %ic %is %ib %ip %iP %ig %iA %ia %iu %il %P\n"); } return 0; } diff --git a/tests/printucddata.c b/tests/printucddata.c index e410480c..2fb87ed8 100644 --- a/tests/printucddata.c +++ b/tests/printucddata.c @@ -160,6 +160,9 @@ void uprintf(FILE *out, codepoint_t c, const char *format) case 'p': // codepoint uprintf_codepoint(out, c, *++format); break; + case 'P': // properties + fprintf(out, "%08x", ucd_properties(c, ucd_lookup_category(c))); + break; case 'i': // is* uprintf_is(out, c, *++format); break; @@ -244,7 +247,7 @@ int main(int argc, char **argv) { for (codepoint_t c = 0; c <= 0x10FFFF; ++c) uprintf(stdout, c, format ? format : - "%pH %s %C %c %UH %LH %TH %id %ix %ic %is %ib %ip %iP %ig %iA %ia %iu %il\n"); + "%pH %s %C %c %UH %LH %TH %id %ix %ic %is %ib %ip %iP %ig %iA %ia %iu %il %P\n"); } return 0; } diff --git a/tests/printucddata_cpp.cpp b/tests/printucddata_cpp.cpp index 03214bae..59128c22 100644 --- a/tests/printucddata_cpp.cpp +++ b/tests/printucddata_cpp.cpp @@ -160,6 +160,9 @@ void uprintf(FILE *out, ucd::codepoint_t c, const char *format) case 'p': // codepoint uprintf_codepoint(out, c, *++format); break; + case 'P': // properties + fprintf(out, "%08x", ucd::properties(c, ucd::lookup_category(c))); + break; case 'i': // is* uprintf_is(out, c, *++format); break; @@ -244,7 +247,7 @@ int main(int argc, char **argv) { for (ucd::codepoint_t c = 0; c <= 0x10FFFF; ++c) uprintf(stdout, c, format ? format : - "%pH %s %C %c %UH %LH %TH %id %ix %ic %is %ib %ip %iP %ig %iA %ia %iu %il\n"); + "%pH %s %C %c %UH %LH %TH %id %ix %ic %is %ib %ip %iP %ig %iA %ia %iu %il %P\n"); } return 0; } diff --git a/tools/printdata.py b/tools/printdata.py index bc724de5..a951e90e 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -113,6 +113,23 @@ def islower(data): else: return 0 +def decomposition_type(data, dtype): + value = data.get('DecompositionType', None) + if value and value.startswith(dtype): + return value + return None + +def properties(data): + props = 0 + props += 1 * data.get('White_Space', 0) + props += 2 * (decomposition_type(data, '') != None) + props += 4 * data.get('Bidi_Control', 0) + props += 8 * data.get('Join_Control', 0) + props += 16 * data.get('Dash', 0) + props += 32 * data.get('Hyphen', 0) + props += 64 * data.get('Quotation_Mark', 0) + return props + if __name__ == '__main__': for codepoint in ucd.CodeRange('000000..10FFFF'): try: @@ -126,10 +143,11 @@ if __name__ == '__main__': if title == null: title = codepoint if upper == null: upper = codepoint if lower == null: lower = codepoint - print('%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s' % ( + print('%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %08x' % ( codepoint, script, data.get('GeneralCategory', 'Cn')[0], data.get('GeneralCategory', 'Cn'), upper, lower, title, isdigit(data), isxdigit(data), iscntrl(data), isspace(data), isblank(data), ispunct(data), - isprint(data), isgraph(data), isalnum(data), isalpha(data), isupper(data), islower(data))) + isprint(data), isgraph(data), isalnum(data), isalpha(data), isupper(data), islower(data), + properties(data))) From ab0a1da5bec43a1236d366472c290ad96a1221d6 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Tue, 11 Apr 2017 08:01:07 +0100 Subject: [PATCH 12/44] Simplify the Dash property classification logic. Pd is the 'Dash' general category. --- src/proplist.c | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/src/proplist.c b/src/proplist.c index 636c609f..19d4c618 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -56,10 +56,6 @@ static int properties_Pd(codepoint_t c) break; case 0x0500: if (c == 0x058A) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; - if (c == 0x05BE) return UCD_PROPERTY_DASH; - break; - case 0x1400: - if (c == 0x1400) return UCD_PROPERTY_DASH; break; case 0x1800: if (c == 0x1806) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; @@ -67,29 +63,18 @@ static int properties_Pd(codepoint_t c) case 0x2000: if (c == 0x2010) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; if (c == 0x2011) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN | UCD_PROPERTY_NO_BREAK; - if (c >= 0x2012 && c <= 0x2015) return UCD_PROPERTY_DASH; break; case 0x2E00: if (c == 0x2E17) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; - if (c == 0x2E1A) return UCD_PROPERTY_DASH; - if (c >= 0x2E3A && c <= 0x2E3B) return UCD_PROPERTY_DASH; - if (c == 0x2E40) return UCD_PROPERTY_DASH; - break; - case 0x3000: - if (c == 0x301C) return UCD_PROPERTY_DASH; - if (c == 0x3030) return UCD_PROPERTY_DASH; - if (c == 0x30A0) return UCD_PROPERTY_DASH; break; case 0xFE00: - if (c >= 0xFE31 && c <= 0xFE32) return UCD_PROPERTY_DASH; - if (c == 0xFE58) return UCD_PROPERTY_DASH; if (c == 0xFE63) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; break; case 0xFF00: if (c == 0xFF0D) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; break; } - return 0; + return UCD_PROPERTY_DASH; } static int properties_Pe(codepoint_t c) From 4226457fb5aed7f9ef173dc0d5c7fabae3219866 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Tue, 11 Apr 2017 21:21:22 +0100 Subject: [PATCH 13/44] Add Terminal_Punctuation support from PropList.txt. --- src/include/ucd/ucd.h | 30 +++---- src/proplist.c | 180 +++++++++++++++++++++++++++++++++++++++++- tools/printdata.py | 17 ++-- 3 files changed, 204 insertions(+), 23 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 294c3386..1ddcf92b 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -329,13 +329,14 @@ ucd_script ucd_lookup_script(codepoint_t c); */ typedef enum ucd_property_ { - UCD_PROPERTY_WHITE_SPACE = 0x00000001, /**< @brief White_Space */ - UCD_PROPERTY_NO_BREAK = 0x00000002, /**< @brief */ - UCD_PROPERTY_BIDI_CONTROL = 0x00000004, /**< @brief Bidi_Control */ - UCD_PROPERTY_JOIN_CONTROL = 0x00000008, /**< @brief Join_Control */ - UCD_PROPERTY_DASH = 0x00000010, /**< @brief Dash */ - UCD_PROPERTY_HYPHEN = 0x00000020, /**< @brief Hyphen */ - UCD_PROPERTY_QUOTATION_MARK = 0x00000040, /**< @brief Quotation_Mark */ + UCD_PROPERTY_WHITE_SPACE = 0x00000001, /**< @brief White_Space PropList */ + UCD_PROPERTY_NO_BREAK = 0x00000002, /**< @brief DispositionType (enabled check only) */ + UCD_PROPERTY_BIDI_CONTROL = 0x00000004, /**< @brief Bidi_Control PropList */ + UCD_PROPERTY_JOIN_CONTROL = 0x00000008, /**< @brief Join_Control PropList */ + UCD_PROPERTY_DASH = 0x00000010, /**< @brief Dash PropList */ + UCD_PROPERTY_HYPHEN = 0x00000020, /**< @brief Hyphen PropList */ + UCD_PROPERTY_QUOTATION_MARK = 0x00000040, /**< @brief Quotation_Mark PropList */ + UCD_PROPERTY_TERMINAL_PUNCTUATION = 0x00000080, /**< @brief Terminal_Punctuation PropList */ } ucd_property; /** @brief Return the properties of the specified codepoint. @@ -795,13 +796,14 @@ namespace ucd */ enum property { - White_Space = UCD_PROPERTY_WHITE_SPACE, /**< @brief White_Space */ - noBreak = UCD_PROPERTY_NO_BREAK, /**< @brief */ - Bidi_Control = UCD_PROPERTY_BIDI_CONTROL, /**< @brief Bidi_Control */ - Join_Control = UCD_PROPERTY_JOIN_CONTROL, /**< @brief Join_Control */ - Dash = UCD_PROPERTY_DASH, /**< @brief Dash */ - Hyphen = UCD_PROPERTY_HYPHEN, /**< @brief Hyphen */ - Quotation_Mark = UCD_PROPERTY_QUOTATION_MARK, /**< @brief Quotation_Mark */ + White_Space = UCD_PROPERTY_WHITE_SPACE, /**< @brief White_Space PropList */ + noBreak = UCD_PROPERTY_NO_BREAK, /**< @brief DispositionType (enabled check only) */ + Bidi_Control = UCD_PROPERTY_BIDI_CONTROL, /**< @brief Bidi_Control PropList */ + Join_Control = UCD_PROPERTY_JOIN_CONTROL, /**< @brief Join_Control PropList */ + Dash = UCD_PROPERTY_DASH, /**< @brief Dash PropList */ + Hyphen = UCD_PROPERTY_HYPHEN, /**< @brief Hyphen PropList */ + Quotation_Mark = UCD_PROPERTY_QUOTATION_MARK, /**< @brief Quotation_Mark PropList */ + Terminal_Punctuation = UCD_PROPERTY_TERMINAL_PUNCTUATION, /**< @brief Terminal_Punctuation PropList */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index 19d4c618..c886e613 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -135,22 +135,200 @@ static int properties_Po(codepoint_t c) switch (c & 0xFFFFFF00) { case 0x0000: + if (c == 0x0021) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c == 0x0022) return UCD_PROPERTY_QUOTATION_MARK; if (c == 0x0027) return UCD_PROPERTY_QUOTATION_MARK; + if (c == 0x002C) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x002E) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x003A && c <= 0x003B) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x003F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x0300: + if (c == 0x037E) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x0387) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x0500: + if (c == 0x0589) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x05C3) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x0600: + if (c == 0x060C) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x061B) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x061F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x06D4) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x0700: + if (c >= 0x0700 && c <= 0x070A) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x070C) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x07F8 && c <= 0x07F9) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x0800: + if (c >= 0x0830 && c <= 0x083E) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x085E) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x0900: + if (c >= 0x0964 && c <= 0x0965) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x0E00: + if (c >= 0x0E5A && c <= 0x0E5B) return UCD_PROPERTY_TERMINAL_PUNCTUATION; break; case 0x0F00: + if (c == 0x0F08) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c == 0x0F0C) return UCD_PROPERTY_NO_BREAK; + if (c >= 0x0F0D && c <= 0x0F12) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x1000: + if (c >= 0x104A && c <= 0x104B) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x1300: + if (c >= 0x1361 && c <= 0x1368) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x1600: + if (c >= 0x166D && c <= 0x166E) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x16EB && c <= 0x16ED) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x1700: + if (c >= 0x1735 && c <= 0x1736) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x17D4 && c <= 0x17D6) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x17DA) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x1800: + if (c >= 0x1802 && c <= 0x1805) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x1808 && c <= 0x1809) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x1900: + if (c >= 0x1944 && c <= 0x1945) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x1A00: + if (c >= 0x1AA8 && c <= 0x1AAB) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x1B00: + if (c >= 0x1B5A && c <= 0x1B5B) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x1B5D && c <= 0x1B5F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x1C00: + if (c >= 0x1C3B && c <= 0x1C3F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x1C7E && c <= 0x1C7F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; break; case 0x2000: + if (c >= 0x203C && c <= 0x203D) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x2047 && c <= 0x2049) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c == 0x2053) return UCD_PROPERTY_DASH; break; + case 0x2E00: + if (c == 0x2E2E) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x2E3C) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x2E41) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; case 0x3000: + if (c >= 0x3001 && c <= 0x3002) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c == 0x30FB) return UCD_PROPERTY_HYPHEN; break; + case 0xA400: + if (c >= 0xA4FE && c <= 0xA4FF) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0xA600: + if (c >= 0xA60D && c <= 0xA60F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0xA6F3 && c <= 0xA6F7) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0xA800: + if (c >= 0xA876 && c <= 0xA877) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0xA8CE && c <= 0xA8CF) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0xA900: + if (c == 0xA92F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0xA9C7 && c <= 0xA9C9) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0xAA00: + if (c >= 0xAA5D && c <= 0xAA5F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0xAADF) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0xAAF0 && c <= 0xAAF1) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0xAB00: + if (c == 0xABEB) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0xFE00: + if (c >= 0xFE50 && c <= 0xFE52) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0xFE54 && c <= 0xFE57) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; case 0xFF00: + if (c == 0xFF01) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c == 0xFF02) return UCD_PROPERTY_QUOTATION_MARK; if (c == 0xFF07) return UCD_PROPERTY_QUOTATION_MARK; + if (c == 0xFF0C) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0xFF0E) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c == 0xFF65) return UCD_PROPERTY_HYPHEN; + if (c >= 0xFF1A && c <= 0xFF1B) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0xFF1F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0xFF61) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0xFF64) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x10300: + if (c == 0x01039F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x0103D0) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x10800: + if (c == 0x010857) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x10900: + if (c == 0x01091F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x10A00: + if (c >= 0x010A56 && c <= 0x010A57) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x010AF0 && c <= 0x010AF5) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x10B00: + if (c >= 0x010B3A && c <= 0x010B3F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x010B99 && c <= 0x010B9C) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x11000: + if (c >= 0x011047 && c <= 0x01104D) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x0110BE && c <= 0x0110C1) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x11100: + if (c >= 0x011141 && c <= 0x011143) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x0111C5 && c <= 0x0111C6) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x0111CD) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x0111DE && c <= 0x0111DF) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x11200: + if (c >= 0x011238 && c <= 0x01123C) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x0112A9) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x11400: + if (c >= 0x01144B && c <= 0x01144D) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x01145B) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x11500: + if (c >= 0x0115C2 && c <= 0x0115C5) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x0115C9 && c <= 0x0115D7) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x11600: + if (c >= 0x011641 && c <= 0x011642) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x11700: + if (c >= 0x01173C && c <= 0x01173E) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x11C00: + if (c >= 0x011C41 && c <= 0x011C43) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x011C71) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x12400: + if (c >= 0x012470 && c <= 0x012474) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x16A00: + if (c >= 0x016A6E && c <= 0x016A6F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x016AF5) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x16B00: + if (c >= 0x016B37 && c <= 0x016B39) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x016B44) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x1BC00: + if (c == 0x01BC9F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + break; + case 0x1DA00: + if (c >= 0x01DA87 && c <= 0x01DA8A) return UCD_PROPERTY_TERMINAL_PUNCTUATION; break; } return 0; @@ -226,6 +404,6 @@ ucd_property ucd_properties(codepoint_t c, ucd_category category) case UCD_CATEGORY_Zl: return UCD_PROPERTY_WHITE_SPACE; case UCD_CATEGORY_Zp: return UCD_PROPERTY_WHITE_SPACE; case UCD_CATEGORY_Zs: return properties_Zs(c); - default: return 0; + default: return 0; // Cn Co Cs Ii L& M& N& Pc Sc Sk So }; } diff --git a/tools/printdata.py b/tools/printdata.py index a951e90e..52c4edf2 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -120,14 +120,15 @@ def decomposition_type(data, dtype): return None def properties(data): - props = 0 - props += 1 * data.get('White_Space', 0) - props += 2 * (decomposition_type(data, '') != None) - props += 4 * data.get('Bidi_Control', 0) - props += 8 * data.get('Join_Control', 0) - props += 16 * data.get('Dash', 0) - props += 32 * data.get('Hyphen', 0) - props += 64 * data.get('Quotation_Mark', 0) + props = 0 + props += 1 * data.get('White_Space', 0) + props += 2 * (decomposition_type(data, '') != None) + props += 4 * data.get('Bidi_Control', 0) + props += 8 * data.get('Join_Control', 0) + props += 16 * data.get('Dash', 0) + props += 32 * data.get('Hyphen', 0) + props += 64 * data.get('Quotation_Mark', 0) + props += 128 * data.get('Terminal_Punctuation', 0) return props if __name__ == '__main__': From 844895d315e44a1802a30e8cd02073a4f809cdef Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Tue, 11 Apr 2017 22:59:22 +0100 Subject: [PATCH 14/44] Add Other_Math support from PropList.txt. --- src/include/ucd/ucd.h | 2 + src/proplist.c | 271 +++++++++++++++++++++++++++++++++++++++++- tools/printdata.py | 1 + 3 files changed, 272 insertions(+), 2 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 1ddcf92b..e4250327 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -337,6 +337,7 @@ typedef enum ucd_property_ UCD_PROPERTY_HYPHEN = 0x00000020, /**< @brief Hyphen PropList */ UCD_PROPERTY_QUOTATION_MARK = 0x00000040, /**< @brief Quotation_Mark PropList */ UCD_PROPERTY_TERMINAL_PUNCTUATION = 0x00000080, /**< @brief Terminal_Punctuation PropList */ + UCD_PROPERTY_OTHER_MATH = 0x00000100, /**< @brief Other_Math PropList */ } ucd_property; /** @brief Return the properties of the specified codepoint. @@ -804,6 +805,7 @@ namespace ucd Hyphen = UCD_PROPERTY_HYPHEN, /**< @brief Hyphen PropList */ Quotation_Mark = UCD_PROPERTY_QUOTATION_MARK, /**< @brief Quotation_Mark PropList */ Terminal_Punctuation = UCD_PROPERTY_TERMINAL_PUNCTUATION, /**< @brief Terminal_Punctuation PropList */ + Other_Math = UCD_PROPERTY_OTHER_MATH, /**< @brief Other_Math PropList */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index c886e613..8547dc51 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -41,12 +41,171 @@ static int properties_Cf(codepoint_t c) if (c >= 0x200C && c <= 0x200D) return UCD_PROPERTY_JOIN_CONTROL; if (c >= 0x200E && c <= 0x200F) return UCD_PROPERTY_BIDI_CONTROL; if (c >= 0x202A && c <= 0x202E) return UCD_PROPERTY_BIDI_CONTROL; + if (c >= 0x2061 && c <= 0x2064) return UCD_PROPERTY_OTHER_MATH; if (c >= 0x2066 && c <= 0x2069) return UCD_PROPERTY_BIDI_CONTROL; break; } return 0; } +static int properties_Ll(codepoint_t c) +{ + switch (c & 0xFFFFFF00) + { + case 0x0300: + if (c >= 0x03D0 && c <= 0x03D2) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c == 0x03D5) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x03F0 && c <= 0x03F1) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x03F4 && c <= 0x03F5) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + break; + case 0x2100: + if (c >= 0x210A && c <= 0x2113) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x212F && c <= 0x2131) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x2133 && c <= 0x2134) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x213C && c <= 0x213F) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x2145 && c <= 0x2149) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + break; + case 0x01D400: + if (c >= 0x01D400 && c <= 0x01D454) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x01D456 && c <= 0x01D49C) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x01D4AE && c <= 0x01D4B9) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c == 0x01D4BB) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x01D4BD && c <= 0x01D4C3) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x01D4C5) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + break; + case 0x01D500: + if (c <= 0x01D505) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x01D51E && c <= 0x01D539) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x01D552) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + break; + case 0x01D600: + if (c <= 0x01D6A5) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x01D6C2 && c <= 0x01D6DA) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x01D6DC && c <= 0x01D6FA) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x01D6FC) return UCD_PROPERTY_OTHER_MATH; + break; + case 0x01D700: + if (c <= 0x01D714) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x01D716 && c <= 0x01D734) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x01D736 && c <= 0x01D74E) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x01D750 && c <= 0x01D76E) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x01D770 && c <= 0x01D788) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x01D78A && c <= 0x01D7A8) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x01D7AA && c <= 0x01D7C2) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x01D7C4 && c <= 0x01D7CB) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + break; + } + return 0; +} + +static int properties_Lo(codepoint_t c) +{ + switch (c & 0xFFFFFF00) + { + case 0x2100: + if (c >= 0x2135 && c <= 0x2138) return UCD_PROPERTY_OTHER_MATH; + break; + case 0x01EE00: + return UCD_PROPERTY_OTHER_MATH; + } + return 0; +} + +static int properties_Lu(codepoint_t c) +{ + switch (c & 0xFFFFFF00) + { + case 0x0300: + if (c >= 0x03D0 && c <= 0x03D2) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x03F4 && c <= 0x03F5) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + break; + case 0x2100: + if (c == 0x2102) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x2107) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x210A && c <= 0x2113) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c == 0x2115) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x2119 && c <= 0x211D) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x2124) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x2128) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x212C && c <= 0x212D) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x212F && c <= 0x2131) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x2133 && c <= 0x2134) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x213C && c <= 0x213F) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x2145 && c <= 0x2149) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + break; + case 0x01D400: + if (c >= 0x01D400 && c <= 0x01D454) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x01D456 && c <= 0x01D49C) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x01D49E && c <= 0x01D49F) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x01D4A2) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x01D4A5 && c <= 0x01D4A6) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x01D4A9 && c <= 0x01D4AC) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x01D4AE && c <= 0x01D4B9) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x01D4C5) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + break; + case 0x01D500: + if (c <= 0x01D505) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x01D507 && c <= 0x01D50A) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x01D50D && c <= 0x01D514) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x01D516 && c <= 0x01D51C) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x01D51E && c <= 0x01D539) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x01D53B && c <= 0x01D53E) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x01D540 && c <= 0x01D544) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x01D546) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x01D54A && c <= 0x01D550) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x01D552) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + break; + case 0x01D600: + if (c <= 0x01D6A5) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x01D6A8 && c <= 0x01D6C0) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x01D6DC && c <= 0x01D6FA) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + break; + case 0x01D700: + if (c >= 0x01D716 && c <= 0x01D734) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x01D750 && c <= 0x01D76E) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x01D78A && c <= 0x01D7A8) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x01D7C4 && c <= 0x01D7CB) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + break; + } + return 0; +} + +static int properties_Mn(codepoint_t c) +{ + switch (c & 0xFFFFFF00) + { + case 0x2000: + if (c >= 0x20D0 && c <= 0x20DC) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x20E1) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x20E5 && c <= 0x20E6) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x20EB && c <= 0x20EF) return UCD_PROPERTY_OTHER_MATH; + break; + } + return 0; +} + +static int properties_Nd(codepoint_t c) +{ + switch (c & 0xFFFFFF00) + { + case 0x01D700: + if (c >= 0x01D7CE && c <= 0x01D7FF) return UCD_PROPERTY_OTHER_MATH; + break; + } + return 0; +} + +static int properties_Pc(codepoint_t c) +{ + switch (c & 0xFFFFFF00) + { + case 0x2000: + if (c == 0x2040) return UCD_PROPERTY_OTHER_MATH; + break; + } + return 0; +} + static int properties_Pd(codepoint_t c) { switch (c & 0xFFFFFF00) @@ -68,7 +227,7 @@ static int properties_Pd(codepoint_t c) if (c == 0x2E17) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; break; case 0xFE00: - if (c == 0xFE63) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; + if (c == 0xFE63) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN | UCD_PROPERTY_OTHER_MATH; break; case 0xFF00: if (c == 0xFF0D) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; @@ -81,6 +240,23 @@ static int properties_Pe(codepoint_t c) { switch (c & 0xFFFFFF00) { + case 0x2000: + if (c == 0x207E) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x208E) return UCD_PROPERTY_OTHER_MATH; + break; + case 0x2300: + if (c == 0x2309) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x230B) return UCD_PROPERTY_OTHER_MATH; + break; + case 0x2700: + if (c == 0x27C6) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x27E6 && c <= 0x27EF) return UCD_PROPERTY_OTHER_MATH; // Pe|Ps + break; + case 0x2900: + if (c >= 0x2983 && c <= 0x2998) return UCD_PROPERTY_OTHER_MATH; // Pe|Ps + if (c >= 0x29D8 && c <= 0x29DB) return UCD_PROPERTY_OTHER_MATH; // Pe|Ps + if (c == 0x29FD) return UCD_PROPERTY_OTHER_MATH; + break; case 0x3000: if (c == 0x300D) return UCD_PROPERTY_QUOTATION_MARK; if (c == 0x300F) return UCD_PROPERTY_QUOTATION_MARK; @@ -211,6 +387,8 @@ static int properties_Po(codepoint_t c) if (c >= 0x1C7E && c <= 0x1C7F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; break; case 0x2000: + if (c == 0x2016) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x2032 && c <= 0x2034) return UCD_PROPERTY_OTHER_MATH; if (c >= 0x203C && c <= 0x203D) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c >= 0x2047 && c <= 0x2049) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c == 0x2053) return UCD_PROPERTY_DASH; @@ -250,6 +428,8 @@ static int properties_Po(codepoint_t c) case 0xFE00: if (c >= 0xFE50 && c <= 0xFE52) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c >= 0xFE54 && c <= 0xFE57) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0xFE61) return UCD_PROPERTY_OTHER_MATH; + if (c == 0xFE68) return UCD_PROPERTY_OTHER_MATH; break; case 0xFF00: if (c == 0xFF01) return UCD_PROPERTY_TERMINAL_PUNCTUATION; @@ -257,6 +437,7 @@ static int properties_Po(codepoint_t c) if (c == 0xFF07) return UCD_PROPERTY_QUOTATION_MARK; if (c == 0xFF0C) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c == 0xFF0E) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0xFF3C) return UCD_PROPERTY_OTHER_MATH; if (c == 0xFF65) return UCD_PROPERTY_HYPHEN; if (c >= 0xFF1A && c <= 0xFF1B) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c == 0xFF1F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; @@ -341,6 +522,21 @@ static int properties_Ps(codepoint_t c) case 0x2000: if (c == 0x201A) return UCD_PROPERTY_QUOTATION_MARK; if (c == 0x201E) return UCD_PROPERTY_QUOTATION_MARK; + if (c == 0x207D) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x208D) return UCD_PROPERTY_OTHER_MATH; + break; + case 0x2300: + if (c == 0x2308) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x230A) return UCD_PROPERTY_OTHER_MATH; + break; + case 0x2700: + if (c == 0x27C5) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x27E6 && c <= 0x27EF) return UCD_PROPERTY_OTHER_MATH; // Pe|Ps + break; + case 0x2900: + if (c >= 0x2983 && c <= 0x2998) return UCD_PROPERTY_OTHER_MATH; // Pe|Ps + if (c >= 0x29D8 && c <= 0x29DB) return UCD_PROPERTY_OTHER_MATH; // Pe|Ps + if (c == 0x29FC) return UCD_PROPERTY_OTHER_MATH; break; case 0x2E00: if (c == 0x2E42) return UCD_PROPERTY_QUOTATION_MARK; @@ -361,6 +557,20 @@ static int properties_Ps(codepoint_t c) return 0; } +static int properties_Sk(codepoint_t c) +{ + switch (c & 0xFFFFFF00) + { + case 0x0000: + if (c == 0x005E) return UCD_PROPERTY_OTHER_MATH; + break; + case 0xFF00: + if (c == 0xFF3E) return UCD_PROPERTY_OTHER_MATH; + break; + } + return 0; +} + static int properties_Sm(codepoint_t c) { switch (c & 0xFFFFFF00) @@ -376,6 +586,55 @@ static int properties_Sm(codepoint_t c) return 0; } +static int properties_So(codepoint_t c) +{ + switch (c & 0xFFFFFF00) + { + case 0x2100: + if (c == 0x2129) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x2195 && c <= 0x2199) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x219C && c <= 0x219F) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x21A1 && c <= 0x21A2) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x21A4 && c <= 0x21A5) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x21A7) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x21A9 && c <= 0x21AD) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x21B0 && c <= 0x21B1) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x21B6 && c <= 0x21B7) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x21BC && c <= 0x21CD) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x21D0 && c <= 0x21D1) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x21D3) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x21D5 && c <= 0x21DB) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x21DD) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x21E4 && c <= 0x21E5) return UCD_PROPERTY_OTHER_MATH; + break; + case 0x2300: + if (c >= 0x23B4 && c <= 0x23B5) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x23B7) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x23D0) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x23E2) return UCD_PROPERTY_OTHER_MATH; + break; + case 0x2500: + if (c >= 0x25A0 && c <= 0x25A1) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x25AE && c <= 0x25B6) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x25BC && c <= 0x25C0) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x25C6 && c <= 0x25C7) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x25CA && c <= 0x25CB) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x25CF && c <= 0x25D3) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x25E2) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x25E4) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x25E7 && c <= 0x25EC) return UCD_PROPERTY_OTHER_MATH; + break; + case 0x2600: + if (c >= 0x2605 && c <= 0x2606) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x2640) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x2642) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x2660 && c <= 0x2663) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x266D && c <= 0x266E) return UCD_PROPERTY_OTHER_MATH; + break; + } + return 0; +} + static int properties_Zs(codepoint_t c) { switch (c) @@ -394,16 +653,24 @@ ucd_property ucd_properties(codepoint_t c, ucd_category category) { case UCD_CATEGORY_Cc: return properties_Cc(c); case UCD_CATEGORY_Cf: return properties_Cf(c); + case UCD_CATEGORY_Ll: return properties_Ll(c); + case UCD_CATEGORY_Lo: return properties_Lo(c); + case UCD_CATEGORY_Lu: return properties_Lu(c); + case UCD_CATEGORY_Mn: return properties_Mn(c); + case UCD_CATEGORY_Nd: return properties_Nd(c); + case UCD_CATEGORY_Pc: return properties_Pc(c); case UCD_CATEGORY_Pd: return properties_Pd(c); case UCD_CATEGORY_Pe: return properties_Pe(c); case UCD_CATEGORY_Pf: return properties_Pf(c); case UCD_CATEGORY_Pi: return properties_Pi(c); case UCD_CATEGORY_Po: return properties_Po(c); case UCD_CATEGORY_Ps: return properties_Ps(c); + case UCD_CATEGORY_Sk: return properties_Sk(c); case UCD_CATEGORY_Sm: return properties_Sm(c); + case UCD_CATEGORY_So: return properties_So(c); case UCD_CATEGORY_Zl: return UCD_PROPERTY_WHITE_SPACE; case UCD_CATEGORY_Zp: return UCD_PROPERTY_WHITE_SPACE; case UCD_CATEGORY_Zs: return properties_Zs(c); - default: return 0; // Cn Co Cs Ii L& M& N& Pc Sc Sk So + default: return 0; // Cn Co Cs Ii Lm Lt Mc Me Nl No Sc }; } diff --git a/tools/printdata.py b/tools/printdata.py index 52c4edf2..7e8ded21 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -129,6 +129,7 @@ def properties(data): props += 32 * data.get('Hyphen', 0) props += 64 * data.get('Quotation_Mark', 0) props += 128 * data.get('Terminal_Punctuation', 0) + props += 256 * data.get('Other_Math', 0) return props if __name__ == '__main__': From a69b99a81b2f06506a045b869a613436398f55c6 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Tue, 11 Apr 2017 23:21:51 +0100 Subject: [PATCH 15/44] Add Hex_Digit support from PropList.txt. --- src/include/ucd/ucd.h | 6 ++++-- src/proplist.c | 18 ++++++++++++++++++ tools/printdata.py | 1 + 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index e4250327..e92d57de 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -326,7 +326,7 @@ const char *ucd_get_script_string(ucd_script s); ucd_script ucd_lookup_script(codepoint_t c); /** @brief Properties - */ + */ typedef enum ucd_property_ { UCD_PROPERTY_WHITE_SPACE = 0x00000001, /**< @brief White_Space PropList */ @@ -338,6 +338,7 @@ typedef enum ucd_property_ UCD_PROPERTY_QUOTATION_MARK = 0x00000040, /**< @brief Quotation_Mark PropList */ UCD_PROPERTY_TERMINAL_PUNCTUATION = 0x00000080, /**< @brief Terminal_Punctuation PropList */ UCD_PROPERTY_OTHER_MATH = 0x00000100, /**< @brief Other_Math PropList */ + UCD_PROPERTY_HEX_DIGIT = 0x00000200, /**< @brief Hex_Digit PropList */ } ucd_property; /** @brief Return the properties of the specified codepoint. @@ -794,7 +795,7 @@ namespace ucd } /** @brief Properties - */ + */ enum property { White_Space = UCD_PROPERTY_WHITE_SPACE, /**< @brief White_Space PropList */ @@ -806,6 +807,7 @@ namespace ucd Quotation_Mark = UCD_PROPERTY_QUOTATION_MARK, /**< @brief Quotation_Mark PropList */ Terminal_Punctuation = UCD_PROPERTY_TERMINAL_PUNCTUATION, /**< @brief Terminal_Punctuation PropList */ Other_Math = UCD_PROPERTY_OTHER_MATH, /**< @brief Other_Math PropList */ + Hex_Digit = UCD_PROPERTY_HEX_DIGIT, /**< @brief Hex_Digit PropList */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index 8547dc51..f6a33a77 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -52,12 +52,18 @@ static int properties_Ll(codepoint_t c) { switch (c & 0xFFFFFF00) { + case 0x0000: + if (c >= 0x0061 && c <= 0x0066) return UCD_PROPERTY_HEX_DIGIT; + break; case 0x0300: if (c >= 0x03D0 && c <= 0x03D2) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu if (c == 0x03D5) return UCD_PROPERTY_OTHER_MATH; if (c >= 0x03F0 && c <= 0x03F1) return UCD_PROPERTY_OTHER_MATH; if (c >= 0x03F4 && c <= 0x03F5) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu break; + case 0xFF00: + if (c >= 0xFF41 && c <= 0xFF46) return UCD_PROPERTY_HEX_DIGIT; + break; case 0x2100: if (c >= 0x210A && c <= 0x2113) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu if (c >= 0x212F && c <= 0x2131) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu @@ -115,10 +121,16 @@ static int properties_Lu(codepoint_t c) { switch (c & 0xFFFFFF00) { + case 0x0000: + if (c >= 0x0041 && c <= 0x0046) return UCD_PROPERTY_HEX_DIGIT; + break; case 0x0300: if (c >= 0x03D0 && c <= 0x03D2) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu if (c >= 0x03F4 && c <= 0x03F5) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu break; + case 0xFF00: + if (c >= 0xFF21 && c <= 0xFF26) return UCD_PROPERTY_HEX_DIGIT; + break; case 0x2100: if (c == 0x2102) return UCD_PROPERTY_OTHER_MATH; if (c == 0x2107) return UCD_PROPERTY_OTHER_MATH; @@ -188,6 +200,12 @@ static int properties_Nd(codepoint_t c) { switch (c & 0xFFFFFF00) { + case 0x0000: + if (c >= 0x0030 && c <= 0x0039) return UCD_PROPERTY_HEX_DIGIT; + break; + case 0xFF00: + if (c >= 0xFF10 && c <= 0xFF19) return UCD_PROPERTY_HEX_DIGIT; + break; case 0x01D700: if (c >= 0x01D7CE && c <= 0x01D7FF) return UCD_PROPERTY_OTHER_MATH; break; diff --git a/tools/printdata.py b/tools/printdata.py index 7e8ded21..81bc5ec4 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -130,6 +130,7 @@ def properties(data): props += 64 * data.get('Quotation_Mark', 0) props += 128 * data.get('Terminal_Punctuation', 0) props += 256 * data.get('Other_Math', 0) + props += 512 * data.get('Hex_Digit', 0) return props if __name__ == '__main__': From 835dfc83a1b1e47f8cc014200984491ecce81145 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Tue, 11 Apr 2017 23:25:02 +0100 Subject: [PATCH 16/44] Add ASCII_Hex_Digit support from PropList.txt. --- src/include/ucd/ucd.h | 2 ++ src/proplist.c | 6 +++--- tools/printdata.py | 23 ++++++++++++----------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index e92d57de..8b079669 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -339,6 +339,7 @@ typedef enum ucd_property_ UCD_PROPERTY_TERMINAL_PUNCTUATION = 0x00000080, /**< @brief Terminal_Punctuation PropList */ UCD_PROPERTY_OTHER_MATH = 0x00000100, /**< @brief Other_Math PropList */ UCD_PROPERTY_HEX_DIGIT = 0x00000200, /**< @brief Hex_Digit PropList */ + UCD_PROPERTY_ASCII_HEX_DIGIT = 0x00000400, /**< @brief ASCII_Hex_Digit PropList */ } ucd_property; /** @brief Return the properties of the specified codepoint. @@ -808,6 +809,7 @@ namespace ucd Terminal_Punctuation = UCD_PROPERTY_TERMINAL_PUNCTUATION, /**< @brief Terminal_Punctuation PropList */ Other_Math = UCD_PROPERTY_OTHER_MATH, /**< @brief Other_Math PropList */ Hex_Digit = UCD_PROPERTY_HEX_DIGIT, /**< @brief Hex_Digit PropList */ + ASCII_Hex_Digit = UCD_PROPERTY_ASCII_HEX_DIGIT, /**< @brief ASCII_Hex_Digit PropList */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index f6a33a77..bd449c4a 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -53,7 +53,7 @@ static int properties_Ll(codepoint_t c) switch (c & 0xFFFFFF00) { case 0x0000: - if (c >= 0x0061 && c <= 0x0066) return UCD_PROPERTY_HEX_DIGIT; + if (c >= 0x0061 && c <= 0x0066) return UCD_PROPERTY_HEX_DIGIT | UCD_PROPERTY_ASCII_HEX_DIGIT; break; case 0x0300: if (c >= 0x03D0 && c <= 0x03D2) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu @@ -122,7 +122,7 @@ static int properties_Lu(codepoint_t c) switch (c & 0xFFFFFF00) { case 0x0000: - if (c >= 0x0041 && c <= 0x0046) return UCD_PROPERTY_HEX_DIGIT; + if (c >= 0x0041 && c <= 0x0046) return UCD_PROPERTY_HEX_DIGIT | UCD_PROPERTY_ASCII_HEX_DIGIT; break; case 0x0300: if (c >= 0x03D0 && c <= 0x03D2) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu @@ -201,7 +201,7 @@ static int properties_Nd(codepoint_t c) switch (c & 0xFFFFFF00) { case 0x0000: - if (c >= 0x0030 && c <= 0x0039) return UCD_PROPERTY_HEX_DIGIT; + if (c >= 0x0030 && c <= 0x0039) return UCD_PROPERTY_HEX_DIGIT | UCD_PROPERTY_ASCII_HEX_DIGIT; break; case 0xFF00: if (c >= 0xFF10 && c <= 0xFF19) return UCD_PROPERTY_HEX_DIGIT; diff --git a/tools/printdata.py b/tools/printdata.py index 81bc5ec4..d22a3246 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -120,17 +120,18 @@ def decomposition_type(data, dtype): return None def properties(data): - props = 0 - props += 1 * data.get('White_Space', 0) - props += 2 * (decomposition_type(data, '') != None) - props += 4 * data.get('Bidi_Control', 0) - props += 8 * data.get('Join_Control', 0) - props += 16 * data.get('Dash', 0) - props += 32 * data.get('Hyphen', 0) - props += 64 * data.get('Quotation_Mark', 0) - props += 128 * data.get('Terminal_Punctuation', 0) - props += 256 * data.get('Other_Math', 0) - props += 512 * data.get('Hex_Digit', 0) + props = 0 + props += 1 * data.get('White_Space', 0) + props += 2 * (decomposition_type(data, '') != None) + props += 4 * data.get('Bidi_Control', 0) + props += 8 * data.get('Join_Control', 0) + props += 16 * data.get('Dash', 0) + props += 32 * data.get('Hyphen', 0) + props += 64 * data.get('Quotation_Mark', 0) + props += 128 * data.get('Terminal_Punctuation', 0) + props += 256 * data.get('Other_Math', 0) + props += 512 * data.get('Hex_Digit', 0) + props += 1024 * data.get('ASCII_Hex_Digit', 0) return props if __name__ == '__main__': From aa32a420af40d33c77e698d7214c124097c6252d Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Wed, 12 Apr 2017 21:30:53 +0100 Subject: [PATCH 17/44] Add Other_Alphabetic support from PropList.txt. --- src/include/ucd/ucd.h | 2 + src/proplist.c | 490 +++++++++++++++++++++++++++++++++++++++++- tools/printdata.py | 1 + 3 files changed, 492 insertions(+), 1 deletion(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 8b079669..81118612 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -340,6 +340,7 @@ typedef enum ucd_property_ UCD_PROPERTY_OTHER_MATH = 0x00000100, /**< @brief Other_Math PropList */ UCD_PROPERTY_HEX_DIGIT = 0x00000200, /**< @brief Hex_Digit PropList */ UCD_PROPERTY_ASCII_HEX_DIGIT = 0x00000400, /**< @brief ASCII_Hex_Digit PropList */ + UCD_PROPERTY_OTHER_ALPHABETIC = 0x00000800, /**< @brief Other_Alphabetic PropList */ } ucd_property; /** @brief Return the properties of the specified codepoint. @@ -810,6 +811,7 @@ namespace ucd Other_Math = UCD_PROPERTY_OTHER_MATH, /**< @brief Other_Math PropList */ Hex_Digit = UCD_PROPERTY_HEX_DIGIT, /**< @brief Hex_Digit PropList */ ASCII_Hex_Digit = UCD_PROPERTY_ASCII_HEX_DIGIT, /**< @brief ASCII_Hex_Digit PropList */ + Other_Alphabetic = UCD_PROPERTY_OTHER_ALPHABETIC, /**< @brief Other_Alphabetic PropList */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index bd449c4a..dccf4b68 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -182,10 +182,489 @@ static int properties_Lu(codepoint_t c) return 0; } +static int properties_Mc(codepoint_t c) +{ + switch (c & 0xFFFFFF00) + { + case 0x0900: + if (c == 0x0903) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x093B) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x093E && c <= 0x0940) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0949 && c <= 0x094C) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x094E && c <= 0x094F) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0982 && c <= 0x0983) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x09BE && c <= 0x09C0) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x09C7 && c <= 0x09C8) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x09CB && c <= 0x09CC) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x09D7) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x0A00: + if (c == 0x0A03) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0A3E && c <= 0x0A40) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0A83) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0ABE && c <= 0x0AC0) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0AC9) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0ACB && c <= 0x0ACC) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x0B00: + if (c >= 0x0B02 && c <= 0x0B03) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0B3E) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0B40) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0B47 && c <= 0x0B48) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0B4B && c <= 0x0B4C) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0B57) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0BBE && c <= 0x0BBF) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0BC1 && c <= 0x0BC2) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0BC6 && c <= 0x0BC8) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0BCA && c <= 0x0BCC) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0BD7) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x0C00: + if (c >= 0x0C01 && c <= 0x0C03) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0C41 && c <= 0x0C44) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0C82 && c <= 0x0C83) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0CBE) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0CC0 && c <= 0x0CC4) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0CC7 && c <= 0x0CC8) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0CCA && c <= 0x0CCB) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0CD5 && c <= 0x0CD6) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x0D00: + if (c >= 0x0D02 && c <= 0x0D03) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0D3E && c <= 0x0D40) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0D46 && c <= 0x0D48) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0D4A && c <= 0x0D4C) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0D57) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0D82 && c <= 0x0D83) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0DCF && c <= 0x0DD1) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0DD8 && c <= 0x0DDF) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0DF2 && c <= 0x0DF3) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x0F00: + if (c == 0x0F7F) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x1000: + if (c >= 0x102B && c <= 0x102C) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1031) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1038) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x103B && c <= 0x103C) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1056 && c <= 0x1057) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1062) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1067 && c <= 0x1068) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1083 && c <= 0x1084) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x109C) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x1700: + if (c == 0x17B6) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x17BE && c <= 0x17C5) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x17C7 && c <= 0x17C8) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x1900: + if (c >= 0x1923 && c <= 0x1926) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1929 && c <= 0x192B) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1930 && c <= 0x1931) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1933 && c <= 0x1938) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x1A00: + if (c >= 0x1A19 && c <= 0x1A1A) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1A55) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1A57) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1A61) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1A63 && c <= 0x1A64) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1A6D && c <= 0x1A72) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x1B00: + if (c == 0x1B04) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1B35) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1B3B) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1B3D && c <= 0x1B41) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1B43) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1B82) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1BA1) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1BA6 && c <= 0x1BA7) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1BE7) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1BEA && c <= 0x1BEC) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1BEE) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x1C00: + if (c >= 0x1C24 && c <= 0x1C2B) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1C34 && c <= 0x1C35) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1CF2 && c <= 0x1CF3) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0xA800: + if (c >= 0xA823 && c <= 0xA824) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0xA827) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0xA880 && c <= 0xA881) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0xA8B4 && c <= 0xA8C3) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0xA900: + if (c == 0xA952) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0xA983) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0xA9B4 && c <= 0xA9B5) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0xA9BA && c <= 0xA9BB) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0xA9BD && c <= 0xA9BF) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0xAA00: + if (c >= 0xAA2F && c <= 0xAA30) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0xAA33 && c <= 0xAA34) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0xAA4D) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0xAAEB) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0xAAEE && c <= 0xAAEF) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0xAAF5) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0xAB00: + if (c >= 0xABE3 && c <= 0xABE4) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0xABE6 && c <= 0xABE7) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0xABE9 && c <= 0xABEA) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x011000: + if (c == 0x011000) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x011002) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x011082) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0110B0 && c <= 0x0110B2) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0110B7 && c <= 0x0110B8) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x011100: + if (c == 0x01112C) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x011182) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0111B3 && c <= 0x0111B5) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0111BF) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x011200: + if (c >= 0x01122C && c <= 0x01122E) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x011232 && c <= 0x011233) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0112E0 && c <= 0x0112E2) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x011300: + if (c >= 0x011302 && c <= 0x011303) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x01133E && c <= 0x01133F) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x011341 && c <= 0x011344) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x011347 && c <= 0x011348) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x01134B && c <= 0x01134C) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x011357) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x011362 && c <= 0x011363) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x011400: + if (c >= 0x011435 && c <= 0x011437) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x011440 && c <= 0x011441) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x011445) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0114B0 && c <= 0x0114B2) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0114B9) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0114BB && c <= 0x0114BE) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0114C1) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x011500: + if (c >= 0x0115AF && c <= 0x0115B1) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0115B8 && c <= 0x0115BB) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0115BE) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x011600: + if (c >= 0x011630 && c <= 0x011632) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x01163B && c <= 0x01163C) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x01163E) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0116AC) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0116AE && c <= 0x0116AF) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x011700: + if (c >= 0x011720 && c <= 0x011721) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x011726) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x011C00: + if (c == 0x011C2F) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x011C3E) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x011CA9) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x011CB1) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x011CB4) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x016F00: + if (c >= 0x016F51 && c <= 0x016F7E) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + } + return 0; +} + static int properties_Mn(codepoint_t c) { switch (c & 0xFFFFFF00) { + case 0x0300: + if (c == 0x0345) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x0500: + if (c >= 0x05B0 && c <= 0x05BD) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x05BF) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x05C1 && c <= 0x05C2) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x05C4 && c <= 0x05C5) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x05C7) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x0600: + if (c >= 0x0610 && c <= 0x061A) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x064B && c <= 0x0657) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0659 && c <= 0x065F) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0670) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x06D6 && c <= 0x06DC) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x06E1 && c <= 0x06E4) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x06E7 && c <= 0x06E8) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x06ED) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x0700: + if (c == 0x0711) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0730 && c <= 0x073F) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x07A6 && c <= 0x07B0) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x0800: + if (c >= 0x0816 && c <= 0x0817) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x081B && c <= 0x0823) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0825 && c <= 0x0827) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0829 && c <= 0x082C) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x08D4 && c <= 0x08DF) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x08E3 && c <= 0x08E9) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x08F0) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x0900: + if (c <= 0x0902) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x093A) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0941 && c <= 0x0948) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0955 && c <= 0x0957) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0962 && c <= 0x0963) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0981) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x09C1 && c <= 0x09C4) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x09E2 && c <= 0x09E3) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x0A00: + if (c >= 0x0A01 && c <= 0x0A02) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0A41 && c <= 0x0A42) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0A47 && c <= 0x0A48) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0A4B && c <= 0x0A4C) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0A51) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0A70 && c <= 0x0A71) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0A75) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0A81 && c <= 0x0A82) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0AC1 && c <= 0x0AC5) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0AC7 && c <= 0x0AC8) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0AE2 && c <= 0x0AE3) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x0B00: + if (c == 0x0B01) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0B3F) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0B41 && c <= 0x0B44) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0B56) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0B62 && c <= 0x0B63) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0B82) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0BC0) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x0C00: + if (c == 0x0C00) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0C3E && c <= 0x0C40) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0C46 && c <= 0x0C48) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0C4A && c <= 0x0C4C) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0C55 && c <= 0x0C56) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0C62 && c <= 0x0C63) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0C81) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0CBF) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0CC6) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0CCC) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0CE2 && c <= 0x0CE3) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x0D00: + if (c == 0x0D01) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0D41 && c <= 0x0D44) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0D62 && c <= 0x0D63) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0DD2 && c <= 0x0DD4) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0DD6) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x0E00: + if (c == 0x0E31) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0E34 && c <= 0x0E3A) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0E4D) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0EB1) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0EB4 && c <= 0x0EB9) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0EBB && c <= 0x0EBC) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0ECD) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x0F00: + if (c >= 0x0F71 && c <= 0x0F7E) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0F80 && c <= 0x0F81) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0F8D && c <= 0x0F97) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0F99 && c <= 0x0FBC) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x1000: + if (c >= 0x102D && c <= 0x1030) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1032 && c <= 0x1036) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x103D && c <= 0x103E) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1058 && c <= 0x1059) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x105E && c <= 0x1060) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1071 && c <= 0x1074) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1082) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1085 && c <= 0x1086) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x109D) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x1300: + if (c == 0x135F) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x1700: + if (c >= 0x1712 && c <= 0x1713) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1732 && c <= 0x1733) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1752 && c <= 0x1753) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1772 && c <= 0x1773) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x17B7 && c <= 0x17BD) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x17C6) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x1800: + if (c >= 0x1885 && c <= 0x1886) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x18A9) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x1900: + if (c >= 0x1920 && c <= 0x1922) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1927 && c <= 0x1928) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1932) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x1A00: + if (c >= 0x1A17 && c <= 0x1A18) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1A1B) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1A56) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1A58 && c <= 0x1A5E) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1A62) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1A65 && c <= 0x1A6C) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1A73 && c <= 0x1A74) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x1B00: + if (c >= 0x1B00 && c <= 0x1B03) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1B36 && c <= 0x1B3A) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1B3C) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1B42) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1B80 && c <= 0x1B81) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1BA2 && c <= 0x1BA5) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1BA8 && c <= 0x1BA9) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1BAC && c <= 0x1BAD) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1BE8 && c <= 0x1BE9) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1BED) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1BEF && c <= 0x1BF1) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x1C00: + if (c >= 0x1C2C && c <= 0x1C33) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x1D00: + if (c >= 0x1DE7 && c <= 0x1DF4) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x2D00: + if (c >= 0x2DE0 && c <= 0x2DFF) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0xA600: + if (c >= 0xA674 && c <= 0xA67B) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0xA69E && c <= 0xA69F) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0xA800: + if (c >= 0xA825 && c <= 0xA826) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0xA8C5) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0xA900: + if (c >= 0xA926 && c <= 0xA92A) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0xA947 && c <= 0xA951) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0xA980 && c <= 0xA982) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0xA9B6 && c <= 0xA9B9) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0xA9BC) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0xAA00: + if (c >= 0xAA29 && c <= 0xAA2E) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0xAA31 && c <= 0xAA32) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0xAA35 && c <= 0xAA36) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0xAA43) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0xAA4C) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0xAAB0) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0xAAB2 && c <= 0xAAB4) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0xAAB7 && c <= 0xAAB8) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0xAABE) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0xAAEC && c <= 0xAAED) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0xAB00: + if (c == 0xABE5) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0xABE8) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0xFB00: + if (c == 0xFB1E) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x010300: + if (c >= 0x010376 && c <= 0x01037A) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x010A00: + if (c >= 0x010A01 && c <= 0x010A03) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x010A05 && c <= 0x010A06) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x010A0C && c <= 0x010A0F) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x011000: + if (c == 0x011001) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x011038 && c <= 0x011045) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0110B3 && c <= 0x0110B6) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x011100: + if (c >= 0x011100 && c <= 0x011102) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x011127 && c <= 0x01112B) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x01112D && c <= 0x011132) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x011180 && c <= 0x011181) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0111B6 && c <= 0x0111BE) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x011200: + if (c >= 0x01122F && c <= 0x011231) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x011234) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x011237) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x01123E) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0112DF) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0112E3 && c <= 0x0112E8) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x011300: + if (c >= 0x011300 && c <= 0x011301) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x011340) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x011400: + if (c >= 0x011438 && c <= 0x01143F) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x011443 && c <= 0x011444) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0114B3 && c <= 0x0114B8) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0114BA) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0114BF && c <= 0x0114C0) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x011500: + if (c >= 0x0115B2 && c <= 0x0115B5) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0115BC && c <= 0x0115BD) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0115DC && c <= 0x0115DD) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x011600: + if (c >= 0x011633 && c <= 0x01163A) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x01163D) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x011640) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0116AB) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0116AD) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0116B0 && c <= 0x0116B5) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x011700: + if (c >= 0x01171D && c <= 0x01171F) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x011722 && c <= 0x011725) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x011727 && c <= 0x01172A) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x011C00: + if (c >= 0x011C30 && c <= 0x011C36) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x011C38 && c <= 0x011C3D) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x011C92 && c <= 0x011CA7) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x011CAA && c <= 0x011CB0) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x011CB2 && c <= 0x011CB3) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x011CB5 && c <= 0x011CB6) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x016B00: + if (c >= 0x016B30 && c <= 0x016B36) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x01BC00: + if (c == 0x01BC9E) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x01E000: + if (c >= 0x01E000 && c <= 0x01E006) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x01E008 && c <= 0x01E018) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x01E01B && c <= 0x01E021) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x01E023 && c <= 0x01E024) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x01E026 && c <= 0x01E02A) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0x01E900: + if (c == 0x01E947) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; case 0x2000: if (c >= 0x20D0 && c <= 0x20DC) return UCD_PROPERTY_OTHER_MATH; if (c == 0x20E1) return UCD_PROPERTY_OTHER_MATH; @@ -631,6 +1110,9 @@ static int properties_So(codepoint_t c) if (c == 0x23D0) return UCD_PROPERTY_OTHER_MATH; if (c == 0x23E2) return UCD_PROPERTY_OTHER_MATH; break; + case 0x2400: + if (c >= 0x24B6 && c <= 0x24E9) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; case 0x2500: if (c >= 0x25A0 && c <= 0x25A1) return UCD_PROPERTY_OTHER_MATH; if (c >= 0x25AE && c <= 0x25B6) return UCD_PROPERTY_OTHER_MATH; @@ -649,6 +1131,11 @@ static int properties_So(codepoint_t c) if (c >= 0x2660 && c <= 0x2663) return UCD_PROPERTY_OTHER_MATH; if (c >= 0x266D && c <= 0x266E) return UCD_PROPERTY_OTHER_MATH; break; + case 0x01F100: + if (c >= 0x01F130 && c <= 0x01F149) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x01F150 && c <= 0x01F169) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x01F170 && c <= 0x01F189) return UCD_PROPERTY_OTHER_ALPHABETIC; + break; } return 0; } @@ -674,6 +1161,7 @@ ucd_property ucd_properties(codepoint_t c, ucd_category category) case UCD_CATEGORY_Ll: return properties_Ll(c); case UCD_CATEGORY_Lo: return properties_Lo(c); case UCD_CATEGORY_Lu: return properties_Lu(c); + case UCD_CATEGORY_Mc: return properties_Mc(c); case UCD_CATEGORY_Mn: return properties_Mn(c); case UCD_CATEGORY_Nd: return properties_Nd(c); case UCD_CATEGORY_Pc: return properties_Pc(c); @@ -689,6 +1177,6 @@ ucd_property ucd_properties(codepoint_t c, ucd_category category) case UCD_CATEGORY_Zl: return UCD_PROPERTY_WHITE_SPACE; case UCD_CATEGORY_Zp: return UCD_PROPERTY_WHITE_SPACE; case UCD_CATEGORY_Zs: return properties_Zs(c); - default: return 0; // Cn Co Cs Ii Lm Lt Mc Me Nl No Sc + default: return 0; // Cn Co Cs Ii Lm Lt Me Nl No Sc }; } diff --git a/tools/printdata.py b/tools/printdata.py index d22a3246..a7ce9436 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -132,6 +132,7 @@ def properties(data): props += 256 * data.get('Other_Math', 0) props += 512 * data.get('Hex_Digit', 0) props += 1024 * data.get('ASCII_Hex_Digit', 0) + props += 2048 * data.get('Other_Alphabetic', 0) return props if __name__ == '__main__': From 3ff6d8ae0f894d765deeaab876b53b149c685fca Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Wed, 12 Apr 2017 21:46:14 +0100 Subject: [PATCH 18/44] Add Ideographic support from PropList.txt. --- src/include/ucd/ucd.h | 2 ++ src/proplist.c | 46 +++++++++++++++++++++++++++++++++++++++++-- tools/printdata.py | 1 + 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 81118612..6c72861f 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -341,6 +341,7 @@ typedef enum ucd_property_ UCD_PROPERTY_HEX_DIGIT = 0x00000200, /**< @brief Hex_Digit PropList */ UCD_PROPERTY_ASCII_HEX_DIGIT = 0x00000400, /**< @brief ASCII_Hex_Digit PropList */ UCD_PROPERTY_OTHER_ALPHABETIC = 0x00000800, /**< @brief Other_Alphabetic PropList */ + UCD_PROPERTY_IDEOGRAPHIC = 0x00001000, /**< @brief Ideographic PropList */ } ucd_property; /** @brief Return the properties of the specified codepoint. @@ -812,6 +813,7 @@ namespace ucd Hex_Digit = UCD_PROPERTY_HEX_DIGIT, /**< @brief Hex_Digit PropList */ ASCII_Hex_Digit = UCD_PROPERTY_ASCII_HEX_DIGIT, /**< @brief ASCII_Hex_Digit PropList */ Other_Alphabetic = UCD_PROPERTY_OTHER_ALPHABETIC, /**< @brief Other_Alphabetic PropList */ + Ideographic = UCD_PROPERTY_IDEOGRAPHIC, /**< @brief Ideographic PropList */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index dccf4b68..1e3b72f8 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -111,12 +111,40 @@ static int properties_Lo(codepoint_t c) case 0x2100: if (c >= 0x2135 && c <= 0x2138) return UCD_PROPERTY_OTHER_MATH; break; + case 0x3000: + if (c == 0x3006) return UCD_PROPERTY_IDEOGRAPHIC; + break; case 0x01EE00: return UCD_PROPERTY_OTHER_MATH; } return 0; } +static int properties_Lo_ideographic(codepoint_t c) +{ + switch (c & 0xFFFF0000) + { + case 0x000000: + if (c >= 0x3400 && c <= 0x4DB5) return UCD_PROPERTY_IDEOGRAPHIC; + if (c >= 0x4E00 && c <= 0x9FD5) return UCD_PROPERTY_IDEOGRAPHIC; + if (c >= 0xF900 && c <= 0xFA6D) return UCD_PROPERTY_IDEOGRAPHIC; + if (c >= 0xFA70 && c <= 0xFAD9) return UCD_PROPERTY_IDEOGRAPHIC; + break; + case 0x010000: + if (c >= 0x017000 && c <= 0x0187EC) return UCD_PROPERTY_IDEOGRAPHIC; + if (c >= 0x018800 && c <= 0x018AF2) return UCD_PROPERTY_IDEOGRAPHIC; + break; + case 0x020000: + if (c >= 0x020000 && c <= 0x02A6D6) return UCD_PROPERTY_IDEOGRAPHIC; + if (c >= 0x02A700 && c <= 0x02B734) return UCD_PROPERTY_IDEOGRAPHIC; + if (c >= 0x02B740 && c <= 0x02B81D) return UCD_PROPERTY_IDEOGRAPHIC; + if (c >= 0x02B820 && c <= 0x02CEA1) return UCD_PROPERTY_IDEOGRAPHIC; + if (c >= 0x02F800 && c <= 0x02FA1D) return UCD_PROPERTY_IDEOGRAPHIC; + break; + } + return 0; +} + static int properties_Lu(codepoint_t c) { switch (c & 0xFFFFFF00) @@ -692,6 +720,19 @@ static int properties_Nd(codepoint_t c) return 0; } +static int properties_Nl(codepoint_t c) +{ + switch (c & 0xFFFFFF00) + { + case 0x3000: + if (c == 0x3007) return UCD_PROPERTY_IDEOGRAPHIC; + if (c >= 0x3021 && c <= 0x3029) return UCD_PROPERTY_IDEOGRAPHIC; + if (c >= 0x3038 && c <= 0x303A) return UCD_PROPERTY_IDEOGRAPHIC; + break; + } + return 0; +} + static int properties_Pc(codepoint_t c) { switch (c & 0xFFFFFF00) @@ -1159,11 +1200,12 @@ ucd_property ucd_properties(codepoint_t c, ucd_category category) case UCD_CATEGORY_Cc: return properties_Cc(c); case UCD_CATEGORY_Cf: return properties_Cf(c); case UCD_CATEGORY_Ll: return properties_Ll(c); - case UCD_CATEGORY_Lo: return properties_Lo(c); + case UCD_CATEGORY_Lo: return properties_Lo(c) | properties_Lo_ideographic(c); case UCD_CATEGORY_Lu: return properties_Lu(c); case UCD_CATEGORY_Mc: return properties_Mc(c); case UCD_CATEGORY_Mn: return properties_Mn(c); case UCD_CATEGORY_Nd: return properties_Nd(c); + case UCD_CATEGORY_Nl: return properties_Nl(c); case UCD_CATEGORY_Pc: return properties_Pc(c); case UCD_CATEGORY_Pd: return properties_Pd(c); case UCD_CATEGORY_Pe: return properties_Pe(c); @@ -1177,6 +1219,6 @@ ucd_property ucd_properties(codepoint_t c, ucd_category category) case UCD_CATEGORY_Zl: return UCD_PROPERTY_WHITE_SPACE; case UCD_CATEGORY_Zp: return UCD_PROPERTY_WHITE_SPACE; case UCD_CATEGORY_Zs: return properties_Zs(c); - default: return 0; // Cn Co Cs Ii Lm Lt Me Nl No Sc + default: return 0; // Cn Co Cs Ii Lm Lt Me No Sc }; } diff --git a/tools/printdata.py b/tools/printdata.py index a7ce9436..2aee1da2 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -133,6 +133,7 @@ def properties(data): props += 512 * data.get('Hex_Digit', 0) props += 1024 * data.get('ASCII_Hex_Digit', 0) props += 2048 * data.get('Other_Alphabetic', 0) + props += 4096 * data.get('Ideographic', 0) return props if __name__ == '__main__': From d0678386843190e4b7da15cde17b93ebdfbf25e5 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Fri, 14 Apr 2017 11:38:47 +0100 Subject: [PATCH 19/44] Add Diacritic support from PropList.txt. --- src/include/ucd/ucd.h | 2 + src/proplist.c | 299 +++++++++++++++++++++++++++++++++++++++--- tools/printdata.py | 1 + 3 files changed, 281 insertions(+), 21 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 6c72861f..9f449601 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -342,6 +342,7 @@ typedef enum ucd_property_ UCD_PROPERTY_ASCII_HEX_DIGIT = 0x00000400, /**< @brief ASCII_Hex_Digit PropList */ UCD_PROPERTY_OTHER_ALPHABETIC = 0x00000800, /**< @brief Other_Alphabetic PropList */ UCD_PROPERTY_IDEOGRAPHIC = 0x00001000, /**< @brief Ideographic PropList */ + UCD_PROPERTY_DIACRITIC = 0x00002000, /**< @brief Diacritic PropList */ } ucd_property; /** @brief Return the properties of the specified codepoint. @@ -814,6 +815,7 @@ namespace ucd ASCII_Hex_Digit = UCD_PROPERTY_ASCII_HEX_DIGIT, /**< @brief ASCII_Hex_Digit PropList */ Other_Alphabetic = UCD_PROPERTY_OTHER_ALPHABETIC, /**< @brief Other_Alphabetic PropList */ Ideographic = UCD_PROPERTY_IDEOGRAPHIC, /**< @brief Ideographic PropList */ + Diacritic = UCD_PROPERTY_DIACRITIC, /**< @brief Diacritic PropList */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index 1e3b72f8..fbbb0915 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -104,6 +104,68 @@ static int properties_Ll(codepoint_t c) return 0; } +static int properties_Lm(codepoint_t c) +{ + switch (c & 0xFFFFFF00) + { + case 0x0200: + if (c >= 0x02B0 && c <= 0x02C1) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x02C6 && c <= 0x02D1) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x02E0 && c <= 0x02E4) return UCD_PROPERTY_DIACRITIC; + if (c == 0x02EC) return UCD_PROPERTY_DIACRITIC; + if (c == 0x02EE) return UCD_PROPERTY_DIACRITIC; + break; + case 0x0300: + if (c == 0x0374) return UCD_PROPERTY_DIACRITIC; + if (c == 0x037A) return UCD_PROPERTY_DIACRITIC; + break; + case 0x0500: + if (c == 0x0559) return UCD_PROPERTY_DIACRITIC; + break; + case 0x0600: + if (c >= 0x06E5 && c <= 0x06E6) return UCD_PROPERTY_DIACRITIC; + break; + case 0x0700: + if (c >= 0x07F4 && c <= 0x07F5) return UCD_PROPERTY_DIACRITIC; + break; + case 0x0900: + if (c == 0x0971) return UCD_PROPERTY_DIACRITIC; + break; + case 0x1C00: + if (c >= 0x1C78 && c <= 0x1C7D) return UCD_PROPERTY_DIACRITIC; + break; + case 0x1D00: + if (c >= 0x1D2C && c <= 0x1D6A) return UCD_PROPERTY_DIACRITIC; + break; + case 0x2E00: + if (c == 0x2E2F) return UCD_PROPERTY_DIACRITIC; + break; + case 0x3000: + if (c == 0x30FC) return UCD_PROPERTY_DIACRITIC; + break; + case 0xA600: + if (c == 0xA67F) return UCD_PROPERTY_DIACRITIC; + if (c >= 0xA69C && c <= 0xA69D) return UCD_PROPERTY_DIACRITIC; + break; + case 0xA700: + if (c >= 0xA717 && c <= 0xA71F) return UCD_PROPERTY_DIACRITIC; + if (c == 0xA788) return UCD_PROPERTY_DIACRITIC; + if (c >= 0xA7F8 && c <= 0xA7F9) return UCD_PROPERTY_DIACRITIC; + break; + case 0xAB00: + if (c >= 0xAB5C && c <= 0xAB5F) return UCD_PROPERTY_DIACRITIC; + break; + case 0xFF00: + if (c == 0xFF70) return UCD_PROPERTY_DIACRITIC; + if (c >= 0xFF9E && c <= 0xFF9F) return UCD_PROPERTY_DIACRITIC; + break; + case 0x016F00: + if (c >= 0x016F93 && c <= 0x016F9F) return UCD_PROPERTY_DIACRITIC; + break; + } + return 0; +} + static int properties_Lo(codepoint_t c) { switch (c & 0xFFFFFF00) @@ -114,6 +176,10 @@ static int properties_Lo(codepoint_t c) case 0x3000: if (c == 0x3006) return UCD_PROPERTY_IDEOGRAPHIC; break; + case 0xAA00: + if (c == 0xAAC0) return UCD_PROPERTY_DIACRITIC; + if (c == 0xAAC2) return UCD_PROPERTY_DIACRITIC; + break; case 0x01EE00: return UCD_PROPERTY_OTHER_MATH; } @@ -269,6 +335,7 @@ static int properties_Mc(codepoint_t c) if (c >= 0x0DF2 && c <= 0x0DF3) return UCD_PROPERTY_OTHER_ALPHABETIC; break; case 0x0F00: + if (c >= 0x0F3E && c <= 0x0F3F) return UCD_PROPERTY_DIACRITIC; if (c == 0x0F7F) return UCD_PROPERTY_OTHER_ALPHABETIC; break; case 0x1000: @@ -280,6 +347,9 @@ static int properties_Mc(codepoint_t c) if (c == 0x1062) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x1067 && c <= 0x1068) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x1083 && c <= 0x1084) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1087 && c <= 0x108C) return UCD_PROPERTY_DIACRITIC; + if (c == 0x108F) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x109A && c <= 0x109B) return UCD_PROPERTY_DIACRITIC; if (c == 0x109C) return UCD_PROPERTY_OTHER_ALPHABETIC; break; case 0x1700: @@ -307,9 +377,11 @@ static int properties_Mc(codepoint_t c) if (c == 0x1B3B) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x1B3D && c <= 0x1B41) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x1B43) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1B44) return UCD_PROPERTY_DIACRITIC; if (c == 0x1B82) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x1BA1) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x1BA6 && c <= 0x1BA7) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1BAA) return UCD_PROPERTY_DIACRITIC; if (c == 0x1BE7) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x1BEA && c <= 0x1BEC) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x1BEE) return UCD_PROPERTY_OTHER_ALPHABETIC; @@ -317,8 +389,12 @@ static int properties_Mc(codepoint_t c) case 0x1C00: if (c >= 0x1C24 && c <= 0x1C2B) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x1C34 && c <= 0x1C35) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1CE1) return UCD_PROPERTY_DIACRITIC; if (c >= 0x1CF2 && c <= 0x1CF3) return UCD_PROPERTY_OTHER_ALPHABETIC; break; + case 0x3000: + if (c >= 0x302E && c <= 0x302F) return UCD_PROPERTY_DIACRITIC; + break; case 0xA800: if (c >= 0xA823 && c <= 0xA824) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0xA827) return UCD_PROPERTY_OTHER_ALPHABETIC; @@ -327,15 +403,19 @@ static int properties_Mc(codepoint_t c) break; case 0xA900: if (c == 0xA952) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0xA953) return UCD_PROPERTY_DIACRITIC; if (c == 0xA983) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0xA9B4 && c <= 0xA9B5) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0xA9BA && c <= 0xA9BB) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0xA9BD && c <= 0xA9BF) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0xA9C0) return UCD_PROPERTY_DIACRITIC; break; case 0xAA00: if (c >= 0xAA2F && c <= 0xAA30) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0xAA33 && c <= 0xAA34) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0xAA4D) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0xAA7B) return UCD_PROPERTY_DIACRITIC; + if (c == 0xAA7D) return UCD_PROPERTY_DIACRITIC; if (c == 0xAAEB) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0xAAEE && c <= 0xAAEF) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0xAAF5) return UCD_PROPERTY_OTHER_ALPHABETIC; @@ -344,6 +424,7 @@ static int properties_Mc(codepoint_t c) if (c >= 0xABE3 && c <= 0xABE4) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0xABE6 && c <= 0xABE7) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0xABE9 && c <= 0xABEA) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0xABEC) return UCD_PROPERTY_DIACRITIC; break; case 0x011000: if (c == 0x011000) return UCD_PROPERTY_OTHER_ALPHABETIC; @@ -357,10 +438,12 @@ static int properties_Mc(codepoint_t c) if (c == 0x011182) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0111B3 && c <= 0x0111B5) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x0111BF) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0111C0) return UCD_PROPERTY_DIACRITIC; break; case 0x011200: if (c >= 0x01122C && c <= 0x01122E) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x011232 && c <= 0x011233) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x011235) return UCD_PROPERTY_DIACRITIC; if (c >= 0x0112E0 && c <= 0x0112E2) return UCD_PROPERTY_OTHER_ALPHABETIC; break; case 0x011300: @@ -369,6 +452,7 @@ static int properties_Mc(codepoint_t c) if (c >= 0x011341 && c <= 0x011344) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x011347 && c <= 0x011348) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x01134B && c <= 0x01134C) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x01134D) return UCD_PROPERTY_DIACRITIC; if (c == 0x011357) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x011362 && c <= 0x011363) return UCD_PROPERTY_OTHER_ALPHABETIC; break; @@ -392,6 +476,7 @@ static int properties_Mc(codepoint_t c) if (c == 0x01163E) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x0116AC) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0116AE && c <= 0x0116AF) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0116B6) return UCD_PROPERTY_DIACRITIC; break; case 0x011700: if (c >= 0x011720 && c <= 0x011721) return UCD_PROPERTY_OTHER_ALPHABETIC; @@ -404,6 +489,9 @@ static int properties_Mc(codepoint_t c) if (c == 0x011CB1) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x011CB4) return UCD_PROPERTY_OTHER_ALPHABETIC; break; + case 0x01D100: + if (c >= 0x01D16D && c <= 0x01D172) return UCD_PROPERTY_DIACRITIC; + break; case 0x016F00: if (c >= 0x016F51 && c <= 0x016F7E) return UCD_PROPERTY_OTHER_ALPHABETIC; break; @@ -416,115 +504,165 @@ static int properties_Mn(codepoint_t c) switch (c & 0xFFFFFF00) { case 0x0300: - if (c == 0x0345) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0300 && c <= 0x0344) return UCD_PROPERTY_DIACRITIC; + if (c == 0x0345) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0346 && c <= 0x034E) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x0350 && c <= 0x0357) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x035D && c <= 0x0362) return UCD_PROPERTY_DIACRITIC; + break; + case 0x0400: + if (c >= 0x0483 && c <= 0x0487) return UCD_PROPERTY_DIACRITIC; break; case 0x0500: - if (c >= 0x05B0 && c <= 0x05BD) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c == 0x05BF) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c >= 0x05C1 && c <= 0x05C2) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c >= 0x05C4 && c <= 0x05C5) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0591 && c <= 0x05A1) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x05A3 && c <= 0x05AF) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x05B0 && c <= 0x05BD) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x05BF) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x05C1 && c <= 0x05C2) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x05C4) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x05C5) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x05C7) return UCD_PROPERTY_OTHER_ALPHABETIC; break; case 0x0600: if (c >= 0x0610 && c <= 0x061A) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c >= 0x064B && c <= 0x0657) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x064B && c <= 0x0652) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0653 && c <= 0x0656) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0657) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0658) return UCD_PROPERTY_DIACRITIC; if (c >= 0x0659 && c <= 0x065F) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x0670) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x06D6 && c <= 0x06DC) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x06DF && c <= 0x06E0) return UCD_PROPERTY_DIACRITIC; if (c >= 0x06E1 && c <= 0x06E4) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x06E7 && c <= 0x06E8) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x06EA && c <= 0x06EC) return UCD_PROPERTY_DIACRITIC; if (c == 0x06ED) return UCD_PROPERTY_OTHER_ALPHABETIC; break; case 0x0700: if (c == 0x0711) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c >= 0x0730 && c <= 0x073F) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c >= 0x07A6 && c <= 0x07B0) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0730 && c <= 0x073F) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0740 && c <= 0x074A) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x07A6 && c <= 0x07B0) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x07EB && c <= 0x07F3) return UCD_PROPERTY_DIACRITIC; break; case 0x0800: if (c >= 0x0816 && c <= 0x0817) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0818 && c <= 0x0819) return UCD_PROPERTY_DIACRITIC; if (c >= 0x081B && c <= 0x0823) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0825 && c <= 0x0827) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0829 && c <= 0x082C) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x08D4 && c <= 0x08DF) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c >= 0x08E3 && c <= 0x08E9) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c >= 0x08F0) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x08E3 && c <= 0x08E9) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x08EA && c <= 0x08EF) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x08F0 && c <= 0x08FE) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x08FF) return UCD_PROPERTY_OTHER_ALPHABETIC; break; case 0x0900: if (c <= 0x0902) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x093A) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x093C) return UCD_PROPERTY_DIACRITIC; if (c >= 0x0941 && c <= 0x0948) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x094D) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x0951 && c <= 0x0954) return UCD_PROPERTY_DIACRITIC; if (c >= 0x0955 && c <= 0x0957) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0962 && c <= 0x0963) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x0981) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x09BC) return UCD_PROPERTY_DIACRITIC; if (c >= 0x09C1 && c <= 0x09C4) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x09CD) return UCD_PROPERTY_DIACRITIC; if (c >= 0x09E2 && c <= 0x09E3) return UCD_PROPERTY_OTHER_ALPHABETIC; break; case 0x0A00: if (c >= 0x0A01 && c <= 0x0A02) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0A3C) return UCD_PROPERTY_DIACRITIC; if (c >= 0x0A41 && c <= 0x0A42) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0A47 && c <= 0x0A48) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0A4B && c <= 0x0A4C) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0A4D) return UCD_PROPERTY_DIACRITIC; if (c == 0x0A51) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0A70 && c <= 0x0A71) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x0A75) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0A81 && c <= 0x0A82) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0ABC) return UCD_PROPERTY_DIACRITIC; if (c >= 0x0AC1 && c <= 0x0AC5) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0AC7 && c <= 0x0AC8) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0ACD) return UCD_PROPERTY_DIACRITIC; if (c >= 0x0AE2 && c <= 0x0AE3) return UCD_PROPERTY_OTHER_ALPHABETIC; break; case 0x0B00: if (c == 0x0B01) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0B3C) return UCD_PROPERTY_DIACRITIC; if (c == 0x0B3F) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0B41 && c <= 0x0B44) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0B4D) return UCD_PROPERTY_DIACRITIC; if (c == 0x0B56) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0B62 && c <= 0x0B63) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x0B82) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x0BC0) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0BCD) return UCD_PROPERTY_DIACRITIC; break; case 0x0C00: if (c == 0x0C00) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0C3E && c <= 0x0C40) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0C46 && c <= 0x0C48) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0C4A && c <= 0x0C4C) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0C4D) return UCD_PROPERTY_DIACRITIC; if (c >= 0x0C55 && c <= 0x0C56) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0C62 && c <= 0x0C63) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x0C81) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0CBC) return UCD_PROPERTY_DIACRITIC; if (c == 0x0CBF) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x0CC6) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x0CCC) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0CCD) return UCD_PROPERTY_DIACRITIC; if (c >= 0x0CE2 && c <= 0x0CE3) return UCD_PROPERTY_OTHER_ALPHABETIC; break; case 0x0D00: if (c == 0x0D01) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0D41 && c <= 0x0D44) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0D4D) return UCD_PROPERTY_DIACRITIC; if (c >= 0x0D62 && c <= 0x0D63) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0DCA) return UCD_PROPERTY_DIACRITIC; if (c >= 0x0DD2 && c <= 0x0DD4) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x0DD6) return UCD_PROPERTY_OTHER_ALPHABETIC; break; case 0x0E00: if (c == 0x0E31) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0E34 && c <= 0x0E3A) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0E47 && c <= 0x0E4C) return UCD_PROPERTY_DIACRITIC; if (c == 0x0E4D) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0E4E) return UCD_PROPERTY_DIACRITIC; if (c == 0x0EB1) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0EB4 && c <= 0x0EB9) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0EBB && c <= 0x0EBC) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0EC8 && c <= 0x0ECC) return UCD_PROPERTY_DIACRITIC; if (c == 0x0ECD) return UCD_PROPERTY_OTHER_ALPHABETIC; break; case 0x0F00: + if (c >= 0x0F18 && c <= 0x0F19) return UCD_PROPERTY_DIACRITIC; + if (c == 0x0F35) return UCD_PROPERTY_DIACRITIC; + if (c == 0x0F37) return UCD_PROPERTY_DIACRITIC; + if (c == 0x0F39) return UCD_PROPERTY_DIACRITIC; if (c >= 0x0F71 && c <= 0x0F7E) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0F80 && c <= 0x0F81) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0F82 && c <= 0x0F84) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x0F86 && c <= 0x0F87) return UCD_PROPERTY_DIACRITIC; if (c >= 0x0F8D && c <= 0x0F97) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0F99 && c <= 0x0FBC) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0FC6) return UCD_PROPERTY_DIACRITIC; break; case 0x1000: if (c >= 0x102D && c <= 0x1030) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x1032 && c <= 0x1036) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1037) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x1039 && c <= 0x103A) return UCD_PROPERTY_DIACRITIC; if (c >= 0x103D && c <= 0x103E) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x1058 && c <= 0x1059) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x105E && c <= 0x1060) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x1071 && c <= 0x1074) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x1082) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x1085 && c <= 0x1086) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x108D) return UCD_PROPERTY_DIACRITIC; if (c == 0x109D) return UCD_PROPERTY_OTHER_ALPHABETIC; break; case 0x1300: @@ -537,6 +675,8 @@ static int properties_Mn(codepoint_t c) if (c >= 0x1772 && c <= 0x1773) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x17B7 && c <= 0x17BD) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x17C6) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x17C9 && c <= 0x17D3) return UCD_PROPERTY_DIACRITIC; + if (c == 0x17DD) return UCD_PROPERTY_DIACRITIC; break; case 0x1800: if (c >= 0x1885 && c <= 0x1886) return UCD_PROPERTY_OTHER_ALPHABETIC; @@ -546,6 +686,7 @@ static int properties_Mn(codepoint_t c) if (c >= 0x1920 && c <= 0x1922) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x1927 && c <= 0x1928) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x1932) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1939 && c <= 0x193B) return UCD_PROPERTY_DIACRITIC; break; case 0x1A00: if (c >= 0x1A17 && c <= 0x1A18) return UCD_PROPERTY_OTHER_ALPHABETIC; @@ -555,15 +696,21 @@ static int properties_Mn(codepoint_t c) if (c == 0x1A62) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x1A65 && c <= 0x1A6C) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x1A73 && c <= 0x1A74) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1A75 && c <= 0x1A7C) return UCD_PROPERTY_DIACRITIC; + if (c == 0x1A7F) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x1AB0 && c <= 0x1ABD) return UCD_PROPERTY_DIACRITIC; break; case 0x1B00: if (c >= 0x1B00 && c <= 0x1B03) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1B34) return UCD_PROPERTY_DIACRITIC; if (c >= 0x1B36 && c <= 0x1B3A) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x1B3C) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x1B42) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1B6B && c <= 0x1B73) return UCD_PROPERTY_DIACRITIC; if (c >= 0x1B80 && c <= 0x1B81) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x1BA2 && c <= 0x1BA5) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x1BA8 && c <= 0x1BA9) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1BAB) return UCD_PROPERTY_DIACRITIC; if (c >= 0x1BAC && c <= 0x1BAD) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x1BE8 && c <= 0x1BE9) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x1BED) return UCD_PROPERTY_OTHER_ALPHABETIC; @@ -571,27 +718,58 @@ static int properties_Mn(codepoint_t c) break; case 0x1C00: if (c >= 0x1C2C && c <= 0x1C33) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1C36 && c <= 0x1C37) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x1CD0 && c <= 0x1CD2) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x1CD4 && c <= 0x1CE0) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x1CE2 && c <= 0x1CE8) return UCD_PROPERTY_DIACRITIC; + if (c == 0x1CED) return UCD_PROPERTY_DIACRITIC; + if (c == 0x1CF4) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x1CF8 && c <= 0x1CF9) return UCD_PROPERTY_DIACRITIC; break; case 0x1D00: + if (c >= 0x1DC4 && c <= 0x1DCF) return UCD_PROPERTY_DIACRITIC; if (c >= 0x1DE7 && c <= 0x1DF4) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x1DF5) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x1DFD && c <= 0x1DFF) return UCD_PROPERTY_DIACRITIC; + break; + case 0x2000: + if (c >= 0x20D0 && c <= 0x20DC) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x20E1) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x20E5 && c <= 0x20E6) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x20EB && c <= 0x20EF) return UCD_PROPERTY_OTHER_MATH; + break; + case 0x2C00: + if (c >= 0x2CEF && c <= 0x2CF1) return UCD_PROPERTY_DIACRITIC; break; case 0x2D00: if (c >= 0x2DE0 && c <= 0x2DFF) return UCD_PROPERTY_OTHER_ALPHABETIC; break; + case 0x3000: + if (c >= 0x302A && c <= 0x302D) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x3099 && c <= 0x309A) return UCD_PROPERTY_DIACRITIC; + break; case 0xA600: + if (c == 0xA66F) return UCD_PROPERTY_DIACRITIC; if (c >= 0xA674 && c <= 0xA67B) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0xA67C && c <= 0xA67D) return UCD_PROPERTY_DIACRITIC; if (c >= 0xA69E && c <= 0xA69F) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0xA6F0 && c <= 0xA6F1) return UCD_PROPERTY_DIACRITIC; break; case 0xA800: if (c >= 0xA825 && c <= 0xA826) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0xA8C4) return UCD_PROPERTY_DIACRITIC; if (c == 0xA8C5) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0xA8E0 && c <= 0xA8F1) return UCD_PROPERTY_DIACRITIC; break; case 0xA900: if (c >= 0xA926 && c <= 0xA92A) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0xA92B && c <= 0xA92D) return UCD_PROPERTY_DIACRITIC; if (c >= 0xA947 && c <= 0xA951) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0xA980 && c <= 0xA982) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0xA9B3) return UCD_PROPERTY_DIACRITIC; if (c >= 0xA9B6 && c <= 0xA9B9) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0xA9BC) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0xA9E5) return UCD_PROPERTY_DIACRITIC; break; case 0xAA00: if (c >= 0xAA29 && c <= 0xAA2E) return UCD_PROPERTY_OTHER_ALPHABETIC; @@ -599,18 +777,29 @@ static int properties_Mn(codepoint_t c) if (c >= 0xAA35 && c <= 0xAA36) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0xAA43) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0xAA4C) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0xAA7C) return UCD_PROPERTY_DIACRITIC; if (c == 0xAAB0) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0xAAB2 && c <= 0xAAB4) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0xAAB7 && c <= 0xAAB8) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0xAABE) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0xAABF) return UCD_PROPERTY_DIACRITIC; + if (c == 0xAAC1) return UCD_PROPERTY_DIACRITIC; if (c >= 0xAAEC && c <= 0xAAED) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0xAAF6) return UCD_PROPERTY_DIACRITIC; break; case 0xAB00: if (c == 0xABE5) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0xABE8) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0xABED) return UCD_PROPERTY_DIACRITIC; break; case 0xFB00: - if (c == 0xFB1E) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0xFB1E) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_ALPHABETIC; + break; + case 0xFE00: + if (c >= 0xFE20 && c <= 0xFE2F) return UCD_PROPERTY_DIACRITIC; + break; + case 0x010200: + if (c == 0x0102E0) return UCD_PROPERTY_DIACRITIC; break; case 0x010300: if (c >= 0x010376 && c <= 0x01037A) return UCD_PROPERTY_OTHER_ALPHABETIC; @@ -619,70 +808,100 @@ static int properties_Mn(codepoint_t c) if (c >= 0x010A01 && c <= 0x010A03) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x010A05 && c <= 0x010A06) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x010A0C && c <= 0x010A0F) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x010AE5 && c <= 0x010AE6) return UCD_PROPERTY_DIACRITIC; break; case 0x011000: if (c == 0x011001) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x011038 && c <= 0x011045) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0110B3 && c <= 0x0110B6) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0110B9 && c <= 0x0110BA) return UCD_PROPERTY_DIACRITIC; break; case 0x011100: if (c >= 0x011100 && c <= 0x011102) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x011127 && c <= 0x01112B) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x01112D && c <= 0x011132) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x011133 && c <= 0x011134) return UCD_PROPERTY_DIACRITIC; + if (c == 0x011173) return UCD_PROPERTY_DIACRITIC; if (c >= 0x011180 && c <= 0x011181) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0111B6 && c <= 0x0111BE) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0111CA && c <= 0x0111CC) return UCD_PROPERTY_DIACRITIC; break; case 0x011200: if (c >= 0x01122F && c <= 0x011231) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x011234) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x011236) return UCD_PROPERTY_DIACRITIC; if (c == 0x011237) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x01123E) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x0112DF) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0112E3 && c <= 0x0112E8) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0112E9 && c <= 0x0112EA) return UCD_PROPERTY_DIACRITIC; break; case 0x011300: if (c >= 0x011300 && c <= 0x011301) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x01133C) return UCD_PROPERTY_DIACRITIC; if (c == 0x011340) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x011366 && c <= 0x01136C) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x011370 && c <= 0x011374) return UCD_PROPERTY_DIACRITIC; break; case 0x011400: if (c >= 0x011438 && c <= 0x01143F) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x011442) return UCD_PROPERTY_DIACRITIC; if (c >= 0x011443 && c <= 0x011444) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x011446) return UCD_PROPERTY_DIACRITIC; if (c >= 0x0114B3 && c <= 0x0114B8) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x0114BA) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0114BF && c <= 0x0114C0) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0114C2 && c <= 0x0114C3) return UCD_PROPERTY_DIACRITIC; break; case 0x011500: if (c >= 0x0115B2 && c <= 0x0115B5) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0115BC && c <= 0x0115BD) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0115BF && c <= 0x0115C0) return UCD_PROPERTY_DIACRITIC; if (c >= 0x0115DC && c <= 0x0115DD) return UCD_PROPERTY_OTHER_ALPHABETIC; break; case 0x011600: if (c >= 0x011633 && c <= 0x01163A) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x01163D) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x01163F) return UCD_PROPERTY_DIACRITIC; if (c == 0x011640) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x0116AB) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x0116AD) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0116B0 && c <= 0x0116B5) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0116B7) return UCD_PROPERTY_DIACRITIC; break; case 0x011700: if (c >= 0x01171D && c <= 0x01171F) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x011722 && c <= 0x011725) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x011727 && c <= 0x01172A) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x01172B) return UCD_PROPERTY_DIACRITIC; break; case 0x011C00: if (c >= 0x011C30 && c <= 0x011C36) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x011C38 && c <= 0x011C3D) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x011C3F) return UCD_PROPERTY_DIACRITIC; if (c >= 0x011C92 && c <= 0x011CA7) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x011CAA && c <= 0x011CB0) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x011CB2 && c <= 0x011CB3) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x011CB5 && c <= 0x011CB6) return UCD_PROPERTY_OTHER_ALPHABETIC; break; + case 0x016A00: + if (c >= 0x016AF0 && c <= 0x016AF4) return UCD_PROPERTY_DIACRITIC; + break; case 0x016B00: if (c >= 0x016B30 && c <= 0x016B36) return UCD_PROPERTY_OTHER_ALPHABETIC; break; + case 0x016F00: + if (c >= 0x016F8F && c <= 0x016F92) return UCD_PROPERTY_DIACRITIC; + break; case 0x01BC00: if (c == 0x01BC9E) return UCD_PROPERTY_OTHER_ALPHABETIC; break; + case 0x01D100: + if (c >= 0x01D167 && c <= 0x01D169) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x01D17B && c <= 0x01D182) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x01D185 && c <= 0x01D18B) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x01D1AA && c <= 0x01D1AD) return UCD_PROPERTY_DIACRITIC; + break; case 0x01E000: if (c >= 0x01E000 && c <= 0x01E006) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x01E008 && c <= 0x01E018) return UCD_PROPERTY_OTHER_ALPHABETIC; @@ -690,14 +909,13 @@ static int properties_Mn(codepoint_t c) if (c >= 0x01E023 && c <= 0x01E024) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x01E026 && c <= 0x01E02A) return UCD_PROPERTY_OTHER_ALPHABETIC; break; - case 0x01E900: - if (c == 0x01E947) return UCD_PROPERTY_OTHER_ALPHABETIC; + case 0x01E800: + if (c >= 0x01E8D0 && c <= 0x01E8D6) return UCD_PROPERTY_DIACRITIC; break; - case 0x2000: - if (c >= 0x20D0 && c <= 0x20DC) return UCD_PROPERTY_OTHER_MATH; - if (c == 0x20E1) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x20E5 && c <= 0x20E6) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x20EB && c <= 0x20EF) return UCD_PROPERTY_OTHER_MATH; + case 0x01E900: + if (c >= 0x01E944 && c <= 0x01E946) return UCD_PROPERTY_DIACRITIC; + if (c == 0x01E947) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x01E948 && c <= 0x01E94A) return UCD_PROPERTY_DIACRITIC; break; } return 0; @@ -856,6 +1074,7 @@ static int properties_Po(codepoint_t c) if (c == 0x002E) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c >= 0x003A && c <= 0x003B) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c == 0x003F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x00B7) return UCD_PROPERTY_DIACRITIC; break; case 0x0300: if (c == 0x037E) return UCD_PROPERTY_TERMINAL_PUNCTUATION; @@ -923,6 +1142,7 @@ static int properties_Po(codepoint_t c) case 0x1C00: if (c >= 0x1C3B && c <= 0x1C3F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c >= 0x1C7E && c <= 0x1C7F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x1CD3) return UCD_PROPERTY_DIACRITIC; break; case 0x2000: if (c == 0x2016) return UCD_PROPERTY_OTHER_MATH; @@ -952,6 +1172,7 @@ static int properties_Po(codepoint_t c) if (c >= 0xA8CE && c <= 0xA8CF) return UCD_PROPERTY_TERMINAL_PUNCTUATION; break; case 0xA900: + if (c == 0xA92E) return UCD_PROPERTY_DIACRITIC; if (c == 0xA92F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c >= 0xA9C7 && c <= 0xA9C9) return UCD_PROPERTY_TERMINAL_PUNCTUATION; break; @@ -1100,10 +1321,45 @@ static int properties_Sk(codepoint_t c) switch (c & 0xFFFFFF00) { case 0x0000: - if (c == 0x005E) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x005E) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_DIACRITIC; + if (c == 0x0060) return UCD_PROPERTY_DIACRITIC; + if (c == 0x00A8) return UCD_PROPERTY_DIACRITIC; + if (c == 0x00AF) return UCD_PROPERTY_DIACRITIC; + if (c == 0x00B4) return UCD_PROPERTY_DIACRITIC; + if (c == 0x00B8) return UCD_PROPERTY_DIACRITIC; + break; + case 0x0200: + if (c >= 0x02C2 && c <= 0x02C5) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x02D2 && c <= 0x02DF) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x02E5 && c <= 0x02EB) return UCD_PROPERTY_DIACRITIC; + if (c == 0x02ED) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x02EF && c <= 0x02FF) return UCD_PROPERTY_DIACRITIC; + break; + case 0x0300: + if (c == 0x0375) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x0384 && c <= 0x0385) return UCD_PROPERTY_DIACRITIC; + break; + case 0x1F00: + if (c == 0x1FBD) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x1FBF && c <= 0x1FC1) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x1FCD && c <= 0x1FCF) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x1FDD && c <= 0x1FDF) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x1FED && c <= 0x1FEF) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x1FFD && c <= 0x1FFE) return UCD_PROPERTY_DIACRITIC; + break; + case 0x3000: + if (c >= 0x309B && c <= 0x309C) return UCD_PROPERTY_DIACRITIC; + break; + case 0xA700: + if (c >= 0xA720 && c <= 0xA721) return UCD_PROPERTY_DIACRITIC; + break; + case 0xAB00: + if (c == 0xAB5B) return UCD_PROPERTY_DIACRITIC; break; case 0xFF00: - if (c == 0xFF3E) return UCD_PROPERTY_OTHER_MATH; + if (c == 0xFF3E) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_MATH; + if (c == 0xFF40) return UCD_PROPERTY_DIACRITIC; + if (c == 0xFFE3) return UCD_PROPERTY_DIACRITIC; break; } return 0; @@ -1200,6 +1456,7 @@ ucd_property ucd_properties(codepoint_t c, ucd_category category) case UCD_CATEGORY_Cc: return properties_Cc(c); case UCD_CATEGORY_Cf: return properties_Cf(c); case UCD_CATEGORY_Ll: return properties_Ll(c); + case UCD_CATEGORY_Lm: return properties_Lm(c); case UCD_CATEGORY_Lo: return properties_Lo(c) | properties_Lo_ideographic(c); case UCD_CATEGORY_Lu: return properties_Lu(c); case UCD_CATEGORY_Mc: return properties_Mc(c); @@ -1219,6 +1476,6 @@ ucd_property ucd_properties(codepoint_t c, ucd_category category) case UCD_CATEGORY_Zl: return UCD_PROPERTY_WHITE_SPACE; case UCD_CATEGORY_Zp: return UCD_PROPERTY_WHITE_SPACE; case UCD_CATEGORY_Zs: return properties_Zs(c); - default: return 0; // Cn Co Cs Ii Lm Lt Me No Sc + default: return 0; // Cn Co Cs Ii Lt Me No Sc }; } diff --git a/tools/printdata.py b/tools/printdata.py index 2aee1da2..eec531bf 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -134,6 +134,7 @@ def properties(data): props += 1024 * data.get('ASCII_Hex_Digit', 0) props += 2048 * data.get('Other_Alphabetic', 0) props += 4096 * data.get('Ideographic', 0) + props += 8192 * data.get('Diacritic', 0) return props if __name__ == '__main__': From 735a6612325a061a942fca51feb3989de7e3f46e Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Fri, 14 Apr 2017 12:03:34 +0100 Subject: [PATCH 20/44] Add Extender support from PropList.txt. --- src/include/ucd/ucd.h | 2 ++ src/proplist.c | 57 +++++++++++++++++++++++++++++++++++++------ tools/printdata.py | 31 +++++++++++------------ 3 files changed, 67 insertions(+), 23 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 9f449601..b5ff2a40 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -343,6 +343,7 @@ typedef enum ucd_property_ UCD_PROPERTY_OTHER_ALPHABETIC = 0x00000800, /**< @brief Other_Alphabetic PropList */ UCD_PROPERTY_IDEOGRAPHIC = 0x00001000, /**< @brief Ideographic PropList */ UCD_PROPERTY_DIACRITIC = 0x00002000, /**< @brief Diacritic PropList */ + UCD_PROPERTY_EXTENDER = 0x00004000, /**< @brief Extender PropList */ } ucd_property; /** @brief Return the properties of the specified codepoint. @@ -816,6 +817,7 @@ namespace ucd Other_Alphabetic = UCD_PROPERTY_OTHER_ALPHABETIC, /**< @brief Other_Alphabetic PropList */ Ideographic = UCD_PROPERTY_IDEOGRAPHIC, /**< @brief Ideographic PropList */ Diacritic = UCD_PROPERTY_DIACRITIC, /**< @brief Diacritic PropList */ + Extender = UCD_PROPERTY_EXTENDER, /**< @brief Extender PropList */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index fbbb0915..46dbdd3a 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -110,7 +110,8 @@ static int properties_Lm(codepoint_t c) { case 0x0200: if (c >= 0x02B0 && c <= 0x02C1) return UCD_PROPERTY_DIACRITIC; - if (c >= 0x02C6 && c <= 0x02D1) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x02C6 && c <= 0x02CF) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x02D0 && c <= 0x02D1) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_EXTENDER; if (c >= 0x02E0 && c <= 0x02E4) return UCD_PROPERTY_DIACRITIC; if (c == 0x02EC) return UCD_PROPERTY_DIACRITIC; if (c == 0x02EE) return UCD_PROPERTY_DIACRITIC; @@ -123,15 +124,28 @@ static int properties_Lm(codepoint_t c) if (c == 0x0559) return UCD_PROPERTY_DIACRITIC; break; case 0x0600: + if (c == 0x0640) return UCD_PROPERTY_EXTENDER; if (c >= 0x06E5 && c <= 0x06E6) return UCD_PROPERTY_DIACRITIC; break; case 0x0700: if (c >= 0x07F4 && c <= 0x07F5) return UCD_PROPERTY_DIACRITIC; + if (c == 0x07FA) return UCD_PROPERTY_EXTENDER; break; case 0x0900: if (c == 0x0971) return UCD_PROPERTY_DIACRITIC; break; + case 0x0E00: + if (c == 0x0E46) return UCD_PROPERTY_EXTENDER; + if (c == 0x0EC6) return UCD_PROPERTY_EXTENDER; + break; + case 0x1800: + if (c == 0x1843) return UCD_PROPERTY_EXTENDER; + break; + case 0x1A00: + if (c == 0x1AA7) return UCD_PROPERTY_EXTENDER; + break; case 0x1C00: + if (c == 0x1C7B) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_EXTENDER; if (c >= 0x1C78 && c <= 0x1C7D) return UCD_PROPERTY_DIACRITIC; break; case 0x1D00: @@ -141,9 +155,17 @@ static int properties_Lm(codepoint_t c) if (c == 0x2E2F) return UCD_PROPERTY_DIACRITIC; break; case 0x3000: - if (c == 0x30FC) return UCD_PROPERTY_DIACRITIC; + if (c == 0x3005) return UCD_PROPERTY_EXTENDER; + if (c >= 0x3031 && c <= 0x3035) return UCD_PROPERTY_EXTENDER; + if (c >= 0x309D && c <= 0x309E) return UCD_PROPERTY_EXTENDER; + if (c == 0x30FC) return UCD_PROPERTY_EXTENDER | UCD_PROPERTY_DIACRITIC; + if (c >= 0x30FD && c <= 0x30FE) return UCD_PROPERTY_EXTENDER; + break; + case 0xA000: + if (c == 0xA015) return UCD_PROPERTY_EXTENDER; break; case 0xA600: + if (c == 0xA60C) return UCD_PROPERTY_EXTENDER; if (c == 0xA67F) return UCD_PROPERTY_DIACRITIC; if (c >= 0xA69C && c <= 0xA69D) return UCD_PROPERTY_DIACRITIC; break; @@ -152,15 +174,28 @@ static int properties_Lm(codepoint_t c) if (c == 0xA788) return UCD_PROPERTY_DIACRITIC; if (c >= 0xA7F8 && c <= 0xA7F9) return UCD_PROPERTY_DIACRITIC; break; + case 0xA900: + if (c == 0xA9CF) return UCD_PROPERTY_EXTENDER; + if (c == 0xA9E6) return UCD_PROPERTY_EXTENDER; + break; + case 0xAA00: + if (c == 0xAA70) return UCD_PROPERTY_EXTENDER; + if (c == 0xAADD) return UCD_PROPERTY_EXTENDER; + if (c >= 0xAAF3 && c <= 0xAAF4) return UCD_PROPERTY_EXTENDER; + break; case 0xAB00: if (c >= 0xAB5C && c <= 0xAB5F) return UCD_PROPERTY_DIACRITIC; break; case 0xFF00: - if (c == 0xFF70) return UCD_PROPERTY_DIACRITIC; + if (c == 0xFF70) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_EXTENDER; if (c >= 0xFF9E && c <= 0xFF9F) return UCD_PROPERTY_DIACRITIC; break; + case 0x016B00: + if (c >= 0x016B42 && c <= 0x016B43) return UCD_PROPERTY_EXTENDER; + break; case 0x016F00: - if (c >= 0x016F93 && c <= 0x016F9F) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x016F93 && c <= 0x016F9F) return UCD_PROPERTY_DIACRITIC; + if (c == 0x016FE0) return UCD_PROPERTY_EXTENDER; break; } return 0; @@ -180,7 +215,10 @@ static int properties_Lo(codepoint_t c) if (c == 0xAAC0) return UCD_PROPERTY_DIACRITIC; if (c == 0xAAC2) return UCD_PROPERTY_DIACRITIC; break; - case 0x01EE00: + case 0x11300: + if (c == 0x01135D) return UCD_PROPERTY_EXTENDER; + break; + case 0x1EE00: return UCD_PROPERTY_OTHER_MATH; } return 0; @@ -718,7 +756,8 @@ static int properties_Mn(codepoint_t c) break; case 0x1C00: if (c >= 0x1C2C && c <= 0x1C33) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c >= 0x1C36 && c <= 0x1C37) return UCD_PROPERTY_DIACRITIC; + if (c == 0x1C36) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_EXTENDER; + if (c == 0x1C37) return UCD_PROPERTY_DIACRITIC; if (c >= 0x1CD0 && c <= 0x1CD2) return UCD_PROPERTY_DIACRITIC; if (c >= 0x1CD4 && c <= 0x1CE0) return UCD_PROPERTY_DIACRITIC; if (c >= 0x1CE2 && c <= 0x1CE8) return UCD_PROPERTY_DIACRITIC; @@ -913,7 +952,7 @@ static int properties_Mn(codepoint_t c) if (c >= 0x01E8D0 && c <= 0x01E8D6) return UCD_PROPERTY_DIACRITIC; break; case 0x01E900: - if (c >= 0x01E944 && c <= 0x01E946) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x01E944 && c <= 0x01E946) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_EXTENDER; if (c == 0x01E947) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x01E948 && c <= 0x01E94A) return UCD_PROPERTY_DIACRITIC; break; @@ -1074,7 +1113,7 @@ static int properties_Po(codepoint_t c) if (c == 0x002E) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c >= 0x003A && c <= 0x003B) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c == 0x003F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0x00B7) return UCD_PROPERTY_DIACRITIC; + if (c == 0x00B7) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_EXTENDER; break; case 0x0300: if (c == 0x037E) return UCD_PROPERTY_TERMINAL_PUNCTUATION; @@ -1128,6 +1167,7 @@ static int properties_Po(codepoint_t c) case 0x1800: if (c >= 0x1802 && c <= 0x1805) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c >= 0x1808 && c <= 0x1809) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x180A) return UCD_PROPERTY_EXTENDER; break; case 0x1900: if (c >= 0x1944 && c <= 0x1945) return UCD_PROPERTY_TERMINAL_PUNCTUATION; @@ -1241,6 +1281,7 @@ static int properties_Po(codepoint_t c) break; case 0x11500: if (c >= 0x0115C2 && c <= 0x0115C5) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x0115C6 && c <= 0x0115C8) return UCD_PROPERTY_EXTENDER; if (c >= 0x0115C9 && c <= 0x0115D7) return UCD_PROPERTY_TERMINAL_PUNCTUATION; break; case 0x11600: diff --git a/tools/printdata.py b/tools/printdata.py index eec531bf..2e63767b 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -120,21 +120,22 @@ def decomposition_type(data, dtype): return None def properties(data): - props = 0 - props += 1 * data.get('White_Space', 0) - props += 2 * (decomposition_type(data, '') != None) - props += 4 * data.get('Bidi_Control', 0) - props += 8 * data.get('Join_Control', 0) - props += 16 * data.get('Dash', 0) - props += 32 * data.get('Hyphen', 0) - props += 64 * data.get('Quotation_Mark', 0) - props += 128 * data.get('Terminal_Punctuation', 0) - props += 256 * data.get('Other_Math', 0) - props += 512 * data.get('Hex_Digit', 0) - props += 1024 * data.get('ASCII_Hex_Digit', 0) - props += 2048 * data.get('Other_Alphabetic', 0) - props += 4096 * data.get('Ideographic', 0) - props += 8192 * data.get('Diacritic', 0) + props = 0 + props += 1 * data.get('White_Space', 0) + props += 2 * (decomposition_type(data, '') != None) + props += 4 * data.get('Bidi_Control', 0) + props += 8 * data.get('Join_Control', 0) + props += 16 * data.get('Dash', 0) + props += 32 * data.get('Hyphen', 0) + props += 64 * data.get('Quotation_Mark', 0) + props += 128 * data.get('Terminal_Punctuation', 0) + props += 256 * data.get('Other_Math', 0) + props += 512 * data.get('Hex_Digit', 0) + props += 1024 * data.get('ASCII_Hex_Digit', 0) + props += 2048 * data.get('Other_Alphabetic', 0) + props += 4096 * data.get('Ideographic', 0) + props += 8192 * data.get('Diacritic', 0) + props += 16384 * data.get('Extender', 0) return props if __name__ == '__main__': From 6a2a87e358af18fdeb189ab7560e6300525c578c Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Fri, 14 Apr 2017 13:00:02 +0100 Subject: [PATCH 21/44] Add Other_Lowercase support from PropList.txt. --- src/include/ucd/ucd.h | 2 ++ src/proplist.c | 39 ++++++++++++++++++++++++++++++--------- tools/printdata.py | 1 + 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index b5ff2a40..83473b3d 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -344,6 +344,7 @@ typedef enum ucd_property_ UCD_PROPERTY_IDEOGRAPHIC = 0x00001000, /**< @brief Ideographic PropList */ UCD_PROPERTY_DIACRITIC = 0x00002000, /**< @brief Diacritic PropList */ UCD_PROPERTY_EXTENDER = 0x00004000, /**< @brief Extender PropList */ + UCD_PROPERTY_OTHER_LOWERCASE = 0x00008000, /**< @brief Other_Lowercase PropList */ } ucd_property; /** @brief Return the properties of the specified codepoint. @@ -818,6 +819,7 @@ namespace ucd Ideographic = UCD_PROPERTY_IDEOGRAPHIC, /**< @brief Ideographic PropList */ Diacritic = UCD_PROPERTY_DIACRITIC, /**< @brief Diacritic PropList */ Extender = UCD_PROPERTY_EXTENDER, /**< @brief Extender PropList */ + Other_Lowercase = UCD_PROPERTY_OTHER_LOWERCASE, /**< @brief Other_Lowercase PropList */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index 46dbdd3a..e88fff65 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -109,16 +109,18 @@ static int properties_Lm(codepoint_t c) switch (c & 0xFFFFFF00) { case 0x0200: - if (c >= 0x02B0 && c <= 0x02C1) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x02B0 && c <= 0x02B8) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_LOWERCASE; + if (c >= 0x02B9 && c <= 0x02BF) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x02C0 && c <= 0x02C1) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_LOWERCASE; if (c >= 0x02C6 && c <= 0x02CF) return UCD_PROPERTY_DIACRITIC; if (c >= 0x02D0 && c <= 0x02D1) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_EXTENDER; - if (c >= 0x02E0 && c <= 0x02E4) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x02E0 && c <= 0x02E4) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_LOWERCASE; if (c == 0x02EC) return UCD_PROPERTY_DIACRITIC; if (c == 0x02EE) return UCD_PROPERTY_DIACRITIC; break; case 0x0300: if (c == 0x0374) return UCD_PROPERTY_DIACRITIC; - if (c == 0x037A) return UCD_PROPERTY_DIACRITIC; + if (c == 0x037A) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_LOWERCASE; break; case 0x0500: if (c == 0x0559) return UCD_PROPERTY_DIACRITIC; @@ -149,7 +151,17 @@ static int properties_Lm(codepoint_t c) if (c >= 0x1C78 && c <= 0x1C7D) return UCD_PROPERTY_DIACRITIC; break; case 0x1D00: - if (c >= 0x1D2C && c <= 0x1D6A) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x1D2C && c <= 0x1D6A) return UCD_PROPERTY_OTHER_LOWERCASE | UCD_PROPERTY_DIACRITIC; + if (c == 0x1D78) return UCD_PROPERTY_OTHER_LOWERCASE; + if (c >= 0x1D9B && c <= 0x1DBF) return UCD_PROPERTY_OTHER_LOWERCASE; + break; + case 0x2000: + if (c == 0x2071) return UCD_PROPERTY_OTHER_LOWERCASE; + if (c == 0x207F) return UCD_PROPERTY_OTHER_LOWERCASE; + if (c >= 0x2090 && c <= 0x209C) return UCD_PROPERTY_OTHER_LOWERCASE; + break; + case 0x2C00: + if (c >= 0x2C7C && c <= 0x2C7D) return UCD_PROPERTY_OTHER_LOWERCASE; break; case 0x2E00: if (c == 0x2E2F) return UCD_PROPERTY_DIACRITIC; @@ -167,12 +179,13 @@ static int properties_Lm(codepoint_t c) case 0xA600: if (c == 0xA60C) return UCD_PROPERTY_EXTENDER; if (c == 0xA67F) return UCD_PROPERTY_DIACRITIC; - if (c >= 0xA69C && c <= 0xA69D) return UCD_PROPERTY_DIACRITIC; + if (c >= 0xA69C && c <= 0xA69D) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_LOWERCASE; break; case 0xA700: if (c >= 0xA717 && c <= 0xA71F) return UCD_PROPERTY_DIACRITIC; + if (c == 0xA770) return UCD_PROPERTY_OTHER_LOWERCASE; if (c == 0xA788) return UCD_PROPERTY_DIACRITIC; - if (c >= 0xA7F8 && c <= 0xA7F9) return UCD_PROPERTY_DIACRITIC; + if (c >= 0xA7F8 && c <= 0xA7F9) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_LOWERCASE; break; case 0xA900: if (c == 0xA9CF) return UCD_PROPERTY_EXTENDER; @@ -184,7 +197,7 @@ static int properties_Lm(codepoint_t c) if (c >= 0xAAF3 && c <= 0xAAF4) return UCD_PROPERTY_EXTENDER; break; case 0xAB00: - if (c >= 0xAB5C && c <= 0xAB5F) return UCD_PROPERTY_DIACRITIC; + if (c >= 0xAB5C && c <= 0xAB5F) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_LOWERCASE; break; case 0xFF00: if (c == 0xFF70) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_EXTENDER; @@ -205,6 +218,10 @@ static int properties_Lo(codepoint_t c) { switch (c & 0xFFFFFF00) { + case 0x0000: + if (c == 0x00AA) return UCD_PROPERTY_OTHER_LOWERCASE; + if (c == 0x00BA) return UCD_PROPERTY_OTHER_LOWERCASE; + break; case 0x2100: if (c >= 0x2135 && c <= 0x2138) return UCD_PROPERTY_OTHER_MATH; break; @@ -543,7 +560,7 @@ static int properties_Mn(codepoint_t c) { case 0x0300: if (c >= 0x0300 && c <= 0x0344) return UCD_PROPERTY_DIACRITIC; - if (c == 0x0345) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0345) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_LOWERCASE; if (c >= 0x0346 && c <= 0x034E) return UCD_PROPERTY_DIACRITIC; if (c >= 0x0350 && c <= 0x0357) return UCD_PROPERTY_DIACRITIC; if (c >= 0x035D && c <= 0x0362) return UCD_PROPERTY_DIACRITIC; @@ -981,6 +998,9 @@ static int properties_Nl(codepoint_t c) { switch (c & 0xFFFFFF00) { + case 0x2100: + if (c >= 0x2170 && c <= 0x217F) return UCD_PROPERTY_OTHER_LOWERCASE; + break; case 0x3000: if (c == 0x3007) return UCD_PROPERTY_IDEOGRAPHIC; if (c >= 0x3021 && c <= 0x3029) return UCD_PROPERTY_IDEOGRAPHIC; @@ -1449,7 +1469,8 @@ static int properties_So(codepoint_t c) if (c == 0x23E2) return UCD_PROPERTY_OTHER_MATH; break; case 0x2400: - if (c >= 0x24B6 && c <= 0x24E9) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x24B6 && c <= 0x24CF) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x24D0 && c <= 0x24E9) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_LOWERCASE; break; case 0x2500: if (c >= 0x25A0 && c <= 0x25A1) return UCD_PROPERTY_OTHER_MATH; diff --git a/tools/printdata.py b/tools/printdata.py index 2e63767b..f139009e 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -136,6 +136,7 @@ def properties(data): props += 4096 * data.get('Ideographic', 0) props += 8192 * data.get('Diacritic', 0) props += 16384 * data.get('Extender', 0) + props += 32768 * data.get('Other_Lowercase', 0) return props if __name__ == '__main__': From a184262d9b5b4215e68893631b669d4b9ed08b3c Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Fri, 14 Apr 2017 13:08:56 +0100 Subject: [PATCH 22/44] Add Other_Uppercase support from PropList.txt. --- src/include/ucd/ucd.h | 2 ++ src/proplist.c | 9 +++++---- tools/printdata.py | 1 + 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 83473b3d..d0ecafbc 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -345,6 +345,7 @@ typedef enum ucd_property_ UCD_PROPERTY_DIACRITIC = 0x00002000, /**< @brief Diacritic PropList */ UCD_PROPERTY_EXTENDER = 0x00004000, /**< @brief Extender PropList */ UCD_PROPERTY_OTHER_LOWERCASE = 0x00008000, /**< @brief Other_Lowercase PropList */ + UCD_PROPERTY_OTHER_UPPERCASE = 0x00010000, /**< @brief Other_Uppercase PropList */ } ucd_property; /** @brief Return the properties of the specified codepoint. @@ -820,6 +821,7 @@ namespace ucd Diacritic = UCD_PROPERTY_DIACRITIC, /**< @brief Diacritic PropList */ Extender = UCD_PROPERTY_EXTENDER, /**< @brief Extender PropList */ Other_Lowercase = UCD_PROPERTY_OTHER_LOWERCASE, /**< @brief Other_Lowercase PropList */ + Other_Uppercase = UCD_PROPERTY_OTHER_UPPERCASE, /**< @brief Other_Uppercase PropList */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index e88fff65..525b45e9 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -999,6 +999,7 @@ static int properties_Nl(codepoint_t c) switch (c & 0xFFFFFF00) { case 0x2100: + if (c >= 0x2160 && c <= 0x216F) return UCD_PROPERTY_OTHER_UPPERCASE; if (c >= 0x2170 && c <= 0x217F) return UCD_PROPERTY_OTHER_LOWERCASE; break; case 0x3000: @@ -1469,7 +1470,7 @@ static int properties_So(codepoint_t c) if (c == 0x23E2) return UCD_PROPERTY_OTHER_MATH; break; case 0x2400: - if (c >= 0x24B6 && c <= 0x24CF) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x24B6 && c <= 0x24CF) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_UPPERCASE; if (c >= 0x24D0 && c <= 0x24E9) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_LOWERCASE; break; case 0x2500: @@ -1491,9 +1492,9 @@ static int properties_So(codepoint_t c) if (c >= 0x266D && c <= 0x266E) return UCD_PROPERTY_OTHER_MATH; break; case 0x01F100: - if (c >= 0x01F130 && c <= 0x01F149) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c >= 0x01F150 && c <= 0x01F169) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c >= 0x01F170 && c <= 0x01F189) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x01F130 && c <= 0x01F149) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_UPPERCASE; + if (c >= 0x01F150 && c <= 0x01F169) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_UPPERCASE; + if (c >= 0x01F170 && c <= 0x01F189) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_UPPERCASE; break; } return 0; diff --git a/tools/printdata.py b/tools/printdata.py index f139009e..58b31fe7 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -137,6 +137,7 @@ def properties(data): props += 8192 * data.get('Diacritic', 0) props += 16384 * data.get('Extender', 0) props += 32768 * data.get('Other_Lowercase', 0) + props += 65536 * data.get('Other_Uppercase', 0) return props if __name__ == '__main__': From 9dc44bf0d4e5b11c50d38d47ee87af6c993fbb80 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Fri, 14 Apr 2017 13:20:22 +0100 Subject: [PATCH 23/44] Add Noncharacter_Code_Point support from PropList.txt. --- src/include/ucd/ucd.h | 38 ++++++++++++++++++++------------------ src/proplist.c | 10 +++++++++- tools/printdata.py | 37 +++++++++++++++++++------------------ 3 files changed, 48 insertions(+), 37 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index d0ecafbc..0fc9cefa 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -329,23 +329,24 @@ ucd_script ucd_lookup_script(codepoint_t c); */ typedef enum ucd_property_ { - UCD_PROPERTY_WHITE_SPACE = 0x00000001, /**< @brief White_Space PropList */ - UCD_PROPERTY_NO_BREAK = 0x00000002, /**< @brief DispositionType (enabled check only) */ - UCD_PROPERTY_BIDI_CONTROL = 0x00000004, /**< @brief Bidi_Control PropList */ - UCD_PROPERTY_JOIN_CONTROL = 0x00000008, /**< @brief Join_Control PropList */ - UCD_PROPERTY_DASH = 0x00000010, /**< @brief Dash PropList */ - UCD_PROPERTY_HYPHEN = 0x00000020, /**< @brief Hyphen PropList */ - UCD_PROPERTY_QUOTATION_MARK = 0x00000040, /**< @brief Quotation_Mark PropList */ - UCD_PROPERTY_TERMINAL_PUNCTUATION = 0x00000080, /**< @brief Terminal_Punctuation PropList */ - UCD_PROPERTY_OTHER_MATH = 0x00000100, /**< @brief Other_Math PropList */ - UCD_PROPERTY_HEX_DIGIT = 0x00000200, /**< @brief Hex_Digit PropList */ - UCD_PROPERTY_ASCII_HEX_DIGIT = 0x00000400, /**< @brief ASCII_Hex_Digit PropList */ - UCD_PROPERTY_OTHER_ALPHABETIC = 0x00000800, /**< @brief Other_Alphabetic PropList */ - UCD_PROPERTY_IDEOGRAPHIC = 0x00001000, /**< @brief Ideographic PropList */ - UCD_PROPERTY_DIACRITIC = 0x00002000, /**< @brief Diacritic PropList */ - UCD_PROPERTY_EXTENDER = 0x00004000, /**< @brief Extender PropList */ - UCD_PROPERTY_OTHER_LOWERCASE = 0x00008000, /**< @brief Other_Lowercase PropList */ - UCD_PROPERTY_OTHER_UPPERCASE = 0x00010000, /**< @brief Other_Uppercase PropList */ + UCD_PROPERTY_WHITE_SPACE = 0x00000001, /**< @brief White_Space PropList */ + UCD_PROPERTY_NO_BREAK = 0x00000002, /**< @brief DispositionType (enabled check) */ + UCD_PROPERTY_BIDI_CONTROL = 0x00000004, /**< @brief Bidi_Control PropList */ + UCD_PROPERTY_JOIN_CONTROL = 0x00000008, /**< @brief Join_Control PropList */ + UCD_PROPERTY_DASH = 0x00000010, /**< @brief Dash PropList */ + UCD_PROPERTY_HYPHEN = 0x00000020, /**< @brief Hyphen PropList */ + UCD_PROPERTY_QUOTATION_MARK = 0x00000040, /**< @brief Quotation_Mark PropList */ + UCD_PROPERTY_TERMINAL_PUNCTUATION = 0x00000080, /**< @brief Terminal_Punctuation PropList */ + UCD_PROPERTY_OTHER_MATH = 0x00000100, /**< @brief Other_Math PropList */ + UCD_PROPERTY_HEX_DIGIT = 0x00000200, /**< @brief Hex_Digit PropList */ + UCD_PROPERTY_ASCII_HEX_DIGIT = 0x00000400, /**< @brief ASCII_Hex_Digit PropList */ + UCD_PROPERTY_OTHER_ALPHABETIC = 0x00000800, /**< @brief Other_Alphabetic PropList */ + UCD_PROPERTY_IDEOGRAPHIC = 0x00001000, /**< @brief Ideographic PropList */ + UCD_PROPERTY_DIACRITIC = 0x00002000, /**< @brief Diacritic PropList */ + UCD_PROPERTY_EXTENDER = 0x00004000, /**< @brief Extender PropList */ + UCD_PROPERTY_OTHER_LOWERCASE = 0x00008000, /**< @brief Other_Lowercase PropList */ + UCD_PROPERTY_OTHER_UPPERCASE = 0x00010000, /**< @brief Other_Uppercase PropList */ + UCD_PROPERTY_NONCHARACTER_CODE_POINT = 0x00020000, /**< @brief Noncharacter_Code_Point PropList */ } ucd_property; /** @brief Return the properties of the specified codepoint. @@ -806,7 +807,7 @@ namespace ucd enum property { White_Space = UCD_PROPERTY_WHITE_SPACE, /**< @brief White_Space PropList */ - noBreak = UCD_PROPERTY_NO_BREAK, /**< @brief DispositionType (enabled check only) */ + noBreak = UCD_PROPERTY_NO_BREAK, /**< @brief DispositionType (enabled check) */ Bidi_Control = UCD_PROPERTY_BIDI_CONTROL, /**< @brief Bidi_Control PropList */ Join_Control = UCD_PROPERTY_JOIN_CONTROL, /**< @brief Join_Control PropList */ Dash = UCD_PROPERTY_DASH, /**< @brief Dash PropList */ @@ -822,6 +823,7 @@ namespace ucd Extender = UCD_PROPERTY_EXTENDER, /**< @brief Extender PropList */ Other_Lowercase = UCD_PROPERTY_OTHER_LOWERCASE, /**< @brief Other_Lowercase PropList */ Other_Uppercase = UCD_PROPERTY_OTHER_UPPERCASE, /**< @brief Other_Uppercase PropList */ + Noncharacter_Code_Point = UCD_PROPERTY_NONCHARACTER_CODE_POINT, /**< @brief Noncharacter_Code_Point PropList */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index 525b45e9..5059bc2f 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -48,6 +48,13 @@ static int properties_Cf(codepoint_t c) return 0; } +static int properties_Cn(codepoint_t c) +{ + if (c >= 0xFDD0 && c <= 0xFDEF) return UCD_PROPERTY_NONCHARACTER_CODE_POINT; + if ((c & 0x0000FFFF) >= 0xFFFE) return UCD_PROPERTY_NONCHARACTER_CODE_POINT; + return 0; +} + static int properties_Ll(codepoint_t c) { switch (c & 0xFFFFFF00) @@ -1518,6 +1525,7 @@ ucd_property ucd_properties(codepoint_t c, ucd_category category) { case UCD_CATEGORY_Cc: return properties_Cc(c); case UCD_CATEGORY_Cf: return properties_Cf(c); + case UCD_CATEGORY_Cn: return properties_Cn(c); case UCD_CATEGORY_Ll: return properties_Ll(c); case UCD_CATEGORY_Lm: return properties_Lm(c); case UCD_CATEGORY_Lo: return properties_Lo(c) | properties_Lo_ideographic(c); @@ -1539,6 +1547,6 @@ ucd_property ucd_properties(codepoint_t c, ucd_category category) case UCD_CATEGORY_Zl: return UCD_PROPERTY_WHITE_SPACE; case UCD_CATEGORY_Zp: return UCD_PROPERTY_WHITE_SPACE; case UCD_CATEGORY_Zs: return properties_Zs(c); - default: return 0; // Cn Co Cs Ii Lt Me No Sc + default: return 0; // Co Cs Ii Lt Me No Sc }; } diff --git a/tools/printdata.py b/tools/printdata.py index 58b31fe7..400911d0 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -120,24 +120,25 @@ def decomposition_type(data, dtype): return None def properties(data): - props = 0 - props += 1 * data.get('White_Space', 0) - props += 2 * (decomposition_type(data, '') != None) - props += 4 * data.get('Bidi_Control', 0) - props += 8 * data.get('Join_Control', 0) - props += 16 * data.get('Dash', 0) - props += 32 * data.get('Hyphen', 0) - props += 64 * data.get('Quotation_Mark', 0) - props += 128 * data.get('Terminal_Punctuation', 0) - props += 256 * data.get('Other_Math', 0) - props += 512 * data.get('Hex_Digit', 0) - props += 1024 * data.get('ASCII_Hex_Digit', 0) - props += 2048 * data.get('Other_Alphabetic', 0) - props += 4096 * data.get('Ideographic', 0) - props += 8192 * data.get('Diacritic', 0) - props += 16384 * data.get('Extender', 0) - props += 32768 * data.get('Other_Lowercase', 0) - props += 65536 * data.get('Other_Uppercase', 0) + props = 0 + props += 1 * data.get('White_Space', 0) + props += 2 * (decomposition_type(data, '') != None) + props += 4 * data.get('Bidi_Control', 0) + props += 8 * data.get('Join_Control', 0) + props += 16 * data.get('Dash', 0) + props += 32 * data.get('Hyphen', 0) + props += 64 * data.get('Quotation_Mark', 0) + props += 128 * data.get('Terminal_Punctuation', 0) + props += 256 * data.get('Other_Math', 0) + props += 512 * data.get('Hex_Digit', 0) + props += 1024 * data.get('ASCII_Hex_Digit', 0) + props += 2048 * data.get('Other_Alphabetic', 0) + props += 4096 * data.get('Ideographic', 0) + props += 8192 * data.get('Diacritic', 0) + props += 16384 * data.get('Extender', 0) + props += 32768 * data.get('Other_Lowercase', 0) + props += 65536 * data.get('Other_Uppercase', 0) + props += 131072 * data.get('Noncharacter_Code_Point', 0) return props if __name__ == '__main__': From 6553494c5cc3e104f0e03229bd1c100793936b7e Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Fri, 14 Apr 2017 13:36:35 +0100 Subject: [PATCH 24/44] Remove noBreak from ucd_properties -- it is a DispositionType, not a PropList type. --- src/ctype.c | 30 +++++++++++++++++-- src/include/ucd/ucd.h | 70 +++++++++++++++++++++---------------------- src/proplist.c | 18 ++--------- tools/printdata.py | 37 +++++++++++------------ 4 files changed, 82 insertions(+), 73 deletions(-) diff --git a/src/ctype.c b/src/ctype.c index 373d15e3..4632b55e 100644 --- a/src/ctype.c +++ b/src/ctype.c @@ -437,8 +437,34 @@ int ucd_ispunct(codepoint_t c) int ucd_isspace(codepoint_t c) { - ucd_property props = ucd_properties(c, ucd_lookup_category(c)); - return (props & (UCD_PROPERTY_WHITE_SPACE | UCD_PROPERTY_NO_BREAK)) == UCD_PROPERTY_WHITE_SPACE; + switch (ucd_lookup_category(c)) + { + case UCD_CATEGORY_Zl: + case UCD_CATEGORY_Zp: + return 1; + case UCD_CATEGORY_Zs: + switch (c) // Exclude characters with the DispositionType + { + case 0x00A0: // U+00A0 : NO-BREAK SPACE + case 0x2007: // U+2007 : FIGURE SPACE + case 0x202F: // U+202F : NARROW NO-BREAK SPACE + return 0; + } + return 1; + case UCD_CATEGORY_Cc: + switch (c) // Include control characters marked as White_Space + { + case 0x09: // U+0009 : CHARACTER TABULATION + case 0x0A: // U+000A : LINE FEED + case 0x0B: // U+000B : LINE TABULATION + case 0x0C: // U+000C : FORM FEED + case 0x0D: // U+000D : CARRIAGE RETURN + case 0x85: // U+0085 : NEXT LINE + return 1; + } + default: + return 0; + } } int ucd_isupper(codepoint_t c) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 0fc9cefa..7296a45b 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -329,24 +329,23 @@ ucd_script ucd_lookup_script(codepoint_t c); */ typedef enum ucd_property_ { - UCD_PROPERTY_WHITE_SPACE = 0x00000001, /**< @brief White_Space PropList */ - UCD_PROPERTY_NO_BREAK = 0x00000002, /**< @brief DispositionType (enabled check) */ - UCD_PROPERTY_BIDI_CONTROL = 0x00000004, /**< @brief Bidi_Control PropList */ - UCD_PROPERTY_JOIN_CONTROL = 0x00000008, /**< @brief Join_Control PropList */ - UCD_PROPERTY_DASH = 0x00000010, /**< @brief Dash PropList */ - UCD_PROPERTY_HYPHEN = 0x00000020, /**< @brief Hyphen PropList */ - UCD_PROPERTY_QUOTATION_MARK = 0x00000040, /**< @brief Quotation_Mark PropList */ - UCD_PROPERTY_TERMINAL_PUNCTUATION = 0x00000080, /**< @brief Terminal_Punctuation PropList */ - UCD_PROPERTY_OTHER_MATH = 0x00000100, /**< @brief Other_Math PropList */ - UCD_PROPERTY_HEX_DIGIT = 0x00000200, /**< @brief Hex_Digit PropList */ - UCD_PROPERTY_ASCII_HEX_DIGIT = 0x00000400, /**< @brief ASCII_Hex_Digit PropList */ - UCD_PROPERTY_OTHER_ALPHABETIC = 0x00000800, /**< @brief Other_Alphabetic PropList */ - UCD_PROPERTY_IDEOGRAPHIC = 0x00001000, /**< @brief Ideographic PropList */ - UCD_PROPERTY_DIACRITIC = 0x00002000, /**< @brief Diacritic PropList */ - UCD_PROPERTY_EXTENDER = 0x00004000, /**< @brief Extender PropList */ - UCD_PROPERTY_OTHER_LOWERCASE = 0x00008000, /**< @brief Other_Lowercase PropList */ - UCD_PROPERTY_OTHER_UPPERCASE = 0x00010000, /**< @brief Other_Uppercase PropList */ - UCD_PROPERTY_NONCHARACTER_CODE_POINT = 0x00020000, /**< @brief Noncharacter_Code_Point PropList */ + UCD_PROPERTY_WHITE_SPACE = 0x00000001, /**< @brief White_Space */ + UCD_PROPERTY_BIDI_CONTROL = 0x00000002, /**< @brief Bidi_Control */ + UCD_PROPERTY_JOIN_CONTROL = 0x00000004, /**< @brief Join_Control */ + UCD_PROPERTY_DASH = 0x00000008, /**< @brief Dash */ + UCD_PROPERTY_HYPHEN = 0x00000010, /**< @brief Hyphen */ + UCD_PROPERTY_QUOTATION_MARK = 0x00000020, /**< @brief Quotation_Mark */ + UCD_PROPERTY_TERMINAL_PUNCTUATION = 0x00000040, /**< @brief Terminal_Punctuation */ + UCD_PROPERTY_OTHER_MATH = 0x00000080, /**< @brief Other_Math */ + UCD_PROPERTY_HEX_DIGIT = 0x00000100, /**< @brief Hex_Digit */ + UCD_PROPERTY_ASCII_HEX_DIGIT = 0x00000200, /**< @brief ASCII_Hex_Digit */ + UCD_PROPERTY_OTHER_ALPHABETIC = 0x00000400, /**< @brief Other_Alphabetic */ + UCD_PROPERTY_IDEOGRAPHIC = 0x00000800, /**< @brief Ideographic */ + UCD_PROPERTY_DIACRITIC = 0x00001000, /**< @brief Diacritic */ + UCD_PROPERTY_EXTENDER = 0x00002000, /**< @brief Extender */ + UCD_PROPERTY_OTHER_LOWERCASE = 0x00004000, /**< @brief Other_Lowercase */ + UCD_PROPERTY_OTHER_UPPERCASE = 0x00008000, /**< @brief Other_Uppercase */ + UCD_PROPERTY_NONCHARACTER_CODE_POINT = 0x00010000, /**< @brief Noncharacter_Code_Point */ } ucd_property; /** @brief Return the properties of the specified codepoint. @@ -806,24 +805,23 @@ namespace ucd */ enum property { - White_Space = UCD_PROPERTY_WHITE_SPACE, /**< @brief White_Space PropList */ - noBreak = UCD_PROPERTY_NO_BREAK, /**< @brief DispositionType (enabled check) */ - Bidi_Control = UCD_PROPERTY_BIDI_CONTROL, /**< @brief Bidi_Control PropList */ - Join_Control = UCD_PROPERTY_JOIN_CONTROL, /**< @brief Join_Control PropList */ - Dash = UCD_PROPERTY_DASH, /**< @brief Dash PropList */ - Hyphen = UCD_PROPERTY_HYPHEN, /**< @brief Hyphen PropList */ - Quotation_Mark = UCD_PROPERTY_QUOTATION_MARK, /**< @brief Quotation_Mark PropList */ - Terminal_Punctuation = UCD_PROPERTY_TERMINAL_PUNCTUATION, /**< @brief Terminal_Punctuation PropList */ - Other_Math = UCD_PROPERTY_OTHER_MATH, /**< @brief Other_Math PropList */ - Hex_Digit = UCD_PROPERTY_HEX_DIGIT, /**< @brief Hex_Digit PropList */ - ASCII_Hex_Digit = UCD_PROPERTY_ASCII_HEX_DIGIT, /**< @brief ASCII_Hex_Digit PropList */ - Other_Alphabetic = UCD_PROPERTY_OTHER_ALPHABETIC, /**< @brief Other_Alphabetic PropList */ - Ideographic = UCD_PROPERTY_IDEOGRAPHIC, /**< @brief Ideographic PropList */ - Diacritic = UCD_PROPERTY_DIACRITIC, /**< @brief Diacritic PropList */ - Extender = UCD_PROPERTY_EXTENDER, /**< @brief Extender PropList */ - Other_Lowercase = UCD_PROPERTY_OTHER_LOWERCASE, /**< @brief Other_Lowercase PropList */ - Other_Uppercase = UCD_PROPERTY_OTHER_UPPERCASE, /**< @brief Other_Uppercase PropList */ - Noncharacter_Code_Point = UCD_PROPERTY_NONCHARACTER_CODE_POINT, /**< @brief Noncharacter_Code_Point PropList */ + White_Space = UCD_PROPERTY_WHITE_SPACE, /**< @brief White_Space */ + Bidi_Control = UCD_PROPERTY_BIDI_CONTROL, /**< @brief Bidi_Control */ + Join_Control = UCD_PROPERTY_JOIN_CONTROL, /**< @brief Join_Control */ + Dash = UCD_PROPERTY_DASH, /**< @brief Dash */ + Hyphen = UCD_PROPERTY_HYPHEN, /**< @brief Hyphen */ + Quotation_Mark = UCD_PROPERTY_QUOTATION_MARK, /**< @brief Quotation_Mark */ + Terminal_Punctuation = UCD_PROPERTY_TERMINAL_PUNCTUATION, /**< @brief Terminal_Punctuation */ + Other_Math = UCD_PROPERTY_OTHER_MATH, /**< @brief Other_Math */ + Hex_Digit = UCD_PROPERTY_HEX_DIGIT, /**< @brief Hex_Digit */ + ASCII_Hex_Digit = UCD_PROPERTY_ASCII_HEX_DIGIT, /**< @brief ASCII_Hex_Digit */ + Other_Alphabetic = UCD_PROPERTY_OTHER_ALPHABETIC, /**< @brief Other_Alphabetic */ + Ideographic = UCD_PROPERTY_IDEOGRAPHIC, /**< @brief Ideographic */ + Diacritic = UCD_PROPERTY_DIACRITIC, /**< @brief Diacritic */ + Extender = UCD_PROPERTY_EXTENDER, /**< @brief Extender */ + Other_Lowercase = UCD_PROPERTY_OTHER_LOWERCASE, /**< @brief Other_Lowercase */ + Other_Uppercase = UCD_PROPERTY_OTHER_UPPERCASE, /**< @brief Other_Uppercase */ + Noncharacter_Code_Point = UCD_PROPERTY_NONCHARACTER_CODE_POINT, /**< @brief Noncharacter_Code_Point */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index 5059bc2f..a1d971ba 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -1043,8 +1043,7 @@ static int properties_Pd(codepoint_t c) if (c == 0x1806) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; break; case 0x2000: - if (c == 0x2010) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; - if (c == 0x2011) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN | UCD_PROPERTY_NO_BREAK; + if (c >= 0x2010 && c <= 0x2011) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; break; case 0x2E00: if (c == 0x2E17) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; @@ -1174,7 +1173,6 @@ static int properties_Po(codepoint_t c) break; case 0x0F00: if (c == 0x0F08) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0x0F0C) return UCD_PROPERTY_NO_BREAK; if (c >= 0x0F0D && c <= 0x0F12) return UCD_PROPERTY_TERMINAL_PUNCTUATION; break; case 0x1000: @@ -1507,18 +1505,6 @@ static int properties_So(codepoint_t c) return 0; } -static int properties_Zs(codepoint_t c) -{ - switch (c) - { - case 0x00A0: // NO-BREAK SPACE - case 0x2007: // FIGURE SPACE - case 0x202F: // NARROW NO-BREAK SPACE - return UCD_PROPERTY_WHITE_SPACE | UCD_PROPERTY_NO_BREAK; - } - return UCD_PROPERTY_WHITE_SPACE; -} - ucd_property ucd_properties(codepoint_t c, ucd_category category) { switch (category) @@ -1546,7 +1532,7 @@ ucd_property ucd_properties(codepoint_t c, ucd_category category) case UCD_CATEGORY_So: return properties_So(c); case UCD_CATEGORY_Zl: return UCD_PROPERTY_WHITE_SPACE; case UCD_CATEGORY_Zp: return UCD_PROPERTY_WHITE_SPACE; - case UCD_CATEGORY_Zs: return properties_Zs(c); + case UCD_CATEGORY_Zs: return UCD_PROPERTY_WHITE_SPACE; default: return 0; // Co Cs Ii Lt Me No Sc }; } diff --git a/tools/printdata.py b/tools/printdata.py index 400911d0..191f9967 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -120,25 +120,24 @@ def decomposition_type(data, dtype): return None def properties(data): - props = 0 - props += 1 * data.get('White_Space', 0) - props += 2 * (decomposition_type(data, '') != None) - props += 4 * data.get('Bidi_Control', 0) - props += 8 * data.get('Join_Control', 0) - props += 16 * data.get('Dash', 0) - props += 32 * data.get('Hyphen', 0) - props += 64 * data.get('Quotation_Mark', 0) - props += 128 * data.get('Terminal_Punctuation', 0) - props += 256 * data.get('Other_Math', 0) - props += 512 * data.get('Hex_Digit', 0) - props += 1024 * data.get('ASCII_Hex_Digit', 0) - props += 2048 * data.get('Other_Alphabetic', 0) - props += 4096 * data.get('Ideographic', 0) - props += 8192 * data.get('Diacritic', 0) - props += 16384 * data.get('Extender', 0) - props += 32768 * data.get('Other_Lowercase', 0) - props += 65536 * data.get('Other_Uppercase', 0) - props += 131072 * data.get('Noncharacter_Code_Point', 0) + props = 0 + props += (2 ** 0) * data.get('White_Space', 0) + props += (2 ** 1) * data.get('Bidi_Control', 0) + props += (2 ** 2) * data.get('Join_Control', 0) + props += (2 ** 3) * data.get('Dash', 0) + props += (2 ** 4) * data.get('Hyphen', 0) + props += (2 ** 5) * data.get('Quotation_Mark', 0) + props += (2 ** 6) * data.get('Terminal_Punctuation', 0) + props += (2 ** 7) * data.get('Other_Math', 0) + props += (2 ** 8) * data.get('Hex_Digit', 0) + props += (2 ** 9) * data.get('ASCII_Hex_Digit', 0) + props += (2 ** 10) * data.get('Other_Alphabetic', 0) + props += (2 ** 11) * data.get('Ideographic', 0) + props += (2 ** 12) * data.get('Diacritic', 0) + props += (2 ** 13) * data.get('Extender', 0) + props += (2 ** 14) * data.get('Other_Lowercase', 0) + props += (2 ** 15) * data.get('Other_Uppercase', 0) + props += (2 ** 16) * data.get('Noncharacter_Code_Point', 0) return props if __name__ == '__main__': From 7fc8ee0e09f8fdf724240e7f2e85d09bf7d7b3d6 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Fri, 14 Apr 2017 13:47:35 +0100 Subject: [PATCH 25/44] Use ucd_properties to implement isalnum, isalpha, isupper and islower. --- src/ctype.c | 298 +++------------------------------------------------- 1 file changed, 13 insertions(+), 285 deletions(-) diff --git a/src/ctype.c b/src/ctype.c index 4632b55e..5c8b025c 100644 --- a/src/ctype.c +++ b/src/ctype.c @@ -20,262 +20,10 @@ #include "ucd/ucd.h" -static int other_alphabetic_MnMcSo(codepoint_t c) -{ - switch (c & 0xFFFFFF00) - { - case 0x0300: - return c == 0x0345; - case 0x0500: - return (c >= 0x05B0 && c <= 0x05BD) - || c == 0x05BF - || (c >= 0x05C1 && c <= 0x05C2) - || (c >= 0x05C4 && c <= 0x05C5) - || c == 0x05C7; - case 0x0600: - return (c >= 0x0610 && c <= 0x061A) - || (c >= 0x064B && c <= 0x0657) - || (c >= 0x0659 && c <= 0x065F) - || c == 0x0670 - || (c >= 0x06D6 && c <= 0x06DC) - || (c >= 0x06E1 && c <= 0x06E4) - || (c >= 0x06E7 && c <= 0x06E8) - || c == 0x06ED; - case 0x0700: - return c == 0x0711 - || (c >= 0x0730 && c <= 0x073F) - || (c >= 0x07A6 && c <= 0x07B0); - case 0x0800: - return (c >= 0x0816 && c <= 0x0817) - || (c >= 0x081B && c <= 0x0823) - || (c >= 0x0825 && c <= 0x0827) - || (c >= 0x0829 && c <= 0x082C) - || (c >= 0x08D4 && c <= 0x08DF) - || (c >= 0x08E3 && c <= 0x08E9) - || c >= 0x08F0; - case 0x0900: - return (c >= 0x0900 && c <= 0x0903) - || (c >= 0x093A && c <= 0x093B) - || (c >= 0x093E && c <= 0x094C) - || (c >= 0x094E && c <= 0x094F) - || (c >= 0x0955 && c <= 0x0957) - || (c >= 0x0962 && c <= 0x0963) - || (c >= 0x0981 && c <= 0x0983) - || (c >= 0x0981 && c <= 0x0983) - || (c >= 0x09BE && c <= 0x09C4) - || (c >= 0x09C7 && c <= 0x09C8) - || (c >= 0x09CB && c <= 0x09CC) - || c == 0x09D7 - || (c >= 0x09E2 && c <= 0x09E3); - case 0x0A00: - return (c >= 0x0A01 && c <= 0x0A03) - || (c >= 0x0A3E && c <= 0x0A42) - || (c >= 0x0A47 && c <= 0x0A48) - || (c >= 0x0A4B && c <= 0x0A4C) - || c == 0x0A51 - || (c >= 0x0A70 && c <= 0x0A71) - || c == 0x0A75 - || (c >= 0x0A81 && c <= 0x0A83) - || (c >= 0x0ABE && c <= 0x0AC5) - || (c >= 0x0AC7 && c <= 0x0AC9) - || (c >= 0x0ACB && c <= 0x0ACC) - || (c >= 0x0AE2 && c <= 0x0AE3); - case 0x0B00: - return (c >= 0x0B01 && c <= 0x0B03) - || (c >= 0x0B3E && c <= 0x0B44) - || (c >= 0x0B47 && c <= 0x0B48) - || (c >= 0x0B4B && c <= 0x0B4C) - || (c >= 0x0B56 && c <= 0x0B57) - || (c >= 0x0B62 && c <= 0x0B63) - || c == 0x0B82 - || (c >= 0x0BBE && c <= 0x0BC2) - || (c >= 0x0BC6 && c <= 0x0BC8) - || (c >= 0x0BCA && c <= 0x0BCC) - || c == 0x0BD7; - case 0x0C00: - return (c >= 0x0C00 && c <= 0x0C03) - || (c >= 0x0C3E && c <= 0x0C44) - || (c >= 0x0C46 && c <= 0x0C48) - || (c >= 0x0C4A && c <= 0x0C4C) - || (c >= 0x0C55 && c <= 0x0C56) - || (c >= 0x0C62 && c <= 0x0C63) - || (c >= 0x0C81 && c <= 0x0C83) - || (c >= 0x0CBE && c <= 0x0CBF) - || (c >= 0x0CC0 && c <= 0x0CC4) - || (c >= 0x0CC6 && c <= 0x0CC8) - || (c >= 0x0CCA && c <= 0x0CCC) - || (c >= 0x0CD5 && c <= 0x0CD6) - || (c >= 0x0CE2 && c <= 0x0CE3); - case 0x0D00: - return (c >= 0x0D01 && c <= 0x0D03) - || (c >= 0x0D3E && c <= 0x0D44) - || (c >= 0x0D46 && c <= 0x0D48) - || (c >= 0x0D4A && c <= 0x0D4C) - || c == 0x0D57 - || (c >= 0x0D62 && c <= 0x0D63) - || (c >= 0x0D82 && c <= 0x0D83) - || (c >= 0x0DCF && c <= 0x0DD4) - || c == 0x0DD6 - || (c >= 0x0DD8 && c <= 0x0DDF) - || (c >= 0x0DF2 && c <= 0x0DF3); - case 0x0E00: - return c == 0x0E31 - || (c >= 0x0E34 && c <= 0x0E3A) - || c == 0x0E4D - || c == 0x0EB1 - || (c >= 0x0EB4 && c <= 0x0EB9) - || (c >= 0x0EBB && c <= 0x0EBD) - || c == 0x0ECD; - case 0x0F00: - return (c >= 0x0F71 && c <= 0x0F7F) - || (c >= 0x0F80 && c <= 0x0F81) - || (c >= 0x0F8D && c <= 0x0F97) - || (c >= 0x0F99 && c <= 0x0FBC); - case 0x1000: - return (c >= 0x102B && c <= 0x1036) - || c == 0x1038 - || (c >= 0x103B && c <= 0x103E) - || (c >= 0x1056 && c <= 0x1059) - || (c >= 0x105E && c <= 0x1060) - || c == 0x1062 - || (c >= 0x1067 && c <= 0x1068) - || (c >= 0x1071 && c <= 0x1074) - || (c >= 0x1082 && c <= 0x1086) - || (c >= 0x109C && c <= 0x109D); - case 0x1300: - return c == 0x135F; - case 0x1700: - return (c >= 0x1712 && c <= 0x1713) - || (c >= 0x1732 && c <= 0x1733) - || (c >= 0x1752 && c <= 0x1753) - || (c >= 0x1772 && c <= 0x1773) - || (c >= 0x17B6 && c <= 0x17C8); - case 0x1800: - return (c >= 0x1885 && c <= 0x1886) - || c == 0x18A9; - case 0x1900: - return (c >= 0x1920 && c <= 0x192B) - || (c >= 0x1930 && c <= 0x1938); - case 0x1A00: - return (c >= 0x1A17 && c <= 0x1A1B) - || (c >= 0x1A55 && c <= 0x1A5E) - || (c >= 0x1A61 && c <= 0x1A74); - case 0x1B00: - return (c >= 0x1B00 && c <= 0x1B04) - || (c >= 0x1B35 && c <= 0x1B43) - || (c >= 0x1B80 && c <= 0x1B82) - || (c >= 0x1BA1 && c <= 0x1BA9) - || (c >= 0x1BAC && c <= 0x1BAD) - || (c >= 0x1BE7 && c <= 0x1BF1); - case 0x1C00: - return (c >= 0x1C24 && c <= 0x1C35) - || (c >= 0x1CF2 && c <= 0x1CF3); - case 0x1D00: - return (c >= 0x1DE7 && c <= 0x1DF4); - case 0x2400: - return (c >= 0x24B6 && c <= 0x24E9); - case 0x2D00: - return (c >= 0x2DE0 && c <= 0x2DFF); - case 0xA600: - return (c >= 0xA674 && c <= 0xA67B) - || (c >= 0xA69E && c <= 0xA69F); - case 0xA800: - return (c >= 0xA823 && c <= 0xA827) - || (c >= 0xA880 && c <= 0xA881) - || (c >= 0xA8B4 && c <= 0xA8C3) - || c == 0xA8C5; - case 0xA900: - return (c >= 0xA926 && c <= 0xA92A) - || (c >= 0xA947 && c <= 0xA952) - || (c >= 0xA980 && c <= 0xA983) - || (c >= 0xA9B4 && c <= 0xA9BF); - case 0xAA00: - return (c >= 0xAA29 && c <= 0xAA36) - || c == 0xAA43 - || (c >= 0xAA4C && c <= 0xAA4D) - || c == 0xAAB0 - || (c >= 0xAAB2 && c <= 0xAAB4) - || (c >= 0xAAB7 && c <= 0xAAB8) - || c == 0xAABE - || (c >= 0xAAEB && c <= 0xAAEF) - || c == 0xAAF5; - case 0xAB00: - return (c >= 0xABE3 && c <= 0xABEA); - case 0xFB00: - return c == 0xFB1E; - case 0x10300: - return (c >= 0x10376 && c <= 0x1037A); - case 0x10A00: - return (c >= 0x10A01 && c <= 0x10A03) - || (c >= 0x10A05 && c <= 0x10A06) - || (c >= 0x10A0C && c <= 0x10A0F); - case 0x11000: - return (c >= 0x11000 && c <= 0x11002) - || (c >= 0x11038 && c <= 0x11045) - || c == 0x11082 - || (c >= 0x110B0 && c <= 0x110B8); - case 0x11100: - return (c >= 0x11100 && c <= 0x11102) - || (c >= 0x11127 && c <= 0x11132) - || (c >= 0x11180 && c <= 0x11182) - || (c >= 0x111B3 && c <= 0x111BF); - case 0x11200: - return (c >= 0x1122C && c <= 0x11234) - || c == 0x11237 - || c == 0x1123E - || (c >= 0x112DF && c <= 0x112E8); - case 0x11300: - return (c >= 0x11300 && c <= 0x11303) - || (c >= 0x1133E && c <= 0x11344) - || (c >= 0x11347 && c <= 0x11348) - || (c >= 0x1134B && c <= 0x1134C) - || c == 0x11357 - || (c >= 0x11362 && c <= 0x11363); - case 0x11400: - return (c >= 0x11435 && c <= 0x11441) - || (c >= 0x11443 && c <= 0x11445) - || (c >= 0x114B0 && c <= 0x114C1); - case 0x11500: - return (c >= 0x115AF && c <= 0x115B5) - || (c >= 0x115B8 && c <= 0x115BE) - || (c >= 0x115DC && c <= 0x115DD); - case 0x11600: - return (c >= 0x11630 && c <= 0x1163E) - || c == 0x11640 - || (c >= 0x116AB && c <= 0x116B5); - case 0x11700: - return (c >= 0x1171D && c <= 0x1172A); - case 0x11C00: - return (c >= 0x11C2F && c <= 0x11C36) - || (c >= 0x11C38 && c <= 0x11C3E) - || (c >= 0x11C92 && c <= 0x11CA7) - || (c >= 0x11CA9 && c <= 0x11CB6); - case 0x16B00: - return (c >= 0x16B30 && c <= 0x16B36); - case 0x16F00: - return (c >= 0x16F51 && c <= 0x16F7E); - case 0x1BC00: - return c == 0x1BC9E; - case 0x1E000: - return (c >= 0x1E000 && c <= 0x1E006) - || (c >= 0x1E008 && c <= 0x1E018) - || (c >= 0x1E01B && c <= 0x1E021) - || (c >= 0x1E023 && c <= 0x1E024) - || (c >= 0x1E026 && c <= 0x1E02A); - case 0x1E900: - return c == 0x1E947; - case 0x1F100: - return (c >= 0x01F130 && c <= 0x01F149) - || (c >= 0x01F150 && c <= 0x01F169) - || (c >= 0x01F170 && c <= 0x01F189); - default: - return 0; - } -} - int ucd_isalnum(codepoint_t c) { - switch (ucd_lookup_category(c)) + ucd_category cat = ucd_lookup_category(c); + switch (cat) { case UCD_CATEGORY_Lu: case UCD_CATEGORY_Ll: @@ -289,7 +37,7 @@ int ucd_isalnum(codepoint_t c) case UCD_CATEGORY_Mn: case UCD_CATEGORY_Mc: case UCD_CATEGORY_So: - return other_alphabetic_MnMcSo(c); + return (ucd_properties(c, cat) & UCD_PROPERTY_OTHER_ALPHABETIC) == UCD_PROPERTY_OTHER_ALPHABETIC; default: return 0; } @@ -297,7 +45,8 @@ int ucd_isalnum(codepoint_t c) int ucd_isalpha(codepoint_t c) { - switch (ucd_lookup_category(c)) + ucd_category cat = ucd_lookup_category(c); + switch (cat) { case UCD_CATEGORY_Lu: case UCD_CATEGORY_Ll: @@ -309,7 +58,7 @@ int ucd_isalpha(codepoint_t c) case UCD_CATEGORY_Mn: case UCD_CATEGORY_Mc: case UCD_CATEGORY_So: - return other_alphabetic_MnMcSo(c); + return (ucd_properties(c, cat) & UCD_PROPERTY_OTHER_ALPHABETIC) == UCD_PROPERTY_OTHER_ALPHABETIC; default: return 0; } @@ -366,37 +115,19 @@ int ucd_isgraph(codepoint_t c) int ucd_islower(codepoint_t c) { - switch (ucd_lookup_category(c)) + ucd_category cat = ucd_lookup_category(c); + switch (cat) { case UCD_CATEGORY_Ll: return 1; case UCD_CATEGORY_Lt: return ucd_toupper(c) != c; - case UCD_CATEGORY_Lo: - return c == 0xAA // Other_Lowercase : FEMININE ORDINAL INDICATOR - || c == 0xBA; // Other_Lowercase : MASCULINE ORDINAL INDICATOR case UCD_CATEGORY_Lm: - return (c >= 0x02B0 && c <= 0x02B8) // Other_Lowercase - || (c >= 0x02C0 && c <= 0x02C1) // Other_Lowercase - || (c >= 0x02E0 && c <= 0x02E4) // Other_Lowercase - || c == 0x037A // Other_Lowercase - || (c >= 0x1D2C && c <= 0x1D6A) // Other_Lowercase - || c == 0x1D78 // Other_Lowercase - || (c >= 0x1D9B && c <= 0x1DBF) // Other_Lowercase - || c == 0x2071 // Other_Lowercase - || c == 0x207F // Other_Lowercase - || (c >= 0x2090 && c <= 0x209C) // Other_Lowercase - || (c >= 0x2C7C && c <= 0x2C7D) // Other_Lowercase - || (c >= 0xA69C && c <= 0xA69D) // Other_Lowercase - || c == 0xA770 // Other_Lowercase - || (c >= 0xA7F8 && c <= 0xA7F9) // Other_Lowercase - || (c >= 0xAB5C && c <= 0xAB5F); // Other_Lowercase + case UCD_CATEGORY_Lo: case UCD_CATEGORY_Mn: - return c == 0x0345; // Other_Lowercase : COMBINING GREEK YPOGEGRAMMENI case UCD_CATEGORY_Nl: - return (c >= 0x2170 && c <= 0x217F); // Other_Lowercase case UCD_CATEGORY_So: - return (c >= 0x24D0 && c <= 0x24E9); // Other_Lowercase + return (ucd_properties(c, cat) & UCD_PROPERTY_OTHER_LOWERCASE) == UCD_PROPERTY_OTHER_LOWERCASE; default: return 0; } @@ -469,19 +200,16 @@ int ucd_isspace(codepoint_t c) int ucd_isupper(codepoint_t c) { - switch (ucd_lookup_category(c)) + ucd_category cat = ucd_lookup_category(c); + switch (cat) { case UCD_CATEGORY_Lu: return 1; case UCD_CATEGORY_Lt: return ucd_tolower(c) != c; case UCD_CATEGORY_Nl: - return (c >= 0x002160 && c <= 0x00216F); // Other_Uppercase case UCD_CATEGORY_So: - return (c >= 0x0024B6 && c <= 0x0024CF) // Other_Uppercase - || (c >= 0x01F130 && c <= 0x01F149) // Other_Uppercase - || (c >= 0x01F150 && c <= 0x01F169) // Other_Uppercase - || (c >= 0x01F170 && c <= 0x01F189); // Other_Uppercase + return (ucd_properties(c, cat) & UCD_PROPERTY_OTHER_UPPERCASE) == UCD_PROPERTY_OTHER_UPPERCASE; default: return 0; } From fdf398ef845dc37b0d2e286041a0349b85c0f3c4 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Fri, 14 Apr 2017 14:44:10 +0100 Subject: [PATCH 26/44] Add Other_Grapheme_Extend support from PropList.txt. --- src/include/ucd/ucd.h | 2 ++ src/proplist.c | 60 ++++++++++++++++++++++++++++--------------- tools/printdata.py | 1 + 3 files changed, 42 insertions(+), 21 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 7296a45b..d19a0f9e 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -346,6 +346,7 @@ typedef enum ucd_property_ UCD_PROPERTY_OTHER_LOWERCASE = 0x00004000, /**< @brief Other_Lowercase */ UCD_PROPERTY_OTHER_UPPERCASE = 0x00008000, /**< @brief Other_Uppercase */ UCD_PROPERTY_NONCHARACTER_CODE_POINT = 0x00010000, /**< @brief Noncharacter_Code_Point */ + UCD_PROPERTY_OTHER_GRAPHEME_EXTEND = 0x00020000, /**< @brief Other_Grapheme_Extend */ } ucd_property; /** @brief Return the properties of the specified codepoint. @@ -822,6 +823,7 @@ namespace ucd Other_Lowercase = UCD_PROPERTY_OTHER_LOWERCASE, /**< @brief Other_Lowercase */ Other_Uppercase = UCD_PROPERTY_OTHER_UPPERCASE, /**< @brief Other_Uppercase */ Noncharacter_Code_Point = UCD_PROPERTY_NONCHARACTER_CODE_POINT, /**< @brief Noncharacter_Code_Point */ + Other_Grapheme_Extend = UCD_PROPERTY_OTHER_GRAPHEME_EXTEND, /**< @brief Other_Grapheme_Extend */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index a1d971ba..429dcc73 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -38,12 +38,16 @@ static int properties_Cf(codepoint_t c) if (c == 0x061C) return UCD_PROPERTY_BIDI_CONTROL; break; case 0x2000: - if (c >= 0x200C && c <= 0x200D) return UCD_PROPERTY_JOIN_CONTROL; + if (c == 0x200C) return UCD_PROPERTY_JOIN_CONTROL | UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; + if (c == 0x200D) return UCD_PROPERTY_JOIN_CONTROL; if (c >= 0x200E && c <= 0x200F) return UCD_PROPERTY_BIDI_CONTROL; if (c >= 0x202A && c <= 0x202E) return UCD_PROPERTY_BIDI_CONTROL; if (c >= 0x2061 && c <= 0x2064) return UCD_PROPERTY_OTHER_MATH; if (c >= 0x2066 && c <= 0x2069) return UCD_PROPERTY_BIDI_CONTROL; break; + case 0x0E0000: + if (c >= 0x0E0020 && c <= 0x0E007F) return UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; + break; } return 0; } @@ -208,7 +212,7 @@ static int properties_Lm(codepoint_t c) break; case 0xFF00: if (c == 0xFF70) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_EXTENDER; - if (c >= 0xFF9E && c <= 0xFF9F) return UCD_PROPERTY_DIACRITIC; + if (c >= 0xFF9E && c <= 0xFF9F) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; break; case 0x016B00: if (c >= 0x016B42 && c <= 0x016B43) return UCD_PROPERTY_EXTENDER; @@ -349,10 +353,11 @@ static int properties_Mc(codepoint_t c) if (c >= 0x0949 && c <= 0x094C) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x094E && c <= 0x094F) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0982 && c <= 0x0983) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c >= 0x09BE && c <= 0x09C0) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x09BE) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; + if (c >= 0x09BF && c <= 0x09C0) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x09C7 && c <= 0x09C8) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x09CB && c <= 0x09CC) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c == 0x09D7) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x09D7) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; break; case 0x0A00: if (c == 0x0A03) return UCD_PROPERTY_OTHER_ALPHABETIC; @@ -364,36 +369,42 @@ static int properties_Mc(codepoint_t c) break; case 0x0B00: if (c >= 0x0B02 && c <= 0x0B03) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c == 0x0B3E) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0B3E) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; if (c == 0x0B40) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0B47 && c <= 0x0B48) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0B4B && c <= 0x0B4C) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c == 0x0B57) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c >= 0x0BBE && c <= 0x0BBF) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0B57) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; + if (c == 0x0BBE) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; + if (c == 0x0BBF) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0BC1 && c <= 0x0BC2) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0BC6 && c <= 0x0BC8) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0BCA && c <= 0x0BCC) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c == 0x0BD7) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0BD7) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; break; case 0x0C00: if (c >= 0x0C01 && c <= 0x0C03) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0C41 && c <= 0x0C44) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0C82 && c <= 0x0C83) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x0CBE) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c >= 0x0CC0 && c <= 0x0CC4) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0CC0 && c <= 0x0CC1) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0CC2) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; + if (c >= 0x0CC3 && c <= 0x0CC4) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0CC7 && c <= 0x0CC8) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0CCA && c <= 0x0CCB) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c >= 0x0CD5 && c <= 0x0CD6) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0CD5 && c <= 0x0CD6) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; break; case 0x0D00: if (c >= 0x0D02 && c <= 0x0D03) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c >= 0x0D3E && c <= 0x0D40) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0D3E) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; + if (c >= 0x0D3F && c <= 0x0D40) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0D46 && c <= 0x0D48) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0D4A && c <= 0x0D4C) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c == 0x0D57) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0D57) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; if (c >= 0x0D82 && c <= 0x0D83) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c >= 0x0DCF && c <= 0x0DD1) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c >= 0x0DD8 && c <= 0x0DDF) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0DCF) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; + if (c >= 0x0DD0 && c <= 0x0DD1) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0DD8 && c <= 0x0DDE) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0DDF) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; if (c >= 0x0DF2 && c <= 0x0DF3) return UCD_PROPERTY_OTHER_ALPHABETIC; break; case 0x0F00: @@ -455,7 +466,7 @@ static int properties_Mc(codepoint_t c) if (c >= 0x1CF2 && c <= 0x1CF3) return UCD_PROPERTY_OTHER_ALPHABETIC; break; case 0x3000: - if (c >= 0x302E && c <= 0x302F) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x302E && c <= 0x302F) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; break; case 0xA800: if (c >= 0xA823 && c <= 0xA824) return UCD_PROPERTY_OTHER_ALPHABETIC; @@ -510,25 +521,30 @@ static int properties_Mc(codepoint_t c) break; case 0x011300: if (c >= 0x011302 && c <= 0x011303) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c >= 0x01133E && c <= 0x01133F) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x01133E) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; + if (c == 0x01133F) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x011341 && c <= 0x011344) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x011347 && c <= 0x011348) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x01134B && c <= 0x01134C) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x01134D) return UCD_PROPERTY_DIACRITIC; - if (c == 0x011357) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x011357) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; if (c >= 0x011362 && c <= 0x011363) return UCD_PROPERTY_OTHER_ALPHABETIC; break; case 0x011400: if (c >= 0x011435 && c <= 0x011437) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x011440 && c <= 0x011441) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x011445) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c >= 0x0114B0 && c <= 0x0114B2) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0114B0) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; + if (c >= 0x0114B1 && c <= 0x0114B2) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x0114B9) return UCD_PROPERTY_OTHER_ALPHABETIC; - if (c >= 0x0114BB && c <= 0x0114BE) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x0114BB && c <= 0x0114BC) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0114BD) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; + if (c == 0x0114BE) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x0114C1) return UCD_PROPERTY_OTHER_ALPHABETIC; break; case 0x011500: - if (c >= 0x0115AF && c <= 0x0115B1) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0115AF) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; + if (c >= 0x0115B0 && c <= 0x0115B1) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0115B8 && c <= 0x0115BB) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x0115BE) return UCD_PROPERTY_OTHER_ALPHABETIC; break; @@ -552,7 +568,9 @@ static int properties_Mc(codepoint_t c) if (c == 0x011CB4) return UCD_PROPERTY_OTHER_ALPHABETIC; break; case 0x01D100: - if (c >= 0x01D16D && c <= 0x01D172) return UCD_PROPERTY_DIACRITIC; + if (c == 0x01D165) return UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; + if (c == 0x01D16D) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x01D16E && c <= 0x01D172) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; break; case 0x016F00: if (c >= 0x016F51 && c <= 0x016F7E) return UCD_PROPERTY_OTHER_ALPHABETIC; diff --git a/tools/printdata.py b/tools/printdata.py index 191f9967..3360b144 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -138,6 +138,7 @@ def properties(data): props += (2 ** 14) * data.get('Other_Lowercase', 0) props += (2 ** 15) * data.get('Other_Uppercase', 0) props += (2 ** 16) * data.get('Noncharacter_Code_Point', 0) + props += (2 ** 17) * data.get('Other_Grapheme_Extend', 0) return props if __name__ == '__main__': From c9af03ee8be53cc90928115e59f251e9f593b851 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Fri, 14 Apr 2017 17:13:28 +0100 Subject: [PATCH 27/44] Add IDS_Binary_Operator support from PropList.txt. --- src/include/ucd/ucd.h | 2 ++ src/proplist.c | 4 ++++ tools/printdata.py | 1 + 3 files changed, 7 insertions(+) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index d19a0f9e..d5aca375 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -347,6 +347,7 @@ typedef enum ucd_property_ UCD_PROPERTY_OTHER_UPPERCASE = 0x00008000, /**< @brief Other_Uppercase */ UCD_PROPERTY_NONCHARACTER_CODE_POINT = 0x00010000, /**< @brief Noncharacter_Code_Point */ UCD_PROPERTY_OTHER_GRAPHEME_EXTEND = 0x00020000, /**< @brief Other_Grapheme_Extend */ + UCD_PROPERTY_IDS_BINARY_OPERATOR = 0x00040000, /**< @brief IDS_Binary_Operator */ } ucd_property; /** @brief Return the properties of the specified codepoint. @@ -824,6 +825,7 @@ namespace ucd Other_Uppercase = UCD_PROPERTY_OTHER_UPPERCASE, /**< @brief Other_Uppercase */ Noncharacter_Code_Point = UCD_PROPERTY_NONCHARACTER_CODE_POINT, /**< @brief Noncharacter_Code_Point */ Other_Grapheme_Extend = UCD_PROPERTY_OTHER_GRAPHEME_EXTEND, /**< @brief Other_Grapheme_Extend */ + IDS_Binary_Operator = UCD_PROPERTY_IDS_BINARY_OPERATOR, /**< @brief IDS_Binary_Operator */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index 429dcc73..00ff48c9 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -1514,6 +1514,10 @@ static int properties_So(codepoint_t c) if (c >= 0x2660 && c <= 0x2663) return UCD_PROPERTY_OTHER_MATH; if (c >= 0x266D && c <= 0x266E) return UCD_PROPERTY_OTHER_MATH; break; + case 0x2F00: + if (c >= 0x2FF0 && c <= 0x2FF1) return UCD_PROPERTY_IDS_BINARY_OPERATOR; + if (c >= 0x2FF4 && c <= 0x2FFB) return UCD_PROPERTY_IDS_BINARY_OPERATOR; + break; case 0x01F100: if (c >= 0x01F130 && c <= 0x01F149) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_UPPERCASE; if (c >= 0x01F150 && c <= 0x01F169) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_UPPERCASE; diff --git a/tools/printdata.py b/tools/printdata.py index 3360b144..8fb31c69 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -139,6 +139,7 @@ def properties(data): props += (2 ** 15) * data.get('Other_Uppercase', 0) props += (2 ** 16) * data.get('Noncharacter_Code_Point', 0) props += (2 ** 17) * data.get('Other_Grapheme_Extend', 0) + props += (2 ** 18) * data.get('IDS_Binary_Operator', 0) return props if __name__ == '__main__': From e6c2c718b453cdc743cc420918048e8846f0b70e Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Fri, 14 Apr 2017 17:40:57 +0100 Subject: [PATCH 28/44] Add IDS_Trinary_Operator support from PropList.txt. --- src/include/ucd/ucd.h | 2 ++ src/proplist.c | 1 + tools/printdata.py | 1 + 3 files changed, 4 insertions(+) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index d5aca375..cd130a5b 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -348,6 +348,7 @@ typedef enum ucd_property_ UCD_PROPERTY_NONCHARACTER_CODE_POINT = 0x00010000, /**< @brief Noncharacter_Code_Point */ UCD_PROPERTY_OTHER_GRAPHEME_EXTEND = 0x00020000, /**< @brief Other_Grapheme_Extend */ UCD_PROPERTY_IDS_BINARY_OPERATOR = 0x00040000, /**< @brief IDS_Binary_Operator */ + UCD_PROPERTY_IDS_TRINARY_OPERATOR = 0x00080000, /**< @brief IDS_Trinary_Operator */ } ucd_property; /** @brief Return the properties of the specified codepoint. @@ -826,6 +827,7 @@ namespace ucd Noncharacter_Code_Point = UCD_PROPERTY_NONCHARACTER_CODE_POINT, /**< @brief Noncharacter_Code_Point */ Other_Grapheme_Extend = UCD_PROPERTY_OTHER_GRAPHEME_EXTEND, /**< @brief Other_Grapheme_Extend */ IDS_Binary_Operator = UCD_PROPERTY_IDS_BINARY_OPERATOR, /**< @brief IDS_Binary_Operator */ + IDS_Trinary_Operator = UCD_PROPERTY_IDS_TRINARY_OPERATOR, /**< @brief IDS_Trinary_Operator */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index 00ff48c9..02493bfa 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -1516,6 +1516,7 @@ static int properties_So(codepoint_t c) break; case 0x2F00: if (c >= 0x2FF0 && c <= 0x2FF1) return UCD_PROPERTY_IDS_BINARY_OPERATOR; + if (c >= 0x2FF2 && c <= 0x2FF3) return UCD_PROPERTY_IDS_TRINARY_OPERATOR; if (c >= 0x2FF4 && c <= 0x2FFB) return UCD_PROPERTY_IDS_BINARY_OPERATOR; break; case 0x01F100: diff --git a/tools/printdata.py b/tools/printdata.py index 8fb31c69..2a2c3add 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -140,6 +140,7 @@ def properties(data): props += (2 ** 16) * data.get('Noncharacter_Code_Point', 0) props += (2 ** 17) * data.get('Other_Grapheme_Extend', 0) props += (2 ** 18) * data.get('IDS_Binary_Operator', 0) + props += (2 ** 19) * data.get('IDS_Trinary_Operator', 0) return props if __name__ == '__main__': From 5b9300b0423c11342517907cd7c66d29720e74f3 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Fri, 14 Apr 2017 17:57:18 +0100 Subject: [PATCH 29/44] Add Radical support from PropList.txt. --- src/include/ucd/ucd.h | 2 ++ src/proplist.c | 5 +++++ tools/printdata.py | 1 + 3 files changed, 8 insertions(+) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index cd130a5b..883ae809 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -349,6 +349,7 @@ typedef enum ucd_property_ UCD_PROPERTY_OTHER_GRAPHEME_EXTEND = 0x00020000, /**< @brief Other_Grapheme_Extend */ UCD_PROPERTY_IDS_BINARY_OPERATOR = 0x00040000, /**< @brief IDS_Binary_Operator */ UCD_PROPERTY_IDS_TRINARY_OPERATOR = 0x00080000, /**< @brief IDS_Trinary_Operator */ + UCD_PROPERTY_RADICAL = 0x00100000, /**< @brief Radical */ } ucd_property; /** @brief Return the properties of the specified codepoint. @@ -828,6 +829,7 @@ namespace ucd Other_Grapheme_Extend = UCD_PROPERTY_OTHER_GRAPHEME_EXTEND, /**< @brief Other_Grapheme_Extend */ IDS_Binary_Operator = UCD_PROPERTY_IDS_BINARY_OPERATOR, /**< @brief IDS_Binary_Operator */ IDS_Trinary_Operator = UCD_PROPERTY_IDS_TRINARY_OPERATOR, /**< @brief IDS_Trinary_Operator */ + Radical = UCD_PROPERTY_RADICAL, /**< @brief Radical */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index 02493bfa..da8b4943 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -1514,7 +1514,12 @@ static int properties_So(codepoint_t c) if (c >= 0x2660 && c <= 0x2663) return UCD_PROPERTY_OTHER_MATH; if (c >= 0x266D && c <= 0x266E) return UCD_PROPERTY_OTHER_MATH; break; + case 0x2E00: + if (c >= 0x2E80 && c <= 0x2E99) return UCD_PROPERTY_RADICAL; + if (c >= 0x2E9B && c <= 0x2EF3) return UCD_PROPERTY_RADICAL; + break; case 0x2F00: + if (c >= 0x2F00 && c <= 0x2FD5) return UCD_PROPERTY_RADICAL; if (c >= 0x2FF0 && c <= 0x2FF1) return UCD_PROPERTY_IDS_BINARY_OPERATOR; if (c >= 0x2FF2 && c <= 0x2FF3) return UCD_PROPERTY_IDS_TRINARY_OPERATOR; if (c >= 0x2FF4 && c <= 0x2FFB) return UCD_PROPERTY_IDS_BINARY_OPERATOR; diff --git a/tools/printdata.py b/tools/printdata.py index 2a2c3add..903c1020 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -141,6 +141,7 @@ def properties(data): props += (2 ** 17) * data.get('Other_Grapheme_Extend', 0) props += (2 ** 18) * data.get('IDS_Binary_Operator', 0) props += (2 ** 19) * data.get('IDS_Trinary_Operator', 0) + props += (2 ** 20) * data.get('Radical', 0) return props if __name__ == '__main__': From 4ce8b611808485a0bd1a18878fdb9cf926ed7e9f Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Fri, 14 Apr 2017 20:12:02 +0100 Subject: [PATCH 30/44] Extend ucd_property to 64-bits to allow all properties to be specified. --- src/include/ucd/ucd.h | 50 ++++++++++++++++++++------------------ tests/printcdata.c | 2 +- tests/printucddata.c | 2 +- tests/printucddata_cpp.cpp | 2 +- tools/printdata.py | 2 +- 5 files changed, 30 insertions(+), 28 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 883ae809..4fb26ee7 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -327,30 +327,31 @@ ucd_script ucd_lookup_script(codepoint_t c); /** @brief Properties */ -typedef enum ucd_property_ +typedef uint64_t ucd_property; +enum { - UCD_PROPERTY_WHITE_SPACE = 0x00000001, /**< @brief White_Space */ - UCD_PROPERTY_BIDI_CONTROL = 0x00000002, /**< @brief Bidi_Control */ - UCD_PROPERTY_JOIN_CONTROL = 0x00000004, /**< @brief Join_Control */ - UCD_PROPERTY_DASH = 0x00000008, /**< @brief Dash */ - UCD_PROPERTY_HYPHEN = 0x00000010, /**< @brief Hyphen */ - UCD_PROPERTY_QUOTATION_MARK = 0x00000020, /**< @brief Quotation_Mark */ - UCD_PROPERTY_TERMINAL_PUNCTUATION = 0x00000040, /**< @brief Terminal_Punctuation */ - UCD_PROPERTY_OTHER_MATH = 0x00000080, /**< @brief Other_Math */ - UCD_PROPERTY_HEX_DIGIT = 0x00000100, /**< @brief Hex_Digit */ - UCD_PROPERTY_ASCII_HEX_DIGIT = 0x00000200, /**< @brief ASCII_Hex_Digit */ - UCD_PROPERTY_OTHER_ALPHABETIC = 0x00000400, /**< @brief Other_Alphabetic */ - UCD_PROPERTY_IDEOGRAPHIC = 0x00000800, /**< @brief Ideographic */ - UCD_PROPERTY_DIACRITIC = 0x00001000, /**< @brief Diacritic */ - UCD_PROPERTY_EXTENDER = 0x00002000, /**< @brief Extender */ - UCD_PROPERTY_OTHER_LOWERCASE = 0x00004000, /**< @brief Other_Lowercase */ - UCD_PROPERTY_OTHER_UPPERCASE = 0x00008000, /**< @brief Other_Uppercase */ - UCD_PROPERTY_NONCHARACTER_CODE_POINT = 0x00010000, /**< @brief Noncharacter_Code_Point */ - UCD_PROPERTY_OTHER_GRAPHEME_EXTEND = 0x00020000, /**< @brief Other_Grapheme_Extend */ - UCD_PROPERTY_IDS_BINARY_OPERATOR = 0x00040000, /**< @brief IDS_Binary_Operator */ - UCD_PROPERTY_IDS_TRINARY_OPERATOR = 0x00080000, /**< @brief IDS_Trinary_Operator */ - UCD_PROPERTY_RADICAL = 0x00100000, /**< @brief Radical */ -} ucd_property; + UCD_PROPERTY_WHITE_SPACE = 0x0000000000000001ull, /**< @brief White_Space */ + UCD_PROPERTY_BIDI_CONTROL = 0x0000000000000002ull, /**< @brief Bidi_Control */ + UCD_PROPERTY_JOIN_CONTROL = 0x0000000000000004ull, /**< @brief Join_Control */ + UCD_PROPERTY_DASH = 0x0000000000000008ull, /**< @brief Dash */ + UCD_PROPERTY_HYPHEN = 0x0000000000000010ull, /**< @brief Hyphen */ + UCD_PROPERTY_QUOTATION_MARK = 0x0000000000000020ull, /**< @brief Quotation_Mark */ + UCD_PROPERTY_TERMINAL_PUNCTUATION = 0x0000000000000040ull, /**< @brief Terminal_Punctuation */ + UCD_PROPERTY_OTHER_MATH = 0x0000000000000080ull, /**< @brief Other_Math */ + UCD_PROPERTY_HEX_DIGIT = 0x0000000000000100ull, /**< @brief Hex_Digit */ + UCD_PROPERTY_ASCII_HEX_DIGIT = 0x0000000000000200ull, /**< @brief ASCII_Hex_Digit */ + UCD_PROPERTY_OTHER_ALPHABETIC = 0x0000000000000400ull, /**< @brief Other_Alphabetic */ + UCD_PROPERTY_IDEOGRAPHIC = 0x0000000000000800ull, /**< @brief Ideographic */ + UCD_PROPERTY_DIACRITIC = 0x0000000000001000ull, /**< @brief Diacritic */ + UCD_PROPERTY_EXTENDER = 0x0000000000002000ull, /**< @brief Extender */ + UCD_PROPERTY_OTHER_LOWERCASE = 0x0000000000004000ull, /**< @brief Other_Lowercase */ + UCD_PROPERTY_OTHER_UPPERCASE = 0x0000000000008000ull, /**< @brief Other_Uppercase */ + UCD_PROPERTY_NONCHARACTER_CODE_POINT = 0x0000000000010000ull, /**< @brief Noncharacter_Code_Point */ + UCD_PROPERTY_OTHER_GRAPHEME_EXTEND = 0x0000000000020000ull, /**< @brief Other_Grapheme_Extend */ + UCD_PROPERTY_IDS_BINARY_OPERATOR = 0x0000000000040000ull, /**< @brief IDS_Binary_Operator */ + UCD_PROPERTY_IDS_TRINARY_OPERATOR = 0x0000000000080000ull, /**< @brief IDS_Trinary_Operator */ + UCD_PROPERTY_RADICAL = 0x0000000000100000ull, /**< @brief Radical */ +}; /** @brief Return the properties of the specified codepoint. * @@ -807,7 +808,8 @@ namespace ucd /** @brief Properties */ - enum property + typedef ucd_property property; + enum { White_Space = UCD_PROPERTY_WHITE_SPACE, /**< @brief White_Space */ Bidi_Control = UCD_PROPERTY_BIDI_CONTROL, /**< @brief Bidi_Control */ diff --git a/tests/printcdata.c b/tests/printcdata.c index 49e29a42..d3e9190c 100644 --- a/tests/printcdata.c +++ b/tests/printcdata.c @@ -164,7 +164,7 @@ void uprintf(FILE *out, codepoint_t c, const char *format) uprintf_codepoint(out, c, *++format); break; case 'P': // properties - fprintf(out, "%08x", ucd_properties(c, ucd_lookup_category(c))); + fprintf(out, "%016x", ucd_properties(c, ucd_lookup_category(c))); break; case 'i': // is* uprintf_is(out, c, *++format); diff --git a/tests/printucddata.c b/tests/printucddata.c index 2fb87ed8..c023141f 100644 --- a/tests/printucddata.c +++ b/tests/printucddata.c @@ -161,7 +161,7 @@ void uprintf(FILE *out, codepoint_t c, const char *format) uprintf_codepoint(out, c, *++format); break; case 'P': // properties - fprintf(out, "%08x", ucd_properties(c, ucd_lookup_category(c))); + fprintf(out, "%016x", ucd_properties(c, ucd_lookup_category(c))); break; case 'i': // is* uprintf_is(out, c, *++format); diff --git a/tests/printucddata_cpp.cpp b/tests/printucddata_cpp.cpp index 59128c22..c5e89479 100644 --- a/tests/printucddata_cpp.cpp +++ b/tests/printucddata_cpp.cpp @@ -161,7 +161,7 @@ void uprintf(FILE *out, ucd::codepoint_t c, const char *format) uprintf_codepoint(out, c, *++format); break; case 'P': // properties - fprintf(out, "%08x", ucd::properties(c, ucd::lookup_category(c))); + fprintf(out, "%016x", ucd::properties(c, ucd::lookup_category(c))); break; case 'i': // is* uprintf_is(out, c, *++format); diff --git a/tools/printdata.py b/tools/printdata.py index 903c1020..2c2c9ad0 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -157,7 +157,7 @@ if __name__ == '__main__': if title == null: title = codepoint if upper == null: upper = codepoint if lower == null: lower = codepoint - print('%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %08x' % ( + print('%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %016x' % ( codepoint, script, data.get('GeneralCategory', 'Cn')[0], data.get('GeneralCategory', 'Cn'), upper, lower, title, From 2459a4fa8f3c006204dabd25c585591734a04021 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Fri, 14 Apr 2017 20:58:31 +0100 Subject: [PATCH 31/44] Add Unified_Ideograph support from PropList.txt. --- src/include/ucd/ucd.h | 2 ++ src/proplist.c | 21 +++++++++++++++------ tools/printdata.py | 1 + 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 4fb26ee7..19815d16 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -351,6 +351,7 @@ enum UCD_PROPERTY_IDS_BINARY_OPERATOR = 0x0000000000040000ull, /**< @brief IDS_Binary_Operator */ UCD_PROPERTY_IDS_TRINARY_OPERATOR = 0x0000000000080000ull, /**< @brief IDS_Trinary_Operator */ UCD_PROPERTY_RADICAL = 0x0000000000100000ull, /**< @brief Radical */ + UCD_PROPERTY_UNIFIED_IDEOGRAPH = 0x0000000000200000ull, /**< @brief Unified_Ideograph */ }; /** @brief Return the properties of the specified codepoint. @@ -832,6 +833,7 @@ namespace ucd IDS_Binary_Operator = UCD_PROPERTY_IDS_BINARY_OPERATOR, /**< @brief IDS_Binary_Operator */ IDS_Trinary_Operator = UCD_PROPERTY_IDS_TRINARY_OPERATOR, /**< @brief IDS_Trinary_Operator */ Radical = UCD_PROPERTY_RADICAL, /**< @brief Radical */ + Unified_Ideograph = UCD_PROPERTY_UNIFIED_IDEOGRAPH, /**< @brief Unified_Ideograph */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index da8b4943..2f4bcd0b 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -243,6 +243,15 @@ static int properties_Lo(codepoint_t c) if (c == 0xAAC0) return UCD_PROPERTY_DIACRITIC; if (c == 0xAAC2) return UCD_PROPERTY_DIACRITIC; break; + case 0xFA00: + if (c >= 0xFA0E && c <= 0xFA0F) return UCD_PROPERTY_UNIFIED_IDEOGRAPH; + if (c == 0xFA11) return UCD_PROPERTY_UNIFIED_IDEOGRAPH; + if (c >= 0xFA13 && c <= 0xFA14) return UCD_PROPERTY_UNIFIED_IDEOGRAPH; + if (c == 0xFA1F) return UCD_PROPERTY_UNIFIED_IDEOGRAPH; + if (c == 0xFA21) return UCD_PROPERTY_UNIFIED_IDEOGRAPH; + if (c >= 0xFA23 && c <= 0xFA24) return UCD_PROPERTY_UNIFIED_IDEOGRAPH; + if (c >= 0xFA27 && c <= 0xFA29) return UCD_PROPERTY_UNIFIED_IDEOGRAPH; + break; case 0x11300: if (c == 0x01135D) return UCD_PROPERTY_EXTENDER; break; @@ -257,8 +266,8 @@ static int properties_Lo_ideographic(codepoint_t c) switch (c & 0xFFFF0000) { case 0x000000: - if (c >= 0x3400 && c <= 0x4DB5) return UCD_PROPERTY_IDEOGRAPHIC; - if (c >= 0x4E00 && c <= 0x9FD5) return UCD_PROPERTY_IDEOGRAPHIC; + if (c >= 0x3400 && c <= 0x4DB5) return UCD_PROPERTY_IDEOGRAPHIC | UCD_PROPERTY_UNIFIED_IDEOGRAPH; + if (c >= 0x4E00 && c <= 0x9FD5) return UCD_PROPERTY_IDEOGRAPHIC | UCD_PROPERTY_UNIFIED_IDEOGRAPH; if (c >= 0xF900 && c <= 0xFA6D) return UCD_PROPERTY_IDEOGRAPHIC; if (c >= 0xFA70 && c <= 0xFAD9) return UCD_PROPERTY_IDEOGRAPHIC; break; @@ -267,10 +276,10 @@ static int properties_Lo_ideographic(codepoint_t c) if (c >= 0x018800 && c <= 0x018AF2) return UCD_PROPERTY_IDEOGRAPHIC; break; case 0x020000: - if (c >= 0x020000 && c <= 0x02A6D6) return UCD_PROPERTY_IDEOGRAPHIC; - if (c >= 0x02A700 && c <= 0x02B734) return UCD_PROPERTY_IDEOGRAPHIC; - if (c >= 0x02B740 && c <= 0x02B81D) return UCD_PROPERTY_IDEOGRAPHIC; - if (c >= 0x02B820 && c <= 0x02CEA1) return UCD_PROPERTY_IDEOGRAPHIC; + if (c >= 0x020000 && c <= 0x02A6D6) return UCD_PROPERTY_IDEOGRAPHIC | UCD_PROPERTY_UNIFIED_IDEOGRAPH; + if (c >= 0x02A700 && c <= 0x02B734) return UCD_PROPERTY_IDEOGRAPHIC | UCD_PROPERTY_UNIFIED_IDEOGRAPH; + if (c >= 0x02B740 && c <= 0x02B81D) return UCD_PROPERTY_IDEOGRAPHIC | UCD_PROPERTY_UNIFIED_IDEOGRAPH; + if (c >= 0x02B820 && c <= 0x02CEA1) return UCD_PROPERTY_IDEOGRAPHIC | UCD_PROPERTY_UNIFIED_IDEOGRAPH; if (c >= 0x02F800 && c <= 0x02FA1D) return UCD_PROPERTY_IDEOGRAPHIC; break; } diff --git a/tools/printdata.py b/tools/printdata.py index 2c2c9ad0..da7bda25 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -142,6 +142,7 @@ def properties(data): props += (2 ** 18) * data.get('IDS_Binary_Operator', 0) props += (2 ** 19) * data.get('IDS_Trinary_Operator', 0) props += (2 ** 20) * data.get('Radical', 0) + props += (2 ** 21) * data.get('Unified_Ideograph', 0) return props if __name__ == '__main__': From e18564899c737aff2c2ab8c56af810357355faa4 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sat, 15 Apr 2017 11:55:23 +0100 Subject: [PATCH 32/44] Add Other_Default_Ignorable_Code_Point support from PropList.txt. --- src/include/ucd/ucd.h | 46 ++++++++++++++++++++++--------------------- src/proplist.c | 32 ++++++++++++++++++++++++++---- tools/printdata.py | 1 + 3 files changed, 53 insertions(+), 26 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 19815d16..390e9509 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -330,28 +330,29 @@ ucd_script ucd_lookup_script(codepoint_t c); typedef uint64_t ucd_property; enum { - UCD_PROPERTY_WHITE_SPACE = 0x0000000000000001ull, /**< @brief White_Space */ - UCD_PROPERTY_BIDI_CONTROL = 0x0000000000000002ull, /**< @brief Bidi_Control */ - UCD_PROPERTY_JOIN_CONTROL = 0x0000000000000004ull, /**< @brief Join_Control */ - UCD_PROPERTY_DASH = 0x0000000000000008ull, /**< @brief Dash */ - UCD_PROPERTY_HYPHEN = 0x0000000000000010ull, /**< @brief Hyphen */ - UCD_PROPERTY_QUOTATION_MARK = 0x0000000000000020ull, /**< @brief Quotation_Mark */ - UCD_PROPERTY_TERMINAL_PUNCTUATION = 0x0000000000000040ull, /**< @brief Terminal_Punctuation */ - UCD_PROPERTY_OTHER_MATH = 0x0000000000000080ull, /**< @brief Other_Math */ - UCD_PROPERTY_HEX_DIGIT = 0x0000000000000100ull, /**< @brief Hex_Digit */ - UCD_PROPERTY_ASCII_HEX_DIGIT = 0x0000000000000200ull, /**< @brief ASCII_Hex_Digit */ - UCD_PROPERTY_OTHER_ALPHABETIC = 0x0000000000000400ull, /**< @brief Other_Alphabetic */ - UCD_PROPERTY_IDEOGRAPHIC = 0x0000000000000800ull, /**< @brief Ideographic */ - UCD_PROPERTY_DIACRITIC = 0x0000000000001000ull, /**< @brief Diacritic */ - UCD_PROPERTY_EXTENDER = 0x0000000000002000ull, /**< @brief Extender */ - UCD_PROPERTY_OTHER_LOWERCASE = 0x0000000000004000ull, /**< @brief Other_Lowercase */ - UCD_PROPERTY_OTHER_UPPERCASE = 0x0000000000008000ull, /**< @brief Other_Uppercase */ - UCD_PROPERTY_NONCHARACTER_CODE_POINT = 0x0000000000010000ull, /**< @brief Noncharacter_Code_Point */ - UCD_PROPERTY_OTHER_GRAPHEME_EXTEND = 0x0000000000020000ull, /**< @brief Other_Grapheme_Extend */ - UCD_PROPERTY_IDS_BINARY_OPERATOR = 0x0000000000040000ull, /**< @brief IDS_Binary_Operator */ - UCD_PROPERTY_IDS_TRINARY_OPERATOR = 0x0000000000080000ull, /**< @brief IDS_Trinary_Operator */ - UCD_PROPERTY_RADICAL = 0x0000000000100000ull, /**< @brief Radical */ - UCD_PROPERTY_UNIFIED_IDEOGRAPH = 0x0000000000200000ull, /**< @brief Unified_Ideograph */ + UCD_PROPERTY_WHITE_SPACE = 0x0000000000000001ull, /**< @brief White_Space */ + UCD_PROPERTY_BIDI_CONTROL = 0x0000000000000002ull, /**< @brief Bidi_Control */ + UCD_PROPERTY_JOIN_CONTROL = 0x0000000000000004ull, /**< @brief Join_Control */ + UCD_PROPERTY_DASH = 0x0000000000000008ull, /**< @brief Dash */ + UCD_PROPERTY_HYPHEN = 0x0000000000000010ull, /**< @brief Hyphen */ + UCD_PROPERTY_QUOTATION_MARK = 0x0000000000000020ull, /**< @brief Quotation_Mark */ + UCD_PROPERTY_TERMINAL_PUNCTUATION = 0x0000000000000040ull, /**< @brief Terminal_Punctuation */ + UCD_PROPERTY_OTHER_MATH = 0x0000000000000080ull, /**< @brief Other_Math */ + UCD_PROPERTY_HEX_DIGIT = 0x0000000000000100ull, /**< @brief Hex_Digit */ + UCD_PROPERTY_ASCII_HEX_DIGIT = 0x0000000000000200ull, /**< @brief ASCII_Hex_Digit */ + UCD_PROPERTY_OTHER_ALPHABETIC = 0x0000000000000400ull, /**< @brief Other_Alphabetic */ + UCD_PROPERTY_IDEOGRAPHIC = 0x0000000000000800ull, /**< @brief Ideographic */ + UCD_PROPERTY_DIACRITIC = 0x0000000000001000ull, /**< @brief Diacritic */ + UCD_PROPERTY_EXTENDER = 0x0000000000002000ull, /**< @brief Extender */ + UCD_PROPERTY_OTHER_LOWERCASE = 0x0000000000004000ull, /**< @brief Other_Lowercase */ + UCD_PROPERTY_OTHER_UPPERCASE = 0x0000000000008000ull, /**< @brief Other_Uppercase */ + UCD_PROPERTY_NONCHARACTER_CODE_POINT = 0x0000000000010000ull, /**< @brief Noncharacter_Code_Point */ + UCD_PROPERTY_OTHER_GRAPHEME_EXTEND = 0x0000000000020000ull, /**< @brief Other_Grapheme_Extend */ + UCD_PROPERTY_IDS_BINARY_OPERATOR = 0x0000000000040000ull, /**< @brief IDS_Binary_Operator */ + UCD_PROPERTY_IDS_TRINARY_OPERATOR = 0x0000000000080000ull, /**< @brief IDS_Trinary_Operator */ + UCD_PROPERTY_RADICAL = 0x0000000000100000ull, /**< @brief Radical */ + UCD_PROPERTY_UNIFIED_IDEOGRAPH = 0x0000000000200000ull, /**< @brief Unified_Ideograph */ + UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT = 0x0000000000400000ull, /**< @brief Other_Default_Ignorable_Code_Point */ }; /** @brief Return the properties of the specified codepoint. @@ -834,6 +835,7 @@ namespace ucd IDS_Trinary_Operator = UCD_PROPERTY_IDS_TRINARY_OPERATOR, /**< @brief IDS_Trinary_Operator */ Radical = UCD_PROPERTY_RADICAL, /**< @brief Radical */ Unified_Ideograph = UCD_PROPERTY_UNIFIED_IDEOGRAPH, /**< @brief Unified_Ideograph */ + Other_Default_Ignorable_Code_Point = UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT, /**< @brief Other_Default_Ignorable_Code_Point */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index 2f4bcd0b..b42ffb01 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -54,7 +54,20 @@ static int properties_Cf(codepoint_t c) static int properties_Cn(codepoint_t c) { - if (c >= 0xFDD0 && c <= 0xFDEF) return UCD_PROPERTY_NONCHARACTER_CODE_POINT; + switch (c & 0xFFFF0000) + { + case 0x0000: + if (c == 0x2065) return UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT; + if (c >= 0xFFF0 && c <= 0xFFF8) return UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT; + if (c >= 0xFDD0 && c <= 0xFDEF) return UCD_PROPERTY_NONCHARACTER_CODE_POINT; + break; + case 0x0E0000: + if (c == 0xE0000) return UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT; + if (c >= 0xE0002 && c <= 0xE001F) return UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT; + if (c >= 0xE0080 && c <= 0xE00FF) return UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT; + if (c >= 0xE01F0 && c <= 0xE0FFF) return UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT; + break; + } if ((c & 0x0000FFFF) >= 0xFFFE) return UCD_PROPERTY_NONCHARACTER_CODE_POINT; return 0; } @@ -72,9 +85,6 @@ static int properties_Ll(codepoint_t c) if (c >= 0x03F0 && c <= 0x03F1) return UCD_PROPERTY_OTHER_MATH; if (c >= 0x03F4 && c <= 0x03F5) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu break; - case 0xFF00: - if (c >= 0xFF41 && c <= 0xFF46) return UCD_PROPERTY_HEX_DIGIT; - break; case 0x2100: if (c >= 0x210A && c <= 0x2113) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu if (c >= 0x212F && c <= 0x2131) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu @@ -82,6 +92,9 @@ static int properties_Ll(codepoint_t c) if (c >= 0x213C && c <= 0x213F) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu if (c >= 0x2145 && c <= 0x2149) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu break; + case 0xFF00: + if (c >= 0xFF41 && c <= 0xFF46) return UCD_PROPERTY_HEX_DIGIT; + break; case 0x01D400: if (c >= 0x01D400 && c <= 0x01D454) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu if (c >= 0x01D456 && c <= 0x01D49C) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu @@ -233,12 +246,18 @@ static int properties_Lo(codepoint_t c) if (c == 0x00AA) return UCD_PROPERTY_OTHER_LOWERCASE; if (c == 0x00BA) return UCD_PROPERTY_OTHER_LOWERCASE; break; + case 0x1100: + if (c >= 0x115F && c <= 0x1160) return UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT; + break; case 0x2100: if (c >= 0x2135 && c <= 0x2138) return UCD_PROPERTY_OTHER_MATH; break; case 0x3000: if (c == 0x3006) return UCD_PROPERTY_IDEOGRAPHIC; break; + case 0x3100: + if (c == 0x3164) return UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT; + break; case 0xAA00: if (c == 0xAAC0) return UCD_PROPERTY_DIACRITIC; if (c == 0xAAC2) return UCD_PROPERTY_DIACRITIC; @@ -252,6 +271,9 @@ static int properties_Lo(codepoint_t c) if (c >= 0xFA23 && c <= 0xFA24) return UCD_PROPERTY_UNIFIED_IDEOGRAPH; if (c >= 0xFA27 && c <= 0xFA29) return UCD_PROPERTY_UNIFIED_IDEOGRAPH; break; + case 0xFF00: + if (c == 0xFFA0) return UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT; + break; case 0x11300: if (c == 0x01135D) return UCD_PROPERTY_EXTENDER; break; @@ -596,6 +618,7 @@ static int properties_Mn(codepoint_t c) if (c >= 0x0300 && c <= 0x0344) return UCD_PROPERTY_DIACRITIC; if (c == 0x0345) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_LOWERCASE; if (c >= 0x0346 && c <= 0x034E) return UCD_PROPERTY_DIACRITIC; + if (c == 0x034F) return UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT; if (c >= 0x0350 && c <= 0x0357) return UCD_PROPERTY_DIACRITIC; if (c >= 0x035D && c <= 0x0362) return UCD_PROPERTY_DIACRITIC; break; @@ -762,6 +785,7 @@ static int properties_Mn(codepoint_t c) if (c >= 0x1732 && c <= 0x1733) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x1752 && c <= 0x1753) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x1772 && c <= 0x1773) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x17B4 && c <= 0x17B5) return UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT; if (c >= 0x17B7 && c <= 0x17BD) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c == 0x17C6) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x17C9 && c <= 0x17D3) return UCD_PROPERTY_DIACRITIC; diff --git a/tools/printdata.py b/tools/printdata.py index da7bda25..fcd1d34d 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -143,6 +143,7 @@ def properties(data): props += (2 ** 19) * data.get('IDS_Trinary_Operator', 0) props += (2 ** 20) * data.get('Radical', 0) props += (2 ** 21) * data.get('Unified_Ideograph', 0) + props += (2 ** 22) * data.get('Other_Default_Ignorable_Code_Point', 0) return props if __name__ == '__main__': From 423aa813aacfc4c1630e488898f714a6c9416c5d Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sat, 15 Apr 2017 12:25:00 +0100 Subject: [PATCH 33/44] Add Deprecated support from PropList.txt. --- src/include/ucd/ucd.h | 2 ++ src/proplist.c | 15 +++++++++++++++ tools/printdata.py | 1 + 3 files changed, 18 insertions(+) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 390e9509..bf9e84df 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -353,6 +353,7 @@ enum UCD_PROPERTY_RADICAL = 0x0000000000100000ull, /**< @brief Radical */ UCD_PROPERTY_UNIFIED_IDEOGRAPH = 0x0000000000200000ull, /**< @brief Unified_Ideograph */ UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT = 0x0000000000400000ull, /**< @brief Other_Default_Ignorable_Code_Point */ + UCD_PROPERTY_DEPRECATED = 0x0000000000800000ull, /**< @brief Deprecated */ }; /** @brief Return the properties of the specified codepoint. @@ -836,6 +837,7 @@ namespace ucd Radical = UCD_PROPERTY_RADICAL, /**< @brief Radical */ Unified_Ideograph = UCD_PROPERTY_UNIFIED_IDEOGRAPH, /**< @brief Unified_Ideograph */ Other_Default_Ignorable_Code_Point = UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT, /**< @brief Other_Default_Ignorable_Code_Point */ + Deprecated = UCD_PROPERTY_DEPRECATED, /**< @brief Deprecated */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index b42ffb01..a3adba60 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -44,8 +44,10 @@ static int properties_Cf(codepoint_t c) if (c >= 0x202A && c <= 0x202E) return UCD_PROPERTY_BIDI_CONTROL; if (c >= 0x2061 && c <= 0x2064) return UCD_PROPERTY_OTHER_MATH; if (c >= 0x2066 && c <= 0x2069) return UCD_PROPERTY_BIDI_CONTROL; + if (c >= 0x206A && c <= 0x206F) return UCD_PROPERTY_DEPRECATED; break; case 0x0E0000: + if (c == 0x0E0001) return UCD_PROPERTY_DEPRECATED; if (c >= 0x0E0020 && c <= 0x0E007F) return UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; break; } @@ -79,6 +81,9 @@ static int properties_Ll(codepoint_t c) case 0x0000: if (c >= 0x0061 && c <= 0x0066) return UCD_PROPERTY_HEX_DIGIT | UCD_PROPERTY_ASCII_HEX_DIGIT; break; + case 0x0100: + if (c == 0x0149) return UCD_PROPERTY_DEPRECATED; + break; case 0x0300: if (c >= 0x03D0 && c <= 0x03D2) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu if (c == 0x03D5) return UCD_PROPERTY_OTHER_MATH; @@ -246,9 +251,15 @@ static int properties_Lo(codepoint_t c) if (c == 0x00AA) return UCD_PROPERTY_OTHER_LOWERCASE; if (c == 0x00BA) return UCD_PROPERTY_OTHER_LOWERCASE; break; + case 0x0600: + if (c == 0x0673) return UCD_PROPERTY_DEPRECATED; + break; case 0x1100: if (c >= 0x115F && c <= 0x1160) return UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT; break; + case 0x1700: + if (c >= 0x17A3 && c <= 0x17A4) return UCD_PROPERTY_DEPRECATED; + break; case 0x2100: if (c >= 0x2135 && c <= 0x2138) return UCD_PROPERTY_OTHER_MATH; break; @@ -755,6 +766,8 @@ static int properties_Mn(codepoint_t c) if (c == 0x0F35) return UCD_PROPERTY_DIACRITIC; if (c == 0x0F37) return UCD_PROPERTY_DIACRITIC; if (c == 0x0F39) return UCD_PROPERTY_DIACRITIC; + if (c == 0x0F77) return UCD_PROPERTY_DEPRECATED | UCD_PROPERTY_OTHER_ALPHABETIC; + if (c == 0x0F79) return UCD_PROPERTY_DEPRECATED | UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0F71 && c <= 0x0F7E) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0F80 && c <= 0x0F81) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x0F82 && c <= 0x0F84) return UCD_PROPERTY_DIACRITIC; @@ -1120,6 +1133,7 @@ static int properties_Pe(codepoint_t c) case 0x2300: if (c == 0x2309) return UCD_PROPERTY_OTHER_MATH; if (c == 0x230B) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x232A) return UCD_PROPERTY_DEPRECATED; break; case 0x2700: if (c == 0x27C6) return UCD_PROPERTY_OTHER_MATH; @@ -1405,6 +1419,7 @@ static int properties_Ps(codepoint_t c) case 0x2300: if (c == 0x2308) return UCD_PROPERTY_OTHER_MATH; if (c == 0x230A) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x2329) return UCD_PROPERTY_DEPRECATED; break; case 0x2700: if (c == 0x27C5) return UCD_PROPERTY_OTHER_MATH; diff --git a/tools/printdata.py b/tools/printdata.py index fcd1d34d..6dd68a95 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -144,6 +144,7 @@ def properties(data): props += (2 ** 20) * data.get('Radical', 0) props += (2 ** 21) * data.get('Unified_Ideograph', 0) props += (2 ** 22) * data.get('Other_Default_Ignorable_Code_Point', 0) + props += (2 ** 23) * data.get('Deprecated', 0) return props if __name__ == '__main__': From 66b2404ce32e08f497ea5cf1f23044be29cf7d8c Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 16 Apr 2017 09:09:03 +0100 Subject: [PATCH 34/44] Add Soft_Dotted support from PropList.txt. --- src/include/ucd/ucd.h | 2 ++ src/proplist.c | 46 +++++++++++++++++++++++++++++++++++++++---- tools/printdata.py | 1 + 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index bf9e84df..6cb75bfa 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -354,6 +354,7 @@ enum UCD_PROPERTY_UNIFIED_IDEOGRAPH = 0x0000000000200000ull, /**< @brief Unified_Ideograph */ UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT = 0x0000000000400000ull, /**< @brief Other_Default_Ignorable_Code_Point */ UCD_PROPERTY_DEPRECATED = 0x0000000000800000ull, /**< @brief Deprecated */ + UCD_PROPERTY_SOFT_DOTTED = 0x0000000001000000ull, /**< @brief Soft_Dotted */ }; /** @brief Return the properties of the specified codepoint. @@ -838,6 +839,7 @@ namespace ucd Unified_Ideograph = UCD_PROPERTY_UNIFIED_IDEOGRAPH, /**< @brief Unified_Ideograph */ Other_Default_Ignorable_Code_Point = UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT, /**< @brief Other_Default_Ignorable_Code_Point */ Deprecated = UCD_PROPERTY_DEPRECATED, /**< @brief Deprecated */ + Soft_Dotted = UCD_PROPERTY_SOFT_DOTTED, /**< @brief Soft_Dotted */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index a3adba60..bba3b31b 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -80,40 +80,73 @@ static int properties_Ll(codepoint_t c) { case 0x0000: if (c >= 0x0061 && c <= 0x0066) return UCD_PROPERTY_HEX_DIGIT | UCD_PROPERTY_ASCII_HEX_DIGIT; + if (c >= 0x0069 && c <= 0x006A) return UCD_PROPERTY_SOFT_DOTTED; break; case 0x0100: + if (c == 0x012F) return UCD_PROPERTY_SOFT_DOTTED; if (c == 0x0149) return UCD_PROPERTY_DEPRECATED; break; + case 0x0200: + if (c == 0x0249) return UCD_PROPERTY_SOFT_DOTTED; + if (c == 0x0268) return UCD_PROPERTY_SOFT_DOTTED; + if (c == 0x029D) return UCD_PROPERTY_SOFT_DOTTED; + break; case 0x0300: if (c >= 0x03D0 && c <= 0x03D2) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu if (c == 0x03D5) return UCD_PROPERTY_OTHER_MATH; if (c >= 0x03F0 && c <= 0x03F1) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x03F3) return UCD_PROPERTY_SOFT_DOTTED; if (c >= 0x03F4 && c <= 0x03F5) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu break; + case 0x0400: + if (c == 0x0456) return UCD_PROPERTY_SOFT_DOTTED; + if (c == 0x0458) return UCD_PROPERTY_SOFT_DOTTED; + break; + case 0x1D00: + if (c == 0x1D96) return UCD_PROPERTY_SOFT_DOTTED; + break; + case 0x1E00: + if (c == 0x1E2D) return UCD_PROPERTY_SOFT_DOTTED; + if (c == 0x1ECB) return UCD_PROPERTY_SOFT_DOTTED; + break; case 0x2100: if (c >= 0x210A && c <= 0x2113) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu if (c >= 0x212F && c <= 0x2131) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu if (c >= 0x2133 && c <= 0x2134) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu if (c >= 0x213C && c <= 0x213F) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu - if (c >= 0x2145 && c <= 0x2149) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x2145 && c <= 0x2147) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x2148 && c <= 0x2149) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_SOFT_DOTTED; break; case 0xFF00: if (c >= 0xFF41 && c <= 0xFF46) return UCD_PROPERTY_HEX_DIGIT; break; case 0x01D400: + if (c >= 0x01D422 && c <= 0x01D423) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_SOFT_DOTTED; if (c >= 0x01D400 && c <= 0x01D454) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu - if (c >= 0x01D456 && c <= 0x01D49C) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x01D456 && c <= 0x01D457) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_SOFT_DOTTED; + if (c >= 0x01D48A && c <= 0x01D48B) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_SOFT_DOTTED; + if (c >= 0x01D458 && c <= 0x01D49C) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu if (c >= 0x01D4AE && c <= 0x01D4B9) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu if (c == 0x01D4BB) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x01D4BE && c <= 0x01D4BF) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_SOFT_DOTTED; if (c >= 0x01D4BD && c <= 0x01D4C3) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x01D4F2 && c <= 0x01D4F3) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_SOFT_DOTTED; if (c >= 0x01D4C5) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu break; case 0x01D500: if (c <= 0x01D505) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x01D526 && c <= 0x01D527) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_SOFT_DOTTED; if (c >= 0x01D51E && c <= 0x01D539) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu + if (c >= 0x01D55A && c <= 0x01D55B) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_SOFT_DOTTED; + if (c >= 0x01D58E && c <= 0x01D58F) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_SOFT_DOTTED; + if (c >= 0x01D5C2 && c <= 0x01D5C3) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_SOFT_DOTTED; + if (c >= 0x01D5F6 && c <= 0x01D5F7) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_SOFT_DOTTED; if (c >= 0x01D552) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu break; case 0x01D600: + if (c >= 0x01D62A && c <= 0x01D62B) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_SOFT_DOTTED; + if (c >= 0x01D65E && c <= 0x01D65F) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_SOFT_DOTTED; + if (c >= 0x01D692 && c <= 0x01D693) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_SOFT_DOTTED; if (c <= 0x01D6A5) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu if (c >= 0x01D6C2 && c <= 0x01D6DA) return UCD_PROPERTY_OTHER_MATH; if (c >= 0x01D6DC && c <= 0x01D6FA) return UCD_PROPERTY_OTHER_MATH; // Ll|Lu @@ -138,6 +171,7 @@ static int properties_Lm(codepoint_t c) switch (c & 0xFFFFFF00) { case 0x0200: + if (c == 0x02B2) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_LOWERCASE | UCD_PROPERTY_SOFT_DOTTED; if (c >= 0x02B0 && c <= 0x02B8) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_LOWERCASE; if (c >= 0x02B9 && c <= 0x02BF) return UCD_PROPERTY_DIACRITIC; if (c >= 0x02C0 && c <= 0x02C1) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_LOWERCASE; @@ -180,17 +214,21 @@ static int properties_Lm(codepoint_t c) if (c >= 0x1C78 && c <= 0x1C7D) return UCD_PROPERTY_DIACRITIC; break; case 0x1D00: + if (c == 0x1D62) return UCD_PROPERTY_OTHER_LOWERCASE | UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_SOFT_DOTTED; if (c >= 0x1D2C && c <= 0x1D6A) return UCD_PROPERTY_OTHER_LOWERCASE | UCD_PROPERTY_DIACRITIC; if (c == 0x1D78) return UCD_PROPERTY_OTHER_LOWERCASE; + if (c == 0x1DA4) return UCD_PROPERTY_OTHER_LOWERCASE | UCD_PROPERTY_SOFT_DOTTED; + if (c == 0x1DA8) return UCD_PROPERTY_OTHER_LOWERCASE | UCD_PROPERTY_SOFT_DOTTED; if (c >= 0x1D9B && c <= 0x1DBF) return UCD_PROPERTY_OTHER_LOWERCASE; break; case 0x2000: - if (c == 0x2071) return UCD_PROPERTY_OTHER_LOWERCASE; + if (c == 0x2071) return UCD_PROPERTY_OTHER_LOWERCASE | UCD_PROPERTY_SOFT_DOTTED; if (c == 0x207F) return UCD_PROPERTY_OTHER_LOWERCASE; if (c >= 0x2090 && c <= 0x209C) return UCD_PROPERTY_OTHER_LOWERCASE; break; case 0x2C00: - if (c >= 0x2C7C && c <= 0x2C7D) return UCD_PROPERTY_OTHER_LOWERCASE; + if (c == 0x2C7C) return UCD_PROPERTY_OTHER_LOWERCASE | UCD_PROPERTY_SOFT_DOTTED; + if (c == 0x2C7D) return UCD_PROPERTY_OTHER_LOWERCASE; break; case 0x2E00: if (c == 0x2E2F) return UCD_PROPERTY_DIACRITIC; diff --git a/tools/printdata.py b/tools/printdata.py index 6dd68a95..740911f8 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -145,6 +145,7 @@ def properties(data): props += (2 ** 21) * data.get('Unified_Ideograph', 0) props += (2 ** 22) * data.get('Other_Default_Ignorable_Code_Point', 0) props += (2 ** 23) * data.get('Deprecated', 0) + props += (2 ** 24) * data.get('Soft_Dotted', 0) return props if __name__ == '__main__': From a3f811aac16a32cc1973b6705b28cad8ca676bfd Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 16 Apr 2017 09:29:02 +0100 Subject: [PATCH 35/44] Add Logical_Order_Exception support from PropList.txt. --- src/include/ucd/ucd.h | 2 ++ src/proplist.c | 11 +++++++++++ tools/printdata.py | 1 + 3 files changed, 14 insertions(+) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 6cb75bfa..341f1e6d 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -355,6 +355,7 @@ enum UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT = 0x0000000000400000ull, /**< @brief Other_Default_Ignorable_Code_Point */ UCD_PROPERTY_DEPRECATED = 0x0000000000800000ull, /**< @brief Deprecated */ UCD_PROPERTY_SOFT_DOTTED = 0x0000000001000000ull, /**< @brief Soft_Dotted */ + UCD_PROPERTY_LOGICAL_ORDER_EXCEPTION = 0x0000000002000000ull, /**< @brief Logical_Order_Exception */ }; /** @brief Return the properties of the specified codepoint. @@ -840,6 +841,7 @@ namespace ucd Other_Default_Ignorable_Code_Point = UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT, /**< @brief Other_Default_Ignorable_Code_Point */ Deprecated = UCD_PROPERTY_DEPRECATED, /**< @brief Deprecated */ Soft_Dotted = UCD_PROPERTY_SOFT_DOTTED, /**< @brief Soft_Dotted */ + Logical_Order_Exception = UCD_PROPERTY_LOGICAL_ORDER_EXCEPTION, /**< @brief Logical_Order_Exception */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index bba3b31b..0a241f5a 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -292,12 +292,20 @@ static int properties_Lo(codepoint_t c) case 0x0600: if (c == 0x0673) return UCD_PROPERTY_DEPRECATED; break; + case 0x0E00: + if (c >= 0x0E40 && c <= 0x0E44) return UCD_PROPERTY_LOGICAL_ORDER_EXCEPTION; + if (c >= 0x0EC0 && c <= 0x0EC4) return UCD_PROPERTY_LOGICAL_ORDER_EXCEPTION; + break; case 0x1100: if (c >= 0x115F && c <= 0x1160) return UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT; break; case 0x1700: if (c >= 0x17A3 && c <= 0x17A4) return UCD_PROPERTY_DEPRECATED; break; + case 0x1900: + if (c >= 0x19B5 && c <= 0x19B7) return UCD_PROPERTY_LOGICAL_ORDER_EXCEPTION; + if (c == 0x19BA) return UCD_PROPERTY_LOGICAL_ORDER_EXCEPTION; + break; case 0x2100: if (c >= 0x2135 && c <= 0x2138) return UCD_PROPERTY_OTHER_MATH; break; @@ -308,6 +316,9 @@ static int properties_Lo(codepoint_t c) if (c == 0x3164) return UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT; break; case 0xAA00: + if (c >= 0xAAB5 && c <= 0xAAB6) return UCD_PROPERTY_LOGICAL_ORDER_EXCEPTION; + if (c == 0xAAB9) return UCD_PROPERTY_LOGICAL_ORDER_EXCEPTION; + if (c >= 0xAABB && c <= 0xAABC) return UCD_PROPERTY_LOGICAL_ORDER_EXCEPTION; if (c == 0xAAC0) return UCD_PROPERTY_DIACRITIC; if (c == 0xAAC2) return UCD_PROPERTY_DIACRITIC; break; diff --git a/tools/printdata.py b/tools/printdata.py index 740911f8..67c65ba9 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -146,6 +146,7 @@ def properties(data): props += (2 ** 22) * data.get('Other_Default_Ignorable_Code_Point', 0) props += (2 ** 23) * data.get('Deprecated', 0) props += (2 ** 24) * data.get('Soft_Dotted', 0) + props += (2 ** 25) * data.get('Logical_Order_Exception', 0) return props if __name__ == '__main__': From 90a3615c48b07220bc3b8f232c69ea07261e5d00 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 16 Apr 2017 09:34:31 +0100 Subject: [PATCH 36/44] Add Other_ID_Start support from PropList.txt. --- src/include/ucd/ucd.h | 2 ++ src/proplist.c | 8 ++++++-- tools/printdata.py | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 341f1e6d..e6bd0ad1 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -356,6 +356,7 @@ enum UCD_PROPERTY_DEPRECATED = 0x0000000000800000ull, /**< @brief Deprecated */ UCD_PROPERTY_SOFT_DOTTED = 0x0000000001000000ull, /**< @brief Soft_Dotted */ UCD_PROPERTY_LOGICAL_ORDER_EXCEPTION = 0x0000000002000000ull, /**< @brief Logical_Order_Exception */ + UCD_PROPERTY_OTHER_ID_START = 0x0000000004000000ull, /**< @brief Other_ID_Start */ }; /** @brief Return the properties of the specified codepoint. @@ -842,6 +843,7 @@ namespace ucd Deprecated = UCD_PROPERTY_DEPRECATED, /**< @brief Deprecated */ Soft_Dotted = UCD_PROPERTY_SOFT_DOTTED, /**< @brief Soft_Dotted */ Logical_Order_Exception = UCD_PROPERTY_LOGICAL_ORDER_EXCEPTION, /**< @brief Logical_Order_Exception */ + Other_ID_Start = UCD_PROPERTY_OTHER_ID_START, /**< @brief Other_ID_Start */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index 0a241f5a..b8b9fdc0 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -854,7 +854,7 @@ static int properties_Mn(codepoint_t c) if (c == 0x17DD) return UCD_PROPERTY_DIACRITIC; break; case 0x1800: - if (c >= 0x1885 && c <= 0x1886) return UCD_PROPERTY_OTHER_ALPHABETIC; + if (c >= 0x1885 && c <= 0x1886) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_ID_START; if (c == 0x18A9) return UCD_PROPERTY_OTHER_ALPHABETIC; break; case 0x1900: @@ -1530,7 +1530,7 @@ static int properties_Sk(codepoint_t c) if (c >= 0x1FFD && c <= 0x1FFE) return UCD_PROPERTY_DIACRITIC; break; case 0x3000: - if (c >= 0x309B && c <= 0x309C) return UCD_PROPERTY_DIACRITIC; + if (c >= 0x309B && c <= 0x309C) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_ID_START; break; case 0xA700: if (c >= 0xA720 && c <= 0xA721) return UCD_PROPERTY_DIACRITIC; @@ -1555,6 +1555,9 @@ static int properties_Sm(codepoint_t c) if (c == 0x207B) return UCD_PROPERTY_DASH; if (c == 0x208B) return UCD_PROPERTY_DASH; break; + case 0x2100: + if (c == 0x2118) return UCD_PROPERTY_OTHER_ID_START; + break; case 0x2200: if (c == 0x2212) return UCD_PROPERTY_DASH; break; @@ -1568,6 +1571,7 @@ static int properties_So(codepoint_t c) { case 0x2100: if (c == 0x2129) return UCD_PROPERTY_OTHER_MATH; + if (c == 0x212E) return UCD_PROPERTY_OTHER_ID_START; if (c >= 0x2195 && c <= 0x2199) return UCD_PROPERTY_OTHER_MATH; if (c >= 0x219C && c <= 0x219F) return UCD_PROPERTY_OTHER_MATH; if (c >= 0x21A1 && c <= 0x21A2) return UCD_PROPERTY_OTHER_MATH; diff --git a/tools/printdata.py b/tools/printdata.py index 67c65ba9..ff275e07 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -147,6 +147,7 @@ def properties(data): props += (2 ** 23) * data.get('Deprecated', 0) props += (2 ** 24) * data.get('Soft_Dotted', 0) props += (2 ** 25) * data.get('Logical_Order_Exception', 0) + props += (2 ** 26) * data.get('Other_ID_Start', 0) return props if __name__ == '__main__': From 628dfb8887a4dabfc0961c7cea0937ebcde25983 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 16 Apr 2017 09:43:03 +0100 Subject: [PATCH 37/44] Add Other_ID_Continue support from PropList.txt. --- src/include/ucd/ucd.h | 2 ++ src/proplist.c | 21 ++++++++++++++++++--- tools/printdata.py | 1 + 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index e6bd0ad1..aceb9f0f 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -357,6 +357,7 @@ enum UCD_PROPERTY_SOFT_DOTTED = 0x0000000001000000ull, /**< @brief Soft_Dotted */ UCD_PROPERTY_LOGICAL_ORDER_EXCEPTION = 0x0000000002000000ull, /**< @brief Logical_Order_Exception */ UCD_PROPERTY_OTHER_ID_START = 0x0000000004000000ull, /**< @brief Other_ID_Start */ + UCD_PROPERTY_OTHER_ID_CONTINUE = 0x0000000008000000ull, /**< @brief Other_ID_Continue */ }; /** @brief Return the properties of the specified codepoint. @@ -844,6 +845,7 @@ namespace ucd Soft_Dotted = UCD_PROPERTY_SOFT_DOTTED, /**< @brief Soft_Dotted */ Logical_Order_Exception = UCD_PROPERTY_LOGICAL_ORDER_EXCEPTION, /**< @brief Logical_Order_Exception */ Other_ID_Start = UCD_PROPERTY_OTHER_ID_START, /**< @brief Other_ID_Start */ + Other_ID_Continue = UCD_PROPERTY_OTHER_ID_CONTINUE, /**< @brief Other_ID_Continue */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index b8b9fdc0..10556e85 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -1131,6 +1131,20 @@ static int properties_Nl(codepoint_t c) return 0; } +static int properties_No(codepoint_t c) +{ + switch (c & 0xFFFFFF00) + { + case 0x1300: + if (c >= 0x1369 && c <= 0x1371) return UCD_PROPERTY_OTHER_ID_CONTINUE; + break; + case 0x1900: + if (c == 0x19DA) return UCD_PROPERTY_OTHER_ID_CONTINUE; + break; + } + return 0; +} + static int properties_Pc(codepoint_t c) { switch (c & 0xFFFFFF00) @@ -1254,11 +1268,11 @@ static int properties_Po(codepoint_t c) if (c == 0x002E) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c >= 0x003A && c <= 0x003B) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c == 0x003F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0x00B7) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_EXTENDER; + if (c == 0x00B7) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_EXTENDER | UCD_PROPERTY_OTHER_ID_CONTINUE; break; case 0x0300: if (c == 0x037E) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0x0387) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x0387) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_OTHER_ID_CONTINUE; break; case 0x0500: if (c == 0x0589) return UCD_PROPERTY_TERMINAL_PUNCTUATION; @@ -1649,6 +1663,7 @@ ucd_property ucd_properties(codepoint_t c, ucd_category category) case UCD_CATEGORY_Mn: return properties_Mn(c); case UCD_CATEGORY_Nd: return properties_Nd(c); case UCD_CATEGORY_Nl: return properties_Nl(c); + case UCD_CATEGORY_No: return properties_No(c); case UCD_CATEGORY_Pc: return properties_Pc(c); case UCD_CATEGORY_Pd: return properties_Pd(c); case UCD_CATEGORY_Pe: return properties_Pe(c); @@ -1662,6 +1677,6 @@ ucd_property ucd_properties(codepoint_t c, ucd_category category) case UCD_CATEGORY_Zl: return UCD_PROPERTY_WHITE_SPACE; case UCD_CATEGORY_Zp: return UCD_PROPERTY_WHITE_SPACE; case UCD_CATEGORY_Zs: return UCD_PROPERTY_WHITE_SPACE; - default: return 0; // Co Cs Ii Lt Me No Sc + default: return 0; // Co Cs Ii Lt Me Sc }; } diff --git a/tools/printdata.py b/tools/printdata.py index ff275e07..c8c6d7a3 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -148,6 +148,7 @@ def properties(data): props += (2 ** 24) * data.get('Soft_Dotted', 0) props += (2 ** 25) * data.get('Logical_Order_Exception', 0) props += (2 ** 26) * data.get('Other_ID_Start', 0) + props += (2 ** 27) * data.get('Other_ID_Continue', 0) return props if __name__ == '__main__': From f7f228bc0c989eef7d2f8f360c58c1b4b13c9069 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 16 Apr 2017 11:05:59 +0100 Subject: [PATCH 38/44] Add Sentence_Terminal support from PropList.txt. --- src/include/ucd/ucd.h | 2 + src/proplist.c | 181 ++++++++++++++++++++++++------------------ tools/printdata.py | 1 + 3 files changed, 106 insertions(+), 78 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index aceb9f0f..259a167b 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -358,6 +358,7 @@ enum UCD_PROPERTY_LOGICAL_ORDER_EXCEPTION = 0x0000000002000000ull, /**< @brief Logical_Order_Exception */ UCD_PROPERTY_OTHER_ID_START = 0x0000000004000000ull, /**< @brief Other_ID_Start */ UCD_PROPERTY_OTHER_ID_CONTINUE = 0x0000000008000000ull, /**< @brief Other_ID_Continue */ + UCD_PROPERTY_SENTENCE_TERMINAL = 0x0000000010000000ull, /**< @brief Sentence_Terminal */ }; /** @brief Return the properties of the specified codepoint. @@ -846,6 +847,7 @@ namespace ucd Logical_Order_Exception = UCD_PROPERTY_LOGICAL_ORDER_EXCEPTION, /**< @brief Logical_Order_Exception */ Other_ID_Start = UCD_PROPERTY_OTHER_ID_START, /**< @brief Other_ID_Start */ Other_ID_Continue = UCD_PROPERTY_OTHER_ID_CONTINUE, /**< @brief Other_ID_Continue */ + Sentence_Terminal = UCD_PROPERTY_SENTENCE_TERMINAL, /**< @brief Sentence_Terminal */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index 10556e85..a09c3ff8 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -1261,13 +1261,13 @@ static int properties_Po(codepoint_t c) switch (c & 0xFFFFFF00) { case 0x0000: - if (c == 0x0021) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x0021) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; if (c == 0x0022) return UCD_PROPERTY_QUOTATION_MARK; if (c == 0x0027) return UCD_PROPERTY_QUOTATION_MARK; if (c == 0x002C) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0x002E) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x002E) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; if (c >= 0x003A && c <= 0x003B) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0x003F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x003F) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; if (c == 0x00B7) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_EXTENDER | UCD_PROPERTY_OTHER_ID_CONTINUE; break; case 0x0300: @@ -1275,26 +1275,28 @@ static int properties_Po(codepoint_t c) if (c == 0x0387) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_OTHER_ID_CONTINUE; break; case 0x0500: - if (c == 0x0589) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x0589) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; if (c == 0x05C3) return UCD_PROPERTY_TERMINAL_PUNCTUATION; break; case 0x0600: if (c == 0x060C) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c == 0x061B) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0x061F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0x06D4) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x061F) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c == 0x06D4) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; break; case 0x0700: - if (c >= 0x0700 && c <= 0x070A) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x0700 && c <= 0x0702) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c >= 0x0703 && c <= 0x070A) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c == 0x070C) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c >= 0x07F8 && c <= 0x07F9) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x07F8) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x07F9) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; break; case 0x0800: if (c >= 0x0830 && c <= 0x083E) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c == 0x085E) return UCD_PROPERTY_TERMINAL_PUNCTUATION; break; case 0x0900: - if (c >= 0x0964 && c <= 0x0965) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x0964 && c <= 0x0965) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; break; case 0x0E00: if (c >= 0x0E5A && c <= 0x0E5B) return UCD_PROPERTY_TERMINAL_PUNCTUATION; @@ -1304,166 +1306,189 @@ static int properties_Po(codepoint_t c) if (c >= 0x0F0D && c <= 0x0F12) return UCD_PROPERTY_TERMINAL_PUNCTUATION; break; case 0x1000: - if (c >= 0x104A && c <= 0x104B) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x104A && c <= 0x104B) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; break; case 0x1300: - if (c >= 0x1361 && c <= 0x1368) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x1361) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x1362) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c >= 0x1363 && c <= 0x1366) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x1367 && c <= 0x1368) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; break; case 0x1600: - if (c >= 0x166D && c <= 0x166E) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x166D) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x166E) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; if (c >= 0x16EB && c <= 0x16ED) return UCD_PROPERTY_TERMINAL_PUNCTUATION; break; case 0x1700: - if (c >= 0x1735 && c <= 0x1736) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x1735 && c <= 0x1736) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; if (c >= 0x17D4 && c <= 0x17D6) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c == 0x17DA) return UCD_PROPERTY_TERMINAL_PUNCTUATION; break; case 0x1800: - if (c >= 0x1802 && c <= 0x1805) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c >= 0x1808 && c <= 0x1809) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x1803) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c >= 0x1802 && c <= 0x1805) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x1808) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x1809) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; if (c == 0x180A) return UCD_PROPERTY_EXTENDER; break; case 0x1900: - if (c >= 0x1944 && c <= 0x1945) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x1944 && c <= 0x1945) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; break; case 0x1A00: - if (c >= 0x1AA8 && c <= 0x1AAB) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x1AA8 && c <= 0x1AAB) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; break; case 0x1B00: - if (c >= 0x1B5A && c <= 0x1B5B) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c >= 0x1B5D && c <= 0x1B5F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x1B5A && c <= 0x1B5B) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c == 0x1B5D) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x1B5E && c <= 0x1B5F) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; break; case 0x1C00: - if (c >= 0x1C3B && c <= 0x1C3F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c >= 0x1C7E && c <= 0x1C7F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x1C3B && c <= 0x1C3C) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c >= 0x1C3D && c <= 0x1C3F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x1C7E && c <= 0x1C7F) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; if (c == 0x1CD3) return UCD_PROPERTY_DIACRITIC; break; case 0x2000: if (c == 0x2016) return UCD_PROPERTY_OTHER_MATH; if (c >= 0x2032 && c <= 0x2034) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x203C && c <= 0x203D) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c >= 0x2047 && c <= 0x2049) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x203C && c <= 0x203D) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c >= 0x2047 && c <= 0x2049) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; if (c == 0x2053) return UCD_PROPERTY_DASH; break; case 0x2E00: - if (c == 0x2E2E) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0x2E3C) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0x2E41) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x2E2E) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c == 0x2E3C) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c == 0x2E41) return UCD_PROPERTY_TERMINAL_PUNCTUATION; break; case 0x3000: - if (c >= 0x3001 && c <= 0x3002) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x3001) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x3002) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; if (c == 0x30FB) return UCD_PROPERTY_HYPHEN; break; case 0xA400: - if (c >= 0xA4FE && c <= 0xA4FF) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0xA4FE) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0xA4FF) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; break; case 0xA600: - if (c >= 0xA60D && c <= 0xA60F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c >= 0xA6F3 && c <= 0xA6F7) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0xA60D) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0xA60E && c <= 0xA60F) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c == 0xA6F3) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c >= 0xA6F4 && c <= 0xA6F6) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0xA6F7) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; break; case 0xA800: - if (c >= 0xA876 && c <= 0xA877) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c >= 0xA8CE && c <= 0xA8CF) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0xA876 && c <= 0xA877) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c >= 0xA8CE && c <= 0xA8CF) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; break; case 0xA900: if (c == 0xA92E) return UCD_PROPERTY_DIACRITIC; - if (c == 0xA92F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c >= 0xA9C7 && c <= 0xA9C9) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0xA92F) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c == 0xA9C7) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0xA9C8 && c <= 0xA9C9) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; break; case 0xAA00: - if (c >= 0xAA5D && c <= 0xAA5F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0xAADF) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c >= 0xAAF0 && c <= 0xAAF1) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0xAA5D && c <= 0xAA5F) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c == 0xAADF) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0xAAF0 && c <= 0xAAF1) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; break; case 0xAB00: - if (c == 0xABEB) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0xABEB) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; break; case 0xFE00: - if (c >= 0xFE50 && c <= 0xFE52) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c >= 0xFE54 && c <= 0xFE57) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0xFE50 && c <= 0xFE51) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0xFE52) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c >= 0xFE54 && c <= 0xFE55) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0xFE56 && c <= 0xFE57) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; if (c == 0xFE61) return UCD_PROPERTY_OTHER_MATH; if (c == 0xFE68) return UCD_PROPERTY_OTHER_MATH; break; case 0xFF00: - if (c == 0xFF01) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0xFF01) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; if (c == 0xFF02) return UCD_PROPERTY_QUOTATION_MARK; if (c == 0xFF07) return UCD_PROPERTY_QUOTATION_MARK; - if (c == 0xFF0C) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0xFF0E) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0xFF0C) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0xFF0E) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; if (c == 0xFF3C) return UCD_PROPERTY_OTHER_MATH; if (c == 0xFF65) return UCD_PROPERTY_HYPHEN; - if (c >= 0xFF1A && c <= 0xFF1B) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0xFF1F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0xFF61) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0xFF64) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0xFF1A && c <= 0xFF1B) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0xFF1F) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c == 0xFF61) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c == 0xFF64) return UCD_PROPERTY_TERMINAL_PUNCTUATION; break; case 0x10300: - if (c == 0x01039F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0x0103D0) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x01039F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x0103D0) return UCD_PROPERTY_TERMINAL_PUNCTUATION; break; case 0x10800: - if (c == 0x010857) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x010857) return UCD_PROPERTY_TERMINAL_PUNCTUATION; break; case 0x10900: - if (c == 0x01091F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x01091F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; break; case 0x10A00: - if (c >= 0x010A56 && c <= 0x010A57) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c >= 0x010AF0 && c <= 0x010AF5) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x010A56 && c <= 0x010A57) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c >= 0x010AF0 && c <= 0x010AF5) return UCD_PROPERTY_TERMINAL_PUNCTUATION; break; case 0x10B00: - if (c >= 0x010B3A && c <= 0x010B3F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c >= 0x010B99 && c <= 0x010B9C) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x010B3A && c <= 0x010B3F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x010B99 && c <= 0x010B9C) return UCD_PROPERTY_TERMINAL_PUNCTUATION; break; case 0x11000: - if (c >= 0x011047 && c <= 0x01104D) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c >= 0x0110BE && c <= 0x0110C1) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x011047 && c <= 0x011048) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c >= 0x011049 && c <= 0x01104D) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x0110BE && c <= 0x0110C1) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; break; case 0x11100: - if (c >= 0x011141 && c <= 0x011143) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c >= 0x0111C5 && c <= 0x0111C6) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0x0111CD) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c >= 0x0111DE && c <= 0x0111DF) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x011141 && c <= 0x011143) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c >= 0x0111C5 && c <= 0x0111C6) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c == 0x0111CD) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c >= 0x0111DE && c <= 0x0111DF) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; break; case 0x11200: - if (c >= 0x011238 && c <= 0x01123C) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0x0112A9) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x01123A) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x011238 && c <= 0x01123C) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c == 0x0112A9) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; break; case 0x11400: - if (c >= 0x01144B && c <= 0x01144D) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0x01145B) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x01144B && c <= 0x01144C) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c == 0x01144D) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x01145B) return UCD_PROPERTY_TERMINAL_PUNCTUATION; break; case 0x11500: - if (c >= 0x0115C2 && c <= 0x0115C5) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x0115C2 && c <= 0x0115C3) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c >= 0x0115C4 && c <= 0x0115C5) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c >= 0x0115C6 && c <= 0x0115C8) return UCD_PROPERTY_EXTENDER; - if (c >= 0x0115C9 && c <= 0x0115D7) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x0115C9 && c <= 0x0115D7) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; break; case 0x11600: - if (c >= 0x011641 && c <= 0x011642) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x011641 && c <= 0x011642) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; break; case 0x11700: - if (c >= 0x01173C && c <= 0x01173E) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x01173C && c <= 0x01173E) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; break; case 0x11C00: - if (c >= 0x011C41 && c <= 0x011C43) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0x011C71) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x011C41 && c <= 0x011C42) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c == 0x011C43) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x011C71) return UCD_PROPERTY_TERMINAL_PUNCTUATION; break; case 0x12400: - if (c >= 0x012470 && c <= 0x012474) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x012470 && c <= 0x012474) return UCD_PROPERTY_TERMINAL_PUNCTUATION; break; case 0x16A00: - if (c >= 0x016A6E && c <= 0x016A6F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0x016AF5) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x016A6E && c <= 0x016A6F) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c == 0x016AF5) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; break; case 0x16B00: - if (c >= 0x016B37 && c <= 0x016B39) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0x016B44) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c >= 0x016B37 && c <= 0x016B38) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c == 0x016B39) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x016B44) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; break; case 0x1BC00: - if (c == 0x01BC9F) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x01BC9F) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; break; case 0x1DA00: - if (c >= 0x01DA87 && c <= 0x01DA8A) return UCD_PROPERTY_TERMINAL_PUNCTUATION; + if (c == 0x01DA88) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c >= 0x01DA87 && c <= 0x01DA8A) return UCD_PROPERTY_TERMINAL_PUNCTUATION; break; } return 0; diff --git a/tools/printdata.py b/tools/printdata.py index c8c6d7a3..6f7bb35b 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -149,6 +149,7 @@ def properties(data): props += (2 ** 25) * data.get('Logical_Order_Exception', 0) props += (2 ** 26) * data.get('Other_ID_Start', 0) props += (2 ** 27) * data.get('Other_ID_Continue', 0) + props += (2 ** 28) * data.get('Sentence_Terminal', 0) return props if __name__ == '__main__': From eecd7984c5c6a72185cd87316e4c512f9ee61311 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 16 Apr 2017 11:26:05 +0100 Subject: [PATCH 39/44] Add Variation_Selector support from PropList.txt. --- src/include/ucd/ucd.h | 2 ++ src/proplist.c | 5 +++++ tools/printdata.py | 1 + 3 files changed, 8 insertions(+) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 259a167b..30962296 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -359,6 +359,7 @@ enum UCD_PROPERTY_OTHER_ID_START = 0x0000000004000000ull, /**< @brief Other_ID_Start */ UCD_PROPERTY_OTHER_ID_CONTINUE = 0x0000000008000000ull, /**< @brief Other_ID_Continue */ UCD_PROPERTY_SENTENCE_TERMINAL = 0x0000000010000000ull, /**< @brief Sentence_Terminal */ + UCD_PROPERTY_VARIATION_SELECTOR = 0x0000000020000000ull, /**< @brief Variation_Selector */ }; /** @brief Return the properties of the specified codepoint. @@ -848,6 +849,7 @@ namespace ucd Other_ID_Start = UCD_PROPERTY_OTHER_ID_START, /**< @brief Other_ID_Start */ Other_ID_Continue = UCD_PROPERTY_OTHER_ID_CONTINUE, /**< @brief Other_ID_Continue */ Sentence_Terminal = UCD_PROPERTY_SENTENCE_TERMINAL, /**< @brief Sentence_Terminal */ + Variation_Selector = UCD_PROPERTY_VARIATION_SELECTOR, /**< @brief Variation_Selector */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index a09c3ff8..be557299 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -854,6 +854,7 @@ static int properties_Mn(codepoint_t c) if (c == 0x17DD) return UCD_PROPERTY_DIACRITIC; break; case 0x1800: + if (c >= 0x180B && c <= 0x180D) return UCD_PROPERTY_VARIATION_SELECTOR; if (c >= 0x1885 && c <= 0x1886) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_ID_START; if (c == 0x18A9) return UCD_PROPERTY_OTHER_ALPHABETIC; break; @@ -972,6 +973,7 @@ static int properties_Mn(codepoint_t c) if (c == 0xFB1E) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_OTHER_ALPHABETIC; break; case 0xFE00: + if (c >= 0xFE00 && c <= 0xFE0F) return UCD_PROPERTY_VARIATION_SELECTOR; if (c >= 0xFE20 && c <= 0xFE2F) return UCD_PROPERTY_DIACRITIC; break; case 0x010200: @@ -1093,6 +1095,9 @@ static int properties_Mn(codepoint_t c) if (c == 0x01E947) return UCD_PROPERTY_OTHER_ALPHABETIC; if (c >= 0x01E948 && c <= 0x01E94A) return UCD_PROPERTY_DIACRITIC; break; + case 0x0E0100: + if (c >= 0x0E0100 && c <= 0x0E01EF) return UCD_PROPERTY_VARIATION_SELECTOR; + break; } return 0; } diff --git a/tools/printdata.py b/tools/printdata.py index 6f7bb35b..ba3e7ae9 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -150,6 +150,7 @@ def properties(data): props += (2 ** 26) * data.get('Other_ID_Start', 0) props += (2 ** 27) * data.get('Other_ID_Continue', 0) props += (2 ** 28) * data.get('Sentence_Terminal', 0) + props += (2 ** 29) * data.get('Variation_Selector', 0) return props if __name__ == '__main__': From d91a249c14ca9af33524c898440d1af0935b137c Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 16 Apr 2017 11:37:14 +0100 Subject: [PATCH 40/44] Add Pattern_White_Space support from PropList.txt. --- src/include/ucd/ucd.h | 2 ++ src/proplist.c | 18 ++++++++++++------ tools/printdata.py | 1 + 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 30962296..6108fefd 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -360,6 +360,7 @@ enum UCD_PROPERTY_OTHER_ID_CONTINUE = 0x0000000008000000ull, /**< @brief Other_ID_Continue */ UCD_PROPERTY_SENTENCE_TERMINAL = 0x0000000010000000ull, /**< @brief Sentence_Terminal */ UCD_PROPERTY_VARIATION_SELECTOR = 0x0000000020000000ull, /**< @brief Variation_Selector */ + UCD_PROPERTY_PATTERN_WHITE_SPACE = 0x0000000040000000ull, /**< @brief Pattern_White_Space */ }; /** @brief Return the properties of the specified codepoint. @@ -850,6 +851,7 @@ namespace ucd Other_ID_Continue = UCD_PROPERTY_OTHER_ID_CONTINUE, /**< @brief Other_ID_Continue */ Sentence_Terminal = UCD_PROPERTY_SENTENCE_TERMINAL, /**< @brief Sentence_Terminal */ Variation_Selector = UCD_PROPERTY_VARIATION_SELECTOR, /**< @brief Variation_Selector */ + Pattern_White_Space = UCD_PROPERTY_PATTERN_WHITE_SPACE, /**< @brief Pattern_White_Space */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index be557299..c35d95d4 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -22,8 +22,8 @@ static int properties_Cc(codepoint_t c) { - if (c >= 0x0009 && c <= 0x000D) return UCD_PROPERTY_WHITE_SPACE; - if (c == 0x0085) return UCD_PROPERTY_WHITE_SPACE; + if (c >= 0x0009 && c <= 0x000D) return UCD_PROPERTY_WHITE_SPACE | UCD_PROPERTY_PATTERN_WHITE_SPACE; + if (c == 0x0085) return UCD_PROPERTY_WHITE_SPACE | UCD_PROPERTY_PATTERN_WHITE_SPACE; return 0; } @@ -40,7 +40,7 @@ static int properties_Cf(codepoint_t c) case 0x2000: if (c == 0x200C) return UCD_PROPERTY_JOIN_CONTROL | UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; if (c == 0x200D) return UCD_PROPERTY_JOIN_CONTROL; - if (c >= 0x200E && c <= 0x200F) return UCD_PROPERTY_BIDI_CONTROL; + if (c >= 0x200E && c <= 0x200F) return UCD_PROPERTY_BIDI_CONTROL | UCD_PROPERTY_PATTERN_WHITE_SPACE; if (c >= 0x202A && c <= 0x202E) return UCD_PROPERTY_BIDI_CONTROL; if (c >= 0x2061 && c <= 0x2064) return UCD_PROPERTY_OTHER_MATH; if (c >= 0x2066 && c <= 0x2069) return UCD_PROPERTY_BIDI_CONTROL; @@ -1678,6 +1678,12 @@ static int properties_So(codepoint_t c) return 0; } +static int properties_Zs(codepoint_t c) +{ + if (c == 0x0020) return UCD_PROPERTY_WHITE_SPACE | UCD_PROPERTY_PATTERN_WHITE_SPACE; + return UCD_PROPERTY_WHITE_SPACE; +} + ucd_property ucd_properties(codepoint_t c, ucd_category category) { switch (category) @@ -1704,9 +1710,9 @@ ucd_property ucd_properties(codepoint_t c, ucd_category category) case UCD_CATEGORY_Sk: return properties_Sk(c); case UCD_CATEGORY_Sm: return properties_Sm(c); case UCD_CATEGORY_So: return properties_So(c); - case UCD_CATEGORY_Zl: return UCD_PROPERTY_WHITE_SPACE; - case UCD_CATEGORY_Zp: return UCD_PROPERTY_WHITE_SPACE; - case UCD_CATEGORY_Zs: return UCD_PROPERTY_WHITE_SPACE; + case UCD_CATEGORY_Zl: return UCD_PROPERTY_WHITE_SPACE | UCD_PROPERTY_PATTERN_WHITE_SPACE; + case UCD_CATEGORY_Zp: return UCD_PROPERTY_WHITE_SPACE | UCD_PROPERTY_PATTERN_WHITE_SPACE; + case UCD_CATEGORY_Zs: return properties_Zs(c); default: return 0; // Co Cs Ii Lt Me Sc }; } diff --git a/tools/printdata.py b/tools/printdata.py index ba3e7ae9..42601df3 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -151,6 +151,7 @@ def properties(data): props += (2 ** 27) * data.get('Other_ID_Continue', 0) props += (2 ** 28) * data.get('Sentence_Terminal', 0) props += (2 ** 29) * data.get('Variation_Selector', 0) + props += (2 ** 30) * data.get('Pattern_White_Space', 0) return props if __name__ == '__main__': From 429b8f36290fbb060335ea74db0741760a958083 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 16 Apr 2017 14:17:00 +0100 Subject: [PATCH 41/44] Add Pattern_Syntax support from PropList.txt. --- src/include/ucd/ucd.h | 2 + src/proplist.c | 319 +++++++++++++++++++++++++++--------------- tools/printdata.py | 1 + 3 files changed, 211 insertions(+), 111 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 6108fefd..d9d1cc86 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -361,6 +361,7 @@ enum UCD_PROPERTY_SENTENCE_TERMINAL = 0x0000000010000000ull, /**< @brief Sentence_Terminal */ UCD_PROPERTY_VARIATION_SELECTOR = 0x0000000020000000ull, /**< @brief Variation_Selector */ UCD_PROPERTY_PATTERN_WHITE_SPACE = 0x0000000040000000ull, /**< @brief Pattern_White_Space */ + UCD_PROPERTY_PATTERN_SYNTAX = 0x0000000080000000ull, /**< @brief Pattern_Syntax */ }; /** @brief Return the properties of the specified codepoint. @@ -852,6 +853,7 @@ namespace ucd Sentence_Terminal = UCD_PROPERTY_SENTENCE_TERMINAL, /**< @brief Sentence_Terminal */ Variation_Selector = UCD_PROPERTY_VARIATION_SELECTOR, /**< @brief Variation_Selector */ Pattern_White_Space = UCD_PROPERTY_PATTERN_WHITE_SPACE, /**< @brief Pattern_White_Space */ + Pattern_Syntax = UCD_PROPERTY_PATTERN_SYNTAX, /**< @brief Pattern_Syntax */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index c35d95d4..2ecadf97 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -56,19 +56,58 @@ static int properties_Cf(codepoint_t c) static int properties_Cn(codepoint_t c) { - switch (c & 0xFFFF0000) + switch (c & 0xFFFFFF00) { - case 0x0000: + case 0x2000: if (c == 0x2065) return UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT; - if (c >= 0xFFF0 && c <= 0xFFF8) return UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT; + break; + case 0x2300: + if (c == 0x23FF) return UCD_PROPERTY_PATTERN_SYNTAX; + break; + case 0x2400: + if (c >= 0x2427 && c <= 0x243F) return UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x244B && c <= 0x245F) return UCD_PROPERTY_PATTERN_SYNTAX; + break; + case 0x2B00: + if (c >= 0x2B74 && c <= 0x2B75) return UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x2B96 && c <= 0x2B97) return UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x2BBA && c <= 0x2BBC) return UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x2BC9) return UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x2BD2 && c <= 0x2BEB) return UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x2BF0 && c <= 0x2BFF) return UCD_PROPERTY_PATTERN_SYNTAX; + break; + case 0x2E00: + if (c >= 0x2E45 && c <= 0x2E7F) return UCD_PROPERTY_PATTERN_SYNTAX; + break; + case 0xFD00: if (c >= 0xFDD0 && c <= 0xFDEF) return UCD_PROPERTY_NONCHARACTER_CODE_POINT; break; + case 0xFF00: + if (c >= 0xFFF0 && c <= 0xFFF8) return UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT; + break; case 0x0E0000: if (c == 0xE0000) return UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT; if (c >= 0xE0002 && c <= 0xE001F) return UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT; if (c >= 0xE0080 && c <= 0xE00FF) return UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT; - if (c >= 0xE01F0 && c <= 0xE0FFF) return UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT; break; + case 0x0E0100: + if (c >= 0xE01F0) return UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT; + break; + case 0x0E0200: + case 0x0E0300: + case 0x0E0400: + case 0x0E0500: + case 0x0E0600: + case 0x0E0700: + case 0x0E0800: + case 0x0E0900: + case 0x0E0A00: + case 0x0E0B00: + case 0x0E0C00: + case 0x0E0D00: + case 0x0E0E00: + case 0x0E0F00: + return UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT; } if ((c & 0x0000FFFF) >= 0xFFFE) return UCD_PROPERTY_NONCHARACTER_CODE_POINT; return 0; @@ -231,7 +270,7 @@ static int properties_Lm(codepoint_t c) if (c == 0x2C7D) return UCD_PROPERTY_OTHER_LOWERCASE; break; case 0x2E00: - if (c == 0x2E2F) return UCD_PROPERTY_DIACRITIC; + if (c == 0x2E2F) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_PATTERN_SYNTAX; break; case 0x3000: if (c == 0x3005) return UCD_PROPERTY_EXTENDER; @@ -1166,8 +1205,7 @@ static int properties_Pd(codepoint_t c) switch (c & 0xFFFFFF00) { case 0x0000: - if (c == 0x002D) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; - break; + return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN | UCD_PROPERTY_PATTERN_SYNTAX; case 0x0500: if (c == 0x058A) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; break; @@ -1175,10 +1213,14 @@ static int properties_Pd(codepoint_t c) if (c == 0x1806) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; break; case 0x2000: - if (c >= 0x2010 && c <= 0x2011) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; - break; + if (c >= 0x2010 && c <= 0x2011) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN | UCD_PROPERTY_PATTERN_SYNTAX; + return UCD_PROPERTY_DASH | UCD_PROPERTY_PATTERN_SYNTAX; case 0x2E00: - if (c == 0x2E17) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN; + if (c == 0x2E17) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN | UCD_PROPERTY_PATTERN_SYNTAX; + return UCD_PROPERTY_DASH | UCD_PROPERTY_PATTERN_SYNTAX; + case 0x3000: + if (c == 0x301C) return UCD_PROPERTY_DASH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x3030) return UCD_PROPERTY_DASH | UCD_PROPERTY_PATTERN_SYNTAX; break; case 0xFE00: if (c == 0xFE63) return UCD_PROPERTY_DASH | UCD_PROPERTY_HYPHEN | UCD_PROPERTY_OTHER_MATH; @@ -1194,28 +1236,33 @@ static int properties_Pe(codepoint_t c) { switch (c & 0xFFFFFF00) { + case 0x0000: + return UCD_PROPERTY_PATTERN_SYNTAX; case 0x2000: + if (c == 0x2046) return UCD_PROPERTY_PATTERN_SYNTAX; if (c == 0x207E) return UCD_PROPERTY_OTHER_MATH; if (c == 0x208E) return UCD_PROPERTY_OTHER_MATH; break; case 0x2300: - if (c == 0x2309) return UCD_PROPERTY_OTHER_MATH; - if (c == 0x230B) return UCD_PROPERTY_OTHER_MATH; - if (c == 0x232A) return UCD_PROPERTY_DEPRECATED; + if (c == 0x2309) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x230B) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x232A) return UCD_PROPERTY_DEPRECATED | UCD_PROPERTY_PATTERN_SYNTAX; break; case 0x2700: - if (c == 0x27C6) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x27E6 && c <= 0x27EF) return UCD_PROPERTY_OTHER_MATH; // Pe|Ps - break; + if (c == 0x27C6) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x27E6 && c <= 0x27EF) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; // Pe|Ps + return UCD_PROPERTY_PATTERN_SYNTAX; case 0x2900: - if (c >= 0x2983 && c <= 0x2998) return UCD_PROPERTY_OTHER_MATH; // Pe|Ps - if (c >= 0x29D8 && c <= 0x29DB) return UCD_PROPERTY_OTHER_MATH; // Pe|Ps - if (c == 0x29FD) return UCD_PROPERTY_OTHER_MATH; - break; + return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; // Pe|Ps + case 0x2E00: + return UCD_PROPERTY_PATTERN_SYNTAX; case 0x3000: - if (c == 0x300D) return UCD_PROPERTY_QUOTATION_MARK; - if (c == 0x300F) return UCD_PROPERTY_QUOTATION_MARK; - if (c >= 0x301E && c <= 0x301F) return UCD_PROPERTY_QUOTATION_MARK; + if (c == 0x300D) return UCD_PROPERTY_PATTERN_SYNTAX | UCD_PROPERTY_QUOTATION_MARK; + if (c == 0x300F) return UCD_PROPERTY_PATTERN_SYNTAX | UCD_PROPERTY_QUOTATION_MARK; + if (c >= 0x301E && c <= 0x301F) return UCD_PROPERTY_PATTERN_SYNTAX | UCD_PROPERTY_QUOTATION_MARK; + return UCD_PROPERTY_PATTERN_SYNTAX; + case 0xFD00: + if (c == 0xFD3E) return UCD_PROPERTY_PATTERN_SYNTAX; break; case 0xFE00: if (c == 0xFE42) return UCD_PROPERTY_QUOTATION_MARK; @@ -1233,13 +1280,10 @@ static int properties_Pf(codepoint_t c) switch (c & 0xFFFFFF00) { case 0x0000: - if (c == 0x00BB) return UCD_PROPERTY_QUOTATION_MARK; - break; case 0x2000: - if (c == 0x2019) return UCD_PROPERTY_QUOTATION_MARK; - if (c == 0x201D) return UCD_PROPERTY_QUOTATION_MARK; - if (c == 0x203A) return UCD_PROPERTY_QUOTATION_MARK; - break; + return UCD_PROPERTY_QUOTATION_MARK | UCD_PROPERTY_PATTERN_SYNTAX; + case 0x2E00: + return UCD_PROPERTY_PATTERN_SYNTAX; } return 0; } @@ -1249,14 +1293,10 @@ static int properties_Pi(codepoint_t c) switch (c & 0xFFFFFF00) { case 0x0000: - if (c == 0x00AB) return UCD_PROPERTY_QUOTATION_MARK; - break; case 0x2000: - if (c == 0x2018) return UCD_PROPERTY_QUOTATION_MARK; - if (c >= 0x201B && c <= 0x201C) return UCD_PROPERTY_QUOTATION_MARK; - if (c == 0x201F) return UCD_PROPERTY_QUOTATION_MARK; - if (c == 0x2039) return UCD_PROPERTY_QUOTATION_MARK; - break; + return UCD_PROPERTY_QUOTATION_MARK | UCD_PROPERTY_PATTERN_SYNTAX; + case 0x2E00: + return UCD_PROPERTY_PATTERN_SYNTAX; } return 0; } @@ -1266,15 +1306,15 @@ static int properties_Po(codepoint_t c) switch (c & 0xFFFFFF00) { case 0x0000: - if (c == 0x0021) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; - if (c == 0x0022) return UCD_PROPERTY_QUOTATION_MARK; - if (c == 0x0027) return UCD_PROPERTY_QUOTATION_MARK; - if (c == 0x002C) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0x002E) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; - if (c >= 0x003A && c <= 0x003B) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0x003F) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c == 0x0021) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x0022) return UCD_PROPERTY_QUOTATION_MARK | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x0027) return UCD_PROPERTY_QUOTATION_MARK | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x002C) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x002E) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x003A && c <= 0x003B) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x003F) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL | UCD_PROPERTY_PATTERN_SYNTAX; if (c == 0x00B7) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_EXTENDER | UCD_PROPERTY_OTHER_ID_CONTINUE; - break; + return UCD_PROPERTY_PATTERN_SYNTAX; case 0x0300: if (c == 0x037E) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c == 0x0387) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_OTHER_ID_CONTINUE; @@ -1354,20 +1394,28 @@ static int properties_Po(codepoint_t c) if (c == 0x1CD3) return UCD_PROPERTY_DIACRITIC; break; case 0x2000: - if (c == 0x2016) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x2032 && c <= 0x2034) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x203C && c <= 0x203D) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; - if (c >= 0x2047 && c <= 0x2049) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; - if (c == 0x2053) return UCD_PROPERTY_DASH; + if (c == 0x2016) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x2017) return UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x2020 && c <= 0x2027) return UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x2032 && c <= 0x2034) return UCD_PROPERTY_PATTERN_SYNTAX | UCD_PROPERTY_OTHER_MATH; + if (c >= 0x2030 && c <= 0x2038) return UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x203C && c <= 0x203D) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x203B && c <= 0x203E) return UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x2041 && c <= 0x2043) return UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x2047 && c <= 0x2049) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x204A && c <= 0x2051) return UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x2053) return UCD_PROPERTY_PATTERN_SYNTAX | UCD_PROPERTY_DASH; + if (c >= 0x2055 && c <= 0x205E) return UCD_PROPERTY_PATTERN_SYNTAX; break; case 0x2E00: - if (c == 0x2E2E) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; - if (c == 0x2E3C) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; - if (c == 0x2E41) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - break; + if (c == 0x2E2E) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x2E3C) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x2E41) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_PATTERN_SYNTAX; + return UCD_PROPERTY_PATTERN_SYNTAX; case 0x3000: - if (c == 0x3001) return UCD_PROPERTY_TERMINAL_PUNCTUATION; - if (c == 0x3002) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; + if (c == 0x3001) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x3002) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x3003) return UCD_PROPERTY_PATTERN_SYNTAX; if (c == 0x30FB) return UCD_PROPERTY_HYPHEN; break; case 0xA400: @@ -1400,6 +1448,7 @@ static int properties_Po(codepoint_t c) if (c == 0xABEB) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; break; case 0xFE00: + if (c >= 0xFE45 && c <= 0xFE46) return UCD_PROPERTY_PATTERN_SYNTAX; if (c >= 0xFE50 && c <= 0xFE51) return UCD_PROPERTY_TERMINAL_PUNCTUATION; if (c == 0xFE52) return UCD_PROPERTY_TERMINAL_PUNCTUATION | UCD_PROPERTY_SENTENCE_TERMINAL; if (c >= 0xFE54 && c <= 0xFE55) return UCD_PROPERTY_TERMINAL_PUNCTUATION; @@ -1503,33 +1552,36 @@ static int properties_Ps(codepoint_t c) { switch (c & 0xFFFFFF00) { + case 0x0000: + return UCD_PROPERTY_PATTERN_SYNTAX; case 0x2000: - if (c == 0x201A) return UCD_PROPERTY_QUOTATION_MARK; - if (c == 0x201E) return UCD_PROPERTY_QUOTATION_MARK; + if (c == 0x201A) return UCD_PROPERTY_QUOTATION_MARK | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x201E) return UCD_PROPERTY_QUOTATION_MARK | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x2045) return UCD_PROPERTY_PATTERN_SYNTAX; if (c == 0x207D) return UCD_PROPERTY_OTHER_MATH; if (c == 0x208D) return UCD_PROPERTY_OTHER_MATH; break; case 0x2300: - if (c == 0x2308) return UCD_PROPERTY_OTHER_MATH; - if (c == 0x230A) return UCD_PROPERTY_OTHER_MATH; - if (c == 0x2329) return UCD_PROPERTY_DEPRECATED; + if (c == 0x2308) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x230A) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x2329) return UCD_PROPERTY_DEPRECATED | UCD_PROPERTY_PATTERN_SYNTAX; break; case 0x2700: - if (c == 0x27C5) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x27E6 && c <= 0x27EF) return UCD_PROPERTY_OTHER_MATH; // Pe|Ps - break; + if (c == 0x27C5) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x27E6 && c <= 0x27EF) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; // Pe|Ps + return UCD_PROPERTY_PATTERN_SYNTAX; case 0x2900: - if (c >= 0x2983 && c <= 0x2998) return UCD_PROPERTY_OTHER_MATH; // Pe|Ps - if (c >= 0x29D8 && c <= 0x29DB) return UCD_PROPERTY_OTHER_MATH; // Pe|Ps - if (c == 0x29FC) return UCD_PROPERTY_OTHER_MATH; - break; + return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; case 0x2E00: - if (c == 0x2E42) return UCD_PROPERTY_QUOTATION_MARK; - break; + if (c == 0x2E42) return UCD_PROPERTY_QUOTATION_MARK | UCD_PROPERTY_PATTERN_SYNTAX; + return UCD_PROPERTY_PATTERN_SYNTAX; case 0x3000: - if (c == 0x300C) return UCD_PROPERTY_QUOTATION_MARK; - if (c == 0x300E) return UCD_PROPERTY_QUOTATION_MARK; - if (c == 0x301D) return UCD_PROPERTY_QUOTATION_MARK; + if (c == 0x300C) return UCD_PROPERTY_QUOTATION_MARK | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x300E) return UCD_PROPERTY_QUOTATION_MARK | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x301D) return UCD_PROPERTY_QUOTATION_MARK | UCD_PROPERTY_PATTERN_SYNTAX; + return UCD_PROPERTY_PATTERN_SYNTAX; + case 0xFD00: + if (c == 0xFD3F) return UCD_PROPERTY_PATTERN_SYNTAX; break; case 0xFE00: if (c == 0xFE41) return UCD_PROPERTY_QUOTATION_MARK; @@ -1542,13 +1594,23 @@ static int properties_Ps(codepoint_t c) return 0; } +static int properties_Sc(codepoint_t c) +{ + switch (c & 0xFFFFFF00) + { + case 0x0000: + return UCD_PROPERTY_PATTERN_SYNTAX; + } + return 0; +} + static int properties_Sk(codepoint_t c) { switch (c & 0xFFFFFF00) { case 0x0000: - if (c == 0x005E) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_DIACRITIC; - if (c == 0x0060) return UCD_PROPERTY_DIACRITIC; + if (c == 0x005E) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x0060) return UCD_PROPERTY_DIACRITIC | UCD_PROPERTY_PATTERN_SYNTAX; if (c == 0x00A8) return UCD_PROPERTY_DIACRITIC; if (c == 0x00AF) return UCD_PROPERTY_DIACRITIC; if (c == 0x00B4) return UCD_PROPERTY_DIACRITIC; @@ -1595,16 +1657,29 @@ static int properties_Sm(codepoint_t c) { switch (c & 0xFFFFFF00) { + case 0x0000: + return UCD_PROPERTY_PATTERN_SYNTAX; case 0x2000: + if (c == 0x2044) return UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x2052) return UCD_PROPERTY_PATTERN_SYNTAX; if (c == 0x207B) return UCD_PROPERTY_DASH; if (c == 0x208B) return UCD_PROPERTY_DASH; break; case 0x2100: if (c == 0x2118) return UCD_PROPERTY_OTHER_ID_START; + if (c >= 0x2190) return UCD_PROPERTY_PATTERN_SYNTAX; break; case 0x2200: - if (c == 0x2212) return UCD_PROPERTY_DASH; - break; + if (c == 0x2212) return UCD_PROPERTY_DASH | UCD_PROPERTY_PATTERN_SYNTAX; + return UCD_PROPERTY_PATTERN_SYNTAX; + case 0x2300: + case 0x2500: + case 0x2600: + case 0x2700: + case 0x2900: + case 0x2A00: + case 0x2B00: + return UCD_PROPERTY_PATTERN_SYNTAX; } return 0; } @@ -1613,52 +1688,69 @@ static int properties_So(codepoint_t c) { switch (c & 0xFFFFFF00) { + case 0x0000: + return UCD_PROPERTY_PATTERN_SYNTAX; case 0x2100: if (c == 0x2129) return UCD_PROPERTY_OTHER_MATH; if (c == 0x212E) return UCD_PROPERTY_OTHER_ID_START; - if (c >= 0x2195 && c <= 0x2199) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x219C && c <= 0x219F) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x21A1 && c <= 0x21A2) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x21A4 && c <= 0x21A5) return UCD_PROPERTY_OTHER_MATH; - if (c == 0x21A7) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x21A9 && c <= 0x21AD) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x21B0 && c <= 0x21B1) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x21B6 && c <= 0x21B7) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x21BC && c <= 0x21CD) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x21D0 && c <= 0x21D1) return UCD_PROPERTY_OTHER_MATH; - if (c == 0x21D3) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x21D5 && c <= 0x21DB) return UCD_PROPERTY_OTHER_MATH; - if (c == 0x21DD) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x21E4 && c <= 0x21E5) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x2195 && c <= 0x2199) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x219C && c <= 0x219F) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x21A1 && c <= 0x21A2) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x21A4 && c <= 0x21A5) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x21A8) return UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x21A7 && c <= 0x21AD) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x21B0 && c <= 0x21B1) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x21B6 && c <= 0x21B7) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x21AF && c <= 0x21BB) return UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x21BC && c <= 0x21CD) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x21D0 && c <= 0x21D1) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x21D3) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x21D5 && c <= 0x21DB) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x21DD) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x21E4 && c <= 0x21E5) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x21D5 && c <= 0x21F3) return UCD_PROPERTY_PATTERN_SYNTAX; break; case 0x2300: - if (c >= 0x23B4 && c <= 0x23B5) return UCD_PROPERTY_OTHER_MATH; - if (c == 0x23B7) return UCD_PROPERTY_OTHER_MATH; - if (c == 0x23D0) return UCD_PROPERTY_OTHER_MATH; - if (c == 0x23E2) return UCD_PROPERTY_OTHER_MATH; + if (c >= 0x2300 && c <= 0x2307) return UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x230C && c <= 0x231F) return UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x2322 && c <= 0x2328) return UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x232B && c <= 0x237B) return UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x237D && c <= 0x239A) return UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x23B4 && c <= 0x23B5) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x23B7) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x23D0) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x23B4 && c <= 0x23DB) return UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x23E2) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x23E3 && c <= 0x23FE) return UCD_PROPERTY_PATTERN_SYNTAX; break; case 0x2400: + if (c >= 0x2400 && c <= 0x2426) return UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x2440 && c <= 0x244A) return UCD_PROPERTY_PATTERN_SYNTAX; if (c >= 0x24B6 && c <= 0x24CF) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_UPPERCASE; if (c >= 0x24D0 && c <= 0x24E9) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_LOWERCASE; break; case 0x2500: - if (c >= 0x25A0 && c <= 0x25A1) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x25AE && c <= 0x25B6) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x25BC && c <= 0x25C0) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x25C6 && c <= 0x25C7) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x25CA && c <= 0x25CB) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x25CF && c <= 0x25D3) return UCD_PROPERTY_OTHER_MATH; - if (c == 0x25E2) return UCD_PROPERTY_OTHER_MATH; - if (c == 0x25E4) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x25E7 && c <= 0x25EC) return UCD_PROPERTY_OTHER_MATH; - break; + if (c >= 0x25A0 && c <= 0x25A1) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x25AE && c <= 0x25B6) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x25BC && c <= 0x25C0) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x25C6 && c <= 0x25C7) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x25CA && c <= 0x25CB) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x25CF && c <= 0x25D3) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x25E2) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x25E4) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x25E7 && c <= 0x25EC) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + return UCD_PROPERTY_PATTERN_SYNTAX; case 0x2600: - if (c >= 0x2605 && c <= 0x2606) return UCD_PROPERTY_OTHER_MATH; - if (c == 0x2640) return UCD_PROPERTY_OTHER_MATH; - if (c == 0x2642) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x2660 && c <= 0x2663) return UCD_PROPERTY_OTHER_MATH; - if (c >= 0x266D && c <= 0x266E) return UCD_PROPERTY_OTHER_MATH; - break; + if (c >= 0x2605 && c <= 0x2606) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x2640) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x2642) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x2660 && c <= 0x2663) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + if (c >= 0x266D && c <= 0x266E) return UCD_PROPERTY_OTHER_MATH | UCD_PROPERTY_PATTERN_SYNTAX; + return UCD_PROPERTY_PATTERN_SYNTAX; + case 0x2700: + case 0x2800: + case 0x2B00: + return UCD_PROPERTY_PATTERN_SYNTAX; case 0x2E00: if (c >= 0x2E80 && c <= 0x2E99) return UCD_PROPERTY_RADICAL; if (c >= 0x2E9B && c <= 0x2EF3) return UCD_PROPERTY_RADICAL; @@ -1669,6 +1761,10 @@ static int properties_So(codepoint_t c) if (c >= 0x2FF2 && c <= 0x2FF3) return UCD_PROPERTY_IDS_TRINARY_OPERATOR; if (c >= 0x2FF4 && c <= 0x2FFB) return UCD_PROPERTY_IDS_BINARY_OPERATOR; break; + case 0x3000: + if (c >= 0x3012 && c <= 0x3013) return UCD_PROPERTY_PATTERN_SYNTAX; + if (c == 0x3020) return UCD_PROPERTY_PATTERN_SYNTAX; + break; case 0x01F100: if (c >= 0x01F130 && c <= 0x01F149) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_UPPERCASE; if (c >= 0x01F150 && c <= 0x01F169) return UCD_PROPERTY_OTHER_ALPHABETIC | UCD_PROPERTY_OTHER_UPPERCASE; @@ -1707,12 +1803,13 @@ ucd_property ucd_properties(codepoint_t c, ucd_category category) case UCD_CATEGORY_Pi: return properties_Pi(c); case UCD_CATEGORY_Po: return properties_Po(c); case UCD_CATEGORY_Ps: return properties_Ps(c); + case UCD_CATEGORY_Sc: return properties_Sc(c); case UCD_CATEGORY_Sk: return properties_Sk(c); case UCD_CATEGORY_Sm: return properties_Sm(c); case UCD_CATEGORY_So: return properties_So(c); case UCD_CATEGORY_Zl: return UCD_PROPERTY_WHITE_SPACE | UCD_PROPERTY_PATTERN_WHITE_SPACE; case UCD_CATEGORY_Zp: return UCD_PROPERTY_WHITE_SPACE | UCD_PROPERTY_PATTERN_WHITE_SPACE; case UCD_CATEGORY_Zs: return properties_Zs(c); - default: return 0; // Co Cs Ii Lt Me Sc + default: return 0; // Co Cs Ii Lt Me }; } diff --git a/tools/printdata.py b/tools/printdata.py index 42601df3..4f30c21c 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -152,6 +152,7 @@ def properties(data): props += (2 ** 28) * data.get('Sentence_Terminal', 0) props += (2 ** 29) * data.get('Variation_Selector', 0) props += (2 ** 30) * data.get('Pattern_White_Space', 0) + props += (2 ** 31) * data.get('Pattern_Syntax', 0) return props if __name__ == '__main__': From 691457e98db34ab29b897ebe01707aff11588697 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 16 Apr 2017 14:36:27 +0100 Subject: [PATCH 42/44] Add Prepended_Concatenation_Mark support from PropList.txt. --- src/include/ucd/ucd.h | 2 ++ src/proplist.c | 63 ++++++++++++++++++++++---------------- tests/printcdata.c | 2 +- tests/printucddata.c | 2 +- tests/printucddata_cpp.cpp | 2 +- tools/printdata.py | 1 + 6 files changed, 43 insertions(+), 29 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index d9d1cc86..e5c046e5 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -362,6 +362,7 @@ enum UCD_PROPERTY_VARIATION_SELECTOR = 0x0000000020000000ull, /**< @brief Variation_Selector */ UCD_PROPERTY_PATTERN_WHITE_SPACE = 0x0000000040000000ull, /**< @brief Pattern_White_Space */ UCD_PROPERTY_PATTERN_SYNTAX = 0x0000000080000000ull, /**< @brief Pattern_Syntax */ + UCD_PROPERTY_PREPENDED_CONCATENATION_MARK = 0x0000000100000000ull, /**< @brief Prepended_Concatenation_Mark */ }; /** @brief Return the properties of the specified codepoint. @@ -854,6 +855,7 @@ namespace ucd Variation_Selector = UCD_PROPERTY_VARIATION_SELECTOR, /**< @brief Variation_Selector */ Pattern_White_Space = UCD_PROPERTY_PATTERN_WHITE_SPACE, /**< @brief Pattern_White_Space */ Pattern_Syntax = UCD_PROPERTY_PATTERN_SYNTAX, /**< @brief Pattern_Syntax */ + Prepended_Concatenation_Mark = UCD_PROPERTY_PREPENDED_CONCATENATION_MARK, /**< @brief Prepended_Concatenation_Mark */ }; /** @brief Return the properties of the specified codepoint. diff --git a/src/proplist.c b/src/proplist.c index 2ecadf97..9cf15f83 100644 --- a/src/proplist.c +++ b/src/proplist.c @@ -20,14 +20,14 @@ #include "ucd/ucd.h" -static int properties_Cc(codepoint_t c) +static ucd_property properties_Cc(codepoint_t c) { if (c >= 0x0009 && c <= 0x000D) return UCD_PROPERTY_WHITE_SPACE | UCD_PROPERTY_PATTERN_WHITE_SPACE; if (c == 0x0085) return UCD_PROPERTY_WHITE_SPACE | UCD_PROPERTY_PATTERN_WHITE_SPACE; return 0; } -static int properties_Cf(codepoint_t c) +static ucd_property properties_Cf(codepoint_t c) { switch (c & 0xFFFFFF00) { @@ -35,7 +35,15 @@ static int properties_Cf(codepoint_t c) if (c == 0x00AD) return UCD_PROPERTY_HYPHEN; break; case 0x0600: + if (c >= 0x0600 && c <= 0x0605) return UCD_PROPERTY_PREPENDED_CONCATENATION_MARK; if (c == 0x061C) return UCD_PROPERTY_BIDI_CONTROL; + if (c == 0x06DD) return UCD_PROPERTY_PREPENDED_CONCATENATION_MARK; + break; + case 0x0700: + if (c == 0x070F) return UCD_PROPERTY_PREPENDED_CONCATENATION_MARK; + break; + case 0x0800: + if (c == 0x08E2) return UCD_PROPERTY_PREPENDED_CONCATENATION_MARK; break; case 0x2000: if (c == 0x200C) return UCD_PROPERTY_JOIN_CONTROL | UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; @@ -46,15 +54,18 @@ static int properties_Cf(codepoint_t c) if (c >= 0x2066 && c <= 0x2069) return UCD_PROPERTY_BIDI_CONTROL; if (c >= 0x206A && c <= 0x206F) return UCD_PROPERTY_DEPRECATED; break; + case 0x011000: + if (c == 0x0110BD) return UCD_PROPERTY_PREPENDED_CONCATENATION_MARK; + break; case 0x0E0000: - if (c == 0x0E0001) return UCD_PROPERTY_DEPRECATED; + if (c == 0x0E0001) return UCD_PROPERTY_DEPRECATED; if (c >= 0x0E0020 && c <= 0x0E007F) return UCD_PROPERTY_OTHER_GRAPHEME_EXTEND; break; } return 0; } -static int properties_Cn(codepoint_t c) +static ucd_property properties_Cn(codepoint_t c) { switch (c & 0xFFFFFF00) { @@ -113,7 +124,7 @@ static int properties_Cn(codepoint_t c) return 0; } -static int properties_Ll(codepoint_t c) +static ucd_property properties_Ll(codepoint_t c) { switch (c & 0xFFFFFF00) { @@ -205,7 +216,7 @@ static int properties_Ll(codepoint_t c) return 0; } -static int properties_Lm(codepoint_t c) +static ucd_property properties_Lm(codepoint_t c) { switch (c & 0xFFFFFF00) { @@ -320,7 +331,7 @@ static int properties_Lm(codepoint_t c) return 0; } -static int properties_Lo(codepoint_t c) +static ucd_property properties_Lo(codepoint_t c) { switch (c & 0xFFFFFF00) { @@ -382,7 +393,7 @@ static int properties_Lo(codepoint_t c) return 0; } -static int properties_Lo_ideographic(codepoint_t c) +static ucd_property properties_Lo_ideographic(codepoint_t c) { switch (c & 0xFFFF0000) { @@ -407,7 +418,7 @@ static int properties_Lo_ideographic(codepoint_t c) return 0; } -static int properties_Lu(codepoint_t c) +static ucd_property properties_Lu(codepoint_t c) { switch (c & 0xFFFFFF00) { @@ -472,7 +483,7 @@ static int properties_Lu(codepoint_t c) return 0; } -static int properties_Mc(codepoint_t c) +static ucd_property properties_Mc(codepoint_t c) { switch (c & 0xFFFFFF00) { @@ -709,7 +720,7 @@ static int properties_Mc(codepoint_t c) return 0; } -static int properties_Mn(codepoint_t c) +static ucd_property properties_Mn(codepoint_t c) { switch (c & 0xFFFFFF00) { @@ -1141,7 +1152,7 @@ static int properties_Mn(codepoint_t c) return 0; } -static int properties_Nd(codepoint_t c) +static ucd_property properties_Nd(codepoint_t c) { switch (c & 0xFFFFFF00) { @@ -1158,7 +1169,7 @@ static int properties_Nd(codepoint_t c) return 0; } -static int properties_Nl(codepoint_t c) +static ucd_property properties_Nl(codepoint_t c) { switch (c & 0xFFFFFF00) { @@ -1175,7 +1186,7 @@ static int properties_Nl(codepoint_t c) return 0; } -static int properties_No(codepoint_t c) +static ucd_property properties_No(codepoint_t c) { switch (c & 0xFFFFFF00) { @@ -1189,7 +1200,7 @@ static int properties_No(codepoint_t c) return 0; } -static int properties_Pc(codepoint_t c) +static ucd_property properties_Pc(codepoint_t c) { switch (c & 0xFFFFFF00) { @@ -1200,7 +1211,7 @@ static int properties_Pc(codepoint_t c) return 0; } -static int properties_Pd(codepoint_t c) +static ucd_property properties_Pd(codepoint_t c) { switch (c & 0xFFFFFF00) { @@ -1232,7 +1243,7 @@ static int properties_Pd(codepoint_t c) return UCD_PROPERTY_DASH; } -static int properties_Pe(codepoint_t c) +static ucd_property properties_Pe(codepoint_t c) { switch (c & 0xFFFFFF00) { @@ -1275,7 +1286,7 @@ static int properties_Pe(codepoint_t c) return 0; } -static int properties_Pf(codepoint_t c) +static ucd_property properties_Pf(codepoint_t c) { switch (c & 0xFFFFFF00) { @@ -1288,7 +1299,7 @@ static int properties_Pf(codepoint_t c) return 0; } -static int properties_Pi(codepoint_t c) +static ucd_property properties_Pi(codepoint_t c) { switch (c & 0xFFFFFF00) { @@ -1301,7 +1312,7 @@ static int properties_Pi(codepoint_t c) return 0; } -static int properties_Po(codepoint_t c) +static ucd_property properties_Po(codepoint_t c) { switch (c & 0xFFFFFF00) { @@ -1548,7 +1559,7 @@ static int properties_Po(codepoint_t c) return 0; } -static int properties_Ps(codepoint_t c) +static ucd_property properties_Ps(codepoint_t c) { switch (c & 0xFFFFFF00) { @@ -1594,7 +1605,7 @@ static int properties_Ps(codepoint_t c) return 0; } -static int properties_Sc(codepoint_t c) +static ucd_property properties_Sc(codepoint_t c) { switch (c & 0xFFFFFF00) { @@ -1604,7 +1615,7 @@ static int properties_Sc(codepoint_t c) return 0; } -static int properties_Sk(codepoint_t c) +static ucd_property properties_Sk(codepoint_t c) { switch (c & 0xFFFFFF00) { @@ -1653,7 +1664,7 @@ static int properties_Sk(codepoint_t c) return 0; } -static int properties_Sm(codepoint_t c) +static ucd_property properties_Sm(codepoint_t c) { switch (c & 0xFFFFFF00) { @@ -1684,7 +1695,7 @@ static int properties_Sm(codepoint_t c) return 0; } -static int properties_So(codepoint_t c) +static ucd_property properties_So(codepoint_t c) { switch (c & 0xFFFFFF00) { @@ -1774,7 +1785,7 @@ static int properties_So(codepoint_t c) return 0; } -static int properties_Zs(codepoint_t c) +static ucd_property properties_Zs(codepoint_t c) { if (c == 0x0020) return UCD_PROPERTY_WHITE_SPACE | UCD_PROPERTY_PATTERN_WHITE_SPACE; return UCD_PROPERTY_WHITE_SPACE; diff --git a/tests/printcdata.c b/tests/printcdata.c index d3e9190c..686ef44a 100644 --- a/tests/printcdata.c +++ b/tests/printcdata.c @@ -164,7 +164,7 @@ void uprintf(FILE *out, codepoint_t c, const char *format) uprintf_codepoint(out, c, *++format); break; case 'P': // properties - fprintf(out, "%016x", ucd_properties(c, ucd_lookup_category(c))); + fprintf(out, "%016llx", ucd_properties(c, ucd_lookup_category(c))); break; case 'i': // is* uprintf_is(out, c, *++format); diff --git a/tests/printucddata.c b/tests/printucddata.c index c023141f..bdcb783f 100644 --- a/tests/printucddata.c +++ b/tests/printucddata.c @@ -161,7 +161,7 @@ void uprintf(FILE *out, codepoint_t c, const char *format) uprintf_codepoint(out, c, *++format); break; case 'P': // properties - fprintf(out, "%016x", ucd_properties(c, ucd_lookup_category(c))); + fprintf(out, "%016llx", ucd_properties(c, ucd_lookup_category(c))); break; case 'i': // is* uprintf_is(out, c, *++format); diff --git a/tests/printucddata_cpp.cpp b/tests/printucddata_cpp.cpp index c5e89479..4f9ed8b7 100644 --- a/tests/printucddata_cpp.cpp +++ b/tests/printucddata_cpp.cpp @@ -161,7 +161,7 @@ void uprintf(FILE *out, ucd::codepoint_t c, const char *format) uprintf_codepoint(out, c, *++format); break; case 'P': // properties - fprintf(out, "%016x", ucd::properties(c, ucd::lookup_category(c))); + fprintf(out, "%016llx", ucd::properties(c, ucd::lookup_category(c))); break; case 'i': // is* uprintf_is(out, c, *++format); diff --git a/tools/printdata.py b/tools/printdata.py index 4f30c21c..40ee24fd 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -153,6 +153,7 @@ def properties(data): props += (2 ** 29) * data.get('Variation_Selector', 0) props += (2 ** 30) * data.get('Pattern_White_Space', 0) props += (2 ** 31) * data.get('Pattern_Syntax', 0) + props += (2 ** 32) * data.get('Prepended_Concatenation_Mark', 0) return props if __name__ == '__main__': From 1a4b9d087b880a63754aa1158811ff474ea332eb Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 16 Apr 2017 14:44:55 +0100 Subject: [PATCH 43/44] Add the new PropList API to the README file. --- CHANGELOG.md | 1 + README.md | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0612bb58..4cc0aae9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * Add `iswblank` and `iswxdigit` compatibility. * Improve ctype compatibility. +* PropList property lookup. ## 9.0.0 - 2016-12-28 diff --git a/README.md b/README.md index 36e06e80..8fa57b39 100644 --- a/README.md +++ b/README.md @@ -65,10 +65,11 @@ Doxygen documentation format. The library exposes the following properties from the UCD data files: -| Property | Description | -|--------------------|-------------| -| `General_Category` | A [General Category Value](http://www.unicode.org/reports/tr44/#General_Category_Values), including the higher-level grouping. | -| `Script` | An [ISO 15924](http://www.unicode.org/iso15924/iso15924-codes.html) script code. | +| C API | C++ API | Data | Description | +|-----------------------|------------------------|-------------|-------------| +| `ucd_lookup_category` | `ucd::lookup_category` | UnicodeData | A [General Category Value](http://www.unicode.org/reports/tr44/#General_Category_Values). | +| `ucd_lookup_script` | `ucd::lookup_script` | Script | An [ISO 15924](http://www.unicode.org/iso15924/iso15924-codes.html) script code. | +| `ucd_properties` | `ucd::properties` | PropList | The code point properties from the PropList Unicode data file. | ### Case Conversion From ff018e33df5763dc0ff1644c9479ee2351bee45e Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 16 Apr 2017 15:01:37 +0100 Subject: [PATCH 44/44] Support compiling ucd.h with a strict C11 compiler. --- src/include/ucd/ucd.h | 69 +++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 36 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index e5c046e5..46c21304 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -328,42 +328,39 @@ ucd_script ucd_lookup_script(codepoint_t c); /** @brief Properties */ typedef uint64_t ucd_property; -enum -{ - UCD_PROPERTY_WHITE_SPACE = 0x0000000000000001ull, /**< @brief White_Space */ - UCD_PROPERTY_BIDI_CONTROL = 0x0000000000000002ull, /**< @brief Bidi_Control */ - UCD_PROPERTY_JOIN_CONTROL = 0x0000000000000004ull, /**< @brief Join_Control */ - UCD_PROPERTY_DASH = 0x0000000000000008ull, /**< @brief Dash */ - UCD_PROPERTY_HYPHEN = 0x0000000000000010ull, /**< @brief Hyphen */ - UCD_PROPERTY_QUOTATION_MARK = 0x0000000000000020ull, /**< @brief Quotation_Mark */ - UCD_PROPERTY_TERMINAL_PUNCTUATION = 0x0000000000000040ull, /**< @brief Terminal_Punctuation */ - UCD_PROPERTY_OTHER_MATH = 0x0000000000000080ull, /**< @brief Other_Math */ - UCD_PROPERTY_HEX_DIGIT = 0x0000000000000100ull, /**< @brief Hex_Digit */ - UCD_PROPERTY_ASCII_HEX_DIGIT = 0x0000000000000200ull, /**< @brief ASCII_Hex_Digit */ - UCD_PROPERTY_OTHER_ALPHABETIC = 0x0000000000000400ull, /**< @brief Other_Alphabetic */ - UCD_PROPERTY_IDEOGRAPHIC = 0x0000000000000800ull, /**< @brief Ideographic */ - UCD_PROPERTY_DIACRITIC = 0x0000000000001000ull, /**< @brief Diacritic */ - UCD_PROPERTY_EXTENDER = 0x0000000000002000ull, /**< @brief Extender */ - UCD_PROPERTY_OTHER_LOWERCASE = 0x0000000000004000ull, /**< @brief Other_Lowercase */ - UCD_PROPERTY_OTHER_UPPERCASE = 0x0000000000008000ull, /**< @brief Other_Uppercase */ - UCD_PROPERTY_NONCHARACTER_CODE_POINT = 0x0000000000010000ull, /**< @brief Noncharacter_Code_Point */ - UCD_PROPERTY_OTHER_GRAPHEME_EXTEND = 0x0000000000020000ull, /**< @brief Other_Grapheme_Extend */ - UCD_PROPERTY_IDS_BINARY_OPERATOR = 0x0000000000040000ull, /**< @brief IDS_Binary_Operator */ - UCD_PROPERTY_IDS_TRINARY_OPERATOR = 0x0000000000080000ull, /**< @brief IDS_Trinary_Operator */ - UCD_PROPERTY_RADICAL = 0x0000000000100000ull, /**< @brief Radical */ - UCD_PROPERTY_UNIFIED_IDEOGRAPH = 0x0000000000200000ull, /**< @brief Unified_Ideograph */ - UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT = 0x0000000000400000ull, /**< @brief Other_Default_Ignorable_Code_Point */ - UCD_PROPERTY_DEPRECATED = 0x0000000000800000ull, /**< @brief Deprecated */ - UCD_PROPERTY_SOFT_DOTTED = 0x0000000001000000ull, /**< @brief Soft_Dotted */ - UCD_PROPERTY_LOGICAL_ORDER_EXCEPTION = 0x0000000002000000ull, /**< @brief Logical_Order_Exception */ - UCD_PROPERTY_OTHER_ID_START = 0x0000000004000000ull, /**< @brief Other_ID_Start */ - UCD_PROPERTY_OTHER_ID_CONTINUE = 0x0000000008000000ull, /**< @brief Other_ID_Continue */ - UCD_PROPERTY_SENTENCE_TERMINAL = 0x0000000010000000ull, /**< @brief Sentence_Terminal */ - UCD_PROPERTY_VARIATION_SELECTOR = 0x0000000020000000ull, /**< @brief Variation_Selector */ - UCD_PROPERTY_PATTERN_WHITE_SPACE = 0x0000000040000000ull, /**< @brief Pattern_White_Space */ - UCD_PROPERTY_PATTERN_SYNTAX = 0x0000000080000000ull, /**< @brief Pattern_Syntax */ - UCD_PROPERTY_PREPENDED_CONCATENATION_MARK = 0x0000000100000000ull, /**< @brief Prepended_Concatenation_Mark */ -}; +static const ucd_property UCD_PROPERTY_WHITE_SPACE = 0x0000000000000001ull; /**< @brief White_Space */ +static const ucd_property UCD_PROPERTY_BIDI_CONTROL = 0x0000000000000002ull; /**< @brief Bidi_Control */ +static const ucd_property UCD_PROPERTY_JOIN_CONTROL = 0x0000000000000004ull; /**< @brief Join_Control */ +static const ucd_property UCD_PROPERTY_DASH = 0x0000000000000008ull; /**< @brief Dash */ +static const ucd_property UCD_PROPERTY_HYPHEN = 0x0000000000000010ull; /**< @brief Hyphen */ +static const ucd_property UCD_PROPERTY_QUOTATION_MARK = 0x0000000000000020ull; /**< @brief Quotation_Mark */ +static const ucd_property UCD_PROPERTY_TERMINAL_PUNCTUATION = 0x0000000000000040ull; /**< @brief Terminal_Punctuation */ +static const ucd_property UCD_PROPERTY_OTHER_MATH = 0x0000000000000080ull; /**< @brief Other_Math */ +static const ucd_property UCD_PROPERTY_HEX_DIGIT = 0x0000000000000100ull; /**< @brief Hex_Digit */ +static const ucd_property UCD_PROPERTY_ASCII_HEX_DIGIT = 0x0000000000000200ull; /**< @brief ASCII_Hex_Digit */ +static const ucd_property UCD_PROPERTY_OTHER_ALPHABETIC = 0x0000000000000400ull; /**< @brief Other_Alphabetic */ +static const ucd_property UCD_PROPERTY_IDEOGRAPHIC = 0x0000000000000800ull; /**< @brief Ideographic */ +static const ucd_property UCD_PROPERTY_DIACRITIC = 0x0000000000001000ull; /**< @brief Diacritic */ +static const ucd_property UCD_PROPERTY_EXTENDER = 0x0000000000002000ull; /**< @brief Extender */ +static const ucd_property UCD_PROPERTY_OTHER_LOWERCASE = 0x0000000000004000ull; /**< @brief Other_Lowercase */ +static const ucd_property UCD_PROPERTY_OTHER_UPPERCASE = 0x0000000000008000ull; /**< @brief Other_Uppercase */ +static const ucd_property UCD_PROPERTY_NONCHARACTER_CODE_POINT = 0x0000000000010000ull; /**< @brief Noncharacter_Code_Point */ +static const ucd_property UCD_PROPERTY_OTHER_GRAPHEME_EXTEND = 0x0000000000020000ull; /**< @brief Other_Grapheme_Extend */ +static const ucd_property UCD_PROPERTY_IDS_BINARY_OPERATOR = 0x0000000000040000ull; /**< @brief IDS_Binary_Operator */ +static const ucd_property UCD_PROPERTY_IDS_TRINARY_OPERATOR = 0x0000000000080000ull; /**< @brief IDS_Trinary_Operator */ +static const ucd_property UCD_PROPERTY_RADICAL = 0x0000000000100000ull; /**< @brief Radical */ +static const ucd_property UCD_PROPERTY_UNIFIED_IDEOGRAPH = 0x0000000000200000ull; /**< @brief Unified_Ideograph */ +static const ucd_property UCD_PROPERTY_OTHER_DEFAULT_IGNORABLE_CODE_POINT = 0x0000000000400000ull; /**< @brief Other_Default_Ignorable_Code_Point */ +static const ucd_property UCD_PROPERTY_DEPRECATED = 0x0000000000800000ull; /**< @brief Deprecated */ +static const ucd_property UCD_PROPERTY_SOFT_DOTTED = 0x0000000001000000ull; /**< @brief Soft_Dotted */ +static const ucd_property UCD_PROPERTY_LOGICAL_ORDER_EXCEPTION = 0x0000000002000000ull; /**< @brief Logical_Order_Exception */ +static const ucd_property UCD_PROPERTY_OTHER_ID_START = 0x0000000004000000ull; /**< @brief Other_ID_Start */ +static const ucd_property UCD_PROPERTY_OTHER_ID_CONTINUE = 0x0000000008000000ull; /**< @brief Other_ID_Continue */ +static const ucd_property UCD_PROPERTY_SENTENCE_TERMINAL = 0x0000000010000000ull; /**< @brief Sentence_Terminal */ +static const ucd_property UCD_PROPERTY_VARIATION_SELECTOR = 0x0000000020000000ull; /**< @brief Variation_Selector */ +static const ucd_property UCD_PROPERTY_PATTERN_WHITE_SPACE = 0x0000000040000000ull; /**< @brief Pattern_White_Space */ +static const ucd_property UCD_PROPERTY_PATTERN_SYNTAX = 0x0000000080000000ull; /**< @brief Pattern_Syntax */ +static const ucd_property UCD_PROPERTY_PREPENDED_CONCATENATION_MARK = 0x0000000100000000ull; /**< @brief Prepended_Concatenation_Mark */ /** @brief Return the properties of the specified codepoint. *