From 372797d2029584f1e6836e708c2cef496767ea2b Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sat, 11 Feb 2017 10:21:38 +0000 Subject: [PATCH 01/24] Support local UCD data using a UCD_SRCDIR variable. --- Makefile.am | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile.am b/Makefile.am index b879b80a..dfb3d16c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -57,22 +57,23 @@ EXTRA_DIST += ChangeLog UCD_VERSION=@UCD_VERSION@ UCD_ROOTDIR=data/ucd +UCD_SRCDIR=http://www.unicode.org/Public data/ucd/PropList.txt: mkdir -pv data/ucd - wget -O $@ http://www.unicode.org/Public/${UCD_VERSION}/ucd/PropList.txt + curl ${UCD_SRCDIR}/${UCD_VERSION}/ucd/PropList.txt > $@ data/ucd/PropertyValueAliases.txt: mkdir -pv data/ucd - wget -O $@ http://www.unicode.org/Public/${UCD_VERSION}/ucd/PropertyValueAliases.txt + curl ${UCD_SRCDIR}/${UCD_VERSION}/ucd/PropertyValueAliases.txt > $@ data/ucd/Scripts.txt: mkdir -pv data/ucd - wget -O $@ http://www.unicode.org/Public/${UCD_VERSION}/ucd/Scripts.txt + curl ${UCD_SRCDIR}/${UCD_VERSION}/ucd/Scripts.txt > $@ data/ucd/UnicodeData.txt: mkdir -pv data/ucd - wget -O $@ http://www.unicode.org/Public/${UCD_VERSION}/ucd/UnicodeData.txt + curl ${UCD_SRCDIR}/${UCD_VERSION}/ucd/UnicodeData.txt > $@ ############################# documentation ################################### From 3aecbfecb376d167960a80ffcf76f57d614bbce3 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sat, 11 Feb 2017 10:41:58 +0000 Subject: [PATCH 02/24] printucddata: check all command-line arguments --- tests/printucddata.c | 29 +++++++++++++++++------------ tests/printucddata_cpp.cpp | 31 ++++++++++++++++++------------- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/tests/printucddata.c b/tests/printucddata.c index 09b5f03a..37cba1b1 100644 --- a/tests/printucddata.c +++ b/tests/printucddata.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2016 Reece H. Dunn + * Copyright (C) 2012-2017 Reece H. Dunn * * This file is part of ucd-tools. * @@ -152,22 +152,27 @@ void print_file(FILE *in) int main(int argc, char **argv) { - if (argc == 2) + FILE *in = NULL; + for (int argn = 1; argn != argc; ++argn) { - if (!strcmp(argv[1], "--stdin") || !strcmp(argv[1], "-")) - print_file(stdin); - else + const char *arg = argv[argn]; + if (!strcmp(arg, "--stdin") || !strcmp(arg, "-")) + in = stdin; + else if (in == NULL) { - FILE *in = fopen(argv[1], "r"); - if (in) - { - print_file(in); - fclose(in); - } - else + in = fopen(arg, "r"); + if (!in) fprintf(stdout, "cannot open `%s`\n", argv[1]); } } + + if (in == stdin) + print_file(stdin); + else if (in != NULL) + { + print_file(in); + fclose(in); + } else { for (codepoint_t c = 0; c <= 0x10FFFF; ++c) diff --git a/tests/printucddata_cpp.cpp b/tests/printucddata_cpp.cpp index 60a2b04f..f34c3fc3 100644 --- a/tests/printucddata_cpp.cpp +++ b/tests/printucddata_cpp.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2016 Reece H. Dunn + * Copyright (C) 2012-2017 Reece H. Dunn * * This file is part of ucd-tools. * @@ -152,25 +152,30 @@ void print_file(FILE *in) int main(int argc, char **argv) { - if (argc == 2) + FILE *in = NULL; + for (int argn = 1; argn != argc; ++argn) { - if (!strcmp(argv[1], "--stdin") || !strcmp(argv[1], "-")) - print_file(stdin); - else + const char *arg = argv[argn]; + if (!strcmp(arg, "--stdin") || !strcmp(arg, "-")) + in = stdin; + else if (in == NULL) { - FILE *in = fopen(argv[1], "r"); - if (in) - { - print_file(in); - fclose(in); - } - else + in = fopen(arg, "r"); + if (!in) fprintf(stdout, "cannot open `%s`\n", argv[1]); } } + + if (in == stdin) + print_file(stdin); + else if (in != NULL) + { + print_file(in); + fclose(in); + } else { - for (ucd::codepoint_t c = 0; c <= 0x10FFFF; ++c) + for (codepoint_t c = 0; c <= 0x10FFFF; ++c) uprintf(stdout, c, "%pH %s %C %c %UH %LH %TH %W\n"); } return 0; From bde2bb187aa3d66fc637e8190aa152ac08f94d4f Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sat, 11 Feb 2017 10:55:39 +0000 Subject: [PATCH 03/24] printucddata: support custom formats via a --format command-line option --- tests/printucddata.c | 35 ++++++++++++++++++++++++++++++----- tests/printucddata_cpp.cpp | 37 +++++++++++++++++++++++++++++++------ 2 files changed, 61 insertions(+), 11 deletions(-) diff --git a/tests/printucddata.c b/tests/printucddata.c index 37cba1b1..81697b48 100644 --- a/tests/printucddata.c +++ b/tests/printucddata.c @@ -136,6 +136,28 @@ void uprintf(FILE *out, codepoint_t c, const char *format) } ++format; break; + case '\\': + switch (*++format) { + case 0: + break; + case 't': + fputc('\t', out); + ++format; + break; + case 'r': + fputc('\r', out); + ++format; + break; + case 'n': + fputc('\n', out); + ++format; + break; + default: + fputc(*format, out); + ++format; + break; + } + break; default: fputc(*format, out); ++format; @@ -143,21 +165,24 @@ void uprintf(FILE *out, codepoint_t c, const char *format) } } -void print_file(FILE *in) +void print_file(FILE *in, const char *format) { codepoint_t c = 0; while (fget_utf8c(in, &c)) - uprintf(stdout, c, "%pc\t%pH\t%s\t%c\t%Uc\t%Lc\t%Tc\t%W\n"); + uprintf(stdout, c, format ? format : "%pc\t%pH\t%s\t%c\t%Uc\t%Lc\t%Tc\t%W\n"); } int main(int argc, char **argv) { FILE *in = NULL; + const char *format = NULL; for (int argn = 1; argn != argc; ++argn) { const char *arg = argv[argn]; if (!strcmp(arg, "--stdin") || !strcmp(arg, "-")) in = stdin; + else if (!strncmp(arg, "--format=", 9)) + format = arg + 9; else if (in == NULL) { in = fopen(arg, "r"); @@ -167,16 +192,16 @@ int main(int argc, char **argv) } if (in == stdin) - print_file(stdin); + print_file(stdin, format); else if (in != NULL) { - print_file(in); + print_file(in, format); fclose(in); } else { for (codepoint_t c = 0; c <= 0x10FFFF; ++c) - uprintf(stdout, c, "%pH %s %C %c %UH %LH %TH %W\n"); + uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %W\n"); } return 0; } diff --git a/tests/printucddata_cpp.cpp b/tests/printucddata_cpp.cpp index f34c3fc3..0411450f 100644 --- a/tests/printucddata_cpp.cpp +++ b/tests/printucddata_cpp.cpp @@ -136,6 +136,28 @@ void uprintf(FILE *out, ucd::codepoint_t c, const char *format) } ++format; break; + case '\\': + switch (*++format) { + case 0: + break; + case 't': + fputc('\t', out); + ++format; + break; + case 'r': + fputc('\r', out); + ++format; + break; + case 'n': + fputc('\n', out); + ++format; + break; + default: + fputc(*format, out); + ++format; + break; + } + break; default: fputc(*format, out); ++format; @@ -143,21 +165,24 @@ void uprintf(FILE *out, ucd::codepoint_t c, const char *format) } } -void print_file(FILE *in) +void print_file(FILE *in, const char *format) { ucd::codepoint_t c = 0; while (fget_utf8c(in, c)) - uprintf(stdout, c, "%pc\t%pH\t%s\t%c\t%Uc\t%Lc\t%Tc\t%W\n"); + uprintf(stdout, c, format ? format : "%pc\t%pH\t%s\t%c\t%Uc\t%Lc\t%Tc\t%W\n"); } int main(int argc, char **argv) { FILE *in = NULL; + const char *format = NULL; for (int argn = 1; argn != argc; ++argn) { const char *arg = argv[argn]; if (!strcmp(arg, "--stdin") || !strcmp(arg, "-")) in = stdin; + else if (!strncmp(arg, "--format=", 9)) + format = arg + 9; else if (in == NULL) { in = fopen(arg, "r"); @@ -167,16 +192,16 @@ int main(int argc, char **argv) } if (in == stdin) - print_file(stdin); + print_file(stdin, format); else if (in != NULL) { - print_file(in); + print_file(in, format); fclose(in); } else { - for (codepoint_t c = 0; c <= 0x10FFFF; ++c) - uprintf(stdout, c, "%pH %s %C %c %UH %LH %TH %W\n"); + for (ucd::codepoint_t c = 0; c <= 0x10FFFF; ++c) + uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %W\n"); } return 0; } From 5e5b15ddf01da89a72374497491b3d61606310fb Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sat, 11 Feb 2017 11:09:03 +0000 Subject: [PATCH 04/24] printucddata: support printing is* ctype data --- tests/printucddata.c | 40 ++++++++++++++++++++++++++++++++++++++ tests/printucddata_cpp.cpp | 40 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/tests/printucddata.c b/tests/printucddata.c index 81697b48..557b756c 100644 --- a/tests/printucddata.c +++ b/tests/printucddata.c @@ -101,6 +101,43 @@ void uprintf_codepoint(FILE *out, codepoint_t c, char mode) } } +void uprintf_is(FILE *out, codepoint_t c, char mode) +{ + switch (mode) + { + case 'A': // alpha-numeric + fputc(ucd_isalnum(c) ? '1' : '0', out); + break; + case 'a': // alpha + fputc(ucd_isalpha(c) ? '1' : '0', out); + break; + case 'c': // control + fputc(ucd_iscntrl(c) ? '1' : '0', out); + break; + case 'd': // numeric + fputc(ucd_isdigit(c) ? '1' : '0', out); + break; + case 'g': // glyph + fputc(ucd_isgraph(c) ? '1' : '0', out); + break; + case 'l': // lower case + fputc(ucd_islower(c) ? '1' : '0', out); + break; + case 'P': // printable + fputc(ucd_isprint(c) ? '1' : '0', out); + break; + case 'p': // punctuation + fputc(ucd_ispunct(c) ? '1' : '0', out); + break; + case 's': // whitespace + fputc(ucd_isspace(c) ? '1' : '0', out); + break; + case 'u': // upper case + fputc(ucd_isupper(c) ? '1' : '0', out); + break; + } +} + void uprintf(FILE *out, codepoint_t c, const char *format) { while (*format) switch (*format) @@ -117,6 +154,9 @@ void uprintf(FILE *out, codepoint_t c, const char *format) case 'p': // codepoint uprintf_codepoint(out, c, *++format); break; + case 'i': // is* + uprintf_is(out, c, *++format); + break; case 'L': // lowercase uprintf_codepoint(out, ucd_tolower(c), *++format); break; diff --git a/tests/printucddata_cpp.cpp b/tests/printucddata_cpp.cpp index 0411450f..77dffa52 100644 --- a/tests/printucddata_cpp.cpp +++ b/tests/printucddata_cpp.cpp @@ -101,6 +101,43 @@ void uprintf_codepoint(FILE *out, ucd::codepoint_t c, char mode) } } +void uprintf_is(FILE *out, ucd::codepoint_t c, char mode) +{ + switch (mode) + { + case 'A': // alpha-numeric + fputc(ucd::isalnum(c) ? '1' : '0', out); + break; + case 'a': // alpha + fputc(ucd::isalpha(c) ? '1' : '0', out); + break; + case 'c': // control + fputc(ucd::iscntrl(c) ? '1' : '0', out); + break; + case 'd': // numeric + fputc(ucd::isdigit(c) ? '1' : '0', out); + break; + case 'g': // glyph + fputc(ucd::isgraph(c) ? '1' : '0', out); + break; + case 'l': // lower case + fputc(ucd::islower(c) ? '1' : '0', out); + break; + case 'P': // printable + fputc(ucd::isprint(c) ? '1' : '0', out); + break; + case 'p': // punctuation + fputc(ucd::ispunct(c) ? '1' : '0', out); + break; + case 's': // whitespace + fputc(ucd::isspace(c) ? '1' : '0', out); + break; + case 'u': // upper case + fputc(ucd::isupper(c) ? '1' : '0', out); + break; + } +} + void uprintf(FILE *out, ucd::codepoint_t c, const char *format) { while (*format) switch (*format) @@ -117,6 +154,9 @@ void uprintf(FILE *out, ucd::codepoint_t c, const char *format) case 'p': // codepoint uprintf_codepoint(out, c, *++format); break; + case 'i': // is* + uprintf_is(out, c, *++format); + break; case 'L': // lowercase uprintf_codepoint(out, ucd::tolower(c), *++format); break; From d10f63b7109f512dec5500c95827a592b942b7c4 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sat, 11 Feb 2017 11:23:15 +0000 Subject: [PATCH 05/24] printcdata: a version of printucddata that uses the C APIs where available --- .gitignore | 1 + Makefile.am | 4 + tests/printcdata.c | 252 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 257 insertions(+) create mode 100644 tests/printcdata.c diff --git a/.gitignore b/.gitignore index 02824e99..058fa1c5 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ data/ucd src/libucd.la +tests/printcdata tests/printucddata tests/printucddata_cpp diff --git a/Makefile.am b/Makefile.am index dfb3d16c..7717564d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -123,6 +123,10 @@ src_libucd_la_SOURCES = \ ############################# tests ########################################### +noinst_bin_PROGRAMS += tests/printcdata +tests_printcdata_SOURCES = tests/printcdata.c +tests_printcdata_LDADD = src/libucd.la + noinst_bin_PROGRAMS += tests/printucddata tests_printucddata_SOURCES = tests/printucddata.c tests_printucddata_LDADD = src/libucd.la diff --git a/tests/printcdata.c b/tests/printcdata.c new file mode 100644 index 00000000..4b3b9b58 --- /dev/null +++ b/tests/printcdata.c @@ -0,0 +1,252 @@ +/* + * Copyright (C) 2012-2017 Reece H. Dunn + * + * This file is part of ucd-tools. + * + * ucd-tools is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * ucd-tools is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with ucd-tools. If not, see . + */ + +#include "ucd/ucd.h" + +#include +#include +#include +#include +#include + +void fput_utf8c(FILE *out, codepoint_t c) +{ + if (c < 0x80) + fputc((uint8_t)c, out); + else if (c < 0x800) + { + fputc(0xC0 | (c >> 6), out); + fputc(0x80 + (c & 0x3F), out); + } + else if (c < 0x10000) + { + fputc(0xE0 | (c >> 12), out); + fputc(0x80 + ((c >> 6) & 0x3F), out); + fputc(0x80 + (c & 0x3F), out); + } + else if (c < 0x200000) + { + fputc(0xF0 | (c >> 18), out); + fputc(0x80 + ((c >> 12) & 0x3F), out); + fputc(0x80 + ((c >> 6) & 0x3F), out); + fputc(0x80 + (c & 0x3F), out); + } +} + +int fget_utf8c(FILE *in, codepoint_t *c) +{ + int ch = EOF; + if ((ch = fgetc(in)) == EOF) return 0; + if ((uint8_t)ch < 0x80) + *c = (uint8_t)ch; + else switch ((uint8_t)ch & 0xF0) + { + default: + *c = (uint8_t)ch & 0x1F; + if ((ch = fgetc(in)) == EOF) return 0; + *c = (*c << 6) + ((uint8_t)ch & 0x3F); + break; + case 0xE0: + *c = (uint8_t)ch & 0x0F; + if ((ch = fgetc(in)) == EOF) return 0; + *c = (*c << 6) + ((uint8_t)ch & 0x3F); + if ((ch = fgetc(in)) == EOF) return 0; + *c = (*c << 6) + ((uint8_t)ch & 0x3F); + break; + case 0xF0: + *c = (uint8_t)ch & 0x07; + if ((ch = fgetc(in)) == EOF) return 0; + *c = (*c << 6) + ((uint8_t)ch & 0x3F); + if ((ch = fgetc(in)) == EOF) return 0; + *c = (*c << 6) + ((uint8_t)ch & 0x3F); + if ((ch = fgetc(in)) == EOF) return 0; + *c = (*c << 6) + ((uint8_t)ch & 0x3F); + break; + } + return 1; +} + +void uprintf_codepoint(FILE *out, codepoint_t c, char mode) +{ + switch (mode) + { + case 'c': // character + switch (c) + { + case '\t': fputs("\\t", out); break; + case '\r': fputs("\\r", out); break; + case '\n': fputs("\\n", out); break; + default: fput_utf8c(out, c); break; + } + break; + case 'h': // hexadecimal (lower) + fprintf(out, "%06x", c); + break; + case 'H': // hexadecimal (upper) + fprintf(out, "%06X", c); + break; + } +} + +void uprintf_is(FILE *out, codepoint_t c, char mode) +{ + switch (mode) + { + case 'A': // alpha-numeric + fputc(iswalnum(c) ? '1' : '0', out); + break; + case 'a': // alpha + fputc(iswalpha(c) ? '1' : '0', out); + break; + case 'c': // control + fputc(iswcntrl(c) ? '1' : '0', out); + break; + case 'd': // numeric + fputc(iswdigit(c) ? '1' : '0', out); + break; + case 'g': // glyph + fputc(iswgraph(c) ? '1' : '0', out); + break; + case 'l': // lower case + fputc(iswlower(c) ? '1' : '0', out); + break; + case 'P': // printable + fputc(iswprint(c) ? '1' : '0', out); + break; + case 'p': // punctuation + fputc(iswpunct(c) ? '1' : '0', out); + break; + case 's': // whitespace + fputc(iswspace(c) ? '1' : '0', out); + break; + case 'u': // upper case + fputc(iswupper(c) ? '1' : '0', out); + break; + } +} + +void uprintf(FILE *out, codepoint_t c, const char *format) +{ + while (*format) switch (*format) + { + case '%': + switch (*++format) + { + case 'c': // category + fputs(ucd_get_category_string(ucd_lookup_category(c)), out); + break; + case 'C': // category group + fputs(ucd_get_category_group_string(ucd_lookup_category_group(c)), out); + break; + case 'p': // codepoint + uprintf_codepoint(out, c, *++format); + break; + case 'i': // is* + uprintf_is(out, c, *++format); + break; + case 'L': // lowercase + uprintf_codepoint(out, towlower(c), *++format); + break; + case 's': // script + fputs(ucd_get_script_string(ucd_lookup_script(c)), out); + break; + case 'T': // titlecase + uprintf_codepoint(out, ucd_totitle(c), *++format); + break; + case 'U': // uppercase + uprintf_codepoint(out, towupper(c), *++format); + break; + case 'W': // whitespace + if (iswspace(c)) + fputs("White_Space", out); + break; + } + ++format; + break; + case '\\': + switch (*++format) { + case 0: + break; + case 't': + fputc('\t', out); + ++format; + break; + case 'r': + fputc('\r', out); + ++format; + break; + case 'n': + fputc('\n', out); + ++format; + break; + default: + fputc(*format, out); + ++format; + break; + } + break; + default: + fputc(*format, out); + ++format; + break; + } +} + +void print_file(FILE *in, const char *format) +{ + codepoint_t c = 0; + while (fget_utf8c(in, &c)) + uprintf(stdout, c, format ? format : "%pc\t%pH\t%s\t%c\t%Uc\t%Lc\t%Tc\t%W\n"); +} + +int main(int argc, char **argv) +{ + FILE *in = NULL; + const char *format = NULL; + for (int argn = 1; argn != argc; ++argn) + { + const char *arg = argv[argn]; + if (!strcmp(arg, "--stdin") || !strcmp(arg, "-")) + in = stdin; + else if (!strncmp(arg, "--format=", 9)) + format = arg + 9; + else if (!strncmp(arg, "--locale=", 9)) + setlocale(LC_CTYPE, arg + 9); + else if (in == NULL) + { + in = fopen(arg, "r"); + if (!in) + fprintf(stdout, "cannot open `%s`\n", argv[1]); + } + } + + if (in == stdin) + print_file(stdin, format); + else if (in != NULL) + { + print_file(in, format); + fclose(in); + } + else + { + for (codepoint_t c = 0; c <= 0x10FFFF; ++c) + uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %W\n"); + } + return 0; +} From d585c778043be1a80095afe78bddf9ff791c9d82 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sat, 11 Feb 2017 17:37:42 +0000 Subject: [PATCH 06/24] Add an iswblank compatibility API. --- CHANGELOG.md | 4 ++++ README.md | 3 ++- src/ctype.c | 13 +++++++++++++ src/include/ucd/ucd.h | 17 +++++++++++++++++ tests/printcdata.c | 3 +++ tests/printucddata.c | 3 +++ tests/printucddata_cpp.cpp | 3 +++ 7 files changed, 45 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99ff59a0..dd5a382a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Change Log +## 9.0.0.1 - (In Progress) + +* Add `iswblank` compatibility. + ## 9.0.0 - 2016-12-28 * Update to Unicode Character Data 9.0.0. diff --git a/README.md b/README.md index 528d0ac8..ee0610f0 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,7 @@ The following character classification functions are provided: |---------------|----------------| | `ucd_isalnum` | `ucd::isalnum` | | `ucd_isalpha` | `ucd::isalpha` | +| `ucd_isblank` | `ucd::isblank` | | `ucd_iscntrl` | `ucd::iscntrl` | | `ucd_isdigit` | `ucd::isdigit` | | `ucd_isgraph` | `ucd::isgraph` | @@ -105,7 +106,7 @@ The following character classification functions are provided: | `ucd_isspace` | `ucd::isspace` | | `ucd_isupper` | `ucd::isupper` | -__NOTE:__ Equivalents for `isblank` and `isxdigit` are not provided. +__NOTE:__ An equivalent for `isxdigit` is not provided. ## Build Dependencies diff --git a/src/ctype.c b/src/ctype.c index c4b63455..f1be7643 100644 --- a/src/ctype.c +++ b/src/ctype.c @@ -53,6 +53,19 @@ int ucd_isalpha(codepoint_t c) } } +int ucd_isblank(codepoint_t c) +{ + switch (ucd_lookup_category(c)) + { + case UCD_CATEGORY_Zs: + return 1; + case UCD_CATEGORY_Cc: + return c == 0x09; // U+0009 : CHARACTER TABULATION + default: + return 0; + } +} + int ucd_iscntrl(codepoint_t c) { return ucd_lookup_category(c) == UCD_CATEGORY_Cc; diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index cb87671b..ee415cc6 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -339,6 +339,13 @@ int ucd_isalnum(codepoint_t c); */ int ucd_isalpha(codepoint_t c); +/** @brief Is the codepoint a tab or Unicode whitespace character? + * + * @param c The Unicode codepoint to check. + * @return Non-zero if the codepoint is a whitespace character, zero otherwise. + */ +int ucd_isblank(codepoint_t c); + /** @brief Is the codepoint a control character? * * @param c The Unicode codepoint to check. @@ -776,6 +783,16 @@ namespace ucd return ucd_isalpha(c); } + /** @brief Is the codepoint a tab or Unicode whitespace character? + * + * @param c The Unicode codepoint to check. + * @return Non-zero if the codepoint is a whitespace character, zero otherwise. + */ + inline int isblank(codepoint_t c) + { + return ucd_isblank(c); + } + /** @brief Is the codepoint a control character? * * @param c The Unicode codepoint to check. diff --git a/tests/printcdata.c b/tests/printcdata.c index 4b3b9b58..8b19749a 100644 --- a/tests/printcdata.c +++ b/tests/printcdata.c @@ -114,6 +114,9 @@ void uprintf_is(FILE *out, codepoint_t c, char mode) case 'a': // alpha fputc(iswalpha(c) ? '1' : '0', out); break; + case 'b': // blank + fputc(iswblank(c) ? '1' : '0', out); + break; case 'c': // control fputc(iswcntrl(c) ? '1' : '0', out); break; diff --git a/tests/printucddata.c b/tests/printucddata.c index 557b756c..9400b547 100644 --- a/tests/printucddata.c +++ b/tests/printucddata.c @@ -111,6 +111,9 @@ void uprintf_is(FILE *out, codepoint_t c, char mode) case 'a': // alpha fputc(ucd_isalpha(c) ? '1' : '0', out); break; + case 'b': // blank + fputc(ucd_isblank(c) ? '1' : '0', out); + break; case 'c': // control fputc(ucd_iscntrl(c) ? '1' : '0', out); break; diff --git a/tests/printucddata_cpp.cpp b/tests/printucddata_cpp.cpp index 77dffa52..41e37605 100644 --- a/tests/printucddata_cpp.cpp +++ b/tests/printucddata_cpp.cpp @@ -111,6 +111,9 @@ void uprintf_is(FILE *out, ucd::codepoint_t c, char mode) case 'a': // alpha fputc(ucd::isalpha(c) ? '1' : '0', out); break; + case 'b': // blank + fputc(ucd::isblank(c) ? '1' : '0', out); + break; case 'c': // control fputc(ucd::iscntrl(c) ? '1' : '0', out); break; From 3e321158e73d8347a863d07fdb61472321191e3d Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sat, 11 Feb 2017 17:54:25 +0000 Subject: [PATCH 07/24] Update the documentation on the is* APIs. --- src/include/ucd/ucd.h | 88 +++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index ee415cc6..6693329c 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -325,80 +325,80 @@ const char *ucd_get_script_string(ucd_script s); */ ucd_script ucd_lookup_script(codepoint_t c); -/** @brief Is the codepoint an alpha-numeric character? +/** @brief Is the codepoint in the 'alnum' class? * * @param c The Unicode codepoint to check. - * @return Non-zero if the codepoint is a letter or number, zero otherwise. + * @return Non-zero if the codepoint is in the 'alnum' class, zero otherwise. */ int ucd_isalnum(codepoint_t c); -/** @brief Is the codepoint a letter? +/** @brief Is the codepoint in the 'alpha' class? * * @param c The Unicode codepoint to check. - * @return Non-zero if the codepoint is a letter, zero otherwise. + * @return Non-zero if the codepoint is in the 'alpha' class, zero otherwise. */ int ucd_isalpha(codepoint_t c); -/** @brief Is the codepoint a tab or Unicode whitespace character? +/** @brief Is the codepoint in the 'blank' class? * * @param c The Unicode codepoint to check. - * @return Non-zero if the codepoint is a whitespace character, zero otherwise. + * @return Non-zero if the codepoint is in the 'blank' class, zero otherwise. */ int ucd_isblank(codepoint_t c); -/** @brief Is the codepoint a control character? +/** @brief Is the codepoint in the 'cntrl' class? * * @param c The Unicode codepoint to check. - * @return Non-zero if the codepoint is a control character, zero otherwise. + * @return Non-zero if the codepoint is in the 'cntrl' class, zero otherwise. */ int ucd_iscntrl(codepoint_t c); -/** @brief Is the codepoint a numeric character? +/** @brief Is the codepoint in the 'digit' class? * * @param c The Unicode codepoint to check. - * @return Non-zero if the codepoint is a number, zero otherwise. + * @return Non-zero if the codepoint is in the 'digit' class, zero otherwise. */ int ucd_isdigit(codepoint_t c); -/** @brief Does the codepoint have a displayable glyph? +/** @brief Is the codepoint in the 'graph' class? * * @param c The Unicode codepoint to check. - * @return Non-zero if the codepoint has a displayable glyph, zero otherwise. + * @return Non-zero if the codepoint is in the 'graph' class, zero otherwise. */ int ucd_isgraph(codepoint_t c); -/** @brief Is the codepoint a lower-case letter? +/** @brief Is the codepoint in the 'lower' class? * * @param c The Unicode codepoint to check. - * @return Non-zero if the codepoint is a lower-case letter, zero otherwise. + * @return Non-zero if the codepoint is in the 'lower' class, zero otherwise. */ int ucd_islower(codepoint_t c); -/** @brief Is the codepoint a printable character? +/** @brief Is the codepoint in the 'print' class? * * @param c The Unicode codepoint to check. - * @return Non-zero if the codepoint is a printable character, zero otherwise. + * @return Non-zero if the codepoint is in the 'print' class, zero otherwise. */ int ucd_isprint(codepoint_t c); -/** @brief Is the codepoint a punctuation character? +/** @brief Is the codepoint in the 'punct' class? * * @param c The Unicode codepoint to check. - * @return Non-zero if the codepoint is a punctuation character, zero otherwise. + * @return Non-zero if the codepoint is in the 'punct' class, zero otherwise. */ int ucd_ispunct(codepoint_t c); -/** @brief Is the codepoint a whitespace character? +/** @brief Is the codepoint in the 'space' class? * * @param c The Unicode codepoint to check. - * @return Non-zero if the codepoint is a whitespace character, zero otherwise. + * @return Non-zero if the codepoint is in the 'space' class, zero otherwise. */ int ucd_isspace(codepoint_t c); -/** @brief Is the codepoint an upper-case letter? +/** @brief Is the codepoint in the 'upper' class? * * @param c The Unicode codepoint to check. - * @return Non-zero if the codepoint is an upper-case letter, zero otherwise. + * @return Non-zero if the codepoint is in the 'upper' class, zero otherwise. */ int ucd_isupper(codepoint_t c); @@ -763,110 +763,110 @@ namespace ucd return (script)ucd_lookup_script(c); } - /** @brief Is the codepoint an alpha-numeric character? + /** @brief Is the codepoint in the 'alnum' class? * * @param c The Unicode codepoint to check. - * @return Non-zero if the codepoint is a letter or number, zero otherwise. + * @return Non-zero if the codepoint is in the 'alnum' class, zero otherwise. */ inline int isalnum(codepoint_t c) { return ucd_isalnum(c); } - /** @brief Is the codepoint a letter? + /** @brief Is the codepoint in the 'alpha' class? * * @param c The Unicode codepoint to check. - * @return Non-zero if the codepoint is a letter, zero otherwise. + * @return Non-zero if the codepoint is in the 'alpha' class, zero otherwise. */ inline int isalpha(codepoint_t c) { return ucd_isalpha(c); } - /** @brief Is the codepoint a tab or Unicode whitespace character? + /** @brief Is the codepoint in the 'blank' class? * * @param c The Unicode codepoint to check. - * @return Non-zero if the codepoint is a whitespace character, zero otherwise. + * @return Non-zero if the codepoint is in the 'blank' class, zero otherwise. */ inline int isblank(codepoint_t c) { return ucd_isblank(c); } - /** @brief Is the codepoint a control character? + /** @brief Is the codepoint in the 'cntrl' class? * * @param c The Unicode codepoint to check. - * @return Non-zero if the codepoint is a control character, zero otherwise. + * @return Non-zero if the codepoint is in the 'cntrl' class, zero otherwise. */ inline int iscntrl(codepoint_t c) { return ucd_iscntrl(c); } - /** @brief Is the codepoint a numeric character? + /** @brief Is the codepoint in the 'digit' class? * * @param c The Unicode codepoint to check. - * @return Non-zero if the codepoint is a number, zero otherwise. + * @return Non-zero if the codepoint is in the 'digit' class, zero otherwise. */ inline int isdigit(codepoint_t c) { return ucd_isdigit(c); } - /** @brief Does the codepoint have a displayable glyph? + /** @brief Is the codepoint in the 'graph' class? * * @param c The Unicode codepoint to check. - * @return Non-zero if the codepoint has a displayable glyph, zero otherwise. + * @return Non-zero if the codepoint is in the 'graph' class, zero otherwise. */ inline int isgraph(codepoint_t c) { return ucd_isgraph(c); } - /** @brief Is the codepoint a lower-case letter? + /** @brief Is the codepoint in the 'lower' class? * * @param c The Unicode codepoint to check. - * @return Non-zero if the codepoint is a lower-case letter, zero otherwise. + * @return Non-zero if the codepoint is in the 'lower' class, zero otherwise. */ inline int islower(codepoint_t c) { return ucd_islower(c); } - /** @brief Is the codepoint a printable character? + /** @brief Is the codepoint in the 'print' class? * * @param c The Unicode codepoint to check. - * @return Non-zero if the codepoint is a printable character, zero otherwise. + * @return Non-zero if the codepoint is in the 'print' class, zero otherwise. */ inline int isprint(codepoint_t c) { return ucd_isprint(c); } - /** @brief Is the codepoint a punctuation character? + /** @brief Is the codepoint in the 'punct' class? * * @param c The Unicode codepoint to check. - * @return Non-zero if the codepoint is a punctuation character, zero otherwise. + * @return Non-zero if the codepoint is in the 'punct' class, zero otherwise. */ inline int ispunct(codepoint_t c) { return ucd_ispunct(c); } - /** @brief Is the codepoint a whitespace character? + /** @brief Is the codepoint in the 'space' class? * * @param c The Unicode codepoint to check. - * @return Non-zero if the codepoint is a whitespace character, zero otherwise. + * @return Non-zero if the codepoint is in the 'space' class, zero otherwise. */ inline int isspace(codepoint_t c) { return ucd_isspace(c); } - /** @brief Is the codepoint an upper-case letter? + /** @brief Is the codepoint in the 'upper' class? * * @param c The Unicode codepoint to check. - * @return Non-zero if the codepoint is an upper-case letter, zero otherwise. + * @return Non-zero if the codepoint is in the 'upper' class, zero otherwise. */ inline int isupper(codepoint_t c) { From 34167771fd106c4f89bf16a905f5244e41189101 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sat, 11 Feb 2017 18:18:38 +0000 Subject: [PATCH 08/24] Add an iswxdigit compatibility API. --- CHANGELOG.md | 2 +- README.md | 29 ++++++++++++++--------------- src/ctype.c | 7 +++++++ src/include/ucd/ucd.h | 17 +++++++++++++++++ tests/printcdata.c | 3 +++ tests/printucddata.c | 3 +++ tests/printucddata_cpp.cpp | 3 +++ 7 files changed, 48 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd5a382a..26289ca2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 9.0.0.1 - (In Progress) -* Add `iswblank` compatibility. +* Add `iswblank` and `iswxdigit` compatibility. ## 9.0.0 - 2016-12-28 diff --git a/README.md b/README.md index ee0610f0..36e06e80 100644 --- a/README.md +++ b/README.md @@ -92,21 +92,20 @@ C library provides a set of APIs that are compatible with `wctype.h`. The following character classification functions are provided: -| C API | C++ API | -|---------------|----------------| -| `ucd_isalnum` | `ucd::isalnum` | -| `ucd_isalpha` | `ucd::isalpha` | -| `ucd_isblank` | `ucd::isblank` | -| `ucd_iscntrl` | `ucd::iscntrl` | -| `ucd_isdigit` | `ucd::isdigit` | -| `ucd_isgraph` | `ucd::isgraph` | -| `ucd_islower` | `ucd::islower` | -| `ucd_isprint` | `ucd::isprint` | -| `ucd_ispunct` | `ucd::ispunct` | -| `ucd_isspace` | `ucd::isspace` | -| `ucd_isupper` | `ucd::isupper` | - -__NOTE:__ An equivalent for `isxdigit` is not provided. +| C API | C++ API | +|----------------|-----------------| +| `ucd_isalnum` | `ucd::isalnum` | +| `ucd_isalpha` | `ucd::isalpha` | +| `ucd_isblank` | `ucd::isblank` | +| `ucd_iscntrl` | `ucd::iscntrl` | +| `ucd_isdigit` | `ucd::isdigit` | +| `ucd_isgraph` | `ucd::isgraph` | +| `ucd_islower` | `ucd::islower` | +| `ucd_isprint` | `ucd::isprint` | +| `ucd_ispunct` | `ucd::ispunct` | +| `ucd_isspace` | `ucd::isspace` | +| `ucd_isupper` | `ucd::isupper` | +| `ucd_isxdigit` | `ucd::isxdigit` | ## Build Dependencies diff --git a/src/ctype.c b/src/ctype.c index f1be7643..30edfb93 100644 --- a/src/ctype.c +++ b/src/ctype.c @@ -169,3 +169,10 @@ int ucd_isupper(codepoint_t c) { return ucd_lookup_category(c) == UCD_CATEGORY_Lu; } + +int ucd_isxdigit(codepoint_t c) +{ + return (c >= 0x30 && c <= 0x39) // [0-9] + || (c >= 0x41 && c <= 0x46) // [A-Z] + || (c >= 0x61 && c <= 0x66); // [a-z] +} diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index 6693329c..b4bff02a 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -402,6 +402,13 @@ int ucd_isspace(codepoint_t c); */ int ucd_isupper(codepoint_t c); +/** @brief Is the codepoint in the 'xdigit' class? + * + * @param c The Unicode codepoint to check. + * @return Non-zero if the codepoint is in the 'xdigit' class, zero otherwise. + */ +int ucd_isxdigit(codepoint_t c); + /** @brief Convert the Unicode codepoint to upper-case. * * This function only uses the simple case mapping present in the @@ -873,6 +880,16 @@ namespace ucd return ucd_isupper(c); } + /** @brief Is the codepoint in the 'xdigit' class? + * + * @param c The Unicode codepoint to check. + * @return Non-zero if the codepoint is in the 'xdigit' class, zero otherwise. + */ + inline int isxdigit(codepoint_t c) + { + return ucd_isxdigit(c); + } + /** @brief Convert the Unicode codepoint to upper-case. * * This function only uses the simple case mapping present in the diff --git a/tests/printcdata.c b/tests/printcdata.c index 8b19749a..24d57c92 100644 --- a/tests/printcdata.c +++ b/tests/printcdata.c @@ -141,6 +141,9 @@ void uprintf_is(FILE *out, codepoint_t c, char mode) case 'u': // upper case fputc(iswupper(c) ? '1' : '0', out); break; + case 'x': // xdigit + fputc(iswxdigit(c) ? '1' : '0', out); + break; } } diff --git a/tests/printucddata.c b/tests/printucddata.c index 9400b547..e397f12e 100644 --- a/tests/printucddata.c +++ b/tests/printucddata.c @@ -138,6 +138,9 @@ void uprintf_is(FILE *out, codepoint_t c, char mode) case 'u': // upper case fputc(ucd_isupper(c) ? '1' : '0', out); break; + case 'x': // xdigit + fputc(ucd_isxdigit(c) ? '1' : '0', out); + break; } } diff --git a/tests/printucddata_cpp.cpp b/tests/printucddata_cpp.cpp index 41e37605..2318b09a 100644 --- a/tests/printucddata_cpp.cpp +++ b/tests/printucddata_cpp.cpp @@ -138,6 +138,9 @@ void uprintf_is(FILE *out, ucd::codepoint_t c, char mode) case 'u': // upper case fputc(ucd::isupper(c) ? '1' : '0', out); break; + case 'x': // xdigit + fputc(ucd::isxdigit(c) ? '1' : '0', out); + break; } } From 01852ee90d6c26f507f926f6110ab0a439270366 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sat, 11 Feb 2017 18:20:08 +0000 Subject: [PATCH 09/24] isdigit: C99 compatibility -- only check [0-9]. --- src/ctype.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/ctype.c b/src/ctype.c index 30edfb93..91aaaff6 100644 --- a/src/ctype.c +++ b/src/ctype.c @@ -73,15 +73,7 @@ int ucd_iscntrl(codepoint_t c) int ucd_isdigit(codepoint_t c) { - switch (ucd_lookup_category(c)) - { - case UCD_CATEGORY_Nd: - case UCD_CATEGORY_Nl: - case UCD_CATEGORY_No: - return 1; - default: - return 0; - } + return (c >= 0x30 && c <= 0x39); // [0-9] } int ucd_isgraph(codepoint_t c) From a2193799e4bfbb7d1ad0aab62d41098eb22f3320 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 12 Feb 2017 08:03:15 +0000 Subject: [PATCH 10/24] printdata: use get to return a default value if the map key is not present --- tools/printdata.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/tools/printdata.py b/tools/printdata.py index 9ea6fde4..8df3e011 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -40,10 +40,6 @@ if '--with-csur' in sys.argv: for csur in ['Klingon']: for data in ucd.parse_ucd_data('data/csur', csur): for codepoint in data['CodePoint']: - if not 'TitleCase' in data: data['TitleCase'] = codepoint - if not 'UpperCase' in data: data['UpperCase'] = codepoint - if not 'LowerCase' in data: data['LowerCase'] = codepoint - if not 'Properties' in data: data['Properties'] = [] unicode_chars[codepoint] = data null = ucd.CodePoint('0000') @@ -52,19 +48,16 @@ if __name__ == '__main__': try: data = unicode_chars[codepoint] except KeyError: - data = {'GeneralCategory': 'Cn', 'TitleCase': codepoint, 'UpperCase': codepoint, 'LowerCase': codepoint, 'Properties': []} - try: - script = data['Script'] - except KeyError: - script = 'Zzzz' - title = data['TitleCase'] - upper = data['UpperCase'] - lower = data['LowerCase'] + data = {} + script = data.get('Script', 'Zzzz') + title = data.get('TitleCase', codepoint) + upper = data.get('UpperCase', codepoint) + lower = data.get('LowerCase', codepoint) if title == null: title = codepoint if upper == null: upper = codepoint if lower == null: lower = codepoint print('%s %s %s %s %s %s %s %s' % ( codepoint, script, - data['GeneralCategory'][0], data['GeneralCategory'], + data.get('GeneralCategory', 'Cn')[0], data.get('GeneralCategory', 'Cn'), upper, lower, title, - ' '.join(data['Properties']))) + ' '.join(data.get('Properties', [])))) From ceda811b12c3330a6000162c5c5bbbdac3c54910 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 12 Feb 2017 08:16:23 +0000 Subject: [PATCH 11/24] printdata: add the properties to the primary data map --- tests/printcdata.c | 8 ++------ tests/printucddata.c | 8 ++------ tests/printucddata_cpp.cpp | 8 ++------ tools/printdata.py | 7 +++---- 4 files changed, 9 insertions(+), 22 deletions(-) diff --git a/tests/printcdata.c b/tests/printcdata.c index 24d57c92..21f668b9 100644 --- a/tests/printcdata.c +++ b/tests/printcdata.c @@ -178,10 +178,6 @@ void uprintf(FILE *out, codepoint_t c, const char *format) case 'U': // uppercase uprintf_codepoint(out, towupper(c), *++format); break; - case 'W': // whitespace - if (iswspace(c)) - fputs("White_Space", out); - break; } ++format; break; @@ -218,7 +214,7 @@ void print_file(FILE *in, const char *format) { codepoint_t c = 0; while (fget_utf8c(in, &c)) - uprintf(stdout, c, format ? format : "%pc\t%pH\t%s\t%c\t%Uc\t%Lc\t%Tc\t%W\n"); + uprintf(stdout, c, format ? format : "%pc\t%pH\t%s\t%c\t%Uc\t%Lc\t%Tc\t%is\n"); } int main(int argc, char **argv) @@ -252,7 +248,7 @@ int main(int argc, char **argv) else { for (codepoint_t c = 0; c <= 0x10FFFF; ++c) - uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %W\n"); + uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %is\n"); } return 0; } diff --git a/tests/printucddata.c b/tests/printucddata.c index e397f12e..47086c52 100644 --- a/tests/printucddata.c +++ b/tests/printucddata.c @@ -175,10 +175,6 @@ void uprintf(FILE *out, codepoint_t c, const char *format) case 'U': // uppercase uprintf_codepoint(out, ucd_toupper(c), *++format); break; - case 'W': // whitespace - if (ucd_isspace(c)) - fputs("White_Space", out); - break; } ++format; break; @@ -215,7 +211,7 @@ void print_file(FILE *in, const char *format) { codepoint_t c = 0; while (fget_utf8c(in, &c)) - uprintf(stdout, c, format ? format : "%pc\t%pH\t%s\t%c\t%Uc\t%Lc\t%Tc\t%W\n"); + uprintf(stdout, c, format ? format : "%pc\t%pH\t%s\t%c\t%Uc\t%Lc\t%Tc\t%is\n"); } int main(int argc, char **argv) @@ -247,7 +243,7 @@ int main(int argc, char **argv) else { for (codepoint_t c = 0; c <= 0x10FFFF; ++c) - uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %W\n"); + uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %is\n"); } return 0; } diff --git a/tests/printucddata_cpp.cpp b/tests/printucddata_cpp.cpp index 2318b09a..8bc8fc71 100644 --- a/tests/printucddata_cpp.cpp +++ b/tests/printucddata_cpp.cpp @@ -175,10 +175,6 @@ void uprintf(FILE *out, ucd::codepoint_t c, const char *format) case 'U': // uppercase uprintf_codepoint(out, ucd::toupper(c), *++format); break; - case 'W': // whitespace - if (ucd::isspace(c)) - fputs("White_Space", out); - break; } ++format; break; @@ -215,7 +211,7 @@ void print_file(FILE *in, const char *format) { ucd::codepoint_t c = 0; while (fget_utf8c(in, c)) - uprintf(stdout, c, format ? format : "%pc\t%pH\t%s\t%c\t%Uc\t%Lc\t%Tc\t%W\n"); + uprintf(stdout, c, format ? format : "%pc\t%pH\t%s\t%c\t%Uc\t%Lc\t%Tc\t%is\n"); } int main(int argc, char **argv) @@ -247,7 +243,7 @@ int main(int argc, char **argv) else { for (ucd::codepoint_t c = 0; c <= 0x10FFFF; ++c) - uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %W\n"); + uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %is\n"); } return 0; } diff --git a/tools/printdata.py b/tools/printdata.py index 8df3e011..e6de9313 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -1,6 +1,6 @@ #!/usr/bin/python -# Copyright (C) 2012 Reece H. Dunn +# Copyright (C) 2012-2017 Reece H. Dunn # # This file is part of ucd-tools. # @@ -28,11 +28,10 @@ unicode_chars = {} for data in ucd.parse_ucd_data(ucd_rootdir, 'UnicodeData'): for codepoint in data['CodePoint']: unicode_chars[codepoint] = data - unicode_chars[codepoint]['Properties'] = [] for data in ucd.parse_ucd_data(ucd_rootdir, 'PropList'): if data['Property'] in ['White_Space']: for codepoint in data['Range']: - unicode_chars[codepoint]['Properties'].append(data['Property']) + unicode_chars[codepoint][data['Property']] = 1 for data in ucd.parse_ucd_data(ucd_rootdir, 'Scripts'): for codepoint in data['Range']: unicode_chars[codepoint]['Script'] = data['Script'] @@ -60,4 +59,4 @@ if __name__ == '__main__': codepoint, script, data.get('GeneralCategory', 'Cn')[0], data.get('GeneralCategory', 'Cn'), upper, lower, title, - ' '.join(data.get('Properties', [])))) + data.get('White_Space', 0))) From e77b7c7b49303df7866d8809b2e66649e6c82fd7 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 12 Feb 2017 08:33:37 +0000 Subject: [PATCH 12/24] printdata: create an isspace helper function --- tools/printdata.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/printdata.py b/tools/printdata.py index e6de9313..7c49c892 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -41,6 +41,9 @@ if '--with-csur' in sys.argv: for codepoint in data['CodePoint']: unicode_chars[codepoint] = data +def isspace(data): + return data.get('White_Space', 0) + null = ucd.CodePoint('0000') if __name__ == '__main__': for codepoint in ucd.CodeRange('000000..10FFFF'): @@ -59,4 +62,4 @@ if __name__ == '__main__': codepoint, script, data.get('GeneralCategory', 'Cn')[0], data.get('GeneralCategory', 'Cn'), upper, lower, title, - data.get('White_Space', 0))) + isspace(data))) From bd71fed013e07f1b794d1c1bd1fc1e5cc078452b Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 12 Feb 2017 09:22:25 +0000 Subject: [PATCH 13/24] ctype: return true in isupper/islower if there is a simple case mapping present --- CHANGELOG.md | 1 + src/ctype.c | 6 ++++-- tests/printcdata.c | 2 +- tests/printucddata.c | 2 +- tests/printucddata_cpp.cpp | 2 +- tools/printdata.py | 24 +++++++++++++++++++++--- 6 files changed, 29 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26289ca2..0612bb58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 9.0.0.1 - (In Progress) * Add `iswblank` and `iswxdigit` compatibility. +* Improve ctype compatibility. ## 9.0.0 - 2016-12-28 diff --git a/src/ctype.c b/src/ctype.c index 91aaaff6..9f9b91b9 100644 --- a/src/ctype.c +++ b/src/ctype.c @@ -97,7 +97,8 @@ int ucd_isgraph(codepoint_t c) int ucd_islower(codepoint_t c) { - return ucd_lookup_category(c) == UCD_CATEGORY_Ll; + return ucd_lookup_category(c) == UCD_CATEGORY_Ll + || ucd_toupper(c) != c; } int ucd_isprint(codepoint_t c) @@ -159,7 +160,8 @@ int ucd_isspace(codepoint_t c) int ucd_isupper(codepoint_t c) { - return ucd_lookup_category(c) == UCD_CATEGORY_Lu; + return ucd_lookup_category(c) == UCD_CATEGORY_Lu + || ucd_tolower(c) != c; } int ucd_isxdigit(codepoint_t c) diff --git a/tests/printcdata.c b/tests/printcdata.c index 21f668b9..6f59f453 100644 --- a/tests/printcdata.c +++ b/tests/printcdata.c @@ -248,7 +248,7 @@ int main(int argc, char **argv) else { for (codepoint_t c = 0; c <= 0x10FFFF; ++c) - uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %is\n"); + uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %is %iu %il\n"); } return 0; } diff --git a/tests/printucddata.c b/tests/printucddata.c index 47086c52..136d88ee 100644 --- a/tests/printucddata.c +++ b/tests/printucddata.c @@ -243,7 +243,7 @@ int main(int argc, char **argv) else { for (codepoint_t c = 0; c <= 0x10FFFF; ++c) - uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %is\n"); + uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %is %iu %il\n"); } return 0; } diff --git a/tests/printucddata_cpp.cpp b/tests/printucddata_cpp.cpp index 8bc8fc71..c19c4fbf 100644 --- a/tests/printucddata_cpp.cpp +++ b/tests/printucddata_cpp.cpp @@ -243,7 +243,7 @@ int main(int argc, char **argv) else { for (ucd::codepoint_t c = 0; c <= 0x10FFFF; ++c) - uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %is\n"); + uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %is %iu %il\n"); } return 0; } diff --git a/tools/printdata.py b/tools/printdata.py index 7c49c892..fd1eb236 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -24,6 +24,8 @@ import ucd ucd_rootdir = sys.argv[1] csur_rootdir = 'data/csur' +null = ucd.CodePoint('0000') + unicode_chars = {} for data in ucd.parse_ucd_data(ucd_rootdir, 'UnicodeData'): for codepoint in data['CodePoint']: @@ -44,7 +46,22 @@ if '--with-csur' in sys.argv: def isspace(data): return data.get('White_Space', 0) -null = ucd.CodePoint('0000') +def isupper(data): + if data.get('LowerCase', null) != null: + return 1 + elif data.get('GeneralCategory', 'Cn') == 'Lu': + return 1 + else: + return 0 + +def islower(data): + if data.get('UpperCase', null) != null: + return 1 + elif data.get('GeneralCategory', 'Cn') == 'Ll': + return 1 + else: + return 0 + if __name__ == '__main__': for codepoint in ucd.CodeRange('000000..10FFFF'): try: @@ -58,8 +75,9 @@ if __name__ == '__main__': if title == null: title = codepoint if upper == null: upper = codepoint if lower == null: lower = codepoint - print('%s %s %s %s %s %s %s %s' % ( + print('%s %s %s %s %s %s %s %s %s %s' % ( codepoint, script, data.get('GeneralCategory', 'Cn')[0], data.get('GeneralCategory', 'Cn'), upper, lower, title, - isspace(data))) + isspace(data), + isupper(data), islower(data))) From 5f9dc111cf9d1441d7bbe4b8c4730b931a610b3b Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 12 Feb 2017 09:53:43 +0000 Subject: [PATCH 14/24] Add tests for the isdigit and isxdigit ctype APIs. --- tests/printcdata.c | 2 +- tests/printucddata.c | 2 +- tests/printucddata_cpp.cpp | 2 +- tools/printdata.py | 11 +++++++++-- tools/ucd.py | 6 ++++++ 5 files changed, 18 insertions(+), 5 deletions(-) diff --git a/tests/printcdata.c b/tests/printcdata.c index 6f59f453..12b52570 100644 --- a/tests/printcdata.c +++ b/tests/printcdata.c @@ -248,7 +248,7 @@ int main(int argc, char **argv) else { for (codepoint_t c = 0; c <= 0x10FFFF; ++c) - uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %is %iu %il\n"); + uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %id %ix %is %iu %il\n"); } return 0; } diff --git a/tests/printucddata.c b/tests/printucddata.c index 136d88ee..7b24580a 100644 --- a/tests/printucddata.c +++ b/tests/printucddata.c @@ -243,7 +243,7 @@ int main(int argc, char **argv) else { for (codepoint_t c = 0; c <= 0x10FFFF; ++c) - uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %is %iu %il\n"); + uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %id %ix %is %iu %il\n"); } return 0; } diff --git a/tests/printucddata_cpp.cpp b/tests/printucddata_cpp.cpp index c19c4fbf..cfbb29be 100644 --- a/tests/printucddata_cpp.cpp +++ b/tests/printucddata_cpp.cpp @@ -243,7 +243,7 @@ int main(int argc, char **argv) else { for (ucd::codepoint_t c = 0; c <= 0x10FFFF; ++c) - uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %is %iu %il\n"); + uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %id %ix %is %iu %il\n"); } return 0; } diff --git a/tools/printdata.py b/tools/printdata.py index fd1eb236..46a2fc1f 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -43,6 +43,12 @@ if '--with-csur' in sys.argv: for codepoint in data['CodePoint']: unicode_chars[codepoint] = data +def isdigit(data): + return 1 if data['CodePoint'].char() in '0123456789' else 0 + +def isxdigit(data): + return 1 if data['CodePoint'].char() in '0123456789ABCDEFabcdef' else 0 + def isspace(data): return data.get('White_Space', 0) @@ -67,7 +73,7 @@ if __name__ == '__main__': try: data = unicode_chars[codepoint] except KeyError: - data = {} + data = {'CodePoint': codepoint} script = data.get('Script', 'Zzzz') title = data.get('TitleCase', codepoint) upper = data.get('UpperCase', codepoint) @@ -75,9 +81,10 @@ if __name__ == '__main__': if title == null: title = codepoint if upper == null: upper = codepoint if lower == null: lower = codepoint - print('%s %s %s %s %s %s %s %s %s %s' % ( + print('%s %s %s %s %s %s %s %s %s %s %s %s' % ( codepoint, script, data.get('GeneralCategory', 'Cn')[0], data.get('GeneralCategory', 'Cn'), upper, lower, title, + isdigit(data), isxdigit(data), isspace(data), isupper(data), islower(data))) diff --git a/tools/ucd.py b/tools/ucd.py index 2006efe7..4d9fe237 100755 --- a/tools/ucd.py +++ b/tools/ucd.py @@ -50,6 +50,9 @@ class CodePoint: def __lt__(self, other): return self.codepoint < other.codepoint + def char(self): + return unichr(self.codepoint) + class CodeRange: def __init__(self, x): f, l = x.split('..') @@ -69,6 +72,9 @@ class CodeRange: def size(self): return self.last.codepoint - self.first.codepoint + 1 + def char(self): + return unichr(self.first.codepoint) + def codepoint(x): if '..' in x[0]: return CodeRange(x[0]), x[1:] From f109bb918f42823df00b052b72bfa8881d24f91f Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 12 Feb 2017 11:11:52 +0000 Subject: [PATCH 15/24] isspace: don't include characters. --- src/ctype.c | 10 +++++++++- tools/printdata.py | 6 +++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/ctype.c b/src/ctype.c index 9f9b91b9..d9818f90 100644 --- a/src/ctype.c +++ b/src/ctype.c @@ -140,10 +140,18 @@ int ucd_isspace(codepoint_t c) { case UCD_CATEGORY_Zl: case UCD_CATEGORY_Zp: + return 1; case UCD_CATEGORY_Zs: + switch (c) // Exclude characters with the DispositionType + { + case 0x00A0: // U+00A0 : NO-BREAK SPACE + case 0x2007: // U+2007 : FIGURE SPACE + case 0x202F: // U+202F : NARROW NO-BREAK SPACE + return 0; + } return 1; case UCD_CATEGORY_Cc: - switch (c) // Some control characters are also whitespace characters: + switch (c) // Include control characters marked as White_Space { case 0x09: // U+0009 : CHARACTER TABULATION case 0x0A: // U+000A : LINE FEED diff --git a/tools/printdata.py b/tools/printdata.py index 46a2fc1f..7c173add 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -50,7 +50,11 @@ def isxdigit(data): return 1 if data['CodePoint'].char() in '0123456789ABCDEFabcdef' else 0 def isspace(data): - return data.get('White_Space', 0) + if data.get('White_Space', 0): + dt = data.get('DecompositionType', '') + return 1 if dt == None or not dt.startswith('') else 0 + else: + return 0 def isupper(data): if data.get('LowerCase', null) != null: From c9f29403737dbcaa701b92103fe22625b2dd32ca Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 12 Feb 2017 11:47:30 +0000 Subject: [PATCH 16/24] isblank: don't include characters, and add tests for this API. --- src/ctype.c | 7 +++++++ tests/printcdata.c | 2 +- tests/printucddata.c | 2 +- tests/printucddata_cpp.cpp | 2 +- tools/printdata.py | 11 +++++++++-- 5 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/ctype.c b/src/ctype.c index d9818f90..9edd8bfa 100644 --- a/src/ctype.c +++ b/src/ctype.c @@ -58,6 +58,13 @@ int ucd_isblank(codepoint_t c) switch (ucd_lookup_category(c)) { case UCD_CATEGORY_Zs: + switch (c) // Exclude characters with the DispositionType + { + case 0x00A0: // U+00A0 : NO-BREAK SPACE + case 0x2007: // U+2007 : FIGURE SPACE + case 0x202F: // U+202F : NARROW NO-BREAK SPACE + return 0; + } return 1; case UCD_CATEGORY_Cc: return c == 0x09; // U+0009 : CHARACTER TABULATION diff --git a/tests/printcdata.c b/tests/printcdata.c index 12b52570..8580ac62 100644 --- a/tests/printcdata.c +++ b/tests/printcdata.c @@ -248,7 +248,7 @@ int main(int argc, char **argv) else { for (codepoint_t c = 0; c <= 0x10FFFF; ++c) - uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %id %ix %is %iu %il\n"); + uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %id %ix %is %ib %iu %il\n"); } return 0; } diff --git a/tests/printucddata.c b/tests/printucddata.c index 7b24580a..62aaab8d 100644 --- a/tests/printucddata.c +++ b/tests/printucddata.c @@ -243,7 +243,7 @@ int main(int argc, char **argv) else { for (codepoint_t c = 0; c <= 0x10FFFF; ++c) - uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %id %ix %is %iu %il\n"); + uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %id %ix %is %ib %iu %il\n"); } return 0; } diff --git a/tests/printucddata_cpp.cpp b/tests/printucddata_cpp.cpp index cfbb29be..e5542b61 100644 --- a/tests/printucddata_cpp.cpp +++ b/tests/printucddata_cpp.cpp @@ -243,7 +243,7 @@ int main(int argc, char **argv) else { for (ucd::codepoint_t c = 0; c <= 0x10FFFF; ++c) - uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %id %ix %is %iu %il\n"); + uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %id %ix %is %ib %iu %il\n"); } return 0; } diff --git a/tools/printdata.py b/tools/printdata.py index 7c173add..ebe61e92 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -56,6 +56,13 @@ def isspace(data): else: return 0 +def isblank(data): # word separator + if data.get('GeneralCategory', 'Cn') == 'Zs' or data['CodePoint'].char() == '\t': + dt = data.get('DecompositionType', '') + return 1 if dt == None or not dt.startswith('') else 0 + else: + return 0 + def isupper(data): if data.get('LowerCase', null) != null: return 1 @@ -85,10 +92,10 @@ if __name__ == '__main__': if title == null: title = codepoint if upper == null: upper = codepoint if lower == null: lower = codepoint - print('%s %s %s %s %s %s %s %s %s %s %s %s' % ( + print('%s %s %s %s %s %s %s %s %s %s %s %s %s' % ( codepoint, script, data.get('GeneralCategory', 'Cn')[0], data.get('GeneralCategory', 'Cn'), upper, lower, title, isdigit(data), isxdigit(data), - isspace(data), + isspace(data), isblank(data), isupper(data), islower(data))) From eb08dc422ddaf971da6f6168107eb5ada41eee8b Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 12 Feb 2017 12:24:18 +0000 Subject: [PATCH 17/24] Update the copyright dates. --- src/ctype.c | 2 +- src/include/ucd/ucd.h | 2 +- tools/ucd.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ctype.c b/src/ctype.c index 9edd8bfa..ad0e0976 100644 --- a/src/ctype.c +++ b/src/ctype.c @@ -1,6 +1,6 @@ /* ctype-style APIs. * - * Copyright (C) 2012-2016 Reece H. Dunn + * Copyright (C) 2012-2017 Reece H. Dunn * * This file is part of ucd-tools. * diff --git a/src/include/ucd/ucd.h b/src/include/ucd/ucd.h index b4bff02a..840b2995 100644 --- a/src/include/ucd/ucd.h +++ b/src/include/ucd/ucd.h @@ -1,6 +1,6 @@ /* Unicode Character Database API * - * Copyright (C) 2012-2016 Reece H. Dunn + * Copyright (C) 2012-2017 Reece H. Dunn * * This file is part of ucd-tools. * diff --git a/tools/ucd.py b/tools/ucd.py index 4d9fe237..e3355407 100755 --- a/tools/ucd.py +++ b/tools/ucd.py @@ -1,6 +1,6 @@ #!/usr/bin/python -# Copyright (C) 2012-2014 Reece H. Dunn +# Copyright (C) 2012-2017 Reece H. Dunn # # This file is part of ucd-tools. # From ac082c94008c11765c050d2f6c3dd5e4f2de937e Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 12 Feb 2017 13:19:43 +0000 Subject: [PATCH 18/24] Add tests for the remaining is* APIs. --- tests/printcdata.c | 3 ++- tests/printucddata.c | 3 ++- tests/printucddata_cpp.cpp | 3 ++- tools/printdata.py | 39 +++++++++++++++++++++++++++++++++++--- 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/tests/printcdata.c b/tests/printcdata.c index 8580ac62..f9af2d08 100644 --- a/tests/printcdata.c +++ b/tests/printcdata.c @@ -248,7 +248,8 @@ int main(int argc, char **argv) else { for (codepoint_t c = 0; c <= 0x10FFFF; ++c) - uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %id %ix %is %ib %iu %il\n"); + uprintf(stdout, c, format ? format : + "%pH %s %C %c %UH %LH %TH %id %ix %ic %is %ib %ip %iP %ig %iA %ia %iu %il\n"); } return 0; } diff --git a/tests/printucddata.c b/tests/printucddata.c index 62aaab8d..e410480c 100644 --- a/tests/printucddata.c +++ b/tests/printucddata.c @@ -243,7 +243,8 @@ int main(int argc, char **argv) else { for (codepoint_t c = 0; c <= 0x10FFFF; ++c) - uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %id %ix %is %ib %iu %il\n"); + uprintf(stdout, c, format ? format : + "%pH %s %C %c %UH %LH %TH %id %ix %ic %is %ib %ip %iP %ig %iA %ia %iu %il\n"); } return 0; } diff --git a/tests/printucddata_cpp.cpp b/tests/printucddata_cpp.cpp index e5542b61..03214bae 100644 --- a/tests/printucddata_cpp.cpp +++ b/tests/printucddata_cpp.cpp @@ -243,7 +243,8 @@ int main(int argc, char **argv) else { for (ucd::codepoint_t c = 0; c <= 0x10FFFF; ++c) - uprintf(stdout, c, format ? format : "%pH %s %C %c %UH %LH %TH %id %ix %is %ib %iu %il\n"); + uprintf(stdout, c, format ? format : + "%pH %s %C %c %UH %LH %TH %id %ix %ic %is %ib %ip %iP %ig %iA %ia %iu %il\n"); } return 0; } diff --git a/tools/printdata.py b/tools/printdata.py index ebe61e92..f9db6191 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -43,6 +43,9 @@ if '--with-csur' in sys.argv: for codepoint in data['CodePoint']: unicode_chars[codepoint] = data +def iscntrl(data): + return 1 if data.get('Name', '') == '' else 0 + def isdigit(data): return 1 if data['CodePoint'].char() in '0123456789' else 0 @@ -63,6 +66,36 @@ def isblank(data): # word separator else: return 0 +def ispunct(data): + if data.get('GeneralCategory', 'Cn')[0] in 'P': + return 1 + else: + return 0 + +def isprint(data): + if data.get('GeneralCategory', 'Cn')[0] in 'LMNPSZ': # not in 'CI' + return 1 + else: + return 0 + +def isgraph(data): + if data.get('GeneralCategory', 'Cn')[0] in 'LMNPS': # not in 'CZI' + return 1 + else: + return 0 + +def isalnum(data): + if data.get('GeneralCategory', 'Cn')[0] in 'LN': + return 1 + else: + return 0 + +def isalpha(data): + if data.get('GeneralCategory', 'Cn')[0] in 'L': + return 1 + else: + return 0 + def isupper(data): if data.get('LowerCase', null) != null: return 1 @@ -92,10 +125,10 @@ if __name__ == '__main__': if title == null: title = codepoint if upper == null: upper = codepoint if lower == null: lower = codepoint - print('%s %s %s %s %s %s %s %s %s %s %s %s %s' % ( + print('%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s' % ( codepoint, script, data.get('GeneralCategory', 'Cn')[0], data.get('GeneralCategory', 'Cn'), upper, lower, title, isdigit(data), isxdigit(data), - isspace(data), isblank(data), - isupper(data), islower(data))) + iscntrl(data), isspace(data), isblank(data), ispunct(data), + isprint(data), isgraph(data), isalnum(data), isalpha(data), isupper(data), islower(data))) From d9565018634248f8c069f5d6078939b3e844b3e6 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 12 Feb 2017 14:26:09 +0000 Subject: [PATCH 19/24] printdata: load all the properties from the PropList data. --- tools/printdata.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/printdata.py b/tools/printdata.py index f9db6191..0bee85e2 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -31,8 +31,11 @@ for data in ucd.parse_ucd_data(ucd_rootdir, 'UnicodeData'): for codepoint in data['CodePoint']: unicode_chars[codepoint] = data for data in ucd.parse_ucd_data(ucd_rootdir, 'PropList'): - if data['Property'] in ['White_Space']: - for codepoint in data['Range']: + for codepoint in data['Range']: + try: + unicode_chars[codepoint][data['Property']] = 1 + except KeyError: + unicode_chars[codepoint] = {'CodePoint': codepoint} unicode_chars[codepoint][data['Property']] = 1 for data in ucd.parse_ucd_data(ucd_rootdir, 'Scripts'): for codepoint in data['Range']: From 1fc20f39f5e3a5bdfe8dbfd10c9f94f730cf5719 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 12 Feb 2017 14:51:24 +0000 Subject: [PATCH 20/24] isupper: include the Other_Uppercase characters. --- src/ctype.c | 18 ++++++++++++++++-- tools/printdata.py | 6 ++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/ctype.c b/src/ctype.c index ad0e0976..1355a4ee 100644 --- a/src/ctype.c +++ b/src/ctype.c @@ -175,8 +175,22 @@ int ucd_isspace(codepoint_t c) int ucd_isupper(codepoint_t c) { - return ucd_lookup_category(c) == UCD_CATEGORY_Lu - || ucd_tolower(c) != c; + switch (ucd_lookup_category(c)) + { + case UCD_CATEGORY_Lu: + return 1; + case UCD_CATEGORY_Lt: + return ucd_tolower(c) != c; + case UCD_CATEGORY_Nl: + return (c >= 0x002160 && c <= 0x00216F); // Other_Uppercase + case UCD_CATEGORY_So: + return (c >= 0x0024B6 && c <= 0x0024CF) // Other_Uppercase + || (c >= 0x01F130 && c <= 0x01F149) // Other_Uppercase + || (c >= 0x01F150 && c <= 0x01F169) // Other_Uppercase + || (c >= 0x01F170 && c <= 0x01F189); // Other_Uppercase + default: + return 0; + } } int ucd_isxdigit(codepoint_t c) diff --git a/tools/printdata.py b/tools/printdata.py index 0bee85e2..6e84c67c 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -100,9 +100,11 @@ def isalpha(data): return 0 def isupper(data): - if data.get('LowerCase', null) != null: + if data.get('GeneralCategory', 'Cn') == 'Lu': return 1 - elif data.get('GeneralCategory', 'Cn') == 'Lu': + elif data.get('Other_Uppercase', 0): + return 1 + elif data.get('LowerCase', null) != null: # Some Lt characters have lowercase forms. return 1 else: return 0 From 9d1469b9567902257e79d67e3b2e233bfaeeb7b4 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 12 Feb 2017 16:03:55 +0000 Subject: [PATCH 21/24] printdata: use the Uppercase property from DerivedCoreProperties. --- Makefile.am | 5 +++++ tools/printdata.py | 19 +++++++++---------- tools/ucd.py | 4 ++++ 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/Makefile.am b/Makefile.am index 7717564d..6ffe4d11 100644 --- a/Makefile.am +++ b/Makefile.am @@ -63,6 +63,10 @@ data/ucd/PropList.txt: mkdir -pv data/ucd curl ${UCD_SRCDIR}/${UCD_VERSION}/ucd/PropList.txt > $@ +data/ucd/DerivedCoreProperties.txt: + mkdir -pv data/ucd + curl ${UCD_SRCDIR}/${UCD_VERSION}/ucd/DerivedCoreProperties.txt > $@ + data/ucd/PropertyValueAliases.txt: mkdir -pv data/ucd curl ${UCD_SRCDIR}/${UCD_VERSION}/ucd/PropertyValueAliases.txt > $@ @@ -138,6 +142,7 @@ tests_printucddata_cpp_LDADD = src/libucd.la tests/unicode-data.expected: tools/printdata.py tools/ucd.py \ data/ucd/UnicodeData.txt \ data/ucd/PropList.txt \ + data/ucd/DerivedCoreProperties.txt \ data/ucd/Scripts.txt tools/printdata.py ${UCD_ROOTDIR} ${UCD_FLAGS} > $@ diff --git a/tools/printdata.py b/tools/printdata.py index 6e84c67c..2c517cc0 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -30,13 +30,14 @@ unicode_chars = {} for data in ucd.parse_ucd_data(ucd_rootdir, 'UnicodeData'): for codepoint in data['CodePoint']: unicode_chars[codepoint] = data -for data in ucd.parse_ucd_data(ucd_rootdir, 'PropList'): - for codepoint in data['Range']: - try: - unicode_chars[codepoint][data['Property']] = 1 - except KeyError: - unicode_chars[codepoint] = {'CodePoint': codepoint} - unicode_chars[codepoint][data['Property']] = 1 +for propfile in ['PropList', 'DerivedCoreProperties']: + for data in ucd.parse_ucd_data(ucd_rootdir, propfile): + for codepoint in data['Range']: + try: + unicode_chars[codepoint][data['Property']] = 1 + except KeyError: + unicode_chars[codepoint] = {'CodePoint': codepoint} + unicode_chars[codepoint][data['Property']] = 1 for data in ucd.parse_ucd_data(ucd_rootdir, 'Scripts'): for codepoint in data['Range']: unicode_chars[codepoint]['Script'] = data['Script'] @@ -100,9 +101,7 @@ def isalpha(data): return 0 def isupper(data): - if data.get('GeneralCategory', 'Cn') == 'Lu': - return 1 - elif data.get('Other_Uppercase', 0): + if data.get('Uppercase', 0): return 1 elif data.get('LowerCase', null) != null: # Some Lt characters have lowercase forms. return 1 diff --git a/tools/ucd.py b/tools/ucd.py index e3355407..361cebd9 100755 --- a/tools/ucd.py +++ b/tools/ucd.py @@ -113,6 +113,10 @@ data_items = { ('Range', codepoint), ('Age', string), ], + 'DerivedCoreProperties': [ + ('Range', codepoint), + ('Property', string), + ], 'PropList': [ ('Range', codepoint), ('Property', string), From b38659582f47e1407af6efab8101bb5c4e832dd9 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 12 Feb 2017 17:28:22 +0000 Subject: [PATCH 22/24] islower: include the Other_Lowercase characters. --- src/ctype.c | 36 ++++++++++++++++++++++++++++++++++-- tools/printdata.py | 4 ++-- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/ctype.c b/src/ctype.c index 1355a4ee..e77d213e 100644 --- a/src/ctype.c +++ b/src/ctype.c @@ -104,8 +104,40 @@ int ucd_isgraph(codepoint_t c) int ucd_islower(codepoint_t c) { - return ucd_lookup_category(c) == UCD_CATEGORY_Ll - || ucd_toupper(c) != c; + switch (ucd_lookup_category(c)) + { + case UCD_CATEGORY_Ll: + return 1; + case UCD_CATEGORY_Lt: + return ucd_toupper(c) != c; + case UCD_CATEGORY_Lo: + return c == 0xAA // Other_Lowercase : FEMININE ORDINAL INDICATOR + || c == 0xBA; // Other_Lowercase : MASCULINE ORDINAL INDICATOR + case UCD_CATEGORY_Lm: + return (c >= 0x02B0 && c <= 0x02B8) // Other_Lowercase + || (c >= 0x02C0 && c <= 0x02C1) // Other_Lowercase + || (c >= 0x02E0 && c <= 0x02E4) // Other_Lowercase + || c == 0x037A // Other_Lowercase + || (c >= 0x1D2C && c <= 0x1D6A) // Other_Lowercase + || c == 0x1D78 // Other_Lowercase + || (c >= 0x1D9B && c <= 0x1DBF) // Other_Lowercase + || c == 0x2071 // Other_Lowercase + || c == 0x207F // Other_Lowercase + || (c >= 0x2090 && c <= 0x209C) // Other_Lowercase + || (c >= 0x2C7C && c <= 0x2C7D) // Other_Lowercase + || (c >= 0xA69C && c <= 0xA69D) // Other_Lowercase + || c == 0xA770 // Other_Lowercase + || (c >= 0xA7F8 && c <= 0xA7F9) // Other_Lowercase + || (c >= 0xAB5C && c <= 0xAB5F); // Other_Lowercase + case UCD_CATEGORY_Mn: + return c == 0x0345; // Other_Lowercase : COMBINING GREEK YPOGEGRAMMENI + case UCD_CATEGORY_Nl: + return (c >= 0x2170 && c <= 0x217F); // Other_Lowercase + case UCD_CATEGORY_So: + return (c >= 0x24D0 && c <= 0x24E9); // Other_Lowercase + default: + return 0; + } } int ucd_isprint(codepoint_t c) diff --git a/tools/printdata.py b/tools/printdata.py index 2c517cc0..8ac48e87 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -109,9 +109,9 @@ def isupper(data): return 0 def islower(data): - if data.get('UpperCase', null) != null: + if data.get('Lowercase', 0): return 1 - elif data.get('GeneralCategory', 'Cn') == 'Ll': + elif data.get('UpperCase', null) != null: return 1 else: return 0 From cbddf092c9b952e86ce1fc72d8a9a377d093bf51 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 12 Feb 2017 21:20:03 +0000 Subject: [PATCH 23/24] isalpha: use the Alphabetic derived core property to check that the character is in the alpha class. --- src/ctype.c | 262 ++++++++++++++++++++++++++++++++++++++++++++- tools/printdata.py | 5 +- 2 files changed, 261 insertions(+), 6 deletions(-) diff --git a/src/ctype.c b/src/ctype.c index e77d213e..6cd20ff7 100644 --- a/src/ctype.c +++ b/src/ctype.c @@ -20,6 +20,259 @@ #include "ucd/ucd.h" +static int other_alphabetic_MnMcSo(codepoint_t c) +{ + switch (c & 0xFFFFFF00) + { + case 0x0300: + return c == 0x0345; + case 0x0500: + return (c >= 0x05B0 && c <= 0x05BD) + || c == 0x05BF + || (c >= 0x05C1 && c <= 0x05C2) + || (c >= 0x05C4 && c <= 0x05C5) + || c == 0x05C7; + case 0x0600: + return (c >= 0x0610 && c <= 0x061A) + || (c >= 0x064B && c <= 0x0657) + || (c >= 0x0659 && c <= 0x065F) + || c == 0x0670 + || (c >= 0x06D6 && c <= 0x06DC) + || (c >= 0x06E1 && c <= 0x06E4) + || (c >= 0x06E7 && c <= 0x06E8) + || c == 0x06ED; + case 0x0700: + return c == 0x0711 + || (c >= 0x0730 && c <= 0x073F) + || (c >= 0x07A6 && c <= 0x07B0); + case 0x0800: + return (c >= 0x0816 && c <= 0x0817) + || (c >= 0x081B && c <= 0x0823) + || (c >= 0x0825 && c <= 0x0827) + || (c >= 0x0829 && c <= 0x082C) + || (c >= 0x08D4 && c <= 0x08DF) + || (c >= 0x08E3 && c <= 0x08E9) + || c >= 0x08F0; + case 0x0900: + return (c >= 0x0900 && c <= 0x0903) + || (c >= 0x093A && c <= 0x093B) + || (c >= 0x093E && c <= 0x094C) + || (c >= 0x094E && c <= 0x094F) + || (c >= 0x0955 && c <= 0x0957) + || (c >= 0x0962 && c <= 0x0963) + || (c >= 0x0981 && c <= 0x0983) + || (c >= 0x0981 && c <= 0x0983) + || (c >= 0x09BE && c <= 0x09C4) + || (c >= 0x09C7 && c <= 0x09C8) + || (c >= 0x09CB && c <= 0x09CC) + || c == 0x09D7 + || (c >= 0x09E2 && c <= 0x09E3); + case 0x0A00: + return (c >= 0x0A01 && c <= 0x0A03) + || (c >= 0x0A3E && c <= 0x0A42) + || (c >= 0x0A47 && c <= 0x0A48) + || (c >= 0x0A4B && c <= 0x0A4C) + || c == 0x0A51 + || (c >= 0x0A70 && c <= 0x0A71) + || c == 0x0A75 + || (c >= 0x0A81 && c <= 0x0A83) + || (c >= 0x0ABE && c <= 0x0AC5) + || (c >= 0x0AC7 && c <= 0x0AC9) + || (c >= 0x0ACB && c <= 0x0ACC) + || (c >= 0x0AE2 && c <= 0x0AE3); + case 0x0B00: + return (c >= 0x0B01 && c <= 0x0B03) + || (c >= 0x0B3E && c <= 0x0B44) + || (c >= 0x0B47 && c <= 0x0B48) + || (c >= 0x0B4B && c <= 0x0B4C) + || (c >= 0x0B56 && c <= 0x0B57) + || (c >= 0x0B62 && c <= 0x0B63) + || c == 0x0B82 + || (c >= 0x0BBE && c <= 0x0BC2) + || (c >= 0x0BC6 && c <= 0x0BC8) + || (c >= 0x0BCA && c <= 0x0BCC) + || c == 0x0BD7; + case 0x0C00: + return (c >= 0x0C00 && c <= 0x0C03) + || (c >= 0x0C3E && c <= 0x0C44) + || (c >= 0x0C46 && c <= 0x0C48) + || (c >= 0x0C4A && c <= 0x0C4C) + || (c >= 0x0C55 && c <= 0x0C56) + || (c >= 0x0C62 && c <= 0x0C63) + || (c >= 0x0C81 && c <= 0x0C83) + || (c >= 0x0CBE && c <= 0x0CBF) + || (c >= 0x0CC0 && c <= 0x0CC4) + || (c >= 0x0CC6 && c <= 0x0CC8) + || (c >= 0x0CCA && c <= 0x0CCC) + || (c >= 0x0CD5 && c <= 0x0CD6) + || (c >= 0x0CE2 && c <= 0x0CE3); + case 0x0D00: + return (c >= 0x0D01 && c <= 0x0D03) + || (c >= 0x0D3E && c <= 0x0D44) + || (c >= 0x0D46 && c <= 0x0D48) + || (c >= 0x0D4A && c <= 0x0D4C) + || c == 0x0D57 + || (c >= 0x0D62 && c <= 0x0D63) + || (c >= 0x0D82 && c <= 0x0D83) + || (c >= 0x0DCF && c <= 0x0DD4) + || c == 0x0DD6 + || (c >= 0x0DD8 && c <= 0x0DDF) + || (c >= 0x0DF2 && c <= 0x0DF3); + case 0x0E00: + return c == 0x0E31 + || (c >= 0x0E34 && c <= 0x0E3A) + || c == 0x0E4D + || c == 0x0EB1 + || (c >= 0x0EB4 && c <= 0x0EB9) + || (c >= 0x0EBB && c <= 0x0EBD) + || c == 0x0ECD; + case 0x0F00: + return (c >= 0x0F71 && c <= 0x0F7F) + || (c >= 0x0F80 && c <= 0x0F81) + || (c >= 0x0F8D && c <= 0x0F97) + || (c >= 0x0F99 && c <= 0x0FBC); + case 0x1000: + return (c >= 0x102B && c <= 0x1036) + || c == 0x1038 + || (c >= 0x103B && c <= 0x103E) + || (c >= 0x1056 && c <= 0x1059) + || (c >= 0x105E && c <= 0x1060) + || c == 0x1062 + || (c >= 0x1067 && c <= 0x1068) + || (c >= 0x1071 && c <= 0x1074) + || (c >= 0x1082 && c <= 0x1086) + || (c >= 0x109C && c <= 0x109D); + case 0x1300: + return c == 0x135F; + case 0x1700: + return (c >= 0x1712 && c <= 0x1713) + || (c >= 0x1732 && c <= 0x1733) + || (c >= 0x1752 && c <= 0x1753) + || (c >= 0x1772 && c <= 0x1773) + || (c >= 0x17B6 && c <= 0x17C8); + case 0x1800: + return (c >= 0x1885 && c <= 0x1886) + || c == 0x18A9; + case 0x1900: + return (c >= 0x1920 && c <= 0x192B) + || (c >= 0x1930 && c <= 0x1938); + case 0x1A00: + return (c >= 0x1A17 && c <= 0x1A1B) + || (c >= 0x1A55 && c <= 0x1A5E) + || (c >= 0x1A61 && c <= 0x1A74); + case 0x1B00: + return (c >= 0x1B00 && c <= 0x1B04) + || (c >= 0x1B35 && c <= 0x1B43) + || (c >= 0x1B80 && c <= 0x1B82) + || (c >= 0x1BA1 && c <= 0x1BA9) + || (c >= 0x1BAC && c <= 0x1BAD) + || (c >= 0x1BE7 && c <= 0x1BF1); + case 0x1C00: + return (c >= 0x1C24 && c <= 0x1C35) + || (c >= 0x1CF2 && c <= 0x1CF3); + case 0x1D00: + return (c >= 0x1DE7 && c <= 0x1DF4); + case 0x2400: + return (c >= 0x24B6 && c <= 0x24E9); + case 0x2D00: + return (c >= 0x2DE0 && c <= 0x2DFF); + case 0xA600: + return (c >= 0xA674 && c <= 0xA67B) + || (c >= 0xA69E && c <= 0xA69F); + case 0xA800: + return (c >= 0xA823 && c <= 0xA827) + || (c >= 0xA880 && c <= 0xA881) + || (c >= 0xA8B4 && c <= 0xA8C3) + || c == 0xA8C5; + case 0xA900: + return (c >= 0xA926 && c <= 0xA92A) + || (c >= 0xA947 && c <= 0xA952) + || (c >= 0xA980 && c <= 0xA983) + || (c >= 0xA9B4 && c <= 0xA9BF); + case 0xAA00: + return (c >= 0xAA29 && c <= 0xAA36) + || c == 0xAA43 + || (c >= 0xAA4C && c <= 0xAA4D) + || c == 0xAAB0 + || (c >= 0xAAB2 && c <= 0xAAB4) + || (c >= 0xAAB7 && c <= 0xAAB8) + || c == 0xAABE + || (c >= 0xAAEB && c <= 0xAAEF) + || c == 0xAAF5; + case 0xAB00: + return (c >= 0xABE3 && c <= 0xABEA); + case 0xFB00: + return c == 0xFB1E; + case 0x10300: + return (c >= 0x10376 && c <= 0x1037A); + case 0x10A00: + return (c >= 0x10A01 && c <= 0x10A03) + || (c >= 0x10A05 && c <= 0x10A06) + || (c >= 0x10A0C && c <= 0x10A0F); + case 0x11000: + return (c >= 0x11000 && c <= 0x11002) + || (c >= 0x11038 && c <= 0x11045) + || c == 0x11082 + || (c >= 0x110B0 && c <= 0x110B8); + case 0x11100: + return (c >= 0x11100 && c <= 0x11102) + || (c >= 0x11127 && c <= 0x11132) + || (c >= 0x11180 && c <= 0x11182) + || (c >= 0x111B3 && c <= 0x111BF); + case 0x11200: + return (c >= 0x1122C && c <= 0x11234) + || c == 0x11237 + || c == 0x1123E + || (c >= 0x112DF && c <= 0x112E8); + case 0x11300: + return (c >= 0x11300 && c <= 0x11303) + || (c >= 0x1133E && c <= 0x11344) + || (c >= 0x11347 && c <= 0x11348) + || (c >= 0x1134B && c <= 0x1134C) + || c == 0x11357 + || (c >= 0x11362 && c <= 0x11363); + case 0x11400: + return (c >= 0x11435 && c <= 0x11441) + || (c >= 0x11443 && c <= 0x11445) + || (c >= 0x114B0 && c <= 0x114C1); + case 0x11500: + return (c >= 0x115AF && c <= 0x115B5) + || (c >= 0x115B8 && c <= 0x115BE) + || (c >= 0x115DC && c <= 0x115DD); + case 0x11600: + return (c >= 0x11630 && c <= 0x1163E) + || c == 0x11640 + || (c >= 0x116AB && c <= 0x116B5); + case 0x11700: + return (c >= 0x1171D && c <= 0x1172A); + case 0x11C00: + return (c >= 0x11C2F && c <= 0x11C36) + || (c >= 0x11C38 && c <= 0x11C3E) + || (c >= 0x11C92 && c <= 0x11CA7) + || (c >= 0x11CA9 && c <= 0x11CB6); + case 0x16B00: + return (c >= 0x16B30 && c <= 0x16B36); + case 0x16F00: + return (c >= 0x16F51 && c <= 0x16F7E); + case 0x1BC00: + return c == 0x1BC9E; + case 0x1E000: + return (c >= 0x1E000 && c <= 0x1E006) + || (c >= 0x1E008 && c <= 0x1E018) + || (c >= 0x1E01B && c <= 0x1E021) + || (c >= 0x1E023 && c <= 0x1E024) + || (c >= 0x1E026 && c <= 0x1E02A); + case 0x1E900: + return c == 0x1E947; + case 0x1F100: + return (c >= 0x01F130 && c <= 0x01F149) + || (c >= 0x01F150 && c <= 0x01F169) + || (c >= 0x01F170 && c <= 0x01F189); + default: + return 0; + } +} + int ucd_isalnum(codepoint_t c) { switch (ucd_lookup_category(c)) @@ -42,12 +295,17 @@ int ucd_isalpha(codepoint_t c) { switch (ucd_lookup_category(c)) { + case UCD_CATEGORY_Lu: case UCD_CATEGORY_Ll: + case UCD_CATEGORY_Lt: case UCD_CATEGORY_Lm: case UCD_CATEGORY_Lo: - case UCD_CATEGORY_Lt: - case UCD_CATEGORY_Lu: + case UCD_CATEGORY_Nl: return 1; + case UCD_CATEGORY_Mn: + case UCD_CATEGORY_Mc: + case UCD_CATEGORY_So: + return other_alphabetic_MnMcSo(c); default: return 0; } diff --git a/tools/printdata.py b/tools/printdata.py index 8ac48e87..d25c3fd2 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -95,10 +95,7 @@ def isalnum(data): return 0 def isalpha(data): - if data.get('GeneralCategory', 'Cn')[0] in 'L': - return 1 - else: - return 0 + return data.get('Alphabetic', 0) def isupper(data): if data.get('Uppercase', 0): From 6346ede604270202b2aad8159cb914b7a2276c34 Mon Sep 17 00:00:00 2001 From: "Reece H. Dunn" Date: Sun, 12 Feb 2017 21:22:01 +0000 Subject: [PATCH 24/24] isalnum: use the Alphabetic derived core property to check that the character is in the alpha class. --- src/ctype.c | 10 +++++++--- tools/printdata.py | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/ctype.c b/src/ctype.c index 6cd20ff7..4632b55e 100644 --- a/src/ctype.c +++ b/src/ctype.c @@ -277,15 +277,19 @@ int ucd_isalnum(codepoint_t c) { switch (ucd_lookup_category(c)) { + case UCD_CATEGORY_Lu: case UCD_CATEGORY_Ll: + case UCD_CATEGORY_Lt: case UCD_CATEGORY_Lm: case UCD_CATEGORY_Lo: - case UCD_CATEGORY_Lt: - case UCD_CATEGORY_Lu: - case UCD_CATEGORY_Nd: case UCD_CATEGORY_Nl: + case UCD_CATEGORY_Nd: case UCD_CATEGORY_No: return 1; + case UCD_CATEGORY_Mn: + case UCD_CATEGORY_Mc: + case UCD_CATEGORY_So: + return other_alphabetic_MnMcSo(c); default: return 0; } diff --git a/tools/printdata.py b/tools/printdata.py index d25c3fd2..bc724de5 100755 --- a/tools/printdata.py +++ b/tools/printdata.py @@ -89,10 +89,10 @@ def isgraph(data): return 0 def isalnum(data): - if data.get('GeneralCategory', 'Cn')[0] in 'LN': + if data.get('GeneralCategory', 'Cn')[0] in 'N': return 1 else: - return 0 + return data.get('Alphabetic', 0) def isalpha(data): return data.get('Alphabetic', 0)