Merge commit '6346ede604270202b2aad8159cb914b7a2276c34'

This commit is contained in:
Reece H. Dunn
2017-02-12 21:32:06 +00:00
11 changed files with 1000 additions and 137 deletions
+86 -21
View File
@@ -1,6 +1,6 @@
#!/usr/bin/python
# Copyright (C) 2012 Reece H. Dunn
# Copyright (C) 2012-2017 Reece H. Dunn
#
# This file is part of ucd-tools.
#
@@ -24,15 +24,20 @@ import ucd
ucd_rootdir = sys.argv[1]
csur_rootdir = 'data/csur'
null = ucd.CodePoint('0000')
unicode_chars = {}
for data in ucd.parse_ucd_data(ucd_rootdir, 'UnicodeData'):
for codepoint in data['CodePoint']:
unicode_chars[codepoint] = data
unicode_chars[codepoint]['Properties'] = []
for data in ucd.parse_ucd_data(ucd_rootdir, 'PropList'):
if data['Property'] in ['White_Space']:
for propfile in ['PropList', 'DerivedCoreProperties']:
for data in ucd.parse_ucd_data(ucd_rootdir, propfile):
for codepoint in data['Range']:
unicode_chars[codepoint]['Properties'].append(data['Property'])
try:
unicode_chars[codepoint][data['Property']] = 1
except KeyError:
unicode_chars[codepoint] = {'CodePoint': codepoint}
unicode_chars[codepoint][data['Property']] = 1
for data in ucd.parse_ucd_data(ucd_rootdir, 'Scripts'):
for codepoint in data['Range']:
unicode_chars[codepoint]['Script'] = data['Script']
@@ -40,31 +45,91 @@ if '--with-csur' in sys.argv:
for csur in ['Klingon']:
for data in ucd.parse_ucd_data('data/csur', csur):
for codepoint in data['CodePoint']:
if not 'TitleCase' in data: data['TitleCase'] = codepoint
if not 'UpperCase' in data: data['UpperCase'] = codepoint
if not 'LowerCase' in data: data['LowerCase'] = codepoint
if not 'Properties' in data: data['Properties'] = []
unicode_chars[codepoint] = data
null = ucd.CodePoint('0000')
def iscntrl(data):
return 1 if data.get('Name', '') == '<control>' else 0
def isdigit(data):
return 1 if data['CodePoint'].char() in '0123456789' else 0
def isxdigit(data):
return 1 if data['CodePoint'].char() in '0123456789ABCDEFabcdef' else 0
def isspace(data):
if data.get('White_Space', 0):
dt = data.get('DecompositionType', '')
return 1 if dt == None or not dt.startswith('<noBreak>') else 0
else:
return 0
def isblank(data): # word separator
if data.get('GeneralCategory', 'Cn') == 'Zs' or data['CodePoint'].char() == '\t':
dt = data.get('DecompositionType', '')
return 1 if dt == None or not dt.startswith('<noBreak>') else 0
else:
return 0
def ispunct(data):
if data.get('GeneralCategory', 'Cn')[0] in 'P':
return 1
else:
return 0
def isprint(data):
if data.get('GeneralCategory', 'Cn')[0] in 'LMNPSZ': # not in 'CI'
return 1
else:
return 0
def isgraph(data):
if data.get('GeneralCategory', 'Cn')[0] in 'LMNPS': # not in 'CZI'
return 1
else:
return 0
def isalnum(data):
if data.get('GeneralCategory', 'Cn')[0] in 'N':
return 1
else:
return data.get('Alphabetic', 0)
def isalpha(data):
return data.get('Alphabetic', 0)
def isupper(data):
if data.get('Uppercase', 0):
return 1
elif data.get('LowerCase', null) != null: # Some Lt characters have lowercase forms.
return 1
else:
return 0
def islower(data):
if data.get('Lowercase', 0):
return 1
elif data.get('UpperCase', null) != null:
return 1
else:
return 0
if __name__ == '__main__':
for codepoint in ucd.CodeRange('000000..10FFFF'):
try:
data = unicode_chars[codepoint]
except KeyError:
data = {'GeneralCategory': 'Cn', 'TitleCase': codepoint, 'UpperCase': codepoint, 'LowerCase': codepoint, 'Properties': []}
try:
script = data['Script']
except KeyError:
script = 'Zzzz'
title = data['TitleCase']
upper = data['UpperCase']
lower = data['LowerCase']
data = {'CodePoint': codepoint}
script = data.get('Script', 'Zzzz')
title = data.get('TitleCase', codepoint)
upper = data.get('UpperCase', codepoint)
lower = data.get('LowerCase', codepoint)
if title == null: title = codepoint
if upper == null: upper = codepoint
if lower == null: lower = codepoint
print('%s %s %s %s %s %s %s %s' % (
print('%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s' % (
codepoint, script,
data['GeneralCategory'][0], data['GeneralCategory'],
data.get('GeneralCategory', 'Cn')[0], data.get('GeneralCategory', 'Cn'),
upper, lower, title,
' '.join(data['Properties'])))
isdigit(data), isxdigit(data),
iscntrl(data), isspace(data), isblank(data), ispunct(data),
isprint(data), isgraph(data), isalnum(data), isalpha(data), isupper(data), islower(data)))
+11 -1
View File
@@ -1,6 +1,6 @@
#!/usr/bin/python
# Copyright (C) 2012-2014 Reece H. Dunn
# Copyright (C) 2012-2017 Reece H. Dunn
#
# This file is part of ucd-tools.
#
@@ -50,6 +50,9 @@ class CodePoint:
def __lt__(self, other):
return self.codepoint < other.codepoint
def char(self):
return unichr(self.codepoint)
class CodeRange:
def __init__(self, x):
f, l = x.split('..')
@@ -69,6 +72,9 @@ class CodeRange:
def size(self):
return self.last.codepoint - self.first.codepoint + 1
def char(self):
return unichr(self.first.codepoint)
def codepoint(x):
if '..' in x[0]:
return CodeRange(x[0]), x[1:]
@@ -107,6 +113,10 @@ data_items = {
('Range', codepoint),
('Age', string),
],
'DerivedCoreProperties': [
('Range', codepoint),
('Property', string),
],
'PropList': [
('Range', codepoint),
('Property', string),