forked from ettorerizza/aat_reconcile
-
Notifications
You must be signed in to change notification settings - Fork 0
/
text.py
68 lines (61 loc) · 1.77 KB
/
text.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
"""
Taken from the Helmut project.
https://github.com/okfn/helmut/blob/master/helmut/text.py
In process of being reviewed, modified, for AAT optimization.
"""
from unicodedata import normalize as ucnorm, category
def normalize(text, PY3):
if PY3:
if not isinstance(text, str):
str(text, 'utf-8')
else:
if not isinstance(text, unicode):
text = unicode(text)
text = text.lower()
decomposed = ucnorm('NFKD', text)
filtered = []
for char in decomposed:
cat = category(char)
if cat.startswith('C'):
filtered.append(' ')
elif cat.startswith('M'):
# marks, such as umlauts
continue
elif cat.startswith('Z'):
# newlines, non-breaking etc.
filtered.append(' ')
# elif cat.startswith('S'):
# symbols, such as currency
continue
else:
filtered.append(char)
text = u''.join(filtered)
while ' ' in text:
text = text.replace(' ', ' ')
text = text.strip()
return ucnorm('NFKC', text)
def url_slug(text, PY3):
text = normalize(text)
text = text.replace(' ', '-')
text = text.replace('.', '_')
return text
def tokenize(text, splits='COPZ'):
token = []
if PY3:
for c in str(text, 'utf-8'):
if category(c)[0] in splits:
if len(token):
yield u''.join(token)
token = []
else:
token.append(c)
else:
for c in unicode(text):
if category(c)[0] in splits:
if len(token):
yield u''.join(token)
token = []
else:
token.append(c)
if len(token):
yield u''.join(token)