-
Notifications
You must be signed in to change notification settings - Fork 0
/
clean_utf8.py
executable file
·249 lines (208 loc) · 8.03 KB
/
clean_utf8.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
#!/usr/bin/env python3
# coding=utf-8
# @file clean_utf8.py
# @brief Clean up spaces, control characters, hyphen and such in utf8 corpora
#
# @author Michel Simard / Samuel Larkin -- adopted from Perl code by Eric Joanis
#
# Multilingual Text Processing / Traitement multilingue de textes
# Digital Technologies Research Centre / Centre de recherche en technologies numériques
# National Research Council Canada / Conseil national de recherches Canada
# Copyright 2019-2022, Her Majesty in Right of Canada /
# Copyright 2019-2022, Sa Majeste la Reine du Chef du Canada
import click
import re
import sys
from typing import (
List,
Union,
)
from unicodedata import normalize
try:
import regex
regex_available = True
except ModuleNotFoundError:
# print("Failed to import regex to filter out Extended Control characters.", file=sys.stderr)
# print("Consider `pip install --user regex`", file=sys.stderr)
regex_available = False
from portage_utils import open
__version__ = "1.1"
class CleanUTF8:
"""
Clean up spaces, control characters, hyphen and such in utf8 corpora.
"""
def __init__(
self,
wide_punct: bool = True,
phrase_table: bool = False,
extended_crtl_character_filtering: bool = False,
normalization_type: str = None,
):
"""
wide_punct: Substitute fullwidth punctuation for their equivalent in ascii.
phrase_table: Escapes phrase table entry separator " ||| " for " ___|||___ "
extended_crtl_character_filtering: filter out all unicode characters and not just the ascii control characters.
normalization_type: perform unicode normalization ( None, "NFD", "NFC", "NFKD", "NFKC" )
"""
self.wide_punct = wide_punct
self.phrase_table = phrase_table
self.normalization_type = normalization_type
self.re_hyphens = re.compile(r"[\u001E\u00AD\u2011]")
self.re_dhyphens = re.compile(r"\x1F")
self.re_space = re.compile(r"[\u2060\uFEFF\u00A0\u2007\u202F\u2028\u2029]")
self.re_ctrl = re.compile(r"[\x01-\x09\x0B\x0C\x0E-\x1D\x7F]")
self.re_crlf = re.compile(r"\x0D$")
self.re_phrase_table = re.compile(r"(^| )\|\|\|(?= |$)")
self.re_wide = re.compile(r"([,。:)(;?﹗.﹪﹡﹟])")
self.re_mspace = re.compile(r"\s+") # \s => [ \t\n\r\f\v]
self.re_ctrl_extended = None
if extended_crtl_character_filtering:
if not regex_available:
assert "Can't perform unicode extended control character filtering since regex is not installed."
self.re_ctrl_extended = regex.compile(r"\p{C}")
def __call__(self, text: Union[str, List[str]]) -> Union[str, List[str]]:
"""
Apply filters to either a string or a list of string.
"""
if isinstance(text, list):
return self.clean_list(text)
else:
return self.clean_line(text)
def clean_list(self, list_of_lines: List[str]) -> List[str]:
"""
Apply filters to either a list of string.
"""
assert isinstance(list_of_lines, list)
return [self.clean_line(line) for line in list_of_lines]
def clean_line(self, line: str) -> str:
"""
Apply filters to either a string.
"""
assert isinstance(line, str)
line = line.rstrip()
if self.normalization_type is not None:
line = normalize(self.normalization_type, line)
# Convert various non-breaking hyphen encodings to -: \xAD and \x1E for MS
# Word, \x2011 for Unicode. Warning: for html documents, \xAD should be
# stripped, rather than converted to -.
line = self.re_hyphens.sub("-", line)
# Strip out the MS Word discretional hyphen, \x1F
line = self.re_dhyphens.sub("", line)
# Replace various special purpose spaces by regular spaces:
# U+2060: Word joiner / WJ, "a zero width non-breaking space (only) intended
# for disambiguation of functions for byte order mark" (Unicode standard);
# typically used to join separate words without displaying a space, but
# for Portage separate words do need a space.
# U+FEFF: BOM, now called zero-width no-break space, used more-or-less like WJ
# (deprecated use) or left when concatenating files that have the BOM;
# in either case, we want to tokenize on it, so we turn it into a space.
# U+A0: The canonical non-break space
# U+2007: Figure space, has the width of a digit
# U+202F: Narrow no-break space (e.g., before : ; ! ? » and after « in French)
# U+2028: Line Separator (LS)
# U+2029: Paragraph Separator (PS)
# LS and PS ought to be turned into a newline, but in Portage we define
# \n as the newline, sometimes with user-defined semantics, and this
# script gets applied to line-aligned text, so the only legal thing we can
# do here is map them to spaces.
line = self.re_space.sub(" ", line)
# replace remaining control characters by spaces.
line = self.re_ctrl.sub(" ", line)
# equivalent to crlf2lf.sh: convert DOS newlines to Linux ones
line = self.re_crlf.sub("", line)
# If ||| appears stand-alone in text, that causes problems with Portage
if self.phrase_table:
line = self.re_phrase_table.sub(" ___|||___", line)
# Basic wide punctuation mapping
if self.wide_punct:
line = self.re_wide.sub(r" \g<1> ", line)
line = line.translate(str.maketrans(",。:)(;?﹗.﹪﹡﹟", ",.:)(;?!.%*#"))
# Collapse multiple spaces to a single space
line = self.re_mspace.sub(" ", line)
line = line.strip()
if self.re_ctrl_extended is not None:
line = self.re_ctrl_extended.sub("", line)
return line
def progress(*args):
"""
A simple progress report.
"""
print("\r", *args, sep="", end="", file=sys.stderr)
@click.command()
@click.option(
"-v",
"--verbose",
"verbose",
is_flag=True,
default=False,
show_default=True,
help="Display progress",
)
@click.option(
"--phrase-table",
"phrase_table",
is_flag=True,
default=False,
show_default=True,
help="Handle ||| (Portage phrase-table separator)",
)
@click.option(
"--wide-punct",
"wide_punct",
is_flag=True,
default=False,
show_default=True,
help="Handle wide punctuation",
)
@click.option(
"-x",
"--extended",
"extended_crtl_character_filtering",
is_flag=True,
default=False,
show_default=True,
help="Filter out all unicode Control Characters",
)
@click.option(
"-n",
"--normalize",
"normalization_type",
type=click.Choice(("NFD", "NFC", "NFKD", "NFKC"), case_sensitive=False),
default=None,
help="Apply unicode normalization",
)
@click.argument("infile", default="-", type=str)
@click.argument("outfile", default="-", type=str)
@click.help_option("-h", "--help")
@click.version_option(
version=__version__, package_name="PortageTextProcessing", prog_name="clean_utf8"
)
def main(
infile: str,
outfile: str,
phrase_table: bool,
wide_punct: bool,
extended_crtl_character_filtering: bool,
normalization_type: str,
verbose: bool,
):
"""
Clean-up / normalize UTF8 text
clean_utf8 [options] [infile [outfile]]
"""
clean = CleanUTF8(
wide_punct=wide_punct,
phrase_table=phrase_table,
extended_crtl_character_filtering=extended_crtl_character_filtering,
normalization_type=normalization_type,
)
with open(str(infile), mode="r", encoding="UTF-8", newline="\n") as cin, open(
str(outfile), mode="w", encoding="UTF-8"
) as cout:
cin = map(str.strip, cin)
for count, line in enumerate(cin, 1):
if verbose and count % 1000 == 0:
progress(f"[{count} lines...]")
print(clean(line), file=cout)
if __name__ == "__main__":
main()