Repository navigation
Expand file tree
/
Copy pathtext.py
More file actions
258 lines (193 loc) · 7.09 KB
/
Copy pathtext.py
File metadata and controls
258 lines (193 loc) · 7.09 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
"""
Utilities for processing text.
"""
import html
import math
import random
import re
from pathlib import Path
import ftfy
from huggingface_hub import hf_hub_download
from unidecode import unidecode
# based on wiki word occurence
person_token = [("a person", 282265), ("someone", 121194), ("somebody", 12219)]
temp_token = "xtokx" # avoid repeating chars
class HashtagProcessor:
# Adapted from wordni
8000
nja library
# We use our wikipedia word count + a good heuristic to make it work
def __init__(self):
wiki_word_frequency = hf_hub_download(
"dalle-mini/dalle-mini", filename="enwiki-words-frequency.txt"
)
self._word_cost = (
l.split()[0] for l in Path(wiki_word_frequency).read_text().splitlines()
)
self._word_cost = {
str(k): math.log(float(i + 1)) for i, k in enumerate(self._word_cost)
}
self._max_word = max(len(x) for x in self._word_cost.keys())
self._SPLIT_RE = re.compile("[^a-zA-Z0-9']+")
def __call__(self, s):
"""Uses dynamic programming to infer the location of spaces in a string without spaces."""
l = [self._split(x) for x in self._SPLIT_RE.split(s)]
return " ".join([item for sublist in l for item in sublist])
def _split(self, s):
# Find the best match for the i first characters, assuming cost has
# been built for the i-1 first characters.
# Returns a pair (match_cost, match_length).
def best_match(i):
candidates = enumerate(reversed(cost[max(0, i - self._max_word) : i]))
return min(
(c + self._word_cost.get(s[i - k - 1 : i].lower(), 9e999), k + 1)
for k, c in candidates
)
# Build the cost array
cost = [0]
for i in range(1, len(s) + 1):
c, k = best_match(i)
cost.append(c)
# Backtrack to recover the minimal-cost string.
out = []
i = len(s)
while i > 0:
c, k = best_match(i)
assert c == cost[i]
newToken = True
if not s[i - k : i] == "'": # ignore a lone apostrophe
if len(out) > 0:
# re-attach split 's and split digits
if out[-1] == "'s" or (
s[i - 1].isdigit() and out[-1][0].isdigit()
): # digit followed by digit
out[-1] = (
s[i - k : i] + out[-1]
) # combine current token with previous token
newToken = False
if newToken:
out.append(s[i - k : i])
i -= k
return reversed(out)
def replace_person_token(t):
"Used for CC12M"
t = re.sub("<person>([,\s]*(and)*[,\s]*<person>)+", " people ", t)
while "<person>" in t:
t = t.replace(
"<person>", f" {random.choices(*tuple(zip(*person_token)))[0]} ", 1
)
return t
def fix_html(t):
# from OpenAI CLIP
return html.unescape(html.unescape(t))
def replace_punctuation_with_commas(t):
return re.sub("[()[\].,|:;?!=+~\-\/{}]", ",", t)
def simplify_quotes(t):
return re.sub("""['"`]""", ' " ', t)
def merge_quotes(t):
return re.sub('(\s*"+\s*)+', ' " ', t)
def remove_comma_numbers(t):
def _f(t):
return re.sub("(\d),(\d{3})", r"\1\2", t)
return _f(_f(t))
def pre_process_dot_numbers(t):
return re.sub("(\w)\.(\w)", rf"\1{temp_token}dot{temp_token}\2", t)
def post_process_dot_numbers(t):
return re.sub(f"{temp_token}dot{temp_token}", ".", t)
def pre_process_quotes(t):
# allows quotes only for 's, 't, 'd, 'm, 'll, 're, 've
return re.sub(
r"'(?=([stdm]|(ll)|(re)|(ve)|(ll))\b)", rf"{temp_token}quote{temp_token}", t
)
def post_process_quotes(t):
return re.sub(f"{temp_token}quote{temp_token}", "'", t)
def pre_process_dates(t):
return re.sub("(\d)/(\d)", rf"\1{temp_token}slash{temp_token}\2", t)
def post_process_dates(t):
return re.sub(f"{temp_token}slash{temp_token}", "/", t)
def merge_commas(t):
return re.sub("(\s*,+\s*)+", ", ", t)
def add_space_after_commas(t):
return re.sub(",", ", ", t)
def handle_special_chars(t):
"Handle special characters"
# replace "-" with a space when between words without space
t = re.sub("(\w)-(\w)", r"\1 \2", t)
# always add space around some characters
return re.sub("([%&\/$*])", r" \1 ", t)
def expand_hashtags(t, hashtag_processor):
"Remove # and try to split words"
return re.sub("#(\w+)", lambda m: hashtag_processor(m.group(1)), t)
_re_ignore_chars = r"[_#\\]"
def ignore_chars(t):
"Ignore useless characters"
return re.sub(_re_ignore_chars, " ", t)
def remove_extra_spaces(t):
"Remove extra spaces (including \t and \n)"
return re.sub("\s+", " ", t)
def remove_repeating_chars(t):
"If the same character is present 4+ times (not 3 because of roman 'VIII'), replace with single instance"
return re.sub(r"(\D)(\1{3,})", r"\1", t)
def remove_urls(t):
return re.sub(r"http\S+", "", t)
def remove_html_tags(t):
return re.sub("<[^<]+?>", "", t)
def remove_first_last_commas(t):
t = t.strip()
t = t[:-1] if t and t[-1] == "," else t
t = t[1:] if t and t[0] == "," else t
return t.strip()
def remove_wiki_ref(t):
t = re.sub(r"\A\s*\[\d+\]", "", t)
return re.sub(r"\[\d+\]\s*\Z", "", t)
class TextNormalizer:
"Normalize text"
def __init__(self):
self._hashtag_processor = HashtagProcessor()
def __call__(self, t):
# fix some characters
t = ftfy.fix_text(t)
# fix html
t = fix_html(t)
# decode and simplify text: see unidecode library
t = unidecode(t)
# lower case
t = t.lower()
# replace <PERSON> (for CC12M)
t = replace_person_token(t)
# remove wiki reference (for WIT)
t = remove_wiki_ref(t)
# remove html tags
t = remove_html_tags(t)
# remove urls
t = remove_urls(t)
# remove commas in numbers
t = remove_comma_numbers(t)
# handle dots in numbers and quotes - Part 1
t = pre_process_dot_numbers(t)
t = pre_process_quotes(t)
t = pre_process_dates(t)
# handle special characters
t = handle_special_chars(t)
# handle hashtags
t = expand_hashtags(t, self._hashtag_processor)
# ignore useless characters
t = ignore_chars(t)
# simplify quotes
t = simplify_quotes(t)
# all punctuation becomes commas
t = replace_punctuation_with_commas(t)
# handle dots in numbers and quotes - Part 2
t = post_process_dot_numbers(t)
t = post_process_quotes(t)
t = post_process_dates(t)
# handle repeating characters
t = remove_repeating_chars(t)
# merge quotes
t = merge_quotes(t)
# merge commas
t = merge_commas(t)
# remove multiple spaces
t = remove_extra_spaces(t)
# remove first and last comma
t = remove_first_last_commas(t)
# always start with a space
return f" {t}"