-
Notifications
You must be signed in to change notification settings - Fork 10
/
update.sh
executable file
·361 lines (269 loc) · 9.27 KB
/
update.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
#!/bin/bash
# The following variables are defined in adsorber.sh or adsorber.sh
# If you run this file independently following variables need to be set:
# ---variable:---------- ---default value:---
# HOSTS_FILE_PATH /etc/hosts
# HOSTS_FILE_BACKUP_PATH /etc/hosts.original
# PRIMARY_LIST blacklist
# REPLY_TO_FORCE_PROMPT Null (not set)
# SCRIPT_DIR_PATH The scripts root directory (e.g., /home/user/Downloads/adsorber)
# SOURCELIST_FILE_PATH SCRIPT_DIR_PATH/sources.list (e.g., /home/user/Downloads/absorber/sources.list)
# TMP_DIR_PATH /tmp/adsorber
# USE_PARTIAL_MATCHING true
updateCleanUp() {
echo "Cleaning up..."
rm -rf "${TMP_DIR_PATH}" 2>/dev/null 1>&2
return 0
}
checkBackupExist() {
if [ ! -f "${HOSTS_FILE_BACKUP_PATH}" ]; then
if [ -z "${REPLY_TO_FORCE_PROMPT}" ]; then
echo "Backup of ${HOSTS_FILE_PATH} does not exist. To backup run '${0} install'." 1>&2
read -p "Ignore issue and continue? (May break your system, not recommended) [YES/n]: " REPLY_TO_FORCE_PROMPT
fi
case "${REPLY_TO_FORCE_PROMPT}" in
[Yy][Ee][Ss] )
return 0
;;
* )
echo "Aborted." 1>&2
updateCleanUp
exit 1
;;
esac
fi
return 0
}
createTmpDir() {
if [ ! -d "${TMP_DIR_PATH}" ]; then
mkdir "${TMP_DIR_PATH}"
elif [ ! -s "${TMP_DIR_PATH}/config-filtered" ]; then
echo "Removing previous tmp folder..."
rm -rf "${TMP_DIR_PATH}"
mkdir "${TMP_DIR_PATH}"
fi
return 0
}
readSourceList() {
if [ ! -s "${SOURCELIST_FILE_PATH}" ]; then
if [ ! -s "${SCRIPT_DIR_PATH}/blacklist" ]; then
echo "Missing 'sources.list' and blacklist. To fix run '${0} install'." 1>&2
exit 1
fi
echo "No sources to fetch from, ignoring..."
return 1
else
# Only read sources with http(s) at the beginning
# Remove inline # comments
cat "${SOURCELIST_FILE_PATH}" \
| sed -n '/^\s*http.*/p' \
| sed 's/\s\+#.*//g' \
> "${TMP_DIR_PATH}/sourceslist-filtered"
if [ ! -s "${TMP_DIR_PATH}/sourceslist-filtered" ]; then
echo "No hosts set in sources.list, ignoring..."
return 1
fi
fi
return 0
}
readWhiteList() {
if [ ! -f "${SCRIPT_DIR_PATH}/whitelist" ]; then
echo "Whitelist does not exist, ignoring..." 1>&2
return 1
else
cp "${SCRIPT_DIR_PATH}/whitelist" "${TMP_DIR_PATH}/whitelist"
filterDomains "whitelist" "whitelist-filtered"
sortDomains "whitelist-filtered" "whitelist-sorted"
fi
return 0
}
readBlackList() {
if [ ! -f "${SCRIPT_DIR_PATH}/blacklist" ]; then
echo "Blacklist does not exist, ignoring..." 1>&2
return 1
else
cp "${SCRIPT_DIR_PATH}/blacklist" "${TMP_DIR_PATH}/blacklist"
filterDomains "blacklist" "blacklist-filtered"
sortDomains "blacklist-filtered" "blacklist-sorted"
fi
return 0
}
fetchSources() {
local total_count=0
local successful_count=0
local domain
while read -r domain; do
(( total_count++ ))
echo "Getting: ${domain}"
if [ $(type -fP curl) ]; then
if curl "${domain}" --progress-bar -L --connect-timeout 30 --fail --retry 1 >> "${TMP_DIR_PATH}/fetched"; then
(( successful_count++ ))
else
echo "curl couldn't fetch ${domain}" 1>&2
fi
elif [ $(type -fP wget) ]; then
printf "wget: "
if wget "${domain}" --show-progress -L --timeout=30 -t 1 -nv -O - >> "${TMP_DIR_PATH}/fetched"; then
(( successful_count++ ))
else
echo "wget couldn't fetch ${domain}" 1>&2
fi
else
echo "Neither curl nor wget installed. Can't continue." 1>&2
updateCleanUp
exit 2
fi
done < "${TMP_DIR_PATH}/sourceslist-filtered"
if [ "${successful_count}" -eq 0 ]; then
echo "Nothing to apply [${successful_count}/${total_count}]." 1>&2
return 1
else
echo "Successfully fetched ${successful_count} out of ${total_count} hosts sources."
fi
return 0
}
filterDomains() {
local input_file="${1}"
local output_file="${2}"
# - replace OSX '\r' and MS-DOS '\r\n' with Unix '\n' (linebreak)
# - replace 127.0.0.1 and 127.0.1.1 with 0.0.0.0
# - only keep lines starting with 0.0.0.0
# - remove inline '#' comments
# - replace tabs and multiple spaces with one space
# - remove domains without a dot (e.g localhost , loopback , ip6-allnodes , etc...)
# - remove domains that are redirecting to *.local
cat "${TMP_DIR_PATH}/${input_file}" \
| sed 's/\r/\n/g' \
| sed 's/^\s*127\.0\.[01]\.1/0\.0\.0\.0/g' \
| sed -n '/^\s*0\.0\.0\.0\s\+.\+/p' \
| sed 's/\s\+#.*//g' \
| sed 's/[[:blank:]]\+/ /g' \
| sed -n '/^0\.0\.0\.0\s.\+\..\+/p' \
| sed -n '/\.local\s*$/!p' \
> "${TMP_DIR_PATH}/${output_file}"
return 0
}
sortDomains() {
local input_file="${1}"
local output_file="${2}"
# Sort the domains by alphabet and also remove duplicates
sort "${TMP_DIR_PATH}/${input_file}" -f -u -o "${TMP_DIR_PATH}/${output_file}"
return 0
}
applyWhiteList() {
local domain
if [ ! -s "${TMP_DIR_PATH}/whitelist-sorted" ]; then
echo "Whitelist is empty, ignoring..."
return 1
else
echo "Applying whitelist..."
sed -i 's/^0\.0\.0\.0\s\+//g' "${TMP_DIR_PATH}/whitelist-sorted"
cp "${TMP_DIR_PATH}/cache" "${TMP_DIR_PATH}/applied-whitelist"
while read -r domain; do
if [ "${USE_PARTIAL_MATCHING}" == "true" ]; then
sed -i "/\.*${domain}$/d" "${TMP_DIR_PATH}/applied-whitelist"
elif [ "${USE_PARTIAL_MATCHING}" == "false" ]; then
sed -i "/\s\+${domain}$/d" "${TMP_DIR_PATH}/applied-whitelist"
else
echo "Wrong USE_PARTIAL_MATCHING set, either set to 'true' or 'false'." 1>&2
updateCleanUp
exit 1
fi
done < "${TMP_DIR_PATH}/whitelist-sorted"
cp "${TMP_DIR_PATH}/applied-whitelist" "${TMP_DIR_PATH}/cache"
fi
return 0
}
mergeBlackList() {
local input_file="${1}"
if [ ! -s "${TMP_DIR_PATH}/blacklist-sorted" ]; then
echo "Blacklist is empty, ignoring..."
return 1
else
echo "Applying blacklist..."
cat "${TMP_DIR_PATH}/cache" "${TMP_DIR_PATH}/blacklist-sorted" >> "${TMP_DIR_PATH}/merged-blacklist"
filterDomains "merged-blacklist" "merged-blacklist-filtered"
sortDomains "merged-blacklist-filtered" "merged-blacklist-sorted"
cp "${TMP_DIR_PATH}/merged-blacklist-sorted" "${TMP_DIR_PATH}/cache"
fi
return 0
}
isCacheEmpty() {
if [ -s "${TMP_DIR_PATH}/cache" ]; then
return 0
else
echo "Nothing to apply. Exiting..." 1>&2
updateCleanUp
exit 1
fi
return 0
}
preBuildHosts() {
# Replace @...@ with the path to the backup hosts
sed "s|@.\+@|${HOSTS_FILE_BACKUP_PATH}|g" "${SCRIPT_DIR_PATH}/bin/components/hosts_header" > "${TMP_DIR_PATH}/hosts"
echo "" >> "${TMP_DIR_PATH}/hosts"
# Add hosts.original
cat "${HOSTS_FILE_BACKUP_PATH}" >> "${TMP_DIR_PATH}/hosts" \
|| echo "You may want to add your hostname to ${HOSTS_FILE_PATH}" 1>&2
echo "" >> "${TMP_DIR_PATH}/hosts"
# Add hosts_title
cat "${SCRIPT_DIR_PATH}/bin/components/hosts_title" >> "${TMP_DIR_PATH}/hosts"
return 0
}
buildHostsFile() {
echo "" >> "${TMP_DIR_PATH}/hosts"
# Add ad-domains to the hosts file
cat "${TMP_DIR_PATH}/cache" >> "${TMP_DIR_PATH}/hosts"
echo "" >> "${TMP_DIR_PATH}/hosts"
sed "s|@.\+@|${HOSTS_FILE_BACKUP_PATH}|g" "${SCRIPT_DIR_PATH}/bin/components/hosts_header" >> "${TMP_DIR_PATH}/hosts"
return 0
}
applyHostsFile() {
echo "Applying new hosts file...."
# Replace systems hosts file with the modified version
cat "${TMP_DIR_PATH}/hosts" > "${HOSTS_FILE_PATH}" \
|| {
echo "Couldn't apply hosts file. Aborting" 1>&2
updateCleanUp
exit 1
}
echo "Successfully applied new hosts file."
return 0
}
update() {
echo "Updating ${HOSTS_FILE_PATH}..."
checkBackupExist
createTmpDir
readBlackList
readWhiteList
if readSourceList; then
fetchSources
filterDomains "fetched" "fetched-filtered"
sortDomains "fetched-filtered" "fetched-sorted"
cp "${TMP_DIR_PATH}/fetched-sorted" "${TMP_DIR_PATH}/cache"
else
# Create empty cache file
printf "" >> "${TMP_DIR_PATH}/cache"
fi
case "${PRIMARY_LIST}" in
whitelist )
mergeBlackList
applyWhiteList
;;
blacklist )
applyWhiteList
mergeBlackList
;;
* )
echo "Wrong PRIMARY_LIST set in adsorber.conf. Choose either 'whitelist' or 'blacklist'" 1>&2
updateCleanUp
exit 1
;;
esac
isCacheEmpty
preBuildHosts
buildHostsFile
applyHostsFile
updateCleanUp
return 0
}