-
Notifications
You must be signed in to change notification settings - Fork 0
/
administration.py
executable file
·136 lines (100 loc) · 4.29 KB
/
administration.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
This is just a module for administration purpose.
The idea is to leave PyFunceble run this script in order to get less headache
while debugging environnements.
Authors:
- @Funilrys, Nissar Chababy <contactTAfunilrysTODcom>
Contributors:
Let's contribute !
@GitHubUsername, Name, Email (optional)
"""
# pylint:disable=bad-continuation
from ultimate_hosts_blacklist.whitelist import clean_list_with_official_whitelist
from update import Helpers, Settings, path, strftime
INFO = {}
PYFUNCEBLE_CONFIGURATION = {"no_whois": True}
PYFUNCEBLE_CONFIGURATION_VOLATILE = {"no_whois": True, "no_special": True}
REGEX_SPECIAL = r"\.blogspot\.|\.liveadvert\.com$|\.skyrock\.com$|\.tumblr\.com$|\.wordpress\.com$|\.doubleclick\.net$" # pylint: disable=line-too-long
def get_administration_file():
"""
Get the administation file content.
"""
if path.isfile(Settings.repository_info):
content = Helpers.File(Settings.repository_info).read()
INFO.update(Helpers.Dict().from_json(content))
else:
raise Exception("Unable to find the administration file.")
def update_adminisation_file():
"""
Update what should be updated.
"""
INFO.update({"currently_under_test": str(int(False)), "last_test": strftime("%s")})
def save_administration_file():
"""
Save the current state of the administration file content.
"""
Helpers.Dict(INFO).to_json(Settings.repository_info)
def generate_extra_files(): # pylint: disable=too-many-branches,too-many-statements
"""
Update/Create `clean.list`, `volatile.list` and `whitelisted.list`.
"""
if bool(int(INFO["clean_original"])): # pylint: disable=too-many-nested-blocks
clean_list = []
volatile_list = []
list_special_content = Helpers.Regex(
Helpers.File(Settings.file_to_test).to_list(), Settings.regex_whitelist
).matching_list()
active = Settings.current_directory + "output/domains/ACTIVE/list"
inactive = Settings.current_directory + "output/domains/INACTIVE/list"
if path.isfile(active):
print("Starting manipulation of `{}`.".format(active))
clean_list.extend(
Helpers.Regex(Helpers.File(active).to_list(), r"^#").not_matching_list()
)
clean_list.extend(list_special_content)
print("Stoping manipulation of `{}`.".format(active))
if path.isfile(inactive):
print("Starting manipulation of `{}`.".format(inactive))
volatile_list.extend(
Helpers.Regex(
Helpers.File(inactive).to_list(), REGEX_SPECIAL
).matching_list()
)
print("Stoping manipulation of `{}`.".format(inactive))
print("Deletion of duplicate from `{}`".format(Settings.clean_list_file))
clean_list = Helpers.List(clean_list).format()
print(
"Whitelisting of `{}` for `{}`".format(
Settings.clean_list_file, Settings.whitelisted_list_file
)
)
whitelisted = clean_list_with_official_whitelist(
clean_list, multiprocessing=True, processes=60
)
print("Completion of {}".format(Settings.volatile_list_file))
volatile_list.extend(clean_list)
print("Whitelisting of `{}".format(Settings.volatile_list_file))
volatile_list = clean_list_with_official_whitelist(
volatile_list, multiprocessing=True, processes=60
)
volatile_list = Helpers.List(volatile_list).format()
print("Writing `{}`".format(Settings.clean_list_file))
Helpers.File(Settings.clean_list_file).write(
"\n".join(clean_list), overwrite=True
)
print("Writing `{}`".format(Settings.whitelisted_list_file))
Helpers.File(Settings.whitelisted_list_file).write(
"\n".join(whitelisted), overwrite=True
)
print("Writing `{}`".format(Settings.volatile_list_file))
Helpers.File(Settings.volatile_list_file).write(
"\n".join(volatile_list), overwrite=True
)
Helpers.File("whitelisting.py").delete()
if __name__ == "__main__":
get_administration_file()
update_adminisation_file()
save_administration_file()
generate_extra_files()