-
Notifications
You must be signed in to change notification settings - Fork 25
/
updater.py
112 lines (81 loc) · 3.77 KB
/
updater.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
from __future__ import annotations
import logging
import aiofiles
import aiohttp
import aiosqlite
import yarl
log = logging.getLogger("updater")
API = yarl.URL("https://api.github.com/")
RAW_API = yarl.URL("https://raw.githubusercontent.com/")
REPO = "Discord-AntiScam/scam-links"
_COMMIT_URL = API / "repos" / REPO / "commits"
_ORIGINAL_RAW_REPO = RAW_API / REPO
COMMIT_URL = _COMMIT_URL
ORIGINAL_REPO = _ORIGINAL_RAW_REPO
with open("_meta.txt") as f:
is_first_run = f.read().strip().lower() == "true"
async def init():
db = await aiosqlite.connect("cached.sqlite", iter_chunk_size=2**8, cached_statements=2**10)
query = """
BEGIN;
CREATE TABLE IF NOT EXISTS scam_links (id INTEGER PRIMARY KEY AUTOINCREMENT, link TEXT NOT NULL, UNIQUE(link));
CREATE TABLE IF NOT EXISTS discord_tokens (id INTEGER PRIMARY KEY AUTOINCREMENT, token TEXT NOT NULL, UNIQUE(token));
CREATE TABLE IF NOT EXISTS nsfw_links (id INTEGER PRIMARY KEY AUTOINCREMENT, link TEXT NOT NULL, UNIQUE(link));
CREATE TABLE IF NOT EXISTS nsfw_links_grouped (id INTEGER PRIMARY KEY AUTOINCREMENT, link TEXT NOT NULL UNIQUE, type TEXT);
CREATE TABLE IF NOT EXISTS logs (id INTEGER PRIMARY KEY AUTOINCREMENT, level INT NOT NULL, message TEXT NOT NULL, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, extra TEXT, UNIQUE(message, created_at));
DELETE FROM logs;
COMMIT;
"""
await db.executescript(query)
await db.commit()
return db
async def insert_all_scams(db: aiosqlite.Connection):
async with aiofiles.open("_meta.txt", "w") as f:
await f.write("false")
url = ORIGINAL_REPO / "main" / "list.json"
async with aiohttp.ClientSession() as session:
log.debug("Downloading Data... %s", url)
response = await session.get(url)
log.debug("Downloaded Data... %s. return code: %s", url, response.status)
if response.status != 200:
log.warning("Failed to download data... exiting...")
return
log.debug("parsing data from %s", url)
data = await response.json(content_type="text/plain")
log.debug("parsed data from %s. Total Links: %s", url, len(data))
query = """INSERT INTO scam_links (link) VALUES (?) ON CONFLICT DO NOTHING"""
for link in data:
await db.execute(query, (link,))
log.info("inserted link: %s", link)
await db.commit()
async def insert_new(db: aiosqlite.Connection):
if is_first_run:
log.info("First Run... Inserting all scams...")
await insert_all_scams(db)
return
async with aiohttp.ClientSession() as session:
log.debug("Downloading Data... %s", COMMIT_URL)
response = await session.get(COMMIT_URL)
log.debug("Downloaded Data... %s. return code: %s", COMMIT_URL, response.status)
if response.status != 200:
log.info("Failed to download data... trying to download all data...")
await insert_all_scams(db)
return
log.debug("parsing data from %s", COMMIT_URL)
data = await response.json()
log.debug("parsed data from %s. Total Links: %s", COMMIT_URL, len(data))
insert_query = """INSERT INTO scam_links (link) VALUES (?) ON CONFLICT DO NOTHING"""
delete_query = """DELETE FROM scam_links WHERE link = ?"""
for commit in data:
message = commit["commit"]["message"]
if message.startswith("+ "):
link = message[2:]
cur = await db.execute(insert_query, (link,))
if cur.rowcount:
log.info("inserted link: %s", link)
elif message.startswith("- "):
link = message[2:]
cur = await db.execute(delete_query, (link,))
if cur.rowcount:
log.info("deleted link: %s", link)
await db.commit()