-
Notifications
You must be signed in to change notification settings - Fork 137
/
knowledge_pipeline.py
403 lines (344 loc) · 16.1 KB
/
knowledge_pipeline.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
"""
Capture your email locally, and parse out the pictures in the email body and the pictures, videos and other files in the attachment. Subsequently, it supports vectorized analysis of your personal data and serves as a knowledge base to enable large language model answers. Better results.
An example of a local file is as follows:
├── data
│ └── [email protected]
│ └── 5de3e52f3a6b90cabe6cbdd4ae3a5c5b
│ ├── email.txt
│ ├── meta.json
│ ├── image
│ │ ├── [email protected]
│ └── body_image
│ ├── 11044884873.jpg
│ ├── 282985198265470.gif
│ └── dd-login-service-min.png
"""
import asyncio
import datetime
import sqlite3
import imaplib
import logging
import mailparser
import hashlib
import json
import base64
import chardet
import aiofiles
from bs4 import BeautifulSoup
import requests
import os
import toml
from .storage import AIStorage, UserConfigItem
from .knowledge_base import KnowledgeBase, ImageObjectBuilder, ObjectID, ObjectType, DocumentObjectBuilder, EmailObjectBuilder, EmailObject
class KnowledgeJournal:
def __init__(self, source_type: str, source_id: str, item_id: str, object_id: str, timestamp=None):
# define a timestamp variable
self.timestamp = datetime.datetime.now() if timestamp is None else timestamp
self.object_id = object_id
self.source_type = source_type
self.source_id = source_id
self.item_id = item_id
def __str__(self) -> str:
if self.source_type == "dir":
object_id = ObjectID.from_base58(self.object_id)
object_type = None
if object_id.get_object_type() == ObjectType.Image:
object_type = "image"
else:
pass
return f"Add {object_type} from {os.path.join(self.source_id, self.item_id)}"
if self.source_type == "email":
object_id = ObjectID.from_base58(self.object_id)
email = EmailObject.decode(KnowledgeBase().store.get_object_store().get_object(object_id))
meta = email.get_meta()
return f'Add email from {os.path.join(self.source_id)} subject {meta["subject"]}'
# init sqlite3 client
class KnowledgeJournalClient:
def __init__(self):
knowledge_dir = os.path.join(AIStorage.get_instance().get_myai_dir(), "knowledge")
if not os.path.exists(knowledge_dir):
os.makedirs(knowledge_dir)
self.journal_path = os.path.join(knowledge_dir, "journal.db")
conn = sqlite3.connect(self.journal_path)
conn.execute(
'''CREATE TABLE IF NOT EXISTS journal (
id INTEGER PRIMARY KEY AUTOINCREMENT,
time DATETIME DEFAULT CURRENT_TIMESTAMP,
source_type TEXT,
source_id TEXT,
item_id TEXT,
object_id TEXT)'''
)
conn.commit()
def insert(self, journal: KnowledgeJournal):
conn = sqlite3.connect(self.journal_path)
conn.execute(
"INSERT INTO journal (time, source_type, source_id, item_id, object_id) VALUES (?, ?, ?, ?, ?)",
(journal.timestamp, journal.source_type, journal.source_id, journal.item_id, journal.object_id),
)
conn.commit()
def latest_journal(self, source_id: str) -> KnowledgeJournal:
conn = sqlite3.connect(self.journal_path)
cursor = conn.cursor()
cursor.execute("SELECT * FROM journal WHERE source_id = ? ORDER BY id DESC LIMIT 1", (source_id,))
result = cursor.fetchone()
if result is None:
return None
else:
(_, timestamp, source_type, sorce_id, item_id, object_id) = result
return KnowledgeJournal(source_type, sorce_id, item_id, object_id, timestamp)
def latest_journals(self, topn) -> [KnowledgeJournal]:
conn = sqlite3.connect(self.journal_path)
cursor = conn.cursor()
cursor.execute("SELECT * FROM journal ORDER BY id DESC LIMIT ?", (topn,))
return [KnowledgeJournal(source_type, sorce_id, item_id, object_id, timestamp) for (_, timestamp, source_type, sorce_id, item_id, object_id) in cursor.fetchall()]
class KnowledgeEmailSource:
def __init__(self, config:dict):
self.config = config
self.config["type"] = "email"
def id(self):
return self.config["address"]
@classmethod
def user_config_items(cls):
return [("address", "email address"),
("password", "email password"),
("imap_server", "imap server"),
("imap_port", "imap port")
]
@classmethod
def local_root(cls):
user_data_dir = AIStorage.get_instance().get_myai_dir()
return os.path.abspath(f"{user_data_dir}/knowledge/email")
async def run_once(self):
# read config from toml file
# and read from config config.local.toml if exists (config.local.toml is ignored by git)
logging.debug(f"knowledge email source {self.id()} run once")
filter = "ALL"
self.client = self.email_client()
await self.read_emails(imap_keyword=filter)
def email_client(self) -> imaplib.IMAP4_SSL:
logging.info(f"read email config from {self.config.get('imap_server')}")
client = imaplib.IMAP4_SSL(
host=self.config.get('imap_server'),
port=self.config.get('imap_port')
)
client.login(self.config.get('address'), self.config.get('password'))
return client
async def read_emails(self, folder: str = 'INBOX', imap_keyword: str = "UNSEEN"):
journal_client = KnowledgeJournalClient()
latest_journal = journal_client.latest_journal(self.id())
latest_uid = 0 if latest_journal is None else int(latest_journal.item_id)
self.client.select(folder)
_, data = self.client.uid('search', None, imap_keyword)
# get email uid list
email_list = data[0].split()
logging.info(f"got {len(email_list)} emails")
journal_client = KnowledgeJournalClient()
for uid in email_list:
_uid = int.from_bytes(uid)
if _uid > latest_uid:
email_dir = self.check_email_saved(uid)
if email_dir is not None:
logging.info(f"email uid {uid} already saved")
else:
email_dir = self.read_and_save_email(uid)
logging.info(f"email uid {uid} saved")
email_object = EmailObjectBuilder({}, email_dir).build()
await KnowledgeBase().insert_object(email_object)
journal_client.insert(KnowledgeJournal("email", self.id(), str(int.from_bytes(uid)), str(email_object.calculate_id())))
def read_and_save_email(self, uid: str) -> str:
message_parts = "(BODY.PEEK[])"
_, email_data = self.client.uid('fetch', uid, message_parts)
mail = mailparser.parse_from_bytes(email_data[0][1])
logging.info(f"got email subject [{mail.subject}]")
self.save_email(mail)
return self.get_local_dir_name(mail)
def get_local_dir_name(self, mail: mailparser.MailParser) -> str:
dir = f"{self.local_root()}/{self.config.get('address')}"
name = f"{mail.subject}__{mail.date}"
name = hashlib.md5(name.encode('utf-8')).hexdigest()
return f"{dir}/{name}"
def check_email_saved(self, uid: str) -> str:
message_parts = "(BODY[HEADER])"
_, email_data = self.client.uid('fetch', uid, message_parts)
mail = mailparser.parse_from_bytes(email_data[0][1])
logging.info(f"[{uid}]check email subject [{mail.subject}]")
dir = self.get_local_dir_name(mail)
logging.info(f"check email saved {dir}")
file = f"{dir}/email.txt"
if os.path.exists(file):
return dir
return None
# save email attachment(images)
def save_email_attachment(self, mail: mailparser.MailParser, email_dir: str):
for attachment in mail.attachments:
if attachment['mail_content_type'] in ['image/png', 'image/jpeg', 'image/gif']:
print('current mail have image attachment')
img_dir = f"{email_dir}/image"
if not os.path.exists(img_dir):
os.makedirs(img_dir)
filename = attachment['filename']
filefullname = f"{img_dir}/{filename}"
image_data = attachment['payload']
try:
image_data = base64.b64decode(image_data)
except base64.binascii.Error:
image_data = image_data.encode()
with open(filefullname, 'wb') as f:
f.write(image_data)
logging.info(f"save email image {filename} success")
# save email body images(html content)
def save_body_images(self, html_content: str, email_dir: str):
# get all image urls
soup = BeautifulSoup(html_content, 'html.parser')
img_tags = soup.find_all('img')
img_urls = [img['src'] for img in img_tags if 'src' in img.attrs]
logging.info(f'Found {len(img_urls)} images in email body')
name_count = 0
if not os.path.exists(email_dir):
os.makedirs(email_dir)
for img_url in img_urls:
# keep the original image filename(last of url)
ext = img_url.split('/')[-1].split('.')[-1]
img_filename = os.path.join(email_dir, f"{name_count}.{ext}")
name_count += 1
# download image
response = requests.get(img_url, stream=True)
if response.status_code == 200:
with open(img_filename, 'wb') as img_file:
for chunk in response.iter_content(1024):
img_file.write(chunk)
logging.info(f'Downloaded {img_url} to {img_filename}')
else:
logging.info(f'Failed to download {img_url}')
# save email content to local dir
def save_email(self, mail: mailparser.MailParser):
dir = f"{self.local_root()}/{self.config.get('address')}"
if not os.path.exists(dir):
os.makedirs(dir)
email_dir = self.get_local_dir_name(mail)
logging.info(f"save email to {email_dir}")
if not os.path.exists(email_dir):
os.makedirs(email_dir)
with open(f"{email_dir}/email.txt", "w", encoding='utf-8') as f:
# soup = BeautifulSoup(mail.body, 'html.parser')
f.write(mail.body)
with open(f"{email_dir}/meta.json", "w", encoding='utf-8') as f:
mail_dict = json.loads(mail.mail_json)
if 'body' in mail_dict:
del mail_dict['body']
json.dump(mail_dict, f, ensure_ascii=False, indent=4)
logging.info(f"save email meta info {f.name}")
self.save_email_attachment(mail, email_dir)
self.save_body_images(mail.body, f"{email_dir}/body_image")
class KnowledgeDirSource:
def __init__(self, config):
self.config = config
self.config["type"] = "dir"
@classmethod
def user_config_items(cls):
return [("path", "local dir path")]
def id(self):
return self.config["path"]
def path(self):
return self.config["path"]
@staticmethod
async def read_txt_file(file_path:str)->str:
cur_encode = "utf-8"
async with aiofiles.open(file_path,'rb') as f:
cur_encode = chardet.detect(await f.read())['encoding']
async with aiofiles.open(file_path,'r',encoding=cur_encode) as f:
return await f.read()
async def run_once(self):
logging.debug(f"knowledge dir source {self.id()} run once")
journal_client = KnowledgeJournalClient()
latest_journal = journal_client.latest_journal(self.id())
if latest_journal is not None:
if os.path.getmtime(self.path()) <= latest_journal.timestamp:
logging.debug(f"knowledge dir source {self.id()} ingnored for no update")
return
file_pathes = sorted(os.listdir(self.path()), key=lambda x: os.path.getctime(os.path.join(self.path(), x)))
for rel_path in file_pathes:
file_path = os.path.join(self.path(), rel_path)
timestamp = os.path.getctime(file_path)
if latest_journal is not None:
if timestamp <= latest_journal.timestamp:
continue
ext = os.path.splitext(file_path)[1].lower()
if ext in ['.jpg', '.jpeg', '.png', '.gif', '.bmp']:
logging.info(f"knowledge dir source {self.id()} found image file {file_path}")
image = ImageObjectBuilder({}, {}, file_path).build()
await KnowledgeBase().insert_object(image)
journal_client.insert(KnowledgeJournal("dir", self.id(), rel_path, str(image.calculate_id()), timestamp))
if ext in ['.txt']:
logging.info(f"knowledge dir source {self.id()} found text file {file_path}")
text = await self.read_txt_file(file_path)
document = DocumentObjectBuilder({}, {}, text).build()
await KnowledgeBase().insert_object(document)
journal_client.insert(KnowledgeJournal("dir", self.id(), rel_path, str(document.calculate_id()), timestamp))
# define singleton class knowledge pipline
class KnowledgePipline:
_instance = None
@classmethod
def get_instance(cls):
if cls._instance is None:
cls._instance = KnowledgePipline()
cls._instance.__singleton_init__()
return cls._instance
def initial(self):
config_path = self.__config_path()
logging.info(f"initial knowledge pipline from {config_path}")
if os.path.exists(config_path):
config = toml.load(self.__config_path())
for source_config in config["sources"]:
if source_config['type'] == 'email':
self.add_email_source(KnowledgeEmailSource(source_config))
if source_config['type'] == 'dir':
self.add_dir_source(KnowledgeDirSource(source_config))
return True
def __singleton_init__(self):
self.knowledge_base = KnowledgeBase()
self.email_sources = dict()
self.dir_sources = dict()
self.source_queue = list()
self.run_lock = asyncio.Lock()
asyncio.create_task(self.run_loop())
def save_config(self):
config = dict()
config["sources"] = [source.config for source in self.source_queue]
with open(self.__config_path(), "w") as f:
toml.dump(config, f)
@classmethod
def __config_path(cls) -> str:
user_data_dir = AIStorage.get_instance().get_myai_dir()
return os.path.abspath(f"{user_data_dir}/etc/knowledge.cfg.toml")
def add_email_source(self, source: KnowledgeEmailSource):
if self.email_sources.get(source.id()) is not None:
return "already exists"
self.email_sources[source.id()] = source
self.source_queue.append(source)
return None
def add_dir_source(self, source: KnowledgeDirSource):
if self.dir_sources.get(source.id()) is not None:
logging.info(f"knowledge add source {source.id()} failed for already exists")
return "already exists"
logging.info(f"knowledge added source {source.id()}")
self.dir_sources[source.id()] = source
self.source_queue.append(source)
return None
def get_latest_journals(self, topn) -> [KnowledgeJournal]:
return KnowledgeJournalClient().latest_journals(topn)
async def run_loop(self):
while True:
await self.run_once()
await asyncio.sleep(5)
async def run_once(self):
logging.info(f"knowledge pipeline started")
# sources = list()
# async with self.run_lock:
# for source in self.source_queue:
# sources.append(source)
# for source in sources:
# await source.run_once()
for source in self.source_queue:
await source.run_once()