-
Notifications
You must be signed in to change notification settings - Fork 0
/
Embedding_Bert.py
71 lines (57 loc) · 1.87 KB
/
Embedding_Bert.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
#encoding=utf-8
import numpy as np
import gensim
from gensim.models import KeyedVectors
import torch;
import re
import tensorflow as tf;
import torch;
import transformers;
from transformers import AutoModel, AutoTokenizer
def split(word):
return [char for char in word]
def divide_text_by_sentence(text):
text = text.strip('\n')
text = re.split('?|。|!|……|:|“|”',text) # ,|
return text # list
def get_sentence_bert(sent,tokenizer,model):
input_ids = tokenizer.encode(sent, return_tensors='pt')
last_hidden_state, _ = model(input_ids)
return last_hidden_state
corpus_os = '/Users/vanellope/Desktop/FYP/textbook_corpus/textbook_HK_6class.txt'
file = open(corpus_os,encoding='utf-8')
texts = file.readlines()
file.close()
temp = []
for text in texts:
temp.append(split(text))
texts = temp
tokenizer = AutoTokenizer.from_pretrained('bert-base-chinese')
model = AutoModel.from_pretrained('bert-base-chinese')
##############################
print('starting to process the corpus')
file = open(corpus_os)
tb = file.readlines()
file.close()
tb = tb[1:]
print(tb[10],'\n',len(tb))
# loop all article
textbook_vec = []
for i,text in enumerate(tb):
article = []
text = divide_text_by_sentence(text)
for sent in text:
if len(sent) == 0:
continue # drop empty sentence
vec = get_sentence_bert(sent,tokenizer,model)
article.append(vec)
# print(len(article)) # 每篇文章句子数
textbook_vec.append(article)
torch.save(textbook_vec,'/Users/vanellope/2020-2021 Final Year Project/FYP_BERT_CORPUS_6CLASS_HK/article{}'.format(i))
textbook_vec = []
print('lets load one article:')
article_batch = torch.load('/Users/vanellope/2020-2021 Final Year Project/FYP_BERT_CORPUS_6CLASS_HK/article10')
article = article_batch[0]
print(len(article))
for i in range(len(article)):
print(article[i].shape)