-
Notifications
You must be signed in to change notification settings - Fork 0
/
TopicModelling.py
63 lines (51 loc) · 1.87 KB
/
TopicModelling.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
from nltk.tokenize import word_tokenize
import re
import string
file = open('D:/Niranjan/Data Science/Semester 2/Python & ML data sets/twitter_stream.txt', mode='r',encoding='utf-8')
twitter_stream = []
file.seek(0)
twitter_stream = file.readlines()
twitter_stream[17]
from nltk.tokenize import word_tokenize
twitter_tokenized = [word_tokenize(word) for word in twitter_stream]
print(len(twitter_stream),len(twitter_tokenized))
twitter_tokenized[1]
from nltk.corpus import stopwords
stopwordsEng = stopwords.words('english')
twitter_withoutStopwords = []
for sent in twitter_tokenized:
new_sent = []
for word in sent:
if word not in stopwordsEng:
new_sent.append(word)
twitter_withoutStopwords.append(new_sent)
from nltk.stem import wordnet
wordnetLemma = wordnet.WordNetLemmatizer()
twitter_lemmatized = []
for sent in twitter_withoutStopwords:
new_sent = []
for word in sent:
new_token = wordnetLemma.lemmatize(word)
if new_token != '':
new_sent.append(new_token)
twitter_lemmatized.append(new_sent)
print(twitter_tokenized[7])
print(twitter_lemmatized[7])
regex = re.compile('[%s]' % re.escape(string.punctuation))
twitter_withoutPunc = []
for sent in twitter_lemmatized:
new_sent = []
for word in sent:
if regex.sub('', word) != '':
new_sent.append(regex.sub('', word))
twitter_withoutPunc.append(new_sent)
print(twitter_tokenized[7])
print(twitter_lemmatized[7])
print(twitter_withoutPunc[7])
import gensim
from gensim import corpora
dictionary = corpora.Dictionary(twitter_withoutPunc)
corpus = [dictionary.doc2bow(doc) for doc in twitter_withoutPunc]
ldamodel = gensim.models.LdaMulticore(corpus,num_topics=1,id2word=dictionary,passes=1,workers=3)
print(ldamodel.print_topics(num_topics=4,num_words=5))
print(ldamodel.id2word)