Fix encoding bug of craweler

fangpenlin · Jan 19, 2011 · 523cd8a · 523cd8a
1 parent 029aa22
commit 523cd8a
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/loso/crawlers/hinet_news.py b/loso/crawlers/hinet_news.py
@@ -1,3 +1,4 @@
+import codecs
 import urllib2
 import cStringIO as StringIO
 
@@ -48,7 +49,7 @@ def crawelCategory(rss_url):
  yield ' '.join(getNewsText(link))
 
 def main():
- with open('hinet_news.txt', 'wt') as file:
+ with codecs.open('hinet_news.txt', 'wt', encoding='utf8') as file:
  for url in rss_urls:
  for text in crawelCategory(url):
  print text