Add scripts in setup.py

loso/lexicon.py

@@ -245,13 +245,15 @@ def splitTerms(self, text, ngram=4):
  for term in util.ngram(n, text):
  count = int(self.get(term) or 0)
  n = len(term)
- v = float(self.getNgramSum(n))/float(self.getNgramCount(n))
- v = v*v
+ if self.getNgramSum(n) is None:
+ v = 1
+ else:
+ v = float(self.getNgramSum(n))/float(self.getNgramCount(n))
+ v = v*v
  score = count/v
  if score == 0:
  score = 0.00000001
 
- head_tail_score = 0
  head = 0
  tail = 0
  head_tail = self.getHeadTail(term)

loso/scripts.py

@@ -0,0 +1,64 @@
+import sys
+import logging
+from distutils.cmd import Command
+from distutils.errors import DistutilsOptionError
+
+import server
+
+class InteractCommand(Command):
+ description = 'provide interact interface for testing splitting terms'
+ user_options = []
+
+ def initialize_options(self):
+ pass
+
+ def finalize_options(self):
+ pass
+
+ def run(self):
+ logging.basicConfig(level=logging.DEBUG)
+ service = server.SegumentService()
+ while True:
+ text = raw_input('Text:').decode(sys.stdin.encoding)
+ terms = service.splitTerms(text)
+ print ' '.join(terms)
+
+class FeedCommand(Command):
+ description = 'feed text data file'
+ user_options = [
+ ('file=', 'f', 'text file to feed'),
+ ('encoding=', 'e', 'encoding of text file'),
+ ]
+
+ def initialize_options(self):
+ self.encoding = 'utf8'
+ self.file = None
+
+ def finalize_options(self):
+ import codecs
+ if not self.file:
+ raise DistutilsOptionError('Must set text file path to feed')
+ self.text_file = codecs.open(self.file, 'rt', encoding=self.encoding)
+ self.text = self.text_file.read()
+
+ def run(self):
+ logging.basicConfig(level=logging.DEBUG)
+ service = server.SegumentService()
+ service.feed(self.text)
+
+
+class ResetCommand(Command):
+ description = 'reset lexicon database'
+ user_options = []
+
+ def initialize_options(self):
+ pass
+
+ def finalize_options(self):
+ pass
+
+ def run(self):
+ logging.basicConfig(level=logging.DEBUG)
+ service = server.SegumentService()
+ service.db.reset()
+ print 'Done.'

setup.py

@@ -1,5 +1,18 @@
 from setuptools import setup
 
+extra = {}
+
+try:
+ from loso import scripts
+except ImportError:
+ pass
+else:
+ extra['cmdclass'] = {
+ 'interact': scripts.InteractCommand,
+ 'feed': scripts.FeedCommand,
+ 'reset': scripts.ResetCommand
+ }
+
 setup(
  name='Plurk_Loso',
  version='0.1',
@@ -12,5 +25,6 @@
  install_requires=[
  'redis',
  'pyyaml'
- ]
+ ],
+ **extra
 )