-
Notifications
You must be signed in to change notification settings - Fork 1
/
startDailySearchHistoryCrawler.py
29 lines (26 loc) · 991 Bytes
/
startDailySearchHistoryCrawler.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
#!/usr/bin/python
# -*- coding: UTF-8 -*-
'''
@author: zyq
'''
import time
import traceback
from app.SearchHistoryCrawler import crawlByDailySearchHistory
from app.shuqiUpdater import updateFromMysql
from local.hotConfigHelper import getHotConfigDict
from util.logHelper import myLogging
# import logging
# logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p',level=logging.INFO)
if __name__ == '__main__':
timeStart = int(time.time() * 1000) - 24 * 3600 * 1000
while 1:
myLogging.info('begin searchHistoryCrawler')
timeBeforeSearch = int(time.time() * 1000)
try:
crawlByDailySearchHistory(timeStart)
except Exception as e:
myLogging.error(traceback.format_exc())
timeStart = timeBeforeSearch
sleepTime = getHotConfigDict()['searchHistoryCrawler']['updateSleep']
myLogging.info(' done one loop, now sleep '+ str(sleepTime) + ' secs')
time.sleep(int(sleepTime))