-
Notifications
You must be signed in to change notification settings - Fork 1
/
startMianFeiFixer.py
108 lines (85 loc) · 3.92 KB
/
startMianFeiFixer.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
#!/usr/bin/python
# -*- coding: UTF-8 -*-
'''
@author: zyq
'''
import json
import traceback
from urllib import quote
import requests
import time
from Config import MianFeiTXTSearchBaseUrl, MianFeiTXTChapBaseUrl
from app.mianFeiFixer import fixUnFinished
from dao.dushuMianFeiTXTService import getMianAllBookObjs, getMianAllBookBaseObjs
from dao.dushuService import getLatestChapByBookId, updateOneFieldByOneField
from util.logHelper import myLogging
from util.networkHelper import getContentWithUA
from util.signHelper import paramMap
def changeSouceIds():
bookObjs = getMianAllBookBaseObjs()
for bookObj in bookObjs:
try:
foundNewId = False
title = bookObj['title']
author = bookObj['author']
source = bookObj['source']
bookId = bookObj['id']
searchUrl = MianFeiTXTSearchBaseUrl + '?' + paramMap().mianfeiTXT()\
.put('keyword', (title + author).encode('utf-8'))\
.put('pageSize', '10').put('pageNum', '1').put('type', '1')\
.mianfeiTXTSign() \
.toUrl()
# time.sleep(random.)
r = requests.get(searchUrl)
searchRes = json.loads(r.text)
for resBook in searchRes['data']['books']:
resTitle = resBook['name']
if resTitle != title:
continue
resAuthor = resBook['author']
if resAuthor != author:
continue
resId = resBook['id']
if str(resId) == str(source):
myLogging.info('WTF: id no change?, bookId: %s, orgSoueceId: %s, newId: %s', bookId, source, resId)
latestChapObj = getLatestChapByBookId(bookId)
if not latestChapObj:
myLogging.error('no chaps in db yet, bookId: %s, new mid: %s', bookId, resId)
updateOneFieldByOneField('source', resId, 'id', bookId)
foundNewId = True
break
cid = latestChapObj['idx']
chapTitle = latestChapObj['title']
capContentUrl = MianFeiTXTChapBaseUrl + '?' + paramMap().mianfeiTXT().mBookId(resId).mChapId(
cid).mianfeiTXTSign().toUrl()
capContent = getContentWithUA(capContentUrl)
if not capContent:
capContent = getContentWithUA(capContentUrl)
# capContent = capContent.replace(r'\r', '').replace(r'\n', '')
capListJsonObj = json.loads(capContent, strict=False)
if not (capListJsonObj['returnCode'] == '0000'):
capListJsonObj = json.loads(capContent)
if not (capListJsonObj['returnCode'] == '0000' and capListJsonObj['returnMsg'] == u'成功'):
myLogging.error('get chap detail fail mid: %s, cid: %s', resId, cid)
continue
chapterName = capListJsonObj['data']['bookChapter']['chapterName']
if chapterName == chapTitle:
myLogging.info('bookId %s change source from %s to %s', bookId, source, resId)
updateOneFieldByOneField('source', resId, 'id', bookId)
foundNewId = True
break
if not foundNewId:
myLogging.error('bookId %s did not find new id !!!,title: %s, author: %s, org source: %s', bookId, title, author,source )
except Exception as e:
myLogging.error(traceback.format_exc())
if __name__ == '__main__':
# fixUnFinished()
# title = '大主宰'
# searchUrl = MianFeiTXTSearchBaseUrl + '?' + paramMap().mianfeiTXT() \
# .put('keyword', title) \
# .put('pageSize', '10').put('pageNum', '1').put('type', '1') \
# .mianfeiTXTSign().toUrl()
#
# r = requests.get(searchUrl)
# searchRes = json.loads(r.text)
changeSouceIds()