Bug fix about history length for some website

ExecuteStage/.vscode/launch.json

@@ -12,7 +12,7 @@
  "justMyCode": false,
  // "args": ["--id", "[7]", "--read_type", "remote", "--headless", "0"]
  // "args": ["--id", "[9]", "--read_type", "remote", "--headless", "0", "--saved_file_name", "YOUTUBE"]
- "args": ["--id", "[36]", "--headless", "0", "--user_data", "0", "--keyboard", "0"]
+ "args": ["--id", "[14]", "--headless", "0", "--user_data", "0", "--keyboard", "0"]
  }
  ]
 }

ExecuteStage/easyspider_executestage.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # import atexit
+import undetected_chromedriver as uc
 from utils import download_image, get_output_code, isnotnull, lowercase_tags_in_xpath, myMySQL, new_line, on_press_creator, on_release_creator, replace_field_values, write_to_csv, write_to_excel, write_to_json
 from myChrome import MyChrome
 from threading import Thread, Event
@@ -41,7 +42,6 @@
 from lxml import etree
 import onnxruntime
 onnxruntime.set_default_logger_severity(3) # 隐藏onnxruntime的日志
-import undetected_chromedriver as uc
 # import pandas as pd
 # import numpy
 # import pytesseract
@@ -157,7 +157,7 @@ def __init__(self, browser_t, id, service, version, event, saveName, config):
  self.OUTPUT.append([]) # 添加表头
  self.writeMode = 0
  elif self.outputFormat == "json":
- self.writeMode = 3 # JSON模式无需判断是否存在文件
+ self.writeMode = 3  # JSON模式无需判断是否存在文件
  elif self.outputFormat == "mysql":
  self.mysql = myMySQL(config["mysql_config_path"])
  self.mysql.create_table(self.saveName, service["outputParameters"])
@@ -409,7 +409,8 @@ def saveData(self, exit=False):
  elif self.outputFormat == "json":
  file_name = "Data/Task_" + \
  str(self.id) + "/" + self.saveName + '.json'
- write_to_json(file_name, self.OUTPUT, self.outputParametersTypes, self.outputParametersRecord, self.outputParameters.keys())
+ write_to_json(file_name, self.OUTPUT, self.outputParametersTypes,
+ self.outputParametersRecord, self.outputParameters.keys())
  elif self.outputFormat == "mysql":
  self.mysql.write_to_mysql(
  self.OUTPUT, self.outputParametersRecord, self.outputParametersTypes)
@@ -647,7 +648,8 @@ def switchSelect(self, para, loopValue):
  optionValue = loopValue
  optionMode = 1
  try:
- xpath = replace_field_values(para["xpath"], self.outputParameters, self)
+ xpath = replace_field_values(
+ para["xpath"], self.outputParameters, self)
  dropdown = Select(self.browser.find_element(
  By.XPATH, xpath, iframe=para["iframe"]))
  try:
@@ -678,7 +680,8 @@ def switchSelect(self, para, loopValue):
  def moveToElement(self, para, loopElement=None, loopPath="", index=0):
  time.sleep(0.1) # 移动之前等待0.1秒
  loopPath = replace_field_values(loopPath, self.outputParameters, self)
- xpath = replace_field_values(para["xpath"], self.outputParameters, self)
+ xpath = replace_field_values(
+ para["xpath"], self.outputParameters, self)
  if para["useLoop"]: # 使用循环的情况下，传入的clickPath就是实际的xpath
  if xpath == "":
  path = loopPath
@@ -873,8 +876,11 @@ def judgeExecute(self, node, loopElement, clickPath="", index=0):
  def loopExecute(self, node, loopValue, clickPath="", index=0):
  time.sleep(0.1) # 第一次执行循环的时候强制等待1秒
  thisHandle = self.browser.current_window_handle # 记录本次循环内的标签页的ID
- thisHistoryLength = self.browser.execute_script(
- 'return history.length') # 记录本次循环内的history的length
+ try:
+ thisHistoryLength = self.browser.execute_script(
+ 'return history.length') # 记录本次循环内的history的length
+ except:
+ thisHistoryLength = 0
  self.history["index"] = thisHistoryLength
  self.history["handle"] = thisHandle
  if int(node["parameters"]["loopType"]) == 0: # 单个元素循环
@@ -1009,7 +1015,7 @@ def loopExecute(self, node, loopValue, clickPath="", index=0):
  # else:
  # time.sleep(2)
  # 切换历史记录等待：
- self.recordLog("Change history back time or: ", 
+ self.recordLog("Change history back time or: ",
  node["parameters"]["historyWait"])
  try:
  self.browser.execute_script('window.stop()')
@@ -1030,7 +1036,8 @@ def loopExecute(self, node, loopValue, clickPath="", index=0):
  # 千万不要忘了分割！！
  for path in node["parameters"]["pathList"].split("\n"):
  try:
- path = replace_field_values(path, self.outputParameters, self)
+ path = replace_field_values(
+ path, self.outputParameters, self)
  element = self.browser.find_element(
  By.XPATH, path, iframe=node["parameters"]["iframe"])
  # self.recordLog("循环元素|Loop element:", path)
@@ -1224,13 +1231,17 @@ def openPage(self, para, loopValue):
  "return history.length")
  except:
  self.history["index"] = 0
+ except Exception as e:
+ self.print_and_log("History Length Error")
+ self.history["index"] = 0
  self.scrollDown(para) # 控制屏幕向下滚动
 
  # 键盘输入事件
  def inputInfo(self, para, loopValue):
  time.sleep(0.1) # 输入之前等待0.1秒
  try:
- xpath = replace_field_values(para["xpath"], self.outputParameters, self)
+ xpath = replace_field_values(
+ para["xpath"], self.outputParameters, self)
  textbox = self.browser.find_element(
  By.XPATH, xpath, iframe=para["iframe"])
  # textbox.send_keys(Keys.CONTROL, 'a')
@@ -1289,8 +1300,10 @@ def clickElement(self, para, loopElement=None, clickPath="", index=0):
  try:
  # element = self.browser.find_element(
  # By.XPATH, path, iframe=para["iframe"])
- clickPath = replace_field_values(clickPath, self.outputParameters, self)
- xpath = replace_field_values(para["xpath"], self.outputParameters, self)
+ clickPath = replace_field_values(
+ clickPath, self.outputParameters, self)
+ xpath = replace_field_values(
+ para["xpath"], self.outputParameters, self)
  if para["useLoop"]: # 使用循环的情况下，传入的clickPath就是实际的xpath
  if xpath == "":
  path = clickPath
@@ -1375,6 +1388,9 @@ def clickElement(self, para, loopElement=None, clickPath="", index=0):
  pass
  self.history["index"] = self.browser.execute_script(
  "return history.length")
+ except Exception as e:
+ self.print_and_log("History Length Error")
+ self.history["index"] = 0
  else:
  try:
  self.history["index"] = self.browser.execute_script(
@@ -1387,6 +1403,9 @@ def clickElement(self, para, loopElement=None, clickPath="", index=0):
  self.history["index"] = self.browser.execute_script(
  "return history.length")
  # 如果打开了新窗口，切换到新窗口
+ except Exception as e:
+ self.print_and_log("History Length Error")
+ self.history["index"] = 0
  self.scrollDown(para) # 根据参数配置向下滚动
  # rt.end()
 
@@ -1556,7 +1575,8 @@ def clearOutputParameters(self):
 
  # 提取数据事件
  def getData(self, para, loopElement, isInLoop=True, parentPath="", index=0):
- parentPath = replace_field_values(parentPath, self.outputParameters, self)
+ parentPath = replace_field_values(
+ parentPath, self.outputParameters, self)
  if para["clear"] == 1:
  self.clearOutputParameters()
  try:
@@ -1762,8 +1782,8 @@ def getData(self, para, loopElement, isInLoop=True, parentPath="", index=0):
 
 if __name__ == '__main__':
  from multiprocessing import freeze_support
- freeze_support() # 防止无限死循环多开
- 
+ freeze_support()  # 防止无限死循环多开
+
  # 如果需要调试程序，请在命令行参数中加入--keyboard 0 来禁用键盘监听以提升调试速度
  # If you need to debug the program, please add --keyboard 0 in the command line parameters to disable keyboard listening to improve debugging speed
  config = {
@@ -1959,13 +1979,17 @@ def getData(self, para, loopElement, isInLoop=True, parentPath="", index=0):
  elif cloudflare == 1:
  if sys.platform == "win32":
  options.binary_location = "C:\\Program Files\\Google\\Chrome Beta\\Application\\chrome.exe" # 需要用自己的浏览器
- # options.add_argument("--auto-open-devtools-for-tabs") 
+ # options.add_argument("--auto-open-devtools-for-tabs")
  # options.binary_location = "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe" # 需要用自己的浏览器
- browser_t = MyUCChrome(options=options, driver_executable_path=driver_path)
+ browser_t = MyUCChrome(
+ options=options, driver_executable_path=driver_path)
  links = list(filter(isnotnull, service["links"].split("\n")))
- browser_t.execute_script('window.open("'+ links[0] +'","_blank");') # open page in new tab
- time.sleep(5) # wait until page has loaded
- browser_t.switch_to.window(browser_t.window_handles[1]) # switch to new tab
+ # open page in new tab
+ browser_t.execute_script(
+ 'window.open("' + links[0] + '","_blank");')
+ time.sleep(5) # wait until page has loaded
+ browser_t.switch_to.window(
+ browser_t.window_handles[1]) # switch to new tab
  # browser_t = uc.Chrome()
  else:
  print("Cloudflare模式只支持Windows x64平台。")