MACOS version

.gitignore

@@ -1,3 +1,5 @@
+.DS_Store/
+.DS_Store
 .idea/
 node_modules/
 ServiceWrapper/
@@ -9,3 +11,4 @@ old_code/
 *.7z
 *.mp4
 *.tar.xz
+*.zip

ElectronJS/config.json

@@ -1 +1 @@
-{"webserver_address":"http:https://localhost","webserver_port":8074,"user_data_folder":"./user_data","absolute_user_data_folder":"/Users/naibowang/Documents/EasySpider/ElectronJS/user_data"}
+{"webserver_address":"http:https://localhost","webserver_port":8074,"user_data_folder":"./user_data123","absolute_user_data_folder":"/Users/naibowang/Documents/EasySpider/ElectronJS/user_data123"}

ElectronJS/execute_macos.sh

@@ -1,3 +1,5 @@
 #!/bin/bash
 
+echo "Executing EasySpider on MacOS"
+
 ./easyspider_executestage $1 $2 $3 $4 $5 $6 $7 $8 $9

ElectronJS/main.js

@@ -34,7 +34,7 @@ if (process.platform === 'win32' && process.arch === 'ia32') {
 } else if (process.platform === 'darwin') {
  driverPath = path.join(__dirname, "chromedriver_mac64");
  chromeBinaryPath = path.join(__dirname, "chrome_mac64.app/Contents/MacOS/Google Chrome");
- execute_path = path.join(__dirname, "execute_macos.sh");
+ execute_path = path.join(__dirname, "");
 } else if (process.platform === 'linux') {
  driverPath = path.join(__dirname, "chrome_linux64/chromedriver_linux64");
  chromeBinaryPath = path.join(__dirname, "chrome_linux64/chrome");
@@ -204,7 +204,7 @@ async function beginInvoke(msg) {
  if (msg.message.user_data_folder == null || msg.message.user_data_folder == undefined || msg.message.user_data_folder == "") {
  parameters = ["--id", msg.message.id, "--server_address", server_address, "--user_data", 0];
  } else {
- let user_data_folder_path = __dirname.indexOf("resources") >= 0 && __dirname.indexOf("app") >= 0 ? path.join(__dirname, "../../..", msg.message.user_data_folder) : path.join(__dirname, msg.message.user_data_folder);
+ let user_data_folder_path = path.join(task_server.getDir(), msg.message.user_data_folder);
  parameters = ["--id", msg.message.id, "--server_address", server_address, "--user_data", 1];
  config.user_data_folder = msg.message.user_data_folder;
  config.absolute_user_data_folder = user_data_folder_path;
@@ -215,10 +215,12 @@ async function beginInvoke(msg) {
  // });
 
  let spawn = require("child_process").spawn;
- let child_process = spawn(execute_path, parameters);
- child_process.stdout.on('data', function (data) {
- console.log(data.toString());
- });
+ if (process.platform != "darwin") {
+ let child_process = spawn(execute_path, parameters);
+ child_process.stdout.on('data', function (data) {
+ console.log(data.toString());
+ });
+ }
  }
 }
 
@@ -282,7 +284,7 @@ async function runBrowser(lang = "en", user_data_folder = '') {
  }
  options.setChromeBinaryPath(chromeBinaryPath);
  if (user_data_folder != "") {
- let dir = __dirname.indexOf("resources") >= 0 && __dirname.indexOf("app") >= 0 ? path.join(__dirname, "../../..", user_data_folder) : path.join(__dirname, user_data_folder);
+ let dir = path.join(task_server.getDir(), user_data_folder);
  console.log(dir);
  options.addArguments("--user-data-dir=" + dir);
  config.user_data_folder = user_data_folder;

ElectronJS/package.json

@@ -1,8 +1,8 @@
 {
  "name": "easy-spider",
  "productName": "EasySpider",
- "version": "0.1.0",
- "icon": "favicon.ico",
+ "version": "0.2.0",
+ "icon": "./favicon",
  "description": "NoCode Visual Web Crawler",
  "main": "main.js",
  "scripts": {
@@ -59,8 +59,8 @@
  }
  ],
  "packagerConfig": {
- "icon": "favicon.ico",
- "appVersion": "0.1.0",
+ "icon": "./favicon",
+ "appVersion": "0.2.0",
  "name": "EasySpider",
  "executableName": "EasySpider",
  "appCopyright": "Naibo Wang ([email protected])",

ElectronJS/package_macos.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# This script is used to build.md the package for Linux 64-bit.
+rm -rf out
+rm -r ../Releases/EasySpider_MacOS_all_arch/EasySpider.app
+npm run make
+unzip out/make/zip/darwin/x64/EasySpider-darwin-x64-* -d ../Releases/EasySpider_MacOS_all_arch/
+rm ../Releases/EasySpider_MacOS_all_arch/EasySpider.app/Contents/Resources/app/VS_BuildTools.exe
+rm -r ../Releases/EasySpider_MacOS_all_arch/EasySpider.app/Contents/Resources/app/chrome_win64
+rm -r ../Releases/EasySpider_MacOS_all_arch/EasySpider.app/Contents/Resources/app/Data
+rm -r ../Releases/EasySpider_MacOS_all_arch/EasySpider.app/Contents/Resources/app/.idea
+rm -r ../Releases/EasySpider_MacOS_all_arch/EasySpider.app/Contents/Resources/app/tasks
+rm -r ../Releases/EasySpider_MacOS_all_arch/EasySpider.app/Contents/Resources/app/execution_instances
+rm -r ../Releases/EasySpider_MacOS_all_arch/EasySpider.app/Contents/Resources/app/user_data

ElectronJS/server.js

@@ -20,9 +20,14 @@ function compare(p){ //这是比较函数
  return b - a; //降序
  }
 }
+
 function getDir(){
- if(__dirname.indexOf("app") >= 0 && __dirname.indexOf("resources") >= 0){
- return path.join(__dirname,"../../..");
+ if(__dirname.indexOf("app") >= 0 && __dirname.indexOf("sources") >= 0){
+ if(process.platform == "darwin"){
+ return path.join(__dirname,"../../../..");
+ } else {
+ return path.join(__dirname,"../../..");
+ }
  } else{
  return __dirname;
  }
@@ -223,7 +228,10 @@ exports.start = function(port = 8074) {
  let file_names = [];
  fs.readdirSync(path.join(getDir(), "execution_instances")).forEach((file)=>{
  try{
- file_names.push(parseInt(file.split(".")[0]));
+ if(file.split(".")[1] == "json"){
+ file_names.push(parseInt(file.split(".")[0]));
+ }
+ console.log(file);
  } catch (error) {
 
  }
@@ -232,6 +240,7 @@ exports.start = function(port = 8074) {
  if (file_names.length != 0) {
  eid = Math.max(...file_names) + 1;
  }
+ console.log(file_names);
  task["id"] = eid;
  task = JSON.stringify(task);
  fs.writeFile(path.join(getDir(), `execution_instances/${eid}.json`), task, (err) => {});

ElectronJS/src/taskGrid/global.js

@@ -5,6 +5,10 @@ function getUrlParam(name) {
  return ""; //返回参数值,默认后台地址
 }
 
+function isMac() { 
+ return /macintosh|mac os x/i.test(navigator.userAgent); 
+};
+
 Vue.filter('lang', function (value) {
  if (getUrlParam("lang") == "zh") {
  return value.split("~")[1];

ElectronJS/src/taskGrid/invokeTask.html

@@ -8,6 +8,7 @@
  name="viewport">
  <meta content="ie=edge" http-equiv="X-UA-Compatible">
  <script src="vue.js"></script>
+ <script src="bootstrap/js/bootstrap.js"></script>
  <link href="bootstrap/css/bootstrap.css" rel="stylesheet"></link>
  <title>Task Invoke</title>
  <style>
@@ -34,8 +35,32 @@
 </head>
 
 <body>
+
 <div class="row" style="margin-top: 40px;">
+
  <div class="col-md-6" id="taskInfo" style="margin:0 auto" v-if="show">
+ <div class="modal fade" id="myModal" tabindex="-1" role="dialog" aria-labelledby="myModalLabel" aria-hidden="true">
+ <div class="modal-dialog">
+ <div class="modal-content">
+ <div class="modal-header">
+ <h4 class="modal-title" id="myModalLabel">{{"Task Invocation Instruction~执行任务说明" | lang}}</h4>
+ <button type="button" class="close" data-dismiss="modal" aria-hidden="true">&times;</button>
+ </div>
+ <div class="modal-body">
+ <input onkeydown="inputDelete(event)" id="serviceId" type="hidden" name="serviceId" value="-1"></input>
+ <input onkeydown="inputDelete(event)" id="url" type="hidden" name="url" value="about:blank"></input>
+ <label>{{"For MacOS, please open a terminal, go to EasySpider's folder, and then type the following command to run the task (EasySpider cannot quit when executing command):~对于MacOS系统，请在EasySpider目录下打开命令行工具Terminal，然后复制和运行以下命令以执行任务(执行命令时不能退出EasySpider)：" | lang}}</label>
+ <textarea class="form-control">{{command}}</textarea>
+ </div>
+ <!-- <div class="modal-footer">
+ <button type="button" id="saveAsButton" class="btn btn-outline-primary">另存为</button>
+ <button type="button" id="saveButton" class="btn btn-primary">保存</button>
+ </div> -->
+ </div>
+ <!-- /.modal-content -->
+ </div>
+ </div>
+
  <nav aria-label="breadcrumb">
  <ol class="breadcrumb" style="padding-left:0;background-color: white">
  <li @click="gotoHome" class="breadcrumb-item"><a href="#">{{"Home~首页" | lang}}</a></li>
@@ -74,7 +99,7 @@ <h4 style="text-align: center;">{{"Task Invocation~任务调用" | lang}}</h4>
  </tbody>
  </table>
  <div class="form-group" style="margin-top: 10px">
- <label>{{"User Data Folder (If you want to load the cookie data from your local browser, please set this folder path, and then cilck the 'Run with (Data Mode)' button to run the task):~用户本地浏览器数据目录（如果需要使用本地的登录信息和cookie，请设置此目录，并点击下方“执行（带用户信息模式）”按钮开始执行任务）：" | lang}}</label>
+ <label>{{"User Data Folder (If you want to load the cookie, data and extension(s) from your local browser, please set this folder path, and then cilck the 'Run with (Data Mode)' button to run the task):~用户本地浏览器数据目录（如果需要使用本地的登录信息，插件和cookie，请设置此目录，并点击下方“执行（带用户信息模式）”按钮开始执行任务）：" | lang}}</label>
  <input type="text" class="form-control" v-model="user_data_folder"></input>
  </div>
  </form>
@@ -127,10 +152,12 @@ <h4 style="text-align: center;">{{"Task Invocation~任务调用" | lang}}</h4>
  user_data_folder:"",
  with_user_data: true,
  backEndAddressServiceWrapper: getUrlParam("backEndAddressServiceWrapper"),
+ command: "./easyspider_executestage "
  }, mounted() {
  $.get(this.backEndAddressServiceWrapper + "/getConfig", function (result) {
  app.$data.user_data_folder = result.user_data_folder;
  });
+ //TODO 翻译 写清楚readme有关浏览器版本的问题 logo更换 提示看文档来运行
  },
  methods: {
  gotoHome: function () {
@@ -195,6 +222,10 @@ <h4 style="text-align: center;">{{"Task Invocation~任务调用" | lang}}</h4>
  }
  };
  ws.send(JSON.stringify(message));
+ if(isMac()){
+ app.$data.command = "./easyspider_executestage --id " + app.$data.ID.toString() + " --user_data " + (app.$data.with_user_data ? "1" : "0") + " --server_address " + app.$data.backEndAddressServiceWrapper;
+ $('#myModal').modal('show');
+ }
  }
  },
  remoteExecute: function () {
@@ -207,6 +238,7 @@ <h4 style="text-align: center;">{{"Task Invocation~任务调用" | lang}}</h4>
  } else {
  text = "确定要立即在本地运行此任务吗？";
  }
+
  this.with_user_data = with_user_data;
  if (confirm(text)) {
  var para = {};
@@ -226,17 +258,23 @@ <h4 style="text-align: center;">{{"Task Invocation~任务调用" | lang}}</h4>
  }
  };
  ws.send(JSON.stringify(message));
+ if(isMac()){
+ app.$data.command = "./easyspider_executestage --id " + result.toString() + " --user_data " + (app.$data.with_user_data ? "1" : "0") + " --server_address " + app.$data.backEndAddressServiceWrapper;
+ $('#myModal').modal('show');
+ }
  });
  }
  },
  remoteExcuteInstant: function () {
  },
  }
  });
+
  $.get(app.$data.backEndAddressServiceWrapper + "/queryTask?id=" + sId, function (result) {
  app.$data.task = result;
  app.$data.show = true;
  });
+
  ws = new WebSocket("ws:https://localhost:"+getUrlParam("wsport"));
  ws.onopen = function () {
  // Web Socket 已连接上，使用 send() 方法发送数据

ExecuteStage/easyspider_executestage.py

@@ -693,7 +693,17 @@ def clean():
  option = webdriver.ChromeOptions()
  if not os.path.exists(os.getcwd()+"/Data"):
  os.mkdir(os.getcwd()+"/Data")
- if os.path.exists(os.getcwd()+"/EasySpider/resources"): # 打包后的路径
+ if sys.platform == "darwin" and platform.architecture()[0] == "64bit":
+ options.binary_location = "EasySpider.app/Contents/Resources/app/chrome_mac64.app/Contents/MacOS/Google Chrome"
+ # MacOS需要用option而不是options！
+ option.binary_location = "EasySpider.app/Contents/Resources/app/chrome_mac64.app/Contents/MacOS/Google Chrome"
+ driver_path = "EasySpider.app/Contents/Resources/app/chromedriver_mac64"
+ # options.binary_location = "chrome_mac64.app/Contents/MacOS/Google Chrome"
+ # # MacOS需要用option而不是options！
+ # option.binary_location = "chrome_mac64.app/Contents/MacOS/Google Chrome"
+ # driver_path = os.getcwd()+ "/chromedriver_mac64"
+ print(driver_path)
+ elif os.path.exists(os.getcwd()+"/EasySpider/resources"): # 打包后的路径
  print("Finding chromedriver in EasySpider",
  os.getcwd()+"/EasySpider")
  if sys.platform == "win32" and platform.architecture()[0] == "32bit":
@@ -709,11 +719,6 @@ def clean():
  elif sys.platform == "linux" and platform.architecture()[0] == "64bit":
  options.binary_location = "EasySpider/resources/app/chrome_linux64/chrome"
  driver_path = "EasySpider/resources/app/chrome_linux64/chromedriver_linux64"
- elif sys.platform == "darwin" and platform.architecture()[0] == "64bit":
- options.binary_location = "EasySpider/resources/app/chrome_mac64/chrome"
- # MacOS需要用option而不是options！
- option.binary_location = "EasySpider/resources/app/chrome_mac64/chrome"
- driver_path = "EasySpider/resources/app/chrome_mac64/chromedriver_mac64"
  else:
  print("Unsupported platform")
  sys.exit()

ExecuteStage/generateExecutable_Macos.sh

@@ -1,5 +1,7 @@
 rm -r build
 rm -r dist
 pyinstaller -F --icon=favicon.ico easyspider_executestage.py
-rm ../ElectronJS/chrome_mac64/easyspider_executestage
-mv dist/easyspider_executestage ../ElectronJS/chrome_linux64/easyspider_executestage
+rm ../Releases/EasySpider_MacOS_all_arch/easyspider_executestage
+rm ../ElectronJS/easyspider_executestage
+mv dist/easyspider_executestage ../Releases/EasySpider_MacOS_all_arch/easyspider_executestage
+# mv dist/easyspider_executestage ../ElectronJS/easyspider_executestage

Releases/.gitignore

@@ -1,2 +1,3 @@
 EasySpider
-*.tmp
+EasySpider.app/
+*.tmp

Releases/EasySpider_MacOS_all_arch/config.json

@@ -0,0 +1 @@
+{"webserver_address":"http:https://localhost","webserver_port":8074,"user_data_folder":"./user_data","absolute_user_data_folder":"/Users/naibowang/Documents/EasySpider/Releases/EasySpider_MacOS_all_arch/user_data"}