diff --git a/ElectronJS/.gitignore b/ElectronJS/.gitignore index 2e1be61..706816f 100644 --- a/ElectronJS/.gitignore +++ b/ElectronJS/.gitignore @@ -8,4 +8,5 @@ chromedriver_mac64 user-data/ user_data/ Data/ +Chrome/ execution_instances/* diff --git a/ElectronJS/EasySpider_en.crx b/ElectronJS/EasySpider_en.crx index 83fbf8f..1a91478 100644 Binary files a/ElectronJS/EasySpider_en.crx and b/ElectronJS/EasySpider_en.crx differ diff --git a/ElectronJS/EasySpider_zh.crx b/ElectronJS/EasySpider_zh.crx index 553def8..4f888b6 100644 Binary files a/ElectronJS/EasySpider_zh.crx and b/ElectronJS/EasySpider_zh.crx differ diff --git a/ElectronJS/XPathHelper.crx b/ElectronJS/XPathHelper.crx new file mode 100644 index 0000000..d24c7a2 Binary files /dev/null and b/ElectronJS/XPathHelper.crx differ diff --git a/ElectronJS/check_update.js b/ElectronJS/check_update.js new file mode 100644 index 0000000..a04b26a --- /dev/null +++ b/ElectronJS/check_update.js @@ -0,0 +1,34 @@ +const https = require('https'); +const fs = require("fs"); +const path = require("path"); + +// 设置GitHub用户名和仓库名 +const owner = 'NaiboWang'; +const repo = 'EasySpider'; +let config = fs.readFileSync(path.join(__dirname, `package.json`), 'utf8'); +config = JSON.parse(config); +const version = config.version; +console.log(`Current version is ${version}`); + +// 发送GET请求获取GitHub的Release API响应 +https.get(`https://api.github.com/repos/${owner}/${repo}/releases/latest`, { + headers: { + 'User-Agent': 'Node.js' + } +}, (res) => { + let data = ''; + res.on('data', (chunk) => { + data += chunk; + }); + res.on('end', () => { + // 解析响应JSON并输出最新版本号 + const release = JSON.parse(data); + const latestVersion = release.tag_name.replace('v', ''); + console.log(`Latest version is ${latestVersion}`); + if(version !== latestVersion) { + console.log('There is a new version of EasySpider, you can download it from github repo: https://github.com/NaiboWang/EasySpider/releases'); + } + }); +}).on('error', (err) => { + console.error(`Error: ${err.message}`); +}); \ No newline at end of file diff --git a/ElectronJS/config.json b/ElectronJS/config.json index 78a2110..d7f6cab 100644 --- a/ElectronJS/config.json +++ b/ElectronJS/config.json @@ -1 +1 @@ -{"webserver_address":"http://localhost","webserver_port":8074,"user_data_folder":"./user_data","absolute_user_data_folder":"/Users/naibowang/Documents/EasySpider/ElectronJS/user_data"} \ No newline at end of file +{"webserver_address":"http://localhost","webserver_port":8074,"user_data_folder":"./user_data","absolute_user_data_folder":"D:\\Documents\\Projects\\EasySpider\\ElectronJS\\user_data"} \ No newline at end of file diff --git a/ElectronJS/main.js b/ElectronJS/main.js index 7507271..bd58c26 100644 --- a/ElectronJS/main.js +++ b/ElectronJS/main.js @@ -1,7 +1,6 @@ // Modules to control application life and create native browser window const {app, BrowserWindow, dialog, ipcMain, screen} = require('electron'); app.commandLine.appendSwitch("--disable-http-cache"); - const {Builder, By, Key, until} = require("selenium-webdriver"); const chrome = require('selenium-webdriver/chrome'); const {ServiceBuilder} = require('selenium-webdriver/chrome'); @@ -123,6 +122,7 @@ async function beginInvoke(msg, ws) { } flowchart_window.show(); + // flowchart_window.openDevTools(); } else if (msg.type == 2) { //keyboard // const robot = require("@jitsi/robotjs"); @@ -215,13 +215,13 @@ async function beginInvoke(msg, ws) { // }); let spawn = require("child_process").spawn; - if (process.platform != "darwin") { + if (process.platform != "darwin" && msg.message.execute_type == 1) { let child_process = spawn(execute_path, parameters); child_process.stdout.on('data', function (data) { console.log(data.toString()); }); } else { - ws.send(task_server.getDir() + "/"); + ws.send(JSON.stringify({"config_folder": task_server.getDir() + "/", "easyspider_location": task_server.getEasySpiderLocation()})); } } } @@ -284,6 +284,7 @@ async function runBrowser(lang = "en", user_data_folder = '') { } else if (lang == "zh") { options.addExtensions(path.join(__dirname, "EasySpider_zh.crx")); } + options.addExtensions(path.join(__dirname, "XPathHelper.crx")); options.setChromeBinaryPath(chromeBinaryPath); if (user_data_folder != "") { let dir = path.join(task_server.getDir(), user_data_folder); diff --git a/ElectronJS/package.json b/ElectronJS/package.json index 3cedaed..b4acc20 100644 --- a/ElectronJS/package.json +++ b/ElectronJS/package.json @@ -1,7 +1,7 @@ { "name": "easy-spider", "productName": "EasySpider", - "version": "0.2.0", + "version": "0.3.0", "icon": "./favicon", "description": "NoCode Visual Web Crawler", "main": "main.js", @@ -60,7 +60,7 @@ ], "packagerConfig": { "icon": "./favicon", - "appVersion": "0.2.0", + "appVersion": "0.3.0", "name": "EasySpider", "executableName": "EasySpider", "appCopyright": "Naibo Wang (naibowang@foxmail.com)", diff --git a/ElectronJS/server.js b/ElectronJS/server.js index 9a6719d..fe0e681 100644 --- a/ElectronJS/server.js +++ b/ElectronJS/server.js @@ -29,7 +29,18 @@ function getDir(){ } else { return path.join(__dirname,"../../.."); } - } else{ + } else { + return __dirname; + } +} +function getEasySpiderLocation(){ + if(__dirname.indexOf("app") >= 0 && __dirname.indexOf("sources") >= 0){ + if(process.platform == "darwin"){ + return path.join(__dirname,"../../../"); + } else { + return path.join(__dirname,"../../../"); + } + } else { return __dirname; } } @@ -44,6 +55,7 @@ if(!fs.existsSync(path.join(getDir(), "config.json"))){ } exports.getDir = getDir; +exports.getEasySpiderLocation = getEasySpiderLocation; FileMimes = JSON.parse(fs.readFileSync(path.join(__dirname,'mime.json')).toString()); exports.start = function(port = 8074) { http.createServer(function(req, res) { diff --git a/ElectronJS/src/img/XPather_helper.png b/ElectronJS/src/img/XPather_helper.png new file mode 100644 index 0000000..002ee82 Binary files /dev/null and b/ElectronJS/src/img/XPather_helper.png differ diff --git a/ElectronJS/src/index.html b/ElectronJS/src/index.html index f6b56d0..89b48e8 100644 --- a/ElectronJS/src/index.html +++ b/ElectronJS/src/index.html @@ -24,6 +24,10 @@

English

+

Github最新版本/Newest Version：{{newest_version}}

+ + +
@@ -144,15 +148,33 @@ return ""; } - createApp({ + var global = createApp({ data() { return { init: true, lang: 'zh', user_data_folder: getUrlParam("user_data_folder"), step: 0, + newest_version: '-', // 最新版本号 } }, + mounted() { + // 发送GET请求获取GitHub的Release API响应 + const request = new XMLHttpRequest(); + request.open('GET', `https://api.github.com/repos/NaiboWang/EasySpider/releases/latest`); + request.setRequestHeader('User-Agent', 'JavaScript'); + request.onload = function() { + // 解析响应JSON并输出最新版本号 + const release = JSON.parse(request.responseText); + const latestVersion = release.tag_name; + global.$data.newest_version = latestVersion; + // alert(`Latest version is ${latestVersion}`); + }; + request.onerror = function() { + console.error('Error: failed to get latest version.'); + }; + request.send(); + }, methods: { changeLang(lang = 'zh') { this.init = false; diff --git a/ElectronJS/src/taskGrid/FlowChart.html b/ElectronJS/src/taskGrid/FlowChart.html index bdb4bf7..3953432 100644 --- a/ElectronJS/src/taskGrid/FlowChart.html +++ b/ElectronJS/src/taskGrid/FlowChart.html @@ -139,7 +139,7 @@

Use relative XPATH

- +
-->
+ +
- +
@@ -75,9 +98,9 @@

使用循环内的链接

- - - + + +
@@ -98,7 +121,9 @@
+

+ +

+ +

+
+
+ + + + + + + + +
+
+ + @@ -135,17 +180,50 @@
- -

使用相对循环内的XPATH

- - +

当前编辑字段参数名（点击字段的“修改”选项切换参数）:

+ +

使用相对循环内的XPATH

+

XPATH: ☺

+ +

+

+ +

+
+
+ + + + + + + + +
+
+
+ + + + +
+ + + + + - + +
@@ -166,7 +250,7 @@
-

使用循环内的文本

+

使用循环内的文本（不勾选则每次输入的文本为下方“输入值”文本框内的文本，勾选后会使用所在循环内设置的文本）

@@ -175,9 +259,45 @@ +

+

+ +

+
+
+ + + + + + + + +
+
+ + + +
+ + +

将执行后的输出/返回值作为字段记录

+ + +
+ + + +
@@ -193,26 +313,35 @@
+

-
+
- +
-
+
- + +
+
+ + + +
- + @@ -239,9 +368,26 @@ + + + - - +
+ + +
+
+ + + + +
+
+ + + + +
@@ -280,6 +426,8 @@
+ + @@ -309,9 +457,10 @@ url = "taskInfo.html?id="+getUrlParam("id")+"&lang=en&type="+getUrlParam("type")+"&wsport="+getUrlParam("wsport")+"&backEndAddressServiceWrapper=" + this.backEndAddressServiceWrapper } window.location.href = url; - }, + } } }); + \ No newline at end of file diff --git a/ElectronJS/src/taskGrid/FlowChart_CN.js b/ElectronJS/src/taskGrid/FlowChart_CN.js index dae3dd8..ec33d00 100644 --- a/ElectronJS/src/taskGrid/FlowChart_CN.js +++ b/ElectronJS/src/taskGrid/FlowChart_CN.js @@ -40,6 +40,7 @@ var app = new Vue({ paras: { "parameters": [] }, //提取数据的参数列表 TClass: -1, //条件分支的条件类别 paraIndex: 0, //当前参数的index + XPaths: "", //xpath列表 }, watch: { nowArrow: { //变量发生变化的时候进行一些操作 @@ -86,6 +87,13 @@ var app = new Vue({ }, }, methods: { + changeXPaths: function (XPaths){ + let result = ""; + for (var i = 0; i < XPaths.length; i++) { + result += XPaths[i] + "\n"; + } + this.XPaths = result; + }, modifyParas: function(i) { //修改第i个参数 this.paraIndex = i; }, diff --git a/ElectronJS/src/taskGrid/global.js b/ElectronJS/src/taskGrid/global.js index fa8aeca..4156019 100644 --- a/ElectronJS/src/taskGrid/global.js +++ b/ElectronJS/src/taskGrid/global.js @@ -1,13 +1,35 @@ function getUrlParam(name) { - var reg = new RegExp("(^|&)" + name + "=([^&]*)(&|$)"); //构造一个含有目标参数的正则表达式对象 - var r = window.location.search.substr(1).match(reg); //匹配目标参数 + let reg = new RegExp("(^|&)" + name + "=([^&]*)(&|$)"); //构造一个含有目标参数的正则表达式对象 + let r = window.location.search.substr(1).match(reg); //匹配目标参数 if (r != null) return unescape(r[2]); return ""; //返回参数值,默认后台地址 } -function isMac() { - return /macintosh|mac os x/i.test(navigator.userAgent); -}; +function getOperatingSystemInfo() { + let platform = navigator.platform; + let osVersion = ""; + let osBit = ""; + let agent = navigator.userAgent.toLowerCase(); + if (platform.startsWith("Win")) { + osVersion = "win"; + if (agent.indexOf("win32") >= 0 || agent.indexOf("wow32") >= 0) { + osBit = 32; + } + if (agent.indexOf("win64") >= 0 || agent.indexOf("wow64") >= 0) { + osBit = 64; + } + } else if (platform.startsWith("Mac")) { + osVersion = "macOS"; + osBit = 64; + } else if (platform.startsWith("Linux")) { + osVersion = "linux"; + osBit = 64; + } + return { + version: osVersion, + bit: osBit + }; +} Vue.filter('lang', function (value) { if (getUrlParam("lang") == "zh") { diff --git a/ElectronJS/src/taskGrid/invokeTask.html b/ElectronJS/src/taskGrid/invokeTask.html index 678cb50..c9e1ced 100644 --- a/ElectronJS/src/taskGrid/invokeTask.html +++ b/ElectronJS/src/taskGrid/invokeTask.html @@ -40,7 +40,7 @@