NaiboWang-Alienware
1 year ago
17 changed files with 1622 additions and 184 deletions
Split View
Diff Options
-
+1 -1.temp_to_pub/EasySpider_Linux_x64/软件使用说明.txt
-
+1 -1.temp_to_pub/EasySpider_MacOS_all_arch/软件使用说明.txt
-
+1 -1.temp_to_pub/EasySpider_windows_x64/软件使用说明.txt
-
+1 -1.temp_to_pub/EasySpider_windows_x86/软件使用说明.txt
-
+1089 -145ElectronJS/tasks/115.json
-
+1 -0ElectronJS/tasks/116.json
-
+1 -0ElectronJS/tasks/117.json
-
+379 -0ElectronJS/tasks/118.json
-
+1 -0ElectronJS/tasks/119.json
-
+1 -0ElectronJS/tasks/120.json
-
+1 -0ElectronJS/tasks/121.json
-
+1 -0ElectronJS/tasks/122.json
-
+1 -0ElectronJS/tasks/123.json
-
+1 -1ExecuteStage/.vscode/launch.json
-
+64 -33ExecuteStage/easyspider_executestage.py
-
+75 -0ExecuteStage/myChrome.py
-
+3 -1Readme.md
+ 1
- 1
.temp_to_pub/EasySpider_Linux_x64/软件使用说明.txt
View File
+ 1
- 1
.temp_to_pub/EasySpider_MacOS_all_arch/软件使用说明.txt
View File
+ 1
- 1
.temp_to_pub/EasySpider_windows_x64/软件使用说明.txt
View File
+ 1
- 1
.temp_to_pub/EasySpider_windows_x86/软件使用说明.txt
View File
+ 1089
- 145
ElectronJS/tasks/115.json
File diff suppressed because it is too large
View File
+ 1
- 0
ElectronJS/tasks/116.json
View File
| @ -0,0 +1 @@ | |||
| {"id":116,"name":"iP地址查询--手机号码查询归属地 | 邮政编码查询 | iP地址归属地查询 | 身份证号码验证在线查询网","url":"https://www.ip138.com","links":"https://www.ip138.com","create_time":"7/4/2023, 8:21:10 AM","version":"0.3.5","saveThreshold":10,"cloudflare":0,"environment":0,"containJudge":false,"desc":"https://www.ip138.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.ip138.com","desc":"要采集的网址列表，多行以\\n分开","type":"string","exampleValue":"https://www.ip138.com"}],"outputParameters":[{"id":0,"name":"参数1_文本","desc":"","type":"string","exampleValue":"502BadGateway"},{"id":1,"name":"参数2_文本","desc":"","type":"string","exampleValue":"nginx"}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2,3],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"https://www.ip138.com","links":"https://www.ip138.com","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1}},{"id":2,"index":2,"parentId":0,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/p[1]/a[1]","iframe":true,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/p[1]/a[1]","//a[contains(., '220.255.29')]","/html/body/p[last()-2]/a[last()-1]"]}},{"id":3,"index":3,"parentId":0,"type":1,"option":8,"title":"循环","sequence":[4],"isInLoop":false,"position":2,"parameters":{"history":3,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/center","iframe":true,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":1,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":["/html/body/center[1]","//center[contains(., '502 Bad Ga')]","/html/body/center[last()-1]"]}},{"id":4,"index":4,"parentId":3,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":3,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":true,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"paras":[{"nodeType":0,"contentType":1,"relative":true,"name":"参数1_文本","desc":"","relativeXPath":"/h1[1]","allXPaths":["/h1[1]","//h1[contains(., '502 Bad Ga')]","/html/body/center[last()-1]/h1"],"exampleValues":[{"num":0,"value":"502BadGateway"}],"unique_index":"/h1[1]","iframe":true,"default":"","beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":0,"contentType":1,"relative":true,"name":"参数2_文本","desc":"","relativeXPath":"","allXPaths":["","//center[contains(., 'nginx')]","/html/body/center"],"exampleValues":[{"num":1,"value":"nginx"}],"unique_index":"","iframe":true,"default":"","beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}],"loopType":1}}]} | |||
+ 1
- 0
ElectronJS/tasks/117.json
File diff suppressed because it is too large
View File
+ 379
- 0
ElectronJS/tasks/118.json
View File
| @ -0,0 +1,379 @@ | |||
| { | |||
| "id": 118, | |||
| "name": "iP地址查询--手机号码查询归属地 | 邮政编码查询 | iP地址归属地查询 | 身份证号码验证在线查询网", | |||
| "url": "https://www.ip138.com", | |||
| "links": "https://www.ip138.com", | |||
| "create_time": "7/4/2023, 8:43:31 AM", | |||
| "version": "0.3.5", | |||
| "saveThreshold": 10, | |||
| "cloudflare": 0, | |||
| "environment": 0, | |||
| "containJudge": false, | |||
| "desc": "https://www.ip138.com", | |||
| "inputParameters": [ | |||
| { | |||
| "id": 0, | |||
| "name": "urlList_0", | |||
| "nodeId": 1, | |||
| "nodeName": "打开网页", | |||
| "value": "https://www.ip138.com", | |||
| "desc": "要采集的网址列表，多行以\\n分开", | |||
| "type": "string", | |||
| "exampleValue": "https://www.ip138.com" | |||
| } | |||
| ], | |||
| "outputParameters": [ | |||
| { | |||
| "id": 0, | |||
| "name": "参数1_文本", | |||
| "desc": "", | |||
| "type": "string", | |||
| "exampleValue": "502 Bad Gateway" | |||
| } | |||
| ], | |||
| "graph": [ | |||
| { | |||
| "index": 0, | |||
| "id": 0, | |||
| "parentId": 0, | |||
| "type": -1, | |||
| "option": 0, | |||
| "title": "root", | |||
| "sequence": [1, 4], | |||
| "parameters": { | |||
| "history": 1, | |||
| "tabIndex": 0, | |||
| "useLoop": false, | |||
| "xpath": "", | |||
| "wait": 0 | |||
| }, | |||
| "isInLoop": false | |||
| }, | |||
| { | |||
| "id": 1, | |||
| "index": 1, | |||
| "parentId": 0, | |||
| "type": 0, | |||
| "option": 1, | |||
| "title": "打开网页", | |||
| "sequence": [], | |||
| "isInLoop": false, | |||
| "position": 0, | |||
| "parameters": { | |||
| "useLoop": false, | |||
| "xpath": "", | |||
| "wait": 0, | |||
| "waitType": 0, | |||
| "beforeJS": "", | |||
| "beforeJSWaitTime": 0, | |||
| "afterJS": "", | |||
| "afterJSWaitTime": 0, | |||
| "url": "https://www.ip138.com", | |||
| "links": "https://www.ip138.com", | |||
| "maxWaitTime": 10, | |||
| "scrollType": 0, | |||
| "scrollCount": 1, | |||
| "scrollWaitTime": 1 | |||
| } | |||
| }, | |||
| { | |||
| "id": -1, | |||
| "index": 2, | |||
| "parentId": 0, | |||
| "type": 1, | |||
| "option": 8, | |||
| "title": "循环", | |||
| "sequence": [3], | |||
| "isInLoop": false, | |||
| "position": 1, | |||
| "parameters": { | |||
| "history": 4, | |||
| "tabIndex": -1, | |||
| "useLoop": false, | |||
| "xpath": "/html/body/p", | |||
| "iframe": true, | |||
| "wait": 0, | |||
| "waitType": 0, | |||
| "beforeJS": "", | |||
| "beforeJSWaitTime": 0, | |||
| "afterJS": "", | |||
| "afterJSWaitTime": 0, | |||
| "scrollType": 0, | |||
| "scrollCount": 1, | |||
| "scrollWaitTime": 1, | |||
| "loopType": 1, | |||
| "pathList": "", | |||
| "textList": "", | |||
| "code": "", | |||
| "waitTime": 0, | |||
| "exitCount": 0, | |||
| "historyWait": 2, | |||
| "breakMode": 0, | |||
| "breakCode": "", | |||
| "breakCodeWaitTime": 0, | |||
| "allXPaths": [ | |||
| "/html/body/p[1]", | |||
| "//p[contains(., '您的iP地址是：[')]", | |||
| "/html/body/p[last()-2]" | |||
| ] | |||
| } | |||
| }, | |||
| { | |||
| "id": -1, | |||
| "index": 3, | |||
| "parentId": 2, | |||
| "type": 0, | |||
| "option": 3, | |||
| "title": "提取数据", | |||
| "sequence": [], | |||
| "isInLoop": true, | |||
| "position": 0, | |||
| "parameters": { | |||
| "history": 4, | |||
| "tabIndex": -1, | |||
| "useLoop": false, | |||
| "xpath": "", | |||
| "iframe": true, | |||
| "wait": 0, | |||
| "waitType": 0, | |||
| "beforeJS": "", | |||
| "beforeJSWaitTime": 0, | |||
| "afterJS": "", | |||
| "afterJSWaitTime": 0, | |||
| "paras": [ | |||
| { | |||
| "nodeType": 0, | |||
| "contentType": 1, | |||
| "relative": true, | |||
| "name": "参数1_文本", | |||
| "desc": "", | |||
| "relativeXPath": "", | |||
| "allXPaths": [ | |||
| "", | |||
| "//p[contains(., '您的iP地址是：[')]", | |||
| "/html/body/p[last()-2]" | |||
| ], | |||
| "exampleValues": [ | |||
| { "num": 0, "value": "您的iP地址是：[]来自：新加坡Singtel" } | |||
| ], | |||
| "unique_index": "", | |||
| "iframe": true, | |||
| "default": "", | |||
| "beforeJS": "", | |||
| "beforeJSWaitTime": 0, | |||
| "JS": "", | |||
| "JSWaitTime": 0, | |||
| "afterJS": "", | |||
| "afterJSWaitTime": 0, | |||
| "downloadPic": 0 | |||
| }, | |||
| { | |||
| "nodeType": 1, | |||
| "contentType": 0, | |||
| "relative": true, | |||
| "name": "参数2_链接文本", | |||
| "desc": "", | |||
| "relativeXPath": "/a[1]", | |||
| "allXPaths": [ | |||
| "/a[1]", | |||
| "//a[contains(., '220.255.29')]", | |||
| "/html/body/p[last()-2]/a[last()-1]" | |||
| ], | |||
| "exampleValues": [{ "num": 0, "value": "220.255.29.208" }], | |||
| "unique_index": "/a[1]", | |||
| "iframe": true, | |||
| "default": "", | |||
| "beforeJS": "", | |||
| "beforeJSWaitTime": 0, | |||
| "JS": "", | |||
| "JSWaitTime": 0, | |||
| "afterJS": "", | |||
| "afterJSWaitTime": 0, | |||
| "downloadPic": 0 | |||
| }, | |||
| { | |||
| "nodeType": 2, | |||
| "contentType": 0, | |||
| "relative": true, | |||
| "name": "参数3_链接地址", | |||
| "desc": "", | |||
| "relativeXPath": "/a[1]", | |||
| "allXPaths": [ | |||
| "/a[1]", | |||
| "//a[contains(., '220.255.29')]", | |||
| "/html/body/p[last()-2]/a[last()-1]" | |||
| ], | |||
| "exampleValues": [ | |||
| { | |||
| "num": 0, | |||
| "value": "https://www.ip138.com/iplookup.php?ip=220.255.29.208&action=2" | |||
| } | |||
| ], | |||
| "unique_index": "/a[1]", | |||
| "iframe": true, | |||
| "default": "", | |||
| "beforeJS": "", | |||
| "beforeJSWaitTime": 0, | |||
| "JS": "", | |||
| "JSWaitTime": 0, | |||
| "afterJS": "", | |||
| "afterJSWaitTime": 0, | |||
| "downloadPic": 0 | |||
| }, | |||
| { | |||
| "nodeType": 1, | |||
| "contentType": 0, | |||
| "relative": true, | |||
| "name": "参数4_链接文本", | |||
| "desc": "", | |||
| "relativeXPath": "/a[2]", | |||
| "allXPaths": [ | |||
| "/a[2]", | |||
| "//a[contains(., '')]", | |||
| "/html/body/p[last()-2]/a" | |||
| ], | |||
| "exampleValues": [{ "num": 0, "value": "" }], | |||
| "unique_index": "/a[2]", | |||
| "iframe": true, | |||
| "default": "", | |||
| "beforeJS": "", | |||
| "beforeJSWaitTime": 0, | |||
| "JS": "", | |||
| "JSWaitTime": 0, | |||
| "afterJS": "", | |||
| "afterJSWaitTime": 0, | |||
| "downloadPic": 0 | |||
| }, | |||
| { | |||
| "nodeType": 2, | |||
| "contentType": 0, | |||
| "relative": true, | |||
| "name": "参数5_链接地址", | |||
| "desc": "", | |||
| "relativeXPath": "/a[2]", | |||
| "allXPaths": [ | |||
| "/a[2]", | |||
| "//a[contains(., '')]", | |||
| "/html/body/p[last()-2]/a" | |||
| ], | |||
| "exampleValues": [ | |||
| { "num": 0, "value": "https://www.ipshudi.com/" } | |||
| ], | |||
| "unique_index": "/a[2]", | |||
| "iframe": true, | |||
| "default": "", | |||
| "beforeJS": "", | |||
| "beforeJSWaitTime": 0, | |||
| "JS": "", | |||
| "JSWaitTime": 0, | |||
| "afterJS": "", | |||
| "afterJSWaitTime": 0, | |||
| "downloadPic": 0 | |||
| }, | |||
| { | |||
| "nodeType": 4, | |||
| "contentType": 1, | |||
| "relative": true, | |||
| "name": "参数6_图片地址", | |||
| "desc": "", | |||
| "relativeXPath": "/a[2]/img[1]", | |||
| "allXPaths": [ | |||
| "/a[2]/img[1]", | |||
| "//img[contains(., '')]", | |||
| "/html/body/p[last()-2]/a/img" | |||
| ], | |||
| "exampleValues": [ | |||
| { "num": 0, "value": "https://6.ipchaxun.net/220.255.29.208.gif" } | |||
| ], | |||
| "unique_index": "/a[2]/img[1]", | |||
| "iframe": true, | |||
| "default": "", | |||
| "beforeJS": "", | |||
| "beforeJSWaitTime": 0, | |||
| "JS": "", | |||
| "JSWaitTime": 0, | |||
| "afterJS": "", | |||
| "afterJSWaitTime": 0, | |||
| "downloadPic": 0 | |||
| }, | |||
| { | |||
| "nodeType": 0, | |||
| "contentType": 1, | |||
| "relative": true, | |||
| "name": "参数7_文本", | |||
| "desc": "", | |||
| "relativeXPath": "/a[1]/font[1]", | |||
| "allXPaths": [ | |||
| "/a[1]/font[1]", | |||
| "//font[contains(., 'ip查询api接口')]", | |||
| "/html/body/p[last()-1]/a/font" | |||
| ], | |||
| "exampleValues": [{ "num": 1, "value": "ip查询api接口" }], | |||
| "unique_index": "/a[1]/font[1]", | |||
| "iframe": true, | |||
| "default": "", | |||
| "beforeJS": "", | |||
| "beforeJSWaitTime": 0, | |||
| "JS": "", | |||
| "JSWaitTime": 0, | |||
| "afterJS": "", | |||
| "afterJSWaitTime": 0, | |||
| "downloadPic": 0 | |||
| } | |||
| ], | |||
| "loopType": 1 | |||
| } | |||
| }, | |||
| { | |||
| "id": 2, | |||
| "index": 4, | |||
| "parentId": 0, | |||
| "type": 0, | |||
| "option": 3, | |||
| "title": "提取数据", | |||
| "sequence": [], | |||
| "isInLoop": false, | |||
| "position": 1, | |||
| "parameters": { | |||
| "history": 3, | |||
| "tabIndex": -1, | |||
| "useLoop": false, | |||
| "xpath": "", | |||
| "iframe": true, | |||
| "wait": 0, | |||
| "waitType": 0, | |||
| "beforeJS": "", | |||
| "beforeJSWaitTime": 0, | |||
| "afterJS": "", | |||
| "afterJSWaitTime": 0, | |||
| "paras": [ | |||
| { | |||
| "nodeType": 0, | |||
| "contentType": 0, | |||
| "relative": false, | |||
| "name": "参数1_文本", | |||
| "desc": "", | |||
| "extractType": 0, | |||
| "relativeXPath": "/html/body/center[1]/h1[1]", | |||
| "allXPaths": [ | |||
| "/html/body/center[1]/h1[1]", | |||
| "//h1[contains(., '502 Bad Ga')]", | |||
| "/html/body/center[last()-1]/h1" | |||
| ], | |||
| "exampleValues": [{ "num": 0, "value": "502 Bad Gateway" }], | |||
| "unique_index": "l37gwwpsg29ljnkgn7r", | |||
| "iframe": true, | |||
| "default": "", | |||
| "beforeJS": "", | |||
| "beforeJSWaitTime": 0, | |||
| "JS": "", | |||
| "JSWaitTime": 0, | |||
| "afterJS": "", | |||
| "afterJSWaitTime": 0, | |||
| "downloadPic": 0 | |||
| } | |||
| ] | |||
| } | |||
| } | |||
| ] | |||
| } | |||
+ 1
- 0
ElectronJS/tasks/119.json
View File
| @ -0,0 +1 @@ | |||
| {"id":119,"name":"","url":"https://lihkg.com/","links":"https://lihkg.com/thread/3433502/page/1","create_time":"7/4/2023, 5:09:13 PM","version":"0.3.5","saveThreshold":10,"cloudflare":0,"environment":0,"containJudge":false,"desc":"https://lihkg.com/","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://lihkg.com/thread/3433502/page/1","desc":"要采集的网址列表，多行以\\n分开","type":"string","exampleValue":"https://lihkg.com/thread/3433502/page/1"}],"outputParameters":[],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,6],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"https://lihkg.com/","links":"https://lihkg.com/thread/3433502/page/1","maxWaitTime":10,"scrollType":"2","scrollCount":3,"scrollWaitTime":1}},{"id":-1,"index":2,"parentId":0,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":3,"tabIndex":-1,"useLoop":false,"xpath":"//*[contains(@class, \"P3e8vKaXmUeXC9dJgjnsu\")]/div[1]","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/div[1]/div[2]/div[2]/div[1]/div[1]","//div[contains(., 'LIHKG 討論區使')]","/html/body/div[last()-4]/div[last()-2]/div/div/div"]}},{"id":-1,"index":3,"parentId":0,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":3,"tabIndex":-1,"useLoop":false,"xpath":"//*[contains(@class, \"P3e8vKaXmUeXC9dJgjnsu\")]","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/div[1]/div[2]/div[2]/div[1]","//div[contains(., 'LIHKG 討論區使')]","//DIV[@class='P3e8vKaXmUeXC9dJgjnsu']","/html/body/div[last()-4]/div[last()-2]/div/div"]}},{"id":-1,"index":4,"parentId":0,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":3,"tabIndex":-1,"useLoop":false,"xpath":"//*[contains(@class, \"_1PdImYJBCsN8lH0MB4tnqV\")]/a[1]","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":"2","scrollCount":5,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/div[1]/div[2]/div[1]/div[1]/ul[1]/li[1]/a[1]","//a[contains(., '最新')]","/html/body/div[last()-4]/div[last()-2]/div[last()-1]/div[last()-1]/ul/li[last()-1]/a"]}},{"id":-1,"index":5,"parentId":0,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":3,"tabIndex":-1,"useLoop":false,"xpath":"//*[contains(@class, \"P3e8vKaXmUeXC9dJgjnsu\")]","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/div[1]/div[2]/div[2]/div[1]","//div[contains(., 'LIHKG 討論區使')]","//DIV[@class='P3e8vKaXmUeXC9dJgjnsu']","/html/body/div[last()-4]/div[last()-2]/div/div"]}},{"id":2,"index":6,"parentId":0,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":5,"tabIndex":-1,"useLoop":false,"xpath":"//*[@id=\"1\"]/div[1]/small[1]","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":"2","scrollCount":4,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/div[1]/div[2]/div[2]/div[1]/div[2]/div[2]/div[2]/div[1]/div[1]/small[1]","//small[contains(., '#1李芯悅•1 小時')]","//SMALL[@class='_1VcuFUmnOEK51TsshmrnJM']","/html/body/div[last()-5]/div[last()-2]/div/div/div[last()-2]/div/div[last()-24]/div[last()-1]/div/small"]}}]} | |||
+ 1
- 0
ElectronJS/tasks/120.json
View File
| @ -0,0 +1 @@ | |||
| {"id":120,"name":"京东全球版-专业的综合网上购物商城","url":"https://www.jd.com","links":"https://www.jd.com","create_time":"7/4/2023, 5:36:05 PM","version":"0.3.5","saveThreshold":10,"cloudflare":0,"environment":0,"containJudge":false,"desc":"https://www.jd.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.jd.com","desc":"要采集的网址列表，多行以\\n分开","type":"string","exampleValue":"https://www.jd.com"},{"id":1,"name":"inputText_1","nodeName":"输入文字","nodeId":3,"desc":"要输入的文本，如京东搜索框输入：电脑","type":"string","exampleValue":"123Field[\"自定义操作\"]456","value":"123Field[\"自定义操作\"]456"}],"outputParameters":[{"id":0,"name":"自定义操作","desc":"自定义操作返回的数据","type":"string","exampleValue":""}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2,3],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"https://www.jd.com","links":"https://www.jd.com","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1}},{"id":2,"index":2,"parentId":0,"type":0,"option":5,"title":"自定义操作","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"codeMode":"1","code":"python D:/test.py","waitTime":0,"recordASField":"1"}},{"id":3,"index":3,"parentId":0,"type":0,"option":4,"title":"输入文字","sequence":[],"isInLoop":false,"position":2,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"//*[@id=\"key\"]","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"value":"123Field[\"自定义操作\"]456","allXPaths":["/html/body/div[4]/div[1]/div[2]/div[1]/input[1]","//input[contains(., '')]","id(\"key\")","//INPUT[@class='text defcolor']","/html/body/div[last()-6]/div/div[last()-2]/div/input"]}}]} | |||
+ 1
- 0
ElectronJS/tasks/121.json
View File
| @ -0,0 +1 @@ | |||
| {"id":121,"name":"京东全球版-专业的综合网上购物商城","url":"https://www.jd.com","links":"https://www.jd.com","create_time":"7/4/2023, 5:55:11 PM","version":"0.3.5","saveThreshold":10,"cloudflare":0,"environment":0,"containJudge":false,"desc":"https://www.jd.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.jd.com","desc":"要采集的网址列表，多行以\\n分开","type":"string","exampleValue":"https://www.jd.com"}],"outputParameters":[],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":15,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"https://www.jd.com","links":"https://www.jd.com","maxWaitTime":10,"scrollType":"2","scrollCount":3,"scrollWaitTime":1}},{"id":-1,"index":2,"parentId":0,"type":1,"option":8,"title":"循环","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":0,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0}},{"id":-1,"index":3,"parentId":0,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[]}}]} | |||
+ 1
- 0
ElectronJS/tasks/122.json
View File
| @ -0,0 +1 @@ | |||
| {"id":122,"name":"bbs","url":"https://lihkg.com/thread/3429557/page/1","links":"https://lihkg.com/thread/3429557/page/1","create_time":"7/4/2023, 5:57:56 PM","version":"0.3.5","saveThreshold":10,"cloudflare":0,"environment":0,"containJudge":false,"desc":"https://lihkg.com/thread/3429557/page/1","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://lihkg.com/thread/3429557/page/1","desc":"要采集的网址列表，多行以\\n分开","type":"string","exampleValue":"https://lihkg.com/thread/3429557/page/1"}],"outputParameters":[],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"https://lihkg.com/thread/3429557/page/1","links":"https://lihkg.com/thread/3429557/page/1","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1}},{"id":2,"index":2,"parentId":0,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"//*[contains(@class, \"_1PdImYJBCsN8lH0MB4tnqV\")]/a[1]","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":"2","scrollCount":3,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/div[1]/div[2]/div[1]/div[1]/ul[1]/li[1]/a[1]","//a[contains(., '最新')]","/html/body/div[last()-5]/div[last()-2]/div[last()-1]/div[last()-1]/ul/li[last()-1]/a"]}}]} | |||
+ 1
- 0
ElectronJS/tasks/123.json
File diff suppressed because it is too large
View File
+ 1
- 1
ExecuteStage/.vscode/launch.json
View File
+ 64
- 33
ExecuteStage/easyspider_executestage.py
View File
+ 75
- 0
ExecuteStage/myChrome.py
View File
| @ -0,0 +1,75 @@ | |||
| from selenium.webdriver.chrome.options import Options | |||
| from selenium.webdriver.common.keys import Keys | |||
| from selenium.webdriver.common.action_chains import ActionChains | |||
| from selenium import webdriver | |||
| from selenium.webdriver.support.ui import WebDriverWait | |||
| from selenium.webdriver.support import expected_conditions as EC | |||
| from selenium.webdriver.common.by import By | |||
| from selenium.common.exceptions import NoSuchElementException | |||
| from selenium.common.exceptions import TimeoutException | |||
| from selenium.common.exceptions import StaleElementReferenceException, InvalidSelectorException | |||
| from selenium.webdriver.common.desired_capabilities import DesiredCapabilities | |||
| from selenium.webdriver.support.ui import Select | |||
| from selenium.webdriver import ActionChains | |||
| from selenium.webdriver.common.by import By | |||
| desired_capabilities = DesiredCapabilities.CHROME | |||
| desired_capabilities["pageLoadStrategy"] = "none" | |||
| class MyChrome(webdriver.Chrome): | |||
| def find_element(self, by=By.ID, value=None, iframe=False): | |||
| # 在这里改变查找元素的行为 | |||
| if iframe: | |||
| # 获取所有的 iframe | |||
| try: | |||
| iframes = super().find_elements(By.XPATH, "//iframe") | |||
| except Exception as e: | |||
| print(e) | |||
| find_element = False | |||
| # 遍历所有的 iframe 并点击里面的元素 | |||
| for iframe in iframes: | |||
| # 切换到 iframe | |||
| super().switch_to.frame(iframe) | |||
| try: | |||
| # 在 iframe 中查找并点击元素 | |||
| # 在这个例子中，我们查找 XPath 为 '//div[1]' 的元素 | |||
| element = super().find_element(by=by, value=value) | |||
| find_element = True | |||
| except NoSuchElementException: | |||
| print("No such element found in the iframe") | |||
| # 完成操作后切回主文档 | |||
| # super().switch_to.default_content() | |||
| if find_element: | |||
| return element | |||
| if not find_element: | |||
| raise NoSuchElementException | |||
| else: | |||
| return super().find_element(by=by, value=value) | |||
| def find_elements(self, by=By.ID, value=None, iframe=False): | |||
| # 在这里改变查找元素的行为 | |||
| if iframe: | |||
| # 获取所有的 iframe | |||
| iframes = iframes = super().find_elements(By.CSS_SELECTOR, "iframe") | |||
| find_element = False | |||
| # 遍历所有的 iframe 并点击里面的元素 | |||
| for iframe in iframes: | |||
| # 切换到 iframe | |||
| super().switch_to.frame(iframe) | |||
| try: | |||
| # 在 iframe 中查找并点击元素 | |||
| # 在这个例子中，我们查找 XPath 为 '//div[1]' 的元素 | |||
| elements = super().find_elements(by=by, value=value) | |||
| find_element = True | |||
| except NoSuchElementException: | |||
| print("No such element found in the iframe") | |||
| # 完成操作后切回主文档 | |||
| # super().switch_to.default_content() | |||
| if find_element: | |||
| return elements | |||
| if not find_element: | |||
| raise NoSuchElementException | |||
| else: | |||
| return super().find_elements(by=by, value=value) | |||