From 37d4aa337fb79b2136855ccc6fd4194a00f5d656 Mon Sep 17 00:00:00 2001 From: naibo Date: Fri, 14 Jul 2023 07:27:55 +0800 Subject: [PATCH] Bug fix --- ElectronJS/tasks/127.json | 2 +- ElectronJS/tasks/177.json | 2 +- ExecuteStage/.vscode/launch.json | 2 +- ExecuteStage/easyspider_executestage.py | 12 ++++++++++++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/ElectronJS/tasks/127.json b/ElectronJS/tasks/127.json index 68dfa66..cc88029 100644 --- a/ElectronJS/tasks/127.json +++ b/ElectronJS/tasks/127.json @@ -1 +1 @@ -{"id":127,"name":"Dynamic Iframe","url":"http://localhost:8074/taskGrid/test_pages/iframe.html?address=http://www.ceic.ac.cn/history","links":"http://localhost:8074/taskGrid/test_pages/iframe.html?address=http://www.ceic.ac.cn/history","create_time":"7/5/2023, 4:45:57 AM","version":"0.3.5","saveThreshold":10,"cloudflare":0,"environment":0,"containJudge":false,"desc":"http://localhost:8074/taskGrid/test_pages/iframe.html?address=http://www.ceic.ac.cn/history","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"http://localhost:8074/taskGrid/test_pages/iframe.html?address=http://www.ceic.ac.cn/history","desc":"要采集的网址列表，多行以\\n分开","type":"string","exampleValue":"http://localhost:8074/taskGrid/test_pages/iframe.html?address=http://www.ceic.ac.cn/history"},{"id":1,"name":"loopText_1","nodeId":2,"nodeName":"循环","desc":"要输入的文本/网址,多行以\\n分开","type":"string","exampleValue":"1\n2\n","value":"1\n2\n"}],"outputParameters":[],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"http://localhost:8074/taskGrid/test_pages/iframe.html?address=http://www.ceic.ac.cn/history","links":"http://localhost:8074/taskGrid/test_pages/iframe.html?address=http://www.ceic.ac.cn/history","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1}},{"id":2,"index":2,"parentId":0,"type":1,"option":8,"title":"循环","sequence":[3,4],"isInLoop":false,"position":1,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":true,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":"3","pathList":"","textList":"1\n2\n","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0}},{"id":3,"index":3,"parentId":2,"type":0,"option":4,"title":"输入文字","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":5,"tabIndex":-1,"useLoop":true,"xpath":"//*[@id=\"weidu1\"]","iframe":true,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"value":"123","allXPaths":["/html/body/div[1]/div[3]/div[1]/div[1]/div[1]/form[1]/div[2]/input[1]","//input[contains(., '')]","id(\"weidu1\")","//INPUT[@class='span1']","//INPUT[@name='weidu1']","/html/body/div[last()-3]/div[last()-1]/div/div/div[last()-1]/form/div[last()-3]/input[last()-1]"]}},{"id":4,"index":4,"parentId":2,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":true,"position":1,"parameters":{"history":5,"tabIndex":-1,"useLoop":false,"xpath":"//*[@id=\"search\"]","iframe":true,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/div[1]/div[3]/div[1]/div[1]/div[1]/form[1]/div[5]/a[1]","//a[contains(., '查询')]","id(\"search\")","//A[@class='check']","/html/body/div[last()-3]/div[last()-1]/div/div/div[last()-1]/form/div/a"]}}]} \ No newline at end of file +{"id":127,"name":"百度地图","url":"https://map.baidu.com","links":"https://map.baidu.com","create_time":"2023/7/12 20:59:16","update_time":"7/14/2023, 7:25:42 AM","version":"0.3.6","saveThreshold":10,"cloudflare":0,"environment":0,"maxViewLength":15,"outputFormat":"xlsx","saveName":"current_time","inputExcel":"","startFromExit":0,"containJudge":false,"desc":"https://map.baidu.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://map.baidu.com","desc":"要采集的网址列表，多行以\\n分开","type":"text","exampleValue":"https://map.baidu.com"},{"id":1,"name":"inputText_1","nodeName":"输入文字","nodeId":2,"desc":"要输入的文本，如京东搜索框输入：电脑","type":"text","exampleValue":"北京市","value":"北京市"},{"id":2,"name":"inputText_2","nodeName":"输入文字","nodeId":4,"desc":"要输入的文本，如京东搜索框输入：电脑","type":"text","exampleValue":"门窗","value":"门窗"},{"id":3,"name":"loopTimes_循环点击下一页_3","nodeId":7,"nodeName":"循环点击下一页","desc":"循环循环点击下一页执行的次数（0代表无限循环）","type":"int","exampleValue":0,"value":0}],"outputParameters":[{"id":0,"name":"参数8_链接文本","desc":"","type":"text","recordASField":1,"exampleValue":"鑫隆门窗"},{"id":1,"name":"参数11_文本","desc":"","type":"text","recordASField":1,"exampleValue":"电话:13717842988"},{"id":2,"name":"参数10_文本","desc":"","type":"text","recordASField":1,"exampleValue":"北京市朝阳区红霞路13号"}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2,3,4,5,6,7],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0,"waitType":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"https://map.baidu.com","links":"https://map.baidu.com","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}},{"id":2,"index":2,"parentId":0,"type":0,"option":4,"title":"输入文字","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"//*[@id=\"sole-input\"]","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"value":"北京市","allXPaths":["/html/body/div[1]/div[2]/div[1]/div[1]/div[1]/input[1]","//input[contains(., '')]","id(\"sole-input\")","//INPUT[@class='searchbox-content-common']","//INPUT[@name='word']","/html/body/div[last()-5]/div[last()-5]/div/div[last()-1]/div/input"]}},{"id":3,"index":3,"parentId":0,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":false,"position":2,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"//*[@id=\"search-button\"]","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/div[1]/div[2]/div[1]/button[1]","//button[contains(., '')]","id(\"search-button\")","/html/body/div[last()-7]/div[last()-5]/div/button"]}},{"id":4,"index":4,"parentId":0,"type":0,"option":4,"title":"输入文字","sequence":[],"isInLoop":false,"position":3,"parameters":{"history":5,"tabIndex":-1,"useLoop":false,"xpath":"//*[@id=\"sole-input\"]","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"value":"门窗","allXPaths":["/html/body/div[1]/div[2]/div[1]/div[1]/div[1]/input[1]","//input[contains(., '')]","id(\"sole-input\")","//INPUT[@class='searchbox-content-common']","//INPUT[@name='word']","/html/body/div[last()-7]/div[last()-5]/div/div[last()-1]/div/input"]}},{"id":5,"index":5,"parentId":0,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":false,"position":4,"parameters":{"history":5,"tabIndex":-1,"useLoop":false,"xpath":"//*[@id=\"search-button\"]","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/div[1]/div[2]/div[1]/button[1]","//button[contains(., '')]","id(\"search-button\")","/html/body/div[last()-8]/div[last()-5]/div/button"]}},{"id":6,"index":6,"parentId":0,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":false,"position":5,"parameters":{"history":6,"tabIndex":-1,"useLoop":false,"xpath":"//*[@id=\"leadDownloadCard\"]/div[1]/div[1]/div[3]","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/div[1]/div[2]/ul[2]/li[1]/div[1]/div[2]/div[1]/div[1]/div[3]","//div[contains(., '×')]","//DIV[@class='close-btn-download-banner']","/html/body/div[last()-8]/div[last()-5]/ul[last()-1]/li/div/div/div/div/div"]}},{"id":7,"index":7,"parentId":0,"type":1,"option":8,"title":"循环点击下一页","sequence":[9,8],"isInLoop":false,"position":6,"parameters":{"history":6,"tabIndex":-1,"useLoop":false,"xpath":"//a[contains(., '下一页')]","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":0,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":["/html/body/div[1]/div[2]/ul[2]/li[1]/div[1]/div[1]/div[6]/p[1]/span[5]/a[1]","//a[contains(., '下一页>')]","/html/body/div[last()-8]/div[last()-5]/ul[last()-1]/li/div/div[last()-1]/div[last()-2]/p/span/a"]}},{"id":9,"index":8,"parentId":7,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":true,"position":1,"parameters":{"history":6,"tabIndex":-1,"useLoop":true,"xpath":"","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/div[1]/div[2]/ul[2]/li[1]/div[1]/div[1]/div[6]/p[1]/span[5]/a[1]","//a[contains(., '下一页>')]","/html/body/div[last()-8]/div[last()-5]/ul[last()-1]/li/div/div[last()-1]/div[last()-2]/p/span/a"],"loopType":0}},{"id":8,"index":9,"parentId":7,"type":1,"option":8,"title":"循环","sequence":[10],"isInLoop":true,"position":0,"parameters":{"history":8,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/div/div[2]/ul[2]/li[1]/div[1]/div[1]/ul/li","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":1,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":["/html/body/div[1]/div[2]/ul[2]/li[1]/div[1]/div[1]/ul[1]/li[1]","//li[contains(., '')]","//LI[@class='search-item base-item']","/html/body/div[last()-8]/div[last()-5]/ul[last()-1]/li/div/div[last()-1]/ul/li[last()-9]"]}},{"id":10,"index":10,"parentId":8,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":8,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"paras":[{"nodeType":1,"contentType":0,"relative":true,"name":"参数8_链接文本","desc":"","relativeXPath":"/div[1]/div[3]/div[1]/span[1]/a[1]","allXPaths":["/div[1]/div[3]/div[1]/span[1]/a[1]","//a[contains(., '鑫隆门窗')]","//A[@class='n-blue']","/html/body/div[last()-8]/div[last()-5]/ul[last()-1]/li/div/div[last()-1]/ul/li[last()-9]/div[last()-1]/div/div[last()-3]/span[last()-1]/a"],"exampleValues":[{"num":0,"value":"鑫隆门窗"}],"unique_index":"/div[1]/div[3]/div[1]/span[1]/a[1]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":0,"contentType":1,"relative":true,"name":"参数11_文本","desc":"","relativeXPath":"/div[1]/div[3]/div[3]","allXPaths":["/div[1]/div[3]/div[3]","//div[contains(., '')]","//DIV[@class='row tel']","/html/body/div[last()-8]/div[last()-5]/ul[last()-1]/li/div/div[last()-1]/ul/li[last()-9]/div[last()-1]/div/div[last()-1]"],"exampleValues":[{"num":0,"value":"电话:13717842988"}],"unique_index":"/div[1]/div[3]/div[3]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":0,"contentType":1,"relative":true,"name":"参数10_文本","desc":"","relativeXPath":"/div[1]/div[3]/div[2]/span[1]","allXPaths":["/div[1]/div[3]/div[2]/span[1]","//span[contains(., '北京市朝阳区红霞路1')]","//SPAN[@class='n-grey']","/html/body/div[last()-8]/div[last()-5]/ul[last()-1]/li/div/div[last()-1]/ul/li[last()-9]/div[last()-1]/div/div[last()-2]/span"],"exampleValues":[{"num":0,"value":"北京市朝阳区红霞路13号"}],"unique_index":"/div[1]/div[3]/div[2]/span[1]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}],"loopType":1}}]} \ No newline at end of file diff --git a/ElectronJS/tasks/177.json b/ElectronJS/tasks/177.json index b4ef1fd..54ee54a 100644 --- a/ElectronJS/tasks/177.json +++ b/ElectronJS/tasks/177.json @@ -1 +1 @@ -{"id":177,"name":"京东全球版-专业的综合网上购物商城","url":"https://www.jd.com","links":"https://www.jd.com","create_time":"7/14/2023, 6:44:54 AM","update_time":"7/14/2023, 6:47:33 AM","version":"0.3.6","saveThreshold":10,"cloudflare":0,"environment":0,"maxViewLength":15,"outputFormat":"xlsx","saveName":"current_time","inputExcel":"","startFromExit":0,"containJudge":false,"desc":"https://www.jd.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.jd.com","desc":"要采集的网址列表，多行以\\n分开","type":"text","exampleValue":"https://www.jd.com"},{"id":1,"name":"loopTimes_循环_1","nodeId":2,"nodeName":"循环","desc":"循环循环执行的次数（0代表无限循环）","type":"int","exampleValue":8,"value":8}],"outputParameters":[{"id":0,"name":"参数1_文本","desc":"","type":"text","recordASField":1,"exampleValue":"/手机/数码/家用电器/电脑/办公/家纺/家居/厨具/家具/家装/灯具/工业品/内衣/男装/女装/童装/箱包/钟表/珠宝/女鞋/运动/户外/男鞋/汽车用品/车载电器/母婴/洗护喂养/玩具乐器/宠物生活/家庭清洁/个人护理/计生情趣/图书/童书/文学"},{"id":1,"name":"自定义操作","desc":"自定义操作返回的数据","type":"text","recordASField":0,"exampleValue":""},{"id":2,"name":"自定义操作","desc":"自定义操作返回的数据","type":"text","recordASField":0,"exampleValue":""},{"id":3,"name":"自定义操作","desc":"自定义操作返回的数据","type":"text","recordASField":0,"exampleValue":""}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,4,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0,"waitType":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"https://www.jd.com","links":"https://www.jd.com","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}},{"id":3,"index":2,"parentId":0,"type":1,"option":8,"title":"循环","sequence":[5,6,3,7],"isInLoop":false,"position":2,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":0,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":8,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0}},{"id":6,"index":3,"parentId":3,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":true,"position":2,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"clear":0,"paras":[{"nodeType":0,"contentType":0,"relative":true,"name":"参数1_文本","desc":"","extractType":0,"relativeXPath":"//a[Field[\"自定义操作\"]]","allXPaths":["/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]","//div[contains(., '/手机/数码/家用电')]","//DIV[@class='LeftSide_menu_list__qXCeM']","/html/body/div[last()-6]/div/div[last()-4]/div/div[last()-2]/div/div/div/div[last()-1]"],"exampleValues":[{"num":0,"value":"/手机/数码/家用电器/电脑/办公/家纺/家居/厨具/家具/家装/灯具/工业品/内衣/男装/女装/童装/箱包/钟表/珠宝/女鞋/运动/户外/男鞋/汽车用品/车载电器/母婴/洗护喂养/玩具乐器/宠物生活/家庭清洁/个人护理/计生情趣/图书/童书/文学"}],"unique_index":"6bzoou5h7glk1qcdmv","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}]}},{"id":2,"index":4,"parentId":0,"type":0,"option":5,"title":"自定义操作","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"clear":0,"codeMode":"5","code":"self.myVar = 0","waitTime":0,"recordASField":0,"paraType":"text"}},{"id":4,"index":5,"parentId":3,"type":0,"option":5,"title":"自定义操作","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"clear":0,"codeMode":"5","code":"self.myVar += 1","waitTime":0,"recordASField":0,"paraType":"text"}},{"id":5,"index":6,"parentId":3,"type":0,"option":5,"title":"自定义操作","sequence":[],"isInLoop":true,"position":1,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"clear":0,"codeMode":"6","code":"self.myVar","waitTime":0,"recordASField":0,"paraType":"text"}},{"id":7,"index":7,"parentId":3,"type":0,"option":7,"title":"移动到元素","sequence":[],"isInLoop":true,"position":3,"parameters":{"history":1,"tabIndex":0,"useLoop":true,"xpath":"//a[Field[\"自定义操作\"]]","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0}}]} \ No newline at end of file +{"id":177,"name":"京东全球版-专业的综合网上购物商城","url":"https://www.jd.com","links":"https://www.jd.com","create_time":"7/14/2023, 6:44:54 AM","update_time":"7/14/2023, 7:01:48 AM","version":"0.3.6","saveThreshold":10,"cloudflare":0,"environment":0,"maxViewLength":15,"outputFormat":"xlsx","saveName":"current_time","inputExcel":"","startFromExit":0,"containJudge":false,"desc":"https://www.jd.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.jd.com","desc":"要采集的网址列表，多行以\\n分开","type":"text","exampleValue":"https://www.jd.com"},{"id":1,"name":"loopTimes_循环_1","nodeId":2,"nodeName":"循环","desc":"循环循环执行的次数（0代表无限循环）","type":"int","exampleValue":0,"value":0}],"outputParameters":[{"id":0,"name":"参数1_文本","desc":"","type":"text","recordASField":1,"exampleValue":"/手机/数码/家用电器/电脑/办公/家纺/家居/厨具/家具/家装/灯具/工业品/内衣/男装/女装/童装/箱包/钟表/珠宝/女鞋/运动/户外/男鞋/汽车用品/车载电器/母婴/洗护喂养/玩具乐器/宠物生活/家庭清洁/个人护理/计生情趣/图书/童书/文学"},{"id":1,"name":"自定义操作","desc":"自定义操作返回的数据","type":"text","recordASField":0,"exampleValue":""},{"id":2,"name":"自定义操作","desc":"自定义操作返回的数据","type":"text","recordASField":0,"exampleValue":""},{"id":3,"name":"自定义操作","desc":"自定义操作返回的数据","type":"text","recordASField":0,"exampleValue":""}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,4,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0,"waitType":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"https://www.jd.com","links":"https://www.jd.com","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}},{"id":3,"index":2,"parentId":0,"type":1,"option":8,"title":"循环","sequence":[5,6,3,7],"isInLoop":false,"position":2,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":0,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0}},{"id":6,"index":3,"parentId":3,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":true,"position":2,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"clear":0,"paras":[{"nodeType":0,"contentType":0,"relative":true,"name":"参数1_文本","desc":"","extractType":0,"relativeXPath":"//a[Field[\"自定义操作\"]]","allXPaths":["/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]","//div[contains(., '/手机/数码/家用电')]","//DIV[@class='LeftSide_menu_list__qXCeM']","/html/body/div[last()-6]/div/div[last()-4]/div/div[last()-2]/div/div/div/div[last()-1]"],"exampleValues":[{"num":0,"value":"/手机/数码/家用电器/电脑/办公/家纺/家居/厨具/家具/家装/灯具/工业品/内衣/男装/女装/童装/箱包/钟表/珠宝/女鞋/运动/户外/男鞋/汽车用品/车载电器/母婴/洗护喂养/玩具乐器/宠物生活/家庭清洁/个人护理/计生情趣/图书/童书/文学"}],"unique_index":"6bzoou5h7glk1qcdmv","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}]}},{"id":2,"index":4,"parentId":0,"type":0,"option":5,"title":"自定义操作","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"clear":0,"codeMode":"5","code":"self.myVar = 0","waitTime":0,"recordASField":0,"paraType":"text"}},{"id":4,"index":5,"parentId":3,"type":0,"option":5,"title":"自定义操作","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"clear":0,"codeMode":"5","code":"self.myVar += 1","waitTime":0,"recordASField":0,"paraType":"text"}},{"id":5,"index":6,"parentId":3,"type":0,"option":5,"title":"自定义操作","sequence":[],"isInLoop":true,"position":1,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"clear":0,"codeMode":"6","code":"self.myVar","waitTime":0,"recordASField":0,"paraType":"text"}},{"id":7,"index":7,"parentId":3,"type":0,"option":7,"title":"移动到元素","sequence":[],"isInLoop":true,"position":3,"parameters":{"history":1,"tabIndex":0,"useLoop":true,"xpath":"//a[Field[\"自定义操作\"]]","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0}}]} \ No newline at end of file diff --git a/ExecuteStage/.vscode/launch.json b/ExecuteStage/.vscode/launch.json index 9069b9c..4fe2475 100644 --- a/ExecuteStage/.vscode/launch.json +++ b/ExecuteStage/.vscode/launch.json @@ -12,7 +12,7 @@ "justMyCode": false, // "args": ["--id", "[7]", "--read_type", "remote", "--headless", "0"] // "args": ["--id", "[9]", "--read_type", "remote", "--headless", "0", "--saved_file_name", "YOUTUBE"] - "args": ["--id", "[65]", "--headless", "0", "--user_data", "1"] + "args": ["--id", "[71]", "--headless", "0", "--user_data", "1"] } ] } \ No newline at end of file diff --git a/ExecuteStage/easyspider_executestage.py b/ExecuteStage/easyspider_executestage.py index 6e8a551..d455f83 100644 --- a/ExecuteStage/easyspider_executestage.py +++ b/ExecuteStage/easyspider_executestage.py @@ -54,6 +54,7 @@ class BrowserThread(Thread): Thread.__init__(self) self.browser = browser_t self.config = config + self.version = version self.totalSteps = 0 self.id = id self.event = event @@ -114,6 +115,7 @@ class BrowserThread(Thread): except: self.outputFormat = "csv" try: + self.task_version = service["version"] # 任务版本 if service["version"] >= "0.3.1": # 0.3.1及以上版本以上的EasySpider兼容从0.3.1版本开始的所有版本 pass else: # 0.3.1以下版本的EasySpider不兼容0.3.1及以上版本的EasySpider @@ -202,6 +204,11 @@ class BrowserThread(Thread): cookies = node["parameters"]["cookies"] except: node["parameters"]["cookies"] = "" + if node["option"] == 2: # 点击操作 + if node["parameters"]["useLoop"]: + if self.task_version <= "0.3.5": + node["parameters"]["xpath"] = "" # 0.3.5及以下版本的EasySpider下的循环点击不支持相对XPath + print("您的任务版本号为" + self.task_version + "，循环点击不支持相对XPath写法，已自动切换为纯循环的XPath") elif node["option"] == 3: # 提取数据操作 node["parameters"]["recordASField"] = 0 paras = node["parameters"]["paras"] @@ -239,6 +246,11 @@ class BrowserThread(Thread): clear = node["parameters"]["clear"] except: node["parameters"]["clear"] = 0 + elif node["option"] == 7: # 移动到元素 + if node["parameters"]["useLoop"]: + if self.task_version <= "0.3.5": + node["parameters"]["xpath"] = "" # 0.3.5及以下版本的EasySpider下的循环点击不支持相对XPath + print("您的任务版本号为" + self.task_version + "，循环点击不支持相对XPath写法，已自动切换为纯循环的XPath") def run(self):