diff --git a/ElectronJS/.gitignore b/ElectronJS/.gitignore index 2ad8bfa..f2d8a6f 100644 --- a/ElectronJS/.gitignore +++ b/ElectronJS/.gitignore @@ -19,3 +19,4 @@ mysql_config.json EasySpider_en/ EasySpider_zh/ TempUserDataFolder/ +secret_tasks/ diff --git a/ElectronJS/src/taskGrid/FlowChart.html b/ElectronJS/src/taskGrid/FlowChart.html index d94aecb..33bcd1e 100644 --- a/ElectronJS/src/taskGrid/FlowChart.html +++ b/ElectronJS/src/taskGrid/FlowChart.html @@ -380,13 +380,8 @@ + -
- -
-
- -
@@ -452,10 +447,29 @@ Please note that this feature does not support assigning values to variables. In
- - - - +
+ +
+
+ +
+
+ + + + + + + + + + + + + + + +
@@ -607,7 +621,7 @@ If the expression returns a value greater than 0 or evaluates to True, the loop
-
Please read the instructions first and then write the specific code in the input box above (not in this box). To execute a large amount of code, you can simply write "outside:myCode.py" and the program will read and execute the code within myCode.py under the EasySpider directory. 
+                        
Please read the instructions first and then write the specific code in the input box above (not in this box). To execute a large amount of code, you can simply write "outside:myCode.py" and the program will read and execute the code within myCode.py under the EasySpider directory.
 Use the expression value of Python code to determine whether a condition is satisfied. Here are some examples:
 1. Return relevant values of the current browser object. Use `self.browser` to refer to the current browser being operated. You can directly use Selenium's API to perform operations, such as `self.browser.find_element(By.CSS_SELECTOR, "body").text=="123"`, which checks whether the current page contains the text "123".
 2. Return the value of a custom global variable: `self.myVar`
diff --git a/ElectronJS/src/taskGrid/FlowChart_CN.html b/ElectronJS/src/taskGrid/FlowChart_CN.html
index ed071e2..19e8f7b 100644
--- a/ElectronJS/src/taskGrid/FlowChart_CN.html
+++ b/ElectronJS/src/taskGrid/FlowChart_CN.html
@@ -380,13 +380,8 @@
                         
                         
                         
+                        
                     
-                    
- -
-
- -
@@ -452,11 +447,30 @@ print(emotlib.emoji()) # 使用其中的函数。
- - - - - +
+ +
+
+ +
+
+ + + + + + + + + + + + + + + +
+
diff --git a/ElectronJS/src/taskGrid/logic.js b/ElectronJS/src/taskGrid/logic.js index a988b14..615802a 100644 --- a/ElectronJS/src/taskGrid/logic.js +++ b/ElectronJS/src/taskGrid/logic.js @@ -211,6 +211,16 @@ function addParameters(t) { t["parameters"]["waitTime"] = 0; //最长等待时间 t["parameters"]["recordASField"] = 0; //是否记录脚本输出 t["parameters"]["paraType"] = "text"; //记录脚本输出的字段索引 + t["parameters"]["emailConfig"] = { + "host": "", + "port": 465, + "username": "", + "password": "", + "from": "", + "to": "", + "subject": "", + "content": "", + } } else if (t.option == 6) { //切换下拉选项 t["parameters"]["optionMode"] = 0; //下拉模式 t["parameters"]["optionValue"] = ""; //下拉值 diff --git a/ElectronJS/tasks/221.json b/ElectronJS/tasks/221.json index 1bb9724..726c09a 100644 --- a/ElectronJS/tasks/221.json +++ b/ElectronJS/tasks/221.json @@ -1 +1 @@ -{"id":221,"name":"不固定元素列表测试:【软科排名】-中国最好学科排名|最权威的大学学科|高校学科排名","url":"https://www.shanghairanking.cn/rankings/bcsr/2023","links":"https://www.shanghairanking.cn/rankings/bcsr/2023","create_time":"12/5/2023, 4:12:11 AM","update_time":"12/9/2023, 7:22:37 PM","version":"0.6.0","saveThreshold":10,"quitWaitTime":60,"environment":0,"maximizeWindow":0,"maxViewLength":15,"recordLog":1,"outputFormat":"xlsx","saveName":"current_time","inputExcel":"","startFromExit":0,"pauseKey":"p","containJudge":false,"desc":"https://www.shanghairanking.cn/rankings/bcsr/2023","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.shanghairanking.cn/rankings/bcsr/2023","desc":"要采集的网址列表,多行以\\n分开","type":"text","exampleValue":"https://www.shanghairanking.cn/rankings/bcsr/2023"},{"id":1,"name":"loopTimes_循环点击单个元素_1","nodeId":6,"nodeName":"循环点击单个元素","desc":"循环循环点击单个元素执行的次数(0代表无限循环)","type":"int","exampleValue":0,"value":0}],"outputParameters":[{"id":0,"name":"参数1_文本","desc":"","type":"text","recordASField":1,"exampleValue":"1"},{"id":1,"name":"参数2_文本","desc":"","type":"text","recordASField":1,"exampleValue":"1"},{"id":2,"name":"参数3_文本","desc":"","type":"text","recordASField":1,"exampleValue":"前3%"},{"id":3,"name":"参数4_图片地址","desc":"","type":"text","recordASField":1,"exampleValue":"https://www.shanghairanking.cn/_uni/logo/28312850.png"},{"id":4,"name":"参数5_链接文本","desc":"","type":"text","recordASField":1,"exampleValue":"复旦大学\n "},{"id":5,"name":"参数6_链接地址","desc":"","type":"text","recordASField":1,"exampleValue":"/institution/fudan-university"},{"id":6,"name":"参数7_图片地址","desc":"","type":"text","recordASField":1,"exampleValue":"/_nuxt/img/uncollection.388fe56.svg"},{"id":7,"name":"参数8_链接文本","desc":"","type":"text","recordASField":1,"exampleValue":"\n "},{"id":8,"name":"参数9_链接地址","desc":"","type":"text","recordASField":1,"exampleValue":"/institution/fudan-university"},{"id":9,"name":"参数10_文本","desc":"","type":"text","recordASField":1,"exampleValue":"1044"}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"url":"https://www.shanghairanking.cn/rankings/bcsr/2023","links":"https://www.shanghairanking.cn/rankings/bcsr/2023","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}},{"id":2,"index":2,"parentId":0,"type":1,"option":8,"title":"循环点击每个元素","sequence":[3,6],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/div[1]/div[1]/div[1]/div[2]/div[2]/div[1]/div/div[2]/div/a[1]/span[2]","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":1,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":""}},{"id":3,"index":3,"parentId":2,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":4,"tabIndex":-1,"useLoop":true,"xpath":"","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"alertHandleType":0,"allXPaths":"","loopType":1}},{"id":5,"index":4,"parentId":4,"type":1,"option":8,"title":"循环采集数据","sequence":[5],"isInLoop":true,"position":0,"parameters":{"history":5,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/div[1]/div[1]/div[1]/div[2]/div[1]/div[3]/div[2]/div[1]/div[1]/div[2]/table[1]/tbody[1]/tr","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":1,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":["/html/body/div[1]/div[1]/div[1]/div[2]/div[1]/div[3]/div[2]/div[1]/div[1]/div[2]/table[1]/tbody[1]/tr[1]","//tr[contains(., '')]","/html/body/div[last()-3]/div/div/div[last()-1]/div/div/div/div[last()-1]/div/div[last()-1]/table/tbody/tr[last()-29]"]}},{"id":7,"index":5,"parentId":5,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":5,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"clear":0,"newLine":1,"paras":[{"nodeType":0,"contentType":1,"relative":true,"name":"参数1_文本","desc":"","relativeXPath":"/td[1]/div[1]","allXPaths":["/td[1]/div[1]","//div[contains(., '')]","//DIV[@class='ranking']","/html/body/div[last()-3]/div/div/div[last()-1]/div/div/div/div[last()-1]/div/div[last()-1]/table/tbody/tr[last()-29]/td[last()-4]/div"],"exampleValues":[{"num":0,"value":"1"}],"unique_index":"/td[1]/div[1]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":0,"contentType":1,"relative":true,"name":"参数2_文本","desc":"","relativeXPath":"/td[2]/span[1]","allXPaths":["/td[2]/span[1]","//span[contains(., '1')]","/html/body/div[last()-3]/div/div/div[last()-1]/div/div/div/div[last()-1]/div/div[last()-1]/table/tbody/tr[last()-29]/td[last()-3]/span"],"exampleValues":[{"num":0,"value":"1"}],"unique_index":"/td[2]/span[1]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":0,"contentType":1,"relative":true,"name":"参数3_文本","desc":"","relativeXPath":"/td[3]","allXPaths":["/td[3]","//td[contains(., '')]","/html/body/div[last()-3]/div/div/div[last()-1]/div/div/div/div[last()-1]/div/div[last()-1]/table/tbody/tr[last()-29]/td[last()-2]"],"exampleValues":[{"num":0,"value":"前3%"}],"unique_index":"/td[3]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":4,"contentType":1,"relative":true,"name":"参数4_图片地址","desc":"","relativeXPath":"/td[4]/div[1]/div[1]/img[1]","allXPaths":["/td[4]/div[1]/div[1]/img[1]","//img[contains(., '')]","//IMG[@class='univ-logo']","//IMG[@alt='复旦大学']","/html/body/div[last()-3]/div/div/div[last()-1]/div/div/div/div[last()-1]/div/div[last()-1]/table/tbody/tr[last()-29]/td[last()-1]/div/div[last()-1]/img"],"exampleValues":[{"num":0,"value":"https://www.shanghairanking.cn/_uni/logo/28312850.png"}],"unique_index":"/td[4]/div[1]/div[1]/img[1]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":1,"contentType":0,"relative":true,"name":"参数5_链接文本","desc":"","relativeXPath":"/td[4]/div[1]/div[2]/div[1]/div[1]/div[1]/a[1]","allXPaths":["/td[4]/div[1]/div[2]/div[1]/div[1]/div[1]/a[1]","//a[contains(., '复旦大学')]","//A[@class='name-cn']","/html/body/div[last()-3]/div/div/div[last()-1]/div/div/div/div[last()-1]/div/div[last()-1]/table/tbody/tr[last()-29]/td[last()-1]/div/div/div[last()-1]/div/div/a"],"exampleValues":[{"num":0,"value":"复旦大学\n "}],"unique_index":"/td[4]/div[1]/div[2]/div[1]/div[1]/div[1]/a[1]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":2,"contentType":0,"relative":true,"name":"参数6_链接地址","desc":"","relativeXPath":"/td[4]/div[1]/div[2]/div[1]/div[1]/div[1]/a[1]","allXPaths":["/td[4]/div[1]/div[2]/div[1]/div[1]/div[1]/a[1]","//a[contains(., '复旦大学')]","//A[@class='name-cn']","/html/body/div[last()-3]/div/div/div[last()-1]/div/div/div/div[last()-1]/div/div[last()-1]/table/tbody/tr[last()-29]/td[last()-1]/div/div/div[last()-1]/div/div/a"],"exampleValues":[{"num":0,"value":"/institution/fudan-university"}],"unique_index":"/td[4]/div[1]/div[2]/div[1]/div[1]/div[1]/a[1]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":4,"contentType":1,"relative":true,"name":"参数7_图片地址","desc":"","relativeXPath":"/td[4]/div[1]/div[2]/div[1]/div[1]/div[1]/div[1]/img[1]","allXPaths":["/td[4]/div[1]/div[2]/div[1]/div[1]/div[1]/div[1]/img[1]","//img[contains(., '')]","/html/body/div[last()-3]/div/div/div[last()-1]/div/div/div/div[last()-1]/div/div[last()-1]/table/tbody/tr[last()-29]/td[last()-1]/div/div/div[last()-1]/div/div/div/img"],"exampleValues":[{"num":0,"value":"/_nuxt/img/uncollection.388fe56.svg"}],"unique_index":"/td[4]/div[1]/div[2]/div[1]/div[1]/div[1]/div[1]/img[1]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":1,"contentType":0,"relative":true,"name":"参数8_链接文本","desc":"","relativeXPath":"/td[4]/div[1]/div[2]/div[2]/div[1]/div[1]/a[1]","allXPaths":["/td[4]/div[1]/div[2]/div[2]/div[1]/div[1]/a[1]","//a[contains(., '')]","//A[@class='name-en']","/html/body/div[last()-3]/div/div/div[last()-1]/div/div/div/div[last()-1]/div/div[last()-1]/table/tbody/tr[last()-29]/td[last()-1]/div/div/div/div/div/a"],"exampleValues":[{"num":0,"value":"\n "}],"unique_index":"/td[4]/div[1]/div[2]/div[2]/div[1]/div[1]/a[1]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":2,"contentType":0,"relative":true,"name":"参数9_链接地址","desc":"","relativeXPath":"/td[4]/div[1]/div[2]/div[2]/div[1]/div[1]/a[1]","allXPaths":["/td[4]/div[1]/div[2]/div[2]/div[1]/div[1]/a[1]","//a[contains(., '')]","//A[@class='name-en']","/html/body/div[last()-3]/div/div/div[last()-1]/div/div/div/div[last()-1]/div/div[last()-1]/table/tbody/tr[last()-29]/td[last()-1]/div/div/div/div/div/a"],"exampleValues":[{"num":0,"value":"/institution/fudan-university"}],"unique_index":"/td[4]/div[1]/div[2]/div[2]/div[1]/div[1]/a[1]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":0,"contentType":1,"relative":true,"name":"参数10_文本","desc":"","relativeXPath":"/td[5]","allXPaths":["/td[5]","//td[contains(., '')]","/html/body/div[last()-3]/div/div/div[last()-1]/div/div/div/div[last()-1]/div/div[last()-1]/table/tbody/tr[last()-29]/td"],"exampleValues":[{"num":0,"value":"1044"}],"unique_index":"/td[5]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}]}},{"id":4,"index":6,"parentId":2,"type":1,"option":8,"title":"循环点击单个元素","sequence":[4,7],"isInLoop":true,"position":1,"parameters":{"history":5,"tabIndex":-1,"useLoop":false,"xpath":"//*[contains(@class, \"ant-pagination-next\")]/a[1]","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":0,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":["/html/body/div[1]/div[1]/div[1]/div[2]/div[1]/div[3]/div[2]/div[1]/div[1]/ul[1]/li[5]/a[1]","//a[contains(., '')]","//A[@class='ant-pagination-item-link']","/html/body/div[last()-3]/div/div/div[last()-2]/div/div/div/div[last()-1]/div/ul/li[last()-1]/a"]}},{"id":6,"index":7,"parentId":4,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":true,"position":1,"parameters":{"history":5,"tabIndex":-1,"useLoop":true,"xpath":"","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"alertHandleType":0,"allXPaths":["/html/body/div[1]/div[1]/div[1]/div[2]/div[1]/div[3]/div[2]/div[1]/div[1]/ul[1]/li[5]/a[1]","//a[contains(., '')]","//A[@class='ant-pagination-item-link']","/html/body/div[last()-3]/div/div/div[last()-2]/div/div/div/div[last()-1]/div/ul/li[last()-1]/a"],"loopType":0}}]} \ No newline at end of file +{"id":221,"name":"不固定元素列表测试:【软科排名】-中国最好学科排名|最权威的大学学科|高校学科排名","url":"https://www.shanghairanking.cn/rankings/bcsr/2023","links":"https://www.shanghairanking.cn/rankings/bcsr/2023","create_time":"12/5/2023, 4:12:11 AM","update_time":"12/10/2023, 3:54:30 AM","version":"0.6.0","saveThreshold":10,"quitWaitTime":60,"environment":0,"maximizeWindow":0,"maxViewLength":15,"recordLog":1,"outputFormat":"xlsx","saveName":"current_time","inputExcel":"","startFromExit":0,"pauseKey":"p","containJudge":false,"desc":"https://www.shanghairanking.cn/rankings/bcsr/2023","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.shanghairanking.cn/rankings/bcsr/2023","desc":"要采集的网址列表,多行以\\n分开","type":"text","exampleValue":"https://www.shanghairanking.cn/rankings/bcsr/2023"},{"id":1,"name":"loopTimes_循环点击单个元素_1","nodeId":6,"nodeName":"循环点击单个元素","desc":"循环循环点击单个元素执行的次数(0代表无限循环)","type":"int","exampleValue":0,"value":0}],"outputParameters":[{"id":0,"name":"参数1_文本","desc":"","type":"text","recordASField":1,"exampleValue":"1"},{"id":1,"name":"参数2_文本","desc":"","type":"text","recordASField":1,"exampleValue":"1"},{"id":2,"name":"参数3_文本","desc":"","type":"text","recordASField":1,"exampleValue":"前3%"},{"id":3,"name":"参数4_图片地址","desc":"","type":"text","recordASField":1,"exampleValue":"https://www.shanghairanking.cn/_uni/logo/28312850.png"},{"id":4,"name":"参数5_链接文本","desc":"","type":"text","recordASField":1,"exampleValue":"复旦大学\n "},{"id":5,"name":"参数6_链接地址","desc":"","type":"text","recordASField":1,"exampleValue":"/institution/fudan-university"},{"id":6,"name":"参数7_图片地址","desc":"","type":"text","recordASField":1,"exampleValue":"/_nuxt/img/uncollection.388fe56.svg"},{"id":7,"name":"参数8_链接文本","desc":"","type":"text","recordASField":1,"exampleValue":"\n "},{"id":8,"name":"参数9_链接地址","desc":"","type":"text","recordASField":1,"exampleValue":"/institution/fudan-university"},{"id":9,"name":"参数10_文本","desc":"","type":"text","recordASField":1,"exampleValue":"1044"},{"id":10,"name":"自定义操作","desc":"自定义操作返回的数据","type":"text","recordASField":0,"exampleValue":""}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,8,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"url":"https://www.shanghairanking.cn/rankings/bcsr/2023","links":"https://www.shanghairanking.cn/rankings/bcsr/2023","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}},{"id":3,"index":2,"parentId":0,"type":1,"option":8,"title":"循环点击每个元素","sequence":[3,6],"isInLoop":false,"position":2,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/div[1]/div[1]/div[1]/div[2]/div[2]/div[1]/div/div[2]/div/a[1]/span[2]","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":1,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":""}},{"id":4,"index":3,"parentId":3,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":4,"tabIndex":-1,"useLoop":true,"xpath":"","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"alertHandleType":0,"allXPaths":"","loopType":1}},{"id":6,"index":4,"parentId":5,"type":1,"option":8,"title":"循环采集数据","sequence":[5],"isInLoop":true,"position":0,"parameters":{"history":5,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/div[1]/div[1]/div[1]/div[2]/div[1]/div[3]/div[2]/div[1]/div[1]/div[2]/table[1]/tbody[1]/tr","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":1,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":["/html/body/div[1]/div[1]/div[1]/div[2]/div[1]/div[3]/div[2]/div[1]/div[1]/div[2]/table[1]/tbody[1]/tr[1]","//tr[contains(., '')]","/html/body/div[last()-3]/div/div/div[last()-1]/div/div/div/div[last()-1]/div/div[last()-1]/table/tbody/tr[last()-29]"]}},{"id":8,"index":5,"parentId":6,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":5,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"clear":0,"newLine":1,"paras":[{"nodeType":0,"contentType":1,"relative":true,"name":"参数1_文本","desc":"","relativeXPath":"/td[1]/div[1]","allXPaths":["/td[1]/div[1]","//div[contains(., '')]","//DIV[@class='ranking']","/html/body/div[last()-3]/div/div/div[last()-1]/div/div/div/div[last()-1]/div/div[last()-1]/table/tbody/tr[last()-29]/td[last()-4]/div"],"exampleValues":[{"num":0,"value":"1"}],"unique_index":"/td[1]/div[1]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":0,"contentType":1,"relative":true,"name":"参数2_文本","desc":"","relativeXPath":"/td[2]/span[1]","allXPaths":["/td[2]/span[1]","//span[contains(., '1')]","/html/body/div[last()-3]/div/div/div[last()-1]/div/div/div/div[last()-1]/div/div[last()-1]/table/tbody/tr[last()-29]/td[last()-3]/span"],"exampleValues":[{"num":0,"value":"1"}],"unique_index":"/td[2]/span[1]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":0,"contentType":1,"relative":true,"name":"参数3_文本","desc":"","relativeXPath":"/td[3]","allXPaths":["/td[3]","//td[contains(., '')]","/html/body/div[last()-3]/div/div/div[last()-1]/div/div/div/div[last()-1]/div/div[last()-1]/table/tbody/tr[last()-29]/td[last()-2]"],"exampleValues":[{"num":0,"value":"前3%"}],"unique_index":"/td[3]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":4,"contentType":1,"relative":true,"name":"参数4_图片地址","desc":"","relativeXPath":"/td[4]/div[1]/div[1]/img[1]","allXPaths":["/td[4]/div[1]/div[1]/img[1]","//img[contains(., '')]","//IMG[@class='univ-logo']","//IMG[@alt='复旦大学']","/html/body/div[last()-3]/div/div/div[last()-1]/div/div/div/div[last()-1]/div/div[last()-1]/table/tbody/tr[last()-29]/td[last()-1]/div/div[last()-1]/img"],"exampleValues":[{"num":0,"value":"https://www.shanghairanking.cn/_uni/logo/28312850.png"}],"unique_index":"/td[4]/div[1]/div[1]/img[1]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":1,"contentType":0,"relative":true,"name":"参数5_链接文本","desc":"","relativeXPath":"/td[4]/div[1]/div[2]/div[1]/div[1]/div[1]/a[1]","allXPaths":["/td[4]/div[1]/div[2]/div[1]/div[1]/div[1]/a[1]","//a[contains(., '复旦大学')]","//A[@class='name-cn']","/html/body/div[last()-3]/div/div/div[last()-1]/div/div/div/div[last()-1]/div/div[last()-1]/table/tbody/tr[last()-29]/td[last()-1]/div/div/div[last()-1]/div/div/a"],"exampleValues":[{"num":0,"value":"复旦大学\n "}],"unique_index":"/td[4]/div[1]/div[2]/div[1]/div[1]/div[1]/a[1]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":2,"contentType":0,"relative":true,"name":"参数6_链接地址","desc":"","relativeXPath":"/td[4]/div[1]/div[2]/div[1]/div[1]/div[1]/a[1]","allXPaths":["/td[4]/div[1]/div[2]/div[1]/div[1]/div[1]/a[1]","//a[contains(., '复旦大学')]","//A[@class='name-cn']","/html/body/div[last()-3]/div/div/div[last()-1]/div/div/div/div[last()-1]/div/div[last()-1]/table/tbody/tr[last()-29]/td[last()-1]/div/div/div[last()-1]/div/div/a"],"exampleValues":[{"num":0,"value":"/institution/fudan-university"}],"unique_index":"/td[4]/div[1]/div[2]/div[1]/div[1]/div[1]/a[1]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":4,"contentType":1,"relative":true,"name":"参数7_图片地址","desc":"","relativeXPath":"/td[4]/div[1]/div[2]/div[1]/div[1]/div[1]/div[1]/img[1]","allXPaths":["/td[4]/div[1]/div[2]/div[1]/div[1]/div[1]/div[1]/img[1]","//img[contains(., '')]","/html/body/div[last()-3]/div/div/div[last()-1]/div/div/div/div[last()-1]/div/div[last()-1]/table/tbody/tr[last()-29]/td[last()-1]/div/div/div[last()-1]/div/div/div/img"],"exampleValues":[{"num":0,"value":"/_nuxt/img/uncollection.388fe56.svg"}],"unique_index":"/td[4]/div[1]/div[2]/div[1]/div[1]/div[1]/div[1]/img[1]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":1,"contentType":0,"relative":true,"name":"参数8_链接文本","desc":"","relativeXPath":"/td[4]/div[1]/div[2]/div[2]/div[1]/div[1]/a[1]","allXPaths":["/td[4]/div[1]/div[2]/div[2]/div[1]/div[1]/a[1]","//a[contains(., '')]","//A[@class='name-en']","/html/body/div[last()-3]/div/div/div[last()-1]/div/div/div/div[last()-1]/div/div[last()-1]/table/tbody/tr[last()-29]/td[last()-1]/div/div/div/div/div/a"],"exampleValues":[{"num":0,"value":"\n "}],"unique_index":"/td[4]/div[1]/div[2]/div[2]/div[1]/div[1]/a[1]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":2,"contentType":0,"relative":true,"name":"参数9_链接地址","desc":"","relativeXPath":"/td[4]/div[1]/div[2]/div[2]/div[1]/div[1]/a[1]","allXPaths":["/td[4]/div[1]/div[2]/div[2]/div[1]/div[1]/a[1]","//a[contains(., '')]","//A[@class='name-en']","/html/body/div[last()-3]/div/div/div[last()-1]/div/div/div/div[last()-1]/div/div[last()-1]/table/tbody/tr[last()-29]/td[last()-1]/div/div/div/div/div/a"],"exampleValues":[{"num":0,"value":"/institution/fudan-university"}],"unique_index":"/td[4]/div[1]/div[2]/div[2]/div[1]/div[1]/a[1]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":0,"contentType":1,"relative":true,"name":"参数10_文本","desc":"","relativeXPath":"/td[5]","allXPaths":["/td[5]","//td[contains(., '')]","/html/body/div[last()-3]/div/div/div[last()-1]/div/div/div/div[last()-1]/div/div[last()-1]/table/tbody/tr[last()-29]/td"],"exampleValues":[{"num":0,"value":"1044"}],"unique_index":"/td[5]","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}]}},{"id":5,"index":6,"parentId":3,"type":1,"option":8,"title":"循环点击单个元素","sequence":[4,7],"isInLoop":true,"position":1,"parameters":{"history":5,"tabIndex":-1,"useLoop":false,"xpath":"//*[contains(@class, \"ant-pagination-next\")]/a[1]","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":0,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":["/html/body/div[1]/div[1]/div[1]/div[2]/div[1]/div[3]/div[2]/div[1]/div[1]/ul[1]/li[5]/a[1]","//a[contains(., '')]","//A[@class='ant-pagination-item-link']","/html/body/div[last()-3]/div/div/div[last()-2]/div/div/div/div[last()-1]/div/ul/li[last()-1]/a"]}},{"id":7,"index":7,"parentId":5,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":true,"position":1,"parameters":{"history":5,"tabIndex":-1,"useLoop":true,"xpath":"","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"alertHandleType":0,"allXPaths":["/html/body/div[1]/div[1]/div[1]/div[2]/div[1]/div[3]/div[2]/div[1]/div[1]/ul[1]/li[5]/a[1]","//a[contains(., '')]","//A[@class='ant-pagination-item-link']","/html/body/div[last()-3]/div/div/div[last()-2]/div/div/div/div[last()-1]/div/ul/li[last()-1]/a"],"loopType":0}},{"id":2,"index":8,"parentId":0,"type":0,"option":5,"title":"自定义操作","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"clear":0,"newLine":1,"codeMode":"9","code":"","waitTime":0,"recordASField":0,"paraType":"text","emailConfig":{"host":"smtp.163.co23m","port":"465","username":"TEST@163.com","password":"asdf","from":"","to":"naibowang@foxmail.com, ids_server_manager@163.com","subject":"123","content":"434"}}}]} \ No newline at end of file diff --git a/ExecuteStage/.vscode/launch.json b/ExecuteStage/.vscode/launch.json index e76fce5..9ed6277 100644 --- a/ExecuteStage/.vscode/launch.json +++ b/ExecuteStage/.vscode/launch.json @@ -12,7 +12,7 @@ "justMyCode": false, // "args": ["--ids", "[7]", "--read_type", "remote", "--headless", "0"] // "args": ["--ids", "[9]", "--read_type", "remote", "--headless", "0", "--saved_file_name", "YOUTUBE"] - "args": ["--ids", "[93]", "--headless", "0", "--user_data", "0", "--keyboard", "0"] + "args": ["--ids", "[95]", "--headless", "0", "--user_data", "0", "--keyboard", "0"] // "args": "--ids '[97]' --user_data 1 --server_address http://localhost:8074 --config_folder '/Users/naibo/Documents/EasySpider/ElectronJS/' --headless 0 --read_type remote --config_file_name config.json --saved_file_name" } ] diff --git a/ExecuteStage/easyspider_executestage.py b/ExecuteStage/easyspider_executestage.py index cb7ce59..26a1b08 100644 --- a/ExecuteStage/easyspider_executestage.py +++ b/ExecuteStage/easyspider_executestage.py @@ -7,7 +7,7 @@ import shutil import string import undetected_chromedriver as uc from utils import detect_optimizable, download_image, get_output_code, isnotnull, lowercase_tags_in_xpath, myMySQL, new_line, \ - on_press_creator, on_release_creator, readCode, replace_field_values, write_to_csv, write_to_excel, write_to_json + on_press_creator, on_release_creator, readCode, replace_field_values, send_email, write_to_csv, write_to_excel, write_to_json from myChrome import MyChrome from threading import Thread, Event from PIL import Image @@ -727,6 +727,8 @@ class BrowserThread(Thread): elif codeMode == 8: # 刷新页面 self.browser.refresh() self.print_and_log("根据设置的自定义操作,任务已刷新页面|Task refreshed page according to custom operation") + elif codeMode == 9: # 发送邮件 + send_email(node["parameters"]["emailConfig"]) else: # 0 1 5 6 output = self.execute_code( codeMode, code, max_wait_time, iframe=paras["iframe"]) diff --git a/ExecuteStage/utils.py b/ExecuteStage/utils.py index b8a1b66..b7a6e52 100644 --- a/ExecuteStage/utils.py +++ b/ExecuteStage/utils.py @@ -1,5 +1,4 @@ -# 控制流程的暂停和继续 - +# 工具库 import csv import datetime import json @@ -14,6 +13,47 @@ import requests from urllib.parse import urlparse import pymysql from lxml import etree +import smtplib +from email.mime.text import MIMEText +from email.header import Header + +def send_email(config): + """ + 发送邮件的函数。 + + :param config: 包含邮件配置信息的字典。 + """ + # 校验配置信息是否完整 + # required_keys = ["host", "port", "username", "password", "from", "to", "subject", "content"] + # missing_keys = [key for key in required_keys if key not in config] + # if missing_keys: + # raise ValueError(f"邮件配置缺少必要的键: {', '.join(missing_keys)}") + try: + print("正在发送邮件到:" + config['to']) + message = MIMEText(config['content'], 'plain', 'utf-8') + message['From'] = Header(f"{config['username'].split('@')[0]} <{config['username']}>") + to_name_list = [] + for address in config['to'].split(','): + address = address.strip() + name = address.split('@')[0] + to_name_list.append(f"{name} <{address}>") + to_name_list = ', '.join(to_name_list) + message['To'] = Header(to_name_list) + message['Subject'] = Header(config['subject'], 'utf-8') + # 使用SSL加密方式连接邮件服务器 + smtp_server = smtplib.SMTP_SSL(config['host'], config['port']) + smtp_server.login(config['username'], config['password']) + to_address_list = config['to'].split(',') + smtp_server.sendmail(config['username'], to_address_list, message.as_string()) + print("邮件发送成功|Email sent successfully") + except Exception as e: + print(f"无法发送邮件,发生错误:{e}") + print(f"Failed to send email, error: {e}") + finally: + try: + smtp_server.quit() + except: + pass def is_valid_url(url):