Browse Source

Email Sent

pull/254/head
naibo 9 months ago
parent
commit
bad6e5bdf5
8 changed files with 108 additions and 27 deletions
  1. +1
    -0
      ElectronJS/.gitignore
  2. +25
    -11
      ElectronJS/src/taskGrid/FlowChart.html
  3. +25
    -11
      ElectronJS/src/taskGrid/FlowChart_CN.html
  4. +10
    -0
      ElectronJS/src/taskGrid/logic.js
  5. +1
    -1
      ElectronJS/tasks/221.json
  6. +1
    -1
      ExecuteStage/.vscode/launch.json
  7. +3
    -1
      ExecuteStage/easyspider_executestage.py
  8. +42
    -2
      ExecuteStage/utils.py

+ 1
- 0
ElectronJS/.gitignore View File

@ -19,3 +19,4 @@ mysql_config.json
EasySpider_en/ EasySpider_en/
EasySpider_zh/ EasySpider_zh/
TempUserDataFolder/ TempUserDataFolder/
secret_tasks/

+ 25
- 11
ElectronJS/src/taskGrid/FlowChart.html View File

@ -380,13 +380,8 @@
<option value = 6>Get value of a Python expression (the "eval" operation)</option> <option value = 6>Get value of a Python expression (the "eval" operation)</option>
<option value = 7>Pause program execution (such as when the captcha box appears)</option> <option value = 7>Pause program execution (such as when the captcha box appears)</option>
<option value = 8>Refresh page</option> <option value = 8>Refresh page</option>
<option value = 9>Send Email</option>
</select> </select>
<div v-if='nowNode["parameters"]["codeMode"] == 7'>
<label>This operation can pause program execution, such as when a captcha box appears, and it will not continue until you manually press and hold the pause/continue shortcut key (default: key p).</label>
</div>
<div v-if='nowNode["parameters"]["codeMode"] == 8'>
<label>This operation can refresh the current page.</label>
</div>
<div v-if='nowNode["parameters"]["codeMode"] < 3 || nowNode["parameters"]["codeMode"] >= 5 && nowNode["parameters"]["codeMode"] <=6'> <div v-if='nowNode["parameters"]["codeMode"] < 3 || nowNode["parameters"]["codeMode"] >= 5 && nowNode["parameters"]["codeMode"] <=6'>
<label>Code (Use Field["FieldName"] to input the lastest value of a field): </label> <label>Code (Use Field["FieldName"] to input the lastest value of a field): </label>
<textarea onkeydown="inputDelete(event)" class="form-control" rows="2" v-model='nowNode["parameters"]["code"]' placeholder="Please input a JavaScript command or a system command. For example, document.body.innerText = '1' is an example of a JavaScript command, and python D:/test.py is an example of a system command. If you choose to execute a JavaScript script for the current iteration, you can represent the element of the current iteration using arguments[0]. For instance, arguments[0].style.color = 'blue' sets the color of the element in the current iteration to blue."></textarea> <textarea onkeydown="inputDelete(event)" class="form-control" rows="2" v-model='nowNode["parameters"]["code"]' placeholder="Please input a JavaScript command or a system command. For example, document.body.innerText = '1' is an example of a JavaScript command, and python D:/test.py is an example of a system command. If you choose to execute a JavaScript script for the current iteration, you can represent the element of the current iteration using arguments[0]. For instance, arguments[0].style.color = 'blue' sets the color of the element in the current iteration to blue."></textarea>
@ -452,10 +447,29 @@ Please note that this feature does not support assigning values to variables. In
<label>Maximum wait time for script execution (0 represents unlimited wait time): </label> <label>Maximum wait time for script execution (0 represents unlimited wait time): </label>
<input onkeydown="inputDelete(event)" required class="form-control" type="number" v-model.number='nowNode["parameters"]["waitTime"]'></input> <input onkeydown="inputDelete(event)" required class="form-control" type="number" v-model.number='nowNode["parameters"]["waitTime"]'></input>
</div> </div>
<div v-if='nowNode["parameters"]["codeMode"] == 7'>
<label>This operation can pause program execution, such as when a captcha box appears, and it will not continue until you manually press and hold the pause/continue shortcut key (default: key p).</label>
</div>
<div v-if='nowNode["parameters"]["codeMode"] == 8'>
<label>This operation can refresh the current page.</label>
</div>
<div v-if='nowNode["parameters"]["codeMode"] == 9'>
<label>This operation can send emails, for example, to notify by email when a web scraping task is completed.</label>
<label>SMTP email server host:</label>
<input onkeydown="inputDelete(event)" class="form-control" v-model='nowNode["parameters"]["emailConfig"]["host"]' placeholder="e.g., smtp.gmail.com"></input>
<label>Email server port:</label>
<input onkeydown="inputDelete(event)" class="form-control" v-model='nowNode["parameters"]["emailConfig"]["port"]' placeholder="e.g., 465"></input>
<label>Sender email username:</label>
<input onkeydown="inputDelete(event)" class="form-control" v-model='nowNode["parameters"]["emailConfig"]["username"]'></input>
<label>Sender email password (Be careful not to leak the task file if a password is set!):</label>
<input onkeydown="inputDelete(event)" type="password" class="form-control" v-model='nowNode["parameters"]["emailConfig"]["password"]' placeholder="Most email servers use authorization codes, not the original password"></input>
<label>Recipient email address:</label>
<input onkeydown="inputDelete(event)" class="form-control" v-model='nowNode["parameters"]["emailConfig"]["to"]' placeholder="Separate multiple recipients with commas"></input>
<label>Email subject:</label>
<input onkeydown="inputDelete(event)" class="form-control" v-model='nowNode["parameters"]["emailConfig"]["subject"]' placeholder="Write the email subject here"></input>
<label>Email content:</label>
<textarea onkeydown="inputDelete(event)" class="form-control" rows="2" v-model='nowNode["parameters"]["emailConfig"]["content"]' placeholder="Write the email content here"></textarea>
</div>
</div> </div>
@ -607,7 +621,7 @@ If the expression returns a value greater than 0 or evaluates to True, the loop
<div v-else-if='TClass > 0 && TClass < 7 || TClass == 8'> <div v-else-if='TClass > 0 && TClass < 7 || TClass == 8'>
<label>Code/Script Content: </label> <label>Code/Script Content: </label>
<textarea onkeydown="inputDelete(event)" class="form-control" rows="3" v-model='nowNode["parameters"]["code"]' placeholder="If the return value is greater than 0 or true, the operations within this branch will be executed; otherwise, they will not be executed. For example: return document.body.scrollWidth > 1000 or python D:/test.py, representing examples of JS command and system command return values."></textarea> <textarea onkeydown="inputDelete(event)" class="form-control" rows="3" v-model='nowNode["parameters"]["code"]' placeholder="If the return value is greater than 0 or true, the operations within this branch will be executed; otherwise, they will not be executed. For example: return document.body.scrollWidth > 1000 or python D:/test.py, representing examples of JS command and system command return values."></textarea>
<pre class="form-control" style="background: white; margin-top: 20px; min-height: 200px; font-size: 15px!important; word-wrap: break-word!important; white-space: pre-wrap; border-radius: 0; border: 1px solid" disabled v-if='TClass == 8'>Please read the instructions first and then write the specific code in the input box above (not in this box). To execute a large amount of code, you can simply write "outside:myCode.py" and the program will read and execute the code within myCode.py under the EasySpider directory.
<pre class="form-control" style="background: white; margin-top: 20px; min-height: 200px; font-size: 15px!important; word-wrap: break-word!important; white-space: pre-wrap; border-radius: 0; border: 1px solid" disabled v-if='TClass == 8'>Please read the instructions first and then write the specific code in the input box above (not in this box). To execute a large amount of code, you can simply write "outside:myCode.py" and the program will read and execute the code within myCode.py under the EasySpider directory.
Use the expression value of Python code to determine whether a condition is satisfied. Here are some examples: Use the expression value of Python code to determine whether a condition is satisfied. Here are some examples:
1. Return relevant values of the current browser object. Use `self.browser` to refer to the current browser being operated. You can directly use Selenium's API to perform operations, such as `self.browser.find_element(By.CSS_SELECTOR, "body").text=="123"`, which checks whether the current page contains the text "123". 1. Return relevant values of the current browser object. Use `self.browser` to refer to the current browser being operated. You can directly use Selenium's API to perform operations, such as `self.browser.find_element(By.CSS_SELECTOR, "body").text=="123"`, which checks whether the current page contains the text "123".
2. Return the value of a custom global variable: `self.myVar` 2. Return the value of a custom global variable: `self.myVar`

+ 25
- 11
ElectronJS/src/taskGrid/FlowChart_CN.html View File

@ -380,13 +380,8 @@
<option value = 6>在执行环境下获得Python表达式值(eval操作)</option> <option value = 6>在执行环境下获得Python表达式值(eval操作)</option>
<option value = 7>暂停程序执行(如检测到验证码框出现时暂停执行)</option> <option value = 7>暂停程序执行(如检测到验证码框出现时暂停执行)</option>
<option value = 8>刷新页面</option> <option value = 8>刷新页面</option>
<option value = 9>发送邮件</option>
</select> </select>
<div v-if='nowNode["parameters"]["codeMode"] == 7'>
<label>此操作可以暂停程序执行,如检测到验证码框出现时暂停执行,直到手动长按键盘自定义暂停/继续快捷键(默认:p键)后才继续执行。</label>
</div>
<div v-if='nowNode["parameters"]["codeMode"] == 8'>
<label>此操作可以刷新当前页面。</label>
</div>
<div v-if='nowNode["parameters"]["codeMode"] < 3 || nowNode["parameters"]["codeMode"] >= 5 && nowNode["parameters"]["codeMode"] <=6'> <div v-if='nowNode["parameters"]["codeMode"] < 3 || nowNode["parameters"]["codeMode"] >= 5 && nowNode["parameters"]["codeMode"] <=6'>
<label>代码/脚本内容(用Field["字段名"]来输入某字段/自定义操作的最新提取/返回值): </label> <label>代码/脚本内容(用Field["字段名"]来输入某字段/自定义操作的最新提取/返回值): </label>
<textarea onkeydown="inputDelete(event)" class="form-control" rows="2" v-model='nowNode["parameters"]["code"]' placeholder="输入JS或系统命令,如:document.body.innerText = '1' 或 python D:/test.py,分别为JS命令和系统命令示例。如选择针对当前循环项的JS脚本,则循环项元素用arguments[0]表示,如arguments[0].style.color = 'blue'"></textarea> <textarea onkeydown="inputDelete(event)" class="form-control" rows="2" v-model='nowNode["parameters"]["code"]' placeholder="输入JS或系统命令,如:document.body.innerText = '1' 或 python D:/test.py,分别为JS命令和系统命令示例。如选择针对当前循环项的JS脚本,则循环项元素用arguments[0]表示,如arguments[0].style.color = 'blue'"></textarea>
@ -452,11 +447,30 @@ print(emotlib.emoji()) # 使用其中的函数。
<label>最长等待脚本执行时间(0代表无限等待): </label> <label>最长等待脚本执行时间(0代表无限等待): </label>
<input onkeydown="inputDelete(event)" required class="form-control" type="number" v-model.number='nowNode["parameters"]["waitTime"]'></input> <input onkeydown="inputDelete(event)" required class="form-control" type="number" v-model.number='nowNode["parameters"]["waitTime"]'></input>
</div> </div>
<div v-if='nowNode["parameters"]["codeMode"] == 7'>
<label>此操作可以暂停程序执行,如检测到验证码框出现时暂停执行,直到手动长按键盘自定义暂停/继续快捷键(默认:p键)后才继续执行。</label>
</div>
<div v-if='nowNode["parameters"]["codeMode"] == 8'>
<label>此操作可以刷新当前页面。</label>
</div>
<div v-if='nowNode["parameters"]["codeMode"] == 9'>
<label>此操作可以发送邮件,如用于爬虫任务完成后发送邮件通知。</label>
<label>STMP邮件服务器主机:</label>
<input onkeydown="inputDelete(event)" class="form-control" v-model='nowNode["parameters"]["emailConfig"]["host"]' placeholder="如smtp.163.com"></input>
<label>邮件服务器端口:</label>
<input onkeydown="inputDelete(event)" class="form-control" v-model='nowNode["parameters"]["emailConfig"]["port"]' placeholder="如465"></input>
<label>发件人邮箱用户名:</label>
<input onkeydown="inputDelete(event)" class="form-control" v-model='nowNode["parameters"]["emailConfig"]["username"]'></input>
<label>发件人邮箱密码(设置了密码注意不要轻易泄露任务文件!):</label>
<input onkeydown="inputDelete(event)" type="password" class="form-control" v-model='nowNode["parameters"]["emailConfig"]["password"]' placeholder="多数邮箱服务器为授权码而不是原密码"></input>
<label>收件人邮箱地址:</label>
<input onkeydown="inputDelete(event)" class="form-control" v-model='nowNode["parameters"]["emailConfig"]["to"]' placeholder="多个收件人用英文逗号隔开"></input>
<label>邮件主题:</label>
<input onkeydown="inputDelete(event)" class="form-control" v-model='nowNode["parameters"]["emailConfig"]["subject"]' placeholder="这里写邮件主题"></input>
<label>邮件内容:</label>
<textarea onkeydown="inputDelete(event)" class="form-control" rows="2" v-model='nowNode["parameters"]["emailConfig"]["content"]' placeholder="这里写邮件内容"></textarea>
</div>
</div> </div>
<div class="elements" v-if="nodeType==6"> <div class="elements" v-if="nodeType==6">

+ 10
- 0
ElectronJS/src/taskGrid/logic.js View File

@ -211,6 +211,16 @@ function addParameters(t) {
t["parameters"]["waitTime"] = 0; //最长等待时间 t["parameters"]["waitTime"] = 0; //最长等待时间
t["parameters"]["recordASField"] = 0; //是否记录脚本输出 t["parameters"]["recordASField"] = 0; //是否记录脚本输出
t["parameters"]["paraType"] = "text"; //记录脚本输出的字段索引 t["parameters"]["paraType"] = "text"; //记录脚本输出的字段索引
t["parameters"]["emailConfig"] = {
"host": "",
"port": 465,
"username": "",
"password": "",
"from": "",
"to": "",
"subject": "",
"content": "",
}
} else if (t.option == 6) { //切换下拉选项 } else if (t.option == 6) { //切换下拉选项
t["parameters"]["optionMode"] = 0; //下拉模式 t["parameters"]["optionMode"] = 0; //下拉模式
t["parameters"]["optionValue"] = ""; //下拉值 t["parameters"]["optionValue"] = ""; //下拉值

+ 1
- 1
ElectronJS/tasks/221.json
File diff suppressed because it is too large
View File


+ 1
- 1
ExecuteStage/.vscode/launch.json View File

@ -12,7 +12,7 @@
"justMyCode": false, "justMyCode": false,
// "args": ["--ids", "[7]", "--read_type", "remote", "--headless", "0"] // "args": ["--ids", "[7]", "--read_type", "remote", "--headless", "0"]
// "args": ["--ids", "[9]", "--read_type", "remote", "--headless", "0", "--saved_file_name", "YOUTUBE"] // "args": ["--ids", "[9]", "--read_type", "remote", "--headless", "0", "--saved_file_name", "YOUTUBE"]
"args": ["--ids", "[93]", "--headless", "0", "--user_data", "0", "--keyboard", "0"]
"args": ["--ids", "[95]", "--headless", "0", "--user_data", "0", "--keyboard", "0"]
// "args": "--ids '[97]' --user_data 1 --server_address http://localhost:8074 --config_folder '/Users/naibo/Documents/EasySpider/ElectronJS/' --headless 0 --read_type remote --config_file_name config.json --saved_file_name" // "args": "--ids '[97]' --user_data 1 --server_address http://localhost:8074 --config_folder '/Users/naibo/Documents/EasySpider/ElectronJS/' --headless 0 --read_type remote --config_file_name config.json --saved_file_name"
} }
] ]

+ 3
- 1
ExecuteStage/easyspider_executestage.py View File

@ -7,7 +7,7 @@ import shutil
import string import string
import undetected_chromedriver as uc import undetected_chromedriver as uc
from utils import detect_optimizable, download_image, get_output_code, isnotnull, lowercase_tags_in_xpath, myMySQL, new_line, \ from utils import detect_optimizable, download_image, get_output_code, isnotnull, lowercase_tags_in_xpath, myMySQL, new_line, \
on_press_creator, on_release_creator, readCode, replace_field_values, write_to_csv, write_to_excel, write_to_json
on_press_creator, on_release_creator, readCode, replace_field_values, send_email, write_to_csv, write_to_excel, write_to_json
from myChrome import MyChrome from myChrome import MyChrome
from threading import Thread, Event from threading import Thread, Event
from PIL import Image from PIL import Image
@ -727,6 +727,8 @@ class BrowserThread(Thread):
elif codeMode == 8: # 刷新页面 elif codeMode == 8: # 刷新页面
self.browser.refresh() self.browser.refresh()
self.print_and_log("根据设置的自定义操作,任务已刷新页面|Task refreshed page according to custom operation") self.print_and_log("根据设置的自定义操作,任务已刷新页面|Task refreshed page according to custom operation")
elif codeMode == 9: # 发送邮件
send_email(node["parameters"]["emailConfig"])
else: # 0 1 5 6 else: # 0 1 5 6
output = self.execute_code( output = self.execute_code(
codeMode, code, max_wait_time, iframe=paras["iframe"]) codeMode, code, max_wait_time, iframe=paras["iframe"])

+ 42
- 2
ExecuteStage/utils.py View File

@ -1,5 +1,4 @@
# 控制流程的暂停和继续
# 工具库
import csv import csv
import datetime import datetime
import json import json
@ -14,6 +13,47 @@ import requests
from urllib.parse import urlparse from urllib.parse import urlparse
import pymysql import pymysql
from lxml import etree from lxml import etree
import smtplib
from email.mime.text import MIMEText
from email.header import Header
def send_email(config):
"""
:param config:
"""
# 校验配置信息是否完整
# required_keys = ["host", "port", "username", "password", "from", "to", "subject", "content"]
# missing_keys = [key for key in required_keys if key not in config]
# if missing_keys:
# raise ValueError(f"邮件配置缺少必要的键: {', '.join(missing_keys)}")
try:
print("正在发送邮件到:" + config['to'])
message = MIMEText(config['content'], 'plain', 'utf-8')
message['From'] = Header(f"{config['username'].split('@')[0]} <{config['username']}>")
to_name_list = []
for address in config['to'].split(','):
address = address.strip()
name = address.split('@')[0]
to_name_list.append(f"{name} <{address}>")
to_name_list = ', '.join(to_name_list)
message['To'] = Header(to_name_list)
message['Subject'] = Header(config['subject'], 'utf-8')
# 使用SSL加密方式连接邮件服务器
smtp_server = smtplib.SMTP_SSL(config['host'], config['port'])
smtp_server.login(config['username'], config['password'])
to_address_list = config['to'].split(',')
smtp_server.sendmail(config['username'], to_address_list, message.as_string())
print("邮件发送成功|Email sent successfully")
except Exception as e:
print(f"无法发送邮件,发生错误:{e}")
print(f"Failed to send email, error: {e}")
finally:
try:
smtp_server.quit()
except:
pass
def is_valid_url(url): def is_valid_url(url):

Loading…
Cancel
Save