Nelze vybrat více než 25 témat Téma musí začínat písmenem nebo číslem, může obsahovat pomlčky („-“) a může být dlouhé až 35 znaků.

174 řádky
5.6 KiB

  1. #!/usr/bin/env python
  2. # -*- coding:utf-8 -*-
  3. #
  4. # Author : XueWeiHan
  5. # E-mail : 595666367@qq.com
  6. # Date : 2020-05-19 15:27
  7. # Desc : 获取最新的 GitHub 相关域名对应 IP
  8. import os
  9. import re
  10. import json
  11. import traceback
  12. from datetime import datetime, timezone, timedelta
  13. from collections import Counter
  14. from retry import retry
  15. import requests
  16. RAW_URL = [
  17. "github.githubassets.com",
  18. "central.github.com",
  19. "desktop.githubusercontent.com",
  20. "assets-cdn.github.com",
  21. "camo.githubusercontent.com",
  22. "github.map.fastly.net",
  23. "github.global.ssl.fastly.net",
  24. "gist.github.com",
  25. "github.io",
  26. "github.com",
  27. "api.github.com",
  28. "raw.githubusercontent.com",
  29. "user-images.githubusercontent.com",
  30. "favicons.githubusercontent.com",
  31. "avatars5.githubusercontent.com",
  32. "avatars4.githubusercontent.com",
  33. "avatars3.githubusercontent.com",
  34. "avatars2.githubusercontent.com",
  35. "avatars1.githubusercontent.com",
  36. "avatars0.githubusercontent.com",
  37. "avatars.githubusercontent.com",
  38. "codeload.github.com",
  39. "github-cloud.s3.amazonaws.com",
  40. "github-com.s3.amazonaws.com",
  41. "github-production-release-asset-2e65be.s3.amazonaws.com",
  42. "github-production-user-asset-6210df.s3.amazonaws.com",
  43. "github-production-repository-file-5c1aeb.s3.amazonaws.com",
  44. "githubstatus.com",
  45. "github.community",
  46. "media.githubusercontent.com"]
  47. IPADDRESS_PREFIX = ".ipaddress.com"
  48. HOSTS_TEMPLATE = """# GitHub520 Host Start
  49. {content}
  50. # Update time: {update_time}
  51. # Star me GitHub url: https://github.com/521xueweihan/GitHub520
  52. # GitHub520 Host End\n"""
  53. def write_file(hosts_content: str, update_time: str):
  54. output_doc_file_path = os.path.join(os.path.dirname(__file__), "README.md")
  55. template_path = os.path.join(os.path.dirname(__file__),
  56. "README_template.md")
  57. # 应该取消 write yaml file,改成 gitee gist 地址同步(国内访问流畅)
  58. write_yaml_file(hosts_content)
  59. with open(output_doc_file_path, "r") as old_readme_fb:
  60. old_content = old_readme_fb.read()
  61. old_hosts = old_content.split("```bash")[1].split("```")[0].strip()
  62. old_hosts = old_hosts.split("# Update time:")[0]
  63. if old_hosts == hosts_content:
  64. print("host not change")
  65. return False
  66. with open(template_path, "r") as temp_fb:
  67. template_str = temp_fb.read()
  68. hosts_content = template_str.format(hosts_str=hosts_content,
  69. update_time=update_time)
  70. with open(output_doc_file_path, "w") as output_fb:
  71. output_fb.write(hosts_content)
  72. return True
  73. def write_yaml_file(hosts_content: str, ):
  74. output_yaml_file_path = os.path.join(os.path.dirname(__file__), 'hosts')
  75. with open(output_yaml_file_path, "w") as output_yaml_fb:
  76. output_yaml_fb.write(hosts_content)
  77. def make_ipaddress_url(raw_url: str):
  78. """
  79. ipaddress url
  80. :param raw_url: url
  81. :return: ipaddress url
  82. """
  83. dot_count = raw_url.count(".")
  84. if dot_count > 1:
  85. raw_url_list = raw_url.split(".")
  86. tmp_url = raw_url_list[-2] + "." + raw_url_list[-1]
  87. ipaddress_url = "https://" + tmp_url + IPADDRESS_PREFIX + "/" + raw_url
  88. else:
  89. ipaddress_url = "https://" + raw_url + IPADDRESS_PREFIX
  90. return ipaddress_url
  91. @retry(tries=3)
  92. def get_ip(session: requests.session, raw_url: str):
  93. url = make_ipaddress_url(raw_url)
  94. try:
  95. rs = session.get(url, timeout=5)
  96. pattern = r"\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b"
  97. ip_list = re.findall(pattern, rs.text)
  98. ip_counter_obj = Counter(ip_list).most_common(1)
  99. if ip_counter_obj:
  100. return raw_url, ip_counter_obj[0][0]
  101. raise Exception("ip address empty")
  102. except Exception as ex:
  103. print("get: {}, error: {}".format(url, ex))
  104. raise Exception
  105. @retry(tries=3)
  106. def update_gitee_gist(session: requests.session, host_content):
  107. gitee_token = os.getenv("gitee_token")
  108. gitee_gist_id = os.getenv("gitee_gist_id")
  109. gist_file_name = os.getenv("gitee_gist_file_name")
  110. url = "https://gitee.com/api/v5/gists/{}".format(gitee_gist_id)
  111. headers = {
  112. "Content-Type": "application/json"}
  113. data = {
  114. "access_token": gitee_token,
  115. "files": {gist_file_name: {"content": host_content}},
  116. "public": "true"}
  117. json_data = json.dumps(data)
  118. try:
  119. response = session.patch(url, data=json_data, headers=headers,
  120. timeout=20)
  121. if response.status_code == 200:
  122. print("update gitee gist success")
  123. else:
  124. print("update gitee gist fail: {} {}".format(response.status_code,
  125. response.content))
  126. except Exception as e:
  127. traceback.print_exc(e)
  128. raise Exception(e)
  129. def main():
  130. session = requests.session()
  131. content = ""
  132. for raw_url in RAW_URL:
  133. try:
  134. host_name, ip = get_ip(session, raw_url)
  135. content += ip.ljust(30) + host_name + "\n"
  136. except Exception:
  137. continue
  138. if not content:
  139. return
  140. update_time = datetime.utcnow().astimezone(
  141. timezone(timedelta(hours=8))).replace(microsecond=0).isoformat()
  142. hosts_content = HOSTS_TEMPLATE.format(content=content, update_time=update_time)
  143. has_change = write_file(hosts_content, update_time)
  144. if has_change:
  145. try:
  146. update_gitee_gist(session, hosts_content)
  147. except Exception as e:
  148. print("update gitee gist fail:{}".format(e))
  149. print(hosts_content)
  150. if __name__ == '__main__':
  151. main()