完私募基金管理人基金产品

Utils.py

@@ -1,5 +1,8 @@
 import re
 import random
+import time
+from datetime import datetime
+from http.client import RemoteDisconnected
 
 
 def camel_to_snake(name: str) -> str:
@@ -19,3 +22,23 @@ def convert_columns_to_snake_case(columns: str) -> str:
 def random_float():
     # 生成一个随机的浮点数并四舍五入到14位小数
     return round(random.random(), 14)
+
+
+def data_time():
+    # 输出当前时间，精确到毫秒
+    return datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]  # `[:-3]` 用于截取前3位保留到毫秒
+
+
+def get_conn_result(conn, file_name=""):
+    max_retries = 3
+    for attempt in range(max_retries):
+        try:
+            result = conn.getresponse()
+            return result
+        except RemoteDisconnected:
+            if attempt < max_retries - 1:
+                print(f"RemoteDisconnected,{file_name} - {data_time()} 重试 {attempt + 1}/{max_retries}...")
+                time.sleep(10)  # 等待 10 秒后重试
+            else:
+                print(f"{file_name} - {data_time()}请求失败，达到最大重试次数")
+                raise

amac/amac_private_fund_manager_api.py

@@ -1,7 +1,9 @@
 import json
 import time
+
 import Utils
 from time import sleep
+from http.client import RemoteDisconnected
 from Config import mysql_pool, conn, headers, page, size
 
 print("开始时间(精确到毫秒)：", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), time.time() * 1000)
@@ -50,14 +52,14 @@ this_size = size
 payload = json.dumps({})
 
 http_url = "/amac-infodisc/api/pof/fund"
+
 print(f"请求url={http_url}?rand={Utils.random_float()}&page={this_page}&size={this_size}")
 
 conn.request("POST",
              f"{http_url}?rand={Utils.random_float()}&page={this_page}&size={this_size}",
              payload, headers)
 
-res = conn.getresponse()
-data = res.read()
+data = Utils.get_conn_result(conn, __file__).read()
 json_str = data.decode("utf-8")
 
 try:

amac/main.py

@@ -6,7 +6,7 @@ print("执行 开始")
 python_files = [
     # 'member_api.py',  # 会员机构综合查询-会员表
     # 'person_org_registration_api.py',  # 基金从业人员资格注册信息-人员机构注册表
-    # 'person_org_registration_user_api.py',  # 基金从业人员资格注册信息-人员机构用户信息表、人员机构用户证书表
+    'person_org_registration_user_api.py',  # 基金从业人员资格注册信息-人员机构用户信息表、人员机构用户证书表
     'amac_private_fund_manager_api.py'  # 私募基金管理人基金产品
 ]
 

amac/person_org_registration_user_api.py

@@ -2,6 +2,7 @@ import json
 import random
 import time
 
+import Utils
 from Config import mysql_pool, conn, headers, page, size
 
 print("开始时间(精确到毫秒)：", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), time.time() * 1000)
@@ -69,7 +70,9 @@ def savetodb(data):
 
 
 # 开始获取amac_person_org_registration表中的数据并进行数据抓取
-amac_org_type = mysql_pool.fetchall("select * from amac_person_org_registration")
+# amac_org_type = mysql_pool.fetchall("select * from amac_person_org_registration")
+amac_org_type = mysql_pool.fetchall("select * from amac_person_org_registration where user_id >= 2201051041561049 "
+                                    "ORDER BY user_id asc")
 # amac_org_type = [{"user_id": "2309140927320704"}]
 
 for amac_org_type_item in amac_org_type:
@@ -88,7 +91,7 @@ for amac_org_type_item in amac_org_type:
                  f"{http_url}?rand={random_float}&page={this_page}&size={this_size}",
                  payload, headers)
 
-    res = conn.getresponse()
+    res = Utils.get_conn_result(conn, __file__).getresponse()
     data = res.read()
     json_str = data.decode("utf-8")
     # print("JSON String", json_str)