amac/amac/amac_private_fund_manager_api.py

import json
import time

import Utils
from time import sleep
from http.client import RemoteDisconnected
from Config import mysql_pool, conn, headers, page, size

print("开始时间(精确到毫秒)：", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), time.time() * 1000)


# 因为这里数据一条人员信息amac_member_user中包含了对应的多个证书，所以需要先循环把证书拿出来
def savetodb(data):
    # 判断是否为空，为空则跳过直接返回
    if data is None: return

    person_record = []
    for item in data:
        # 构建人员信息记录
        this_data = {
            'id': item.get("id"),
            'manager_id': "",
            'fund_no': item.get("fundNo", ""),
            'fund_name': item.get("fundName"),
            'manager_name': item.get("managerName"),
            'manager_type': item.get("managerType"),
            'working_state': item.get("workingState"),
            'put_on_record_date': item.get("putOnRecordDate"),
            'last_quarter_update': item.get("lastQuarterUpdate"),
            'is_depute_manage': item.get("isDeputeManage"),
            'url': item.get("url"),
            'establish_date': item.get("establishDate"),
            'manager_url': item.get("managerUrl"),
            'mandator_name': item.get("mandatorName")
        }

        managers_info = item.get("managersInfo", [])

        # 找到到对应的managerId
        if managers_info:
            for info in managers_info:
                this_data['manager_id'] = info.get("managerId", "")
                break

        person_record.append(this_data)
    # 批量插入
    mysql_pool.insert('amac_private_fund_manager', person_record)


this_page = page
this_size = size
payload = json.dumps({})

http_url = "/amac-infodisc/api/pof/fund"

print(f"请求url={http_url}?rand={Utils.random_float()}&page={this_page}&size={this_size}")

conn.request("POST",
             f"{http_url}?rand={Utils.random_float()}&page={this_page}&size={this_size}",
             payload, headers)

data = Utils.get_conn_result(conn, __file__).read()
json_str = data.decode("utf-8")

try:
    json_obj = json.loads(json_str)
    print("获取到总条数为:", json_obj.get("totalPages"))

    data_to_insert = json_obj.get("content")

    if data_to_insert is not None:
        # 先睡一秒，确保峰值不会太高
        sleep(1)

    print(f"\n开始插入数据==>{this_page}*{this_size}\n")
    savetodb(data_to_insert)

    if json_obj.get("totalPages") > 1:
        for i in range(1, json_obj.get("totalPages")):
            this_page = i
            print(f"请求url={http_url}?rand={Utils.random_float()}&page={this_page}&size={this_size}")
            conn.request("POST",
                         f"{http_url}?rand={Utils.random_float()}&page={this_page}&size={this_size}",
                         payload, headers)
            res = conn.getresponse()
            data = res.read()
            json_str = data.decode("utf-8")
            print("JSON String", json_str)
            try:
                json_obj_arr = json.loads(json_str)
                print("获取到总条数为:", json_obj_arr.get("totalPages"))
                data_to_insert = json_obj_arr.get("content")
                if data_to_insert is not None:
                    # 先睡一秒，确保峰值不会太高
                    time.sleep(1)
                print(f"\n开始插入数据==>{i}*{this_size}\n")
                savetodb(data_to_insert)
            except json.JSONDecodeError as e:
                print(f"循环 Error decoding JSON: {e}")
except json.JSONDecodeError as e:
    print(f"请求 Error decoding JSON: {e}")

print("结束时间(精确到毫秒)：", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), time.time() * 1000)