获取所有股票历史行情数据

第一段 所有股票 周期：天  复权：不复权

# encoding:utf-8
import os
import sys
import pandas as qh_pd
import datetime
import time
import random
if __name__ == "__main__":
    print("本文件引用")
    qh_path = os.path.abspath(os.path.join(os.getcwd(), ".."))  # 返回当前工作目录   先到达 QH_网络配置 的父文件夹路径
    sys.path.append(qh_path)  # 添加自己指定的搜索路径
else:
    print("外文件引用")
    qh_path = os.path.abspath(os.path.join(os.getcwd(), ".."))  # 返回当前工作目录   先到达 QH_网络配置 的父文件夹路径
    sys.path.append(qh_path)  # 添加自己指定的搜索路径
#引入自己的文件模块
import QH001_Spider_Object.QH_DongFang_Object as Qh_Req_Obj           # 东方财富
import QH001_Spider_Object.QH_SouHu_Home_Object as Qh_SHH_Obj         # 搜狐交易日
import QH_Stock_Treasure_Box.qh_sqlite_orm_json as qh_db              # 数据库
import QH_Stock_Treasure_Box.qh_Log as qh_log                         # 日志
import QH_Stock_Treasure_Box.qh_parameter_chuanru as qh_chcan         # 传入参数
import QH_DF_Interface as qh_df_interface
import QH_IP_KDL as qh_ip_m                                           #ip

if __name__ == "__main__":

    qh_Log = qh_log.Qh_log_write(qh_path, "Qh_DF_His_D_Log")
    qh_Trading_day = Qh_SHH_Obj.QH_Get_Dangri_Date()  # 同一个文件的函数  日期会自动传入
    qh_Log.qh_Log("获取交易日期-搜狐，交易日期：{}".format(qh_Trading_day))
    qh_all_stock_list_sql = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00005")
    qh_Log.qh_Log("获取所有股票代码，代码列表：{}".format(qh_all_stock_list_sql))
    qh_ip_list = qh_ip_m.Qh_Get_Ip(qh_path, qh_flag="B")
    qh_Log.qh_Log("获取所有HTTP代理IP，HTTP代理IP列表：{}".format(qh_ip_list))
    qh_wangzhan0 = "DF"  # 网站代码
    qh_Log.qh_Log("网站代码：{}".format(qh_wangzhan0))
    qh_klt_dic = {"天": "D", "周": "W", "月": "M"}
    qh_Log.qh_Log("周期字典：{}".format(qh_klt_dic))
    qh_fqt_dic = {"不复权": "0", "前复权": "1", "后复权": "2"}
    qh_Log.qh_Log("是否复权字典：{}".format(qh_fqt_dic))
    qh_db_canshu = qh_chcan.Qh_InF_To_DB(qh_path, [], qh_spider_id="QH_Spider_00008", qh_canshu=True)
    qh_Log.qh_Log("获取所有数据库参数：{}".format(qh_db_canshu))
    qh_table_00 = qh_db_canshu['QH_Table']  # 需补充参数的组合表名
    qh_Log.qh_Log("需补充参数的组合表名：{}".format(qh_table_00))
    qh_sql0 = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00015", qh_sql0="", qh_ret_sql=True)
    qh_Log.qh_Log("获取需要补充参数的股票流记录SQL：{}".format(qh_sql0))
    #循环所有股票列表
    for qh_i, qh_row in enumerate(qh_all_stock_list_sql[1][1]):
        qh_code = qh_row[15]   # 股票代码
        qh_zjs = qh_row[16]    # 证交所
        qh_name = qh_row[17]   # 股票名称
        # 组合股票代码
        if str(qh_zjs) == "0":
            qh_code = "SZ{}".format(qh_code)
        else:
            qh_code = "SH{}".format(qh_code)
        qh_Log.qh_Log("正在获取第{}支股票，股票代码：{}，证交所：{}，股票名称：{}".format(qh_i, qh_code, qh_zjs, qh_name))
        qh_klt = "天"
        qh_fqt = "不复权"
        qh_Log.qh_Log("周期：{}，复权：{}".format(qh_klt, qh_fqt))
        qh_df_interface.Qh_Get_His_Flow(qh_code, qh_name, qh_Trading_day, qh_klt, qh_fqt,
                                        qh_wangzhan0, qh_klt_dic, qh_fqt_dic,
                                        qh_table_00, qh_sql0, qh_Log, qh_ip_list)

第二段 所有股票 所有周期：天/周/月  所有复权：不复权/前复权/后复权

# encoding:utf-8
import os
import sys
import pandas as qh_pd
import datetime
import time
import random
if __name__ == "__main__":
    print("本文件引用")
    qh_path = os.path.abspath(os.path.join(os.getcwd(), ".."))  # 返回当前工作目录   先到达 QH_网络配置 的父文件夹路径
    sys.path.append(qh_path)  # 添加自己指定的搜索路径
else:
    print("外文件引用")
    qh_path = os.path.abspath(os.path.join(os.getcwd(), ".."))  # 返回当前工作目录   先到达 QH_网络配置 的父文件夹路径
    sys.path.append(qh_path)  # 添加自己指定的搜索路径
#引入自己的文件模块
import QH001_Spider_Object.QH_DongFang_Object as Qh_Req_Obj           # 东方财富
import QH001_Spider_Object.QH_SouHu_Home_Object as Qh_SHH_Obj         # 搜狐交易日
import QH_Stock_Treasure_Box.qh_sqlite_orm_json as qh_db              # 数据库
import QH_Stock_Treasure_Box.qh_Log as qh_log                         # 日志
import QH_Stock_Treasure_Box.qh_parameter_chuanru as qh_chcan         # 传入参数
import QH_DF_Interface as qh_df_interface
import QH_IP_KDL as qh_ip_m                                           #ip


if __name__ == "__main__":

    qh_Log = qh_log.Qh_log_write(qh_path, "Qh_DF_His_DWM_Log")
    qh_Trading_day = Qh_SHH_Obj.QH_Get_Dangri_Date()                  # 同一个文件的函数  日期会自动传入
    qh_Log.qh_Log("获取交易日期-搜狐，交易日期：{}".format(qh_Trading_day))
    qh_all_stock_list_sql = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00005")
    qh_Log.qh_Log("获取所有股票代码，代码列表：{}".format(qh_all_stock_list_sql))
    qh_ip_list = qh_ip_m.Qh_Get_Ip(qh_path, qh_flag="B")
    qh_Log.qh_Log("获取所有HTTP代理IP，HTTP代理IP列表：{}".format(qh_ip_list))
    qh_wangzhan0 = "DF"  # 网站代码
    qh_Log.qh_Log("网站代码：{}".format(qh_wangzhan0))
    qh_klt_dic = {"天": "D", "周": "W", "月": "M"}
    qh_Log.qh_Log("周期字典：{}".format(qh_klt_dic))
    qh_fqt_dic = {"不复权": "0", "前复权": "1", "后复权": "2"}
    qh_Log.qh_Log("是否复权字典：{}".format(qh_fqt_dic))
    qh_db_canshu = qh_chcan.Qh_InF_To_DB(qh_path, [], qh_spider_id="QH_Spider_00008", qh_canshu=True)
    qh_Log.qh_Log("获取所有数据库参数：{}".format(qh_db_canshu))
    qh_table_00 = qh_db_canshu['QH_Table']  # 需补充参数的组合表名
    qh_Log.qh_Log("需补充参数的组合表名：{}".format(qh_table_00))
    qh_sql0 = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00015", qh_sql0="", qh_ret_sql=True)
    qh_Log.qh_Log("获取需要补充参数的股票流记录SQL：{}".format(qh_sql0))
    # 循环所有股票列表
    for qh_i,qh_row in enumerate(qh_all_stock_list_sql[1][1]):
        qh_code = qh_row[15]  # 股票代码
        qh_zjs = qh_row[16]  # 证交所
        qh_name = qh_row[17]  # 股票名称
        # 组合股票代码
        if str(qh_zjs) == "0":
            qh_code = "SZ{}".format(qh_code)
        else:
            qh_code = "SH{}".format(qh_code)
        qh_Log.qh_Log("正在获取第{}支股票，股票代码：{}，证交所：{}，股票名称：{}".format(qh_i,qh_code,qh_zjs,qh_name))
        for qh_key in qh_klt_dic:
            qh_klt = qh_key          #周期
            for qh_key01 in qh_fqt_dic:
                qh_fqt = qh_key01    #复权
                qh_Log.qh_Log("周期：{}，复权：{}".format(qh_klt, qh_fqt))
                qh_df_interface.Qh_Get_His_Flow(qh_code, qh_name, qh_Trading_day, qh_klt, qh_fqt,
                                                qh_wangzhan0, qh_klt_dic, qh_fqt_dic,
                                                qh_table_00, qh_sql0, qh_Log,qh_ip_list)

第三段：调用函数

def Qh_Get_His_Flow(qh_code,qh_name,qh_Trading_day, qh_klt, qh_fqt,qh_wangzhan0,qh_klt_dic,qh_fqt_dic,qh_table_00,qh_sql0,qh_Log,qh_ip_list=[]):
    """
    获取所有历史交易数据函数         作者：阙辉
    :param qh_code:         股票代码  带证交所 SZ000001
    :param qh_name:         股票名称
    :param qh_Trading_day:  最近交易日期
    :param qh_klt:          周期
    :param qh_fqt:          复权
    :param qh_wangzhan0:    网站
    :param qh_klt_dic:      周期字典
    :param qh_fqt_dic:      复权字典
    :param qh_table_00:     需传参数的表名
    :param qh_sql0:         需传参数的SQL
    :param qh_ip_list:      代理IP
    :return:
    """
    qh_flow_jilu = []      #初始化记录列表
    qh_wangzhan = qh_wangzhan0 + "_" + str(qh_klt_dic[qh_klt]) + "_" + str(qh_fqt_dic[qh_fqt])   #拼接网站id  关键代码，隔离了周期和复权
    qh_flow_jilu.append(qh_wangzhan)     # qh_flow_jilu 0 网站
    qh_flow_jilu.append(qh_code)         # qh_flow_jilu 1 股票代码
    qh_flow_jilu.append(qh_name)         # qh_flow_jilu 2 股票名称
    qh_flow_jilu.append(qh_Trading_day)  # qh_flow_jilu 3 交易日期
    # 爬取历史股票
    qh_type_gg = "QH_JY_HIS"         # 获取历史数据类型  历史交易数据
    qh_flow_jilu.append(qh_type_gg)  # #qh_flow_jilu 4 交易类型   历史交易
    qh_end_date = qh_Trading_day     # 最近一个交易日期
    # 判断最近交易日期的历史数据是否爬取
    try:
        qh_sql = qh_sql0.format(qh_wangzhan, qh_code, qh_type_gg)
        qh_list_qq = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00015", qh_sql0=qh_sql, qh_ret_sql=False,qh_filed=False)
        qh_list_qq = qh_list_qq[1][1]
        if len(qh_list_qq) > 0:
            qh_list_qq = qh_list_qq[0]      # 取第一行
            qh_pq_yy = qh_list_qq[8]        # 爬取状态
            qh_zong_count = qh_list_qq[10]  # 上一次总数量
            qh_stare_date = qh_list_qq[6]   # 上一次爬取交易日期
            qh_Log.qh_Log("股票流有记录，记录为：爬取状态：{}，上一次总数量：{}，上一次爬取交易日期：{}".format(qh_pq_yy, qh_zong_count, qh_stare_date))
        else:
            qh_pq_yy = "N"  # 爬取状态
            qh_zong_count = 0  # 总数量
            qh_stare_date = "1989-01-01"    # 默认起始日期
            qh_Log.qh_Log("股票流无记录，默认记录为：爬取状态：{}，上一次总数量：{}，上一次爬取交易日期：{}".format(qh_pq_yy, qh_zong_count, qh_stare_date))
    except:
        qh_pq_yy = "N"  # 爬取状态
        qh_zong_count = 0  # 总数量
        qh_stare_date = "1989-01-01"  # 默认起始日期
        qh_Log.qh_Log("股票流表未创建，默认记录为：爬取状态：{}，上一次总数量：{}，上一次爬取交易日期：{}".format(qh_pq_yy, qh_zong_count, qh_stare_date))

    # 判断是否要请求数据
    if qh_pq_yy == "Y" and str(qh_end_date) == str(qh_stare_date):
        qh_Log.qh_Log("股票代码（{}）已爬取，最后交易日期{}，阙辉！".format(qh_code, qh_end_date))
    else:
        # 随机从IP列表中获取一个代理IP
        if len(qh_ip_list) > 0:
            qh_ip_kk = random.choice(qh_ip_list)
            qh_htt = qh_ip_kk[1]
            qh_htt = "https"
            qh_ip = qh_ip_kk[0]
            qh_htt_ip = "{}\\:{}".format(qh_htt, qh_ip)
            qh_proxy = {
                qh_htt: qh_htt_ip
            }
            qh_Log.qh_Log("随机获取代理IP（{}），阙辉！".format(qh_proxy))
        else:
            qh_proxy = {}
            qh_Log.qh_Log("不使用代理IP，阙辉！")
        # 请求数据
        qh_Log.qh_Log("开始请求（{}），阙辉！".format(qh_code))
        qh_his_jiaoyi_list = Qh_DF_His_Flow_Interface01(qh_Trading_day, qh_code,
                                                                        qh_stare_date=qh_stare_date,
                                                                        qh_end_date=qh_end_date, qh_klt=qh_klt,
                                                                        qh_fqt=qh_fqt,
                                                                        qh_spider_id="QH_Spider_00008",
                                                                        qh_out="qh_list", qh_day_t=False,
                                                                        qh_proxy=qh_proxy)
        qh_his_jiaoyi_list_len = len(qh_his_jiaoyi_list)  # 统计本次的数据量
        qh_new_table = qh_table_00.format(qh_code)  # 生成表格名
        qh_Log.qh_Log("组合存储表格名（{}），阙辉！".format(qh_new_table))
        # 存储数据 返回存储状态
        if qh_his_jiaoyi_list_len > 0:
            qh_Log.qh_Log("请求（{}）数据成功，本次共{}条，阙辉！".format(qh_code,qh_his_jiaoyi_list_len))
            qh_cc_star = qh_chcan.Qh_InF_To_DB(qh_path, qh_his_jiaoyi_list, qh_spider_id="QH_Spider_00008_D",
                                               qh_canshu=False, qh_new_table=qh_new_table)
        else:
            qh_cc_star = False
        #判断数据是否更新成功，更新成功则
        if qh_cc_star:
            qh_Log.qh_Log("存储（{}）数据成功，本次共{}条，阙辉！".format(qh_code, qh_his_jiaoyi_list_len))
            qh_flow_jilu.append("Y")  # #qh_flow_jilu 5 是否成功 爬取状态
            qh_flow_jilu.append(qh_his_jiaoyi_list_len)  # #qh_flow_jilu 6 本次数量
            qh_flow_jilu.append(qh_his_jiaoyi_list_len + int(qh_zong_count))  # #qh_flow_jilu 7 总数量
            qh_flow_jilu = qh_chcan.Qh_TC_List(qh_flow_jilu, 18)  # 补充不满18的字段为空  #qh_flow_jilu 8以后为空的字段
            if qh_chcan.Qh_InF_To_DB(qh_path, [qh_flow_jilu], qh_spider_id="QH_Spider_99999"):  # 更新数据存储状态
                qh_Log.qh_Log("更新股票记录流成功，股票记录流为：{}，阙辉！".format(qh_flow_jilu))