Get the historical quotation data of all stocks

The first paragraph All shares cycle ： God   Reversion ： Non reversion

# encoding:utf-8
import os
import sys
import pandas as qh_pd
import datetime
import time
import random
if __name__ == "__main__":
    print(" References in this document ")
    qh_path = os.path.abspath(os.path.join(os.getcwd(), ".."))  #  Return to the current working directory     First arrive  QH_ The network configuration   The path of the parent folder 
    sys.path.append(qh_path)  #  Add your own search path 
else:
    print(" External document references ")
    qh_path = os.path.abspath(os.path.join(os.getcwd(), ".."))  #  Return to the current working directory     First arrive  QH_ The network configuration   The path of the parent folder 
    sys.path.append(qh_path)  #  Add your own search path 
# Introduce your own file module 
import QH001_Spider_Object.QH_DongFang_Object as Qh_Req_Obj           #  Oriental wealth 
import QH001_Spider_Object.QH_SouHu_Home_Object as Qh_SHH_Obj         #  Sohu trading day 
import QH_Stock_Treasure_Box.qh_sqlite_orm_json as qh_db              #  database 
import QH_Stock_Treasure_Box.qh_Log as qh_log                         #  journal 
import QH_Stock_Treasure_Box.qh_parameter_chuanru as qh_chcan         #  Pass in the parameter 
import QH_DF_Interface as qh_df_interface
import QH_IP_KDL as qh_ip_m                                           #ip

if __name__ == "__main__":

    qh_Log = qh_log.Qh_log_write(qh_path, "Qh_DF_His_D_Log")
    qh_Trading_day = Qh_SHH_Obj.QH_Get_Dangri_Date()  #  Functions of the same file    The date will be automatically passed in 
    qh_Log.qh_Log(" Get transaction date - sohu , Transaction date ：{}".format(qh_Trading_day))
    qh_all_stock_list_sql = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00005")
    qh_Log.qh_Log(" Get all stock codes , Code list ：{}".format(qh_all_stock_list_sql))
    qh_ip_list = qh_ip_m.Qh_Get_Ip(qh_path, qh_flag="B")
    qh_Log.qh_Log(" Get all HTTP agent IP,HTTP agent IP list ：{}".format(qh_ip_list))
    qh_wangzhan0 = "DF"  #  Website code 
    qh_Log.qh_Log(" Website code ：{}".format(qh_wangzhan0))
    qh_klt_dic = {" God ": "D", " Zhou ": "W", " month ": "M"}
    qh_Log.qh_Log(" Cycle dictionary ：{}".format(qh_klt_dic))
    qh_fqt_dic = {" Non reversion ": "0", " The former restoration right ": "1", " Post restoration right ": "2"}
    qh_Log.qh_Log(" Whether to reply to the right dictionary ：{}".format(qh_fqt_dic))
    qh_db_canshu = qh_chcan.Qh_InF_To_DB(qh_path, [], qh_spider_id="QH_Spider_00008", qh_canshu=True)
    qh_Log.qh_Log(" Get all database parameters ：{}".format(qh_db_canshu))
    qh_table_00 = qh_db_canshu['QH_Table']  #  The name of the combined table with supplementary parameters 
    qh_Log.qh_Log(" The name of the combined table with supplementary parameters ：{}".format(qh_table_00))
    qh_sql0 = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00015", qh_sql0="", qh_ret_sql=True)
    qh_Log.qh_Log(" Get the stock flow record that requires supplementary parameters SQL：{}".format(qh_sql0))
    # Cycle through the list of all stocks 
    for qh_i, qh_row in enumerate(qh_all_stock_list_sql[1][1]):
        qh_code = qh_row[15]   #  Stock code 
        qh_zjs = qh_row[16]    #  The stock exchange 
        qh_name = qh_row[17]   #  The name of the stock 
        #  Combined stock code 
        if str(qh_zjs) == "0":
            qh_code = "SZ{}".format(qh_code)
        else:
            qh_code = "SH{}".format(qh_code)
        qh_Log.qh_Log(" Getting the {} Shares , Stock code ：{}, The stock exchange ：{}, The name of the stock ：{}".format(qh_i, qh_code, qh_zjs, qh_name))
        qh_klt = " God "
        qh_fqt = " Non reversion "
        qh_Log.qh_Log(" cycle ：{}, Reversion ：{}".format(qh_klt, qh_fqt))
        qh_df_interface.Qh_Get_His_Flow(qh_code, qh_name, qh_Trading_day, qh_klt, qh_fqt,
                                        qh_wangzhan0, qh_klt_dic, qh_fqt_dic,
                                        qh_table_00, qh_sql0, qh_Log, qh_ip_list)

The second paragraph All shares All cycles ： God / Zhou / month   All rights restored ： Non reversion / The former restoration right / Post restoration right

# encoding:utf-8
import os
import sys
import pandas as qh_pd
import datetime
import time
import random
if __name__ == "__main__":
    print(" References in this document ")
    qh_path = os.path.abspath(os.path.join(os.getcwd(), ".."))  #  Return to the current working directory     First arrive  QH_ The network configuration   The path of the parent folder 
    sys.path.append(qh_path)  #  Add your own search path 
else:
    print(" External document references ")
    qh_path = os.path.abspath(os.path.join(os.getcwd(), ".."))  #  Return to the current working directory     First arrive  QH_ The network configuration   The path of the parent folder 
    sys.path.append(qh_path)  #  Add your own search path 
# Introduce your own file module 
import QH001_Spider_Object.QH_DongFang_Object as Qh_Req_Obj           #  Oriental wealth 
import QH001_Spider_Object.QH_SouHu_Home_Object as Qh_SHH_Obj         #  Sohu trading day 
import QH_Stock_Treasure_Box.qh_sqlite_orm_json as qh_db              #  database 
import QH_Stock_Treasure_Box.qh_Log as qh_log                         #  journal 
import QH_Stock_Treasure_Box.qh_parameter_chuanru as qh_chcan         #  Pass in the parameter 
import QH_DF_Interface as qh_df_interface
import QH_IP_KDL as qh_ip_m                                           #ip


if __name__ == "__main__":

    qh_Log = qh_log.Qh_log_write(qh_path, "Qh_DF_His_DWM_Log")
    qh_Trading_day = Qh_SHH_Obj.QH_Get_Dangri_Date()                  #  Functions of the same file    The date will be automatically passed in 
    qh_Log.qh_Log(" Get transaction date - sohu , Transaction date ：{}".format(qh_Trading_day))
    qh_all_stock_list_sql = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00005")
    qh_Log.qh_Log(" Get all stock codes , Code list ：{}".format(qh_all_stock_list_sql))
    qh_ip_list = qh_ip_m.Qh_Get_Ip(qh_path, qh_flag="B")
    qh_Log.qh_Log(" Get all HTTP agent IP,HTTP agent IP list ：{}".format(qh_ip_list))
    qh_wangzhan0 = "DF"  #  Website code 
    qh_Log.qh_Log(" Website code ：{}".format(qh_wangzhan0))
    qh_klt_dic = {" God ": "D", " Zhou ": "W", " month ": "M"}
    qh_Log.qh_Log(" Cycle dictionary ：{}".format(qh_klt_dic))
    qh_fqt_dic = {" Non reversion ": "0", " The former restoration right ": "1", " Post restoration right ": "2"}
    qh_Log.qh_Log(" Whether to reply to the right dictionary ：{}".format(qh_fqt_dic))
    qh_db_canshu = qh_chcan.Qh_InF_To_DB(qh_path, [], qh_spider_id="QH_Spider_00008", qh_canshu=True)
    qh_Log.qh_Log(" Get all database parameters ：{}".format(qh_db_canshu))
    qh_table_00 = qh_db_canshu['QH_Table']  #  The name of the combined table with supplementary parameters 
    qh_Log.qh_Log(" The name of the combined table with supplementary parameters ：{}".format(qh_table_00))
    qh_sql0 = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00015", qh_sql0="", qh_ret_sql=True)
    qh_Log.qh_Log(" Get the stock flow record that requires supplementary parameters SQL：{}".format(qh_sql0))
    #  Cycle through the list of all stocks 
    for qh_i,qh_row in enumerate(qh_all_stock_list_sql[1][1]):
        qh_code = qh_row[15]  #  Stock code 
        qh_zjs = qh_row[16]  #  The stock exchange 
        qh_name = qh_row[17]  #  The name of the stock 
        #  Combined stock code 
        if str(qh_zjs) == "0":
            qh_code = "SZ{}".format(qh_code)
        else:
            qh_code = "SH{}".format(qh_code)
        qh_Log.qh_Log(" Getting the {} Shares , Stock code ：{}, The stock exchange ：{}, The name of the stock ：{}".format(qh_i,qh_code,qh_zjs,qh_name))
        for qh_key in qh_klt_dic:
            qh_klt = qh_key          # cycle 
            for qh_key01 in qh_fqt_dic:
                qh_fqt = qh_key01    # Reversion 
                qh_Log.qh_Log(" cycle ：{}, Reversion ：{}".format(qh_klt, qh_fqt))
                qh_df_interface.Qh_Get_His_Flow(qh_code, qh_name, qh_Trading_day, qh_klt, qh_fqt,
                                                qh_wangzhan0, qh_klt_dic, qh_fqt_dic,
                                                qh_table_00, qh_sql0, qh_Log,qh_ip_list)

The third paragraph ： Call function

def Qh_Get_His_Flow(qh_code,qh_name,qh_Trading_day, qh_klt, qh_fqt,qh_wangzhan0,qh_klt_dic,qh_fqt_dic,qh_table_00,qh_sql0,qh_Log,qh_ip_list=[]):
    """
     Get all historical transaction data functions           author ： Que Hui 
    :param qh_code:          Stock code    With stock exchange  SZ000001
    :param qh_name:          The name of the stock 
    :param qh_Trading_day:   Latest transaction date 
    :param qh_klt:           cycle 
    :param qh_fqt:           Reversion 
    :param qh_wangzhan0:     Website 
    :param qh_klt_dic:       Cycle dictionary 
    :param qh_fqt_dic:       Reply to the dictionary 
    :param qh_table_00:      The table name of the parameter to be transferred 
    :param qh_sql0:          Parameters to be transmitted SQL
    :param qh_ip_list:       agent IP
    :return:
    """
    qh_flow_jilu = []      # Initialize the record list 
    qh_wangzhan = qh_wangzhan0 + "_" + str(qh_klt_dic[qh_klt]) + "_" + str(qh_fqt_dic[qh_fqt])   # Splicing websites id   Key code , Isolate the cycle and restoration 
    qh_flow_jilu.append(qh_wangzhan)     # qh_flow_jilu 0  Website 
    qh_flow_jilu.append(qh_code)         # qh_flow_jilu 1  Stock code 
    qh_flow_jilu.append(qh_name)         # qh_flow_jilu 2  The name of the stock 
    qh_flow_jilu.append(qh_Trading_day)  # qh_flow_jilu 3  Transaction date 
    #  Crawl past stocks 
    qh_type_gg = "QH_JY_HIS"         #  Get historical data type    Historical transaction data 
    qh_flow_jilu.append(qh_type_gg)  # #qh_flow_jilu 4  Type of transaction     Historical transactions 
    qh_end_date = qh_Trading_day     #  The latest trading date 
    #  Judge whether the historical data of the latest trading date is crawled 
    try:
        qh_sql = qh_sql0.format(qh_wangzhan, qh_code, qh_type_gg)
        qh_list_qq = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00015", qh_sql0=qh_sql, qh_ret_sql=False,qh_filed=False)
        qh_list_qq = qh_list_qq[1][1]
        if len(qh_list_qq) > 0:
            qh_list_qq = qh_list_qq[0]      #  Take the first line 
            qh_pq_yy = qh_list_qq[8]        #  Crawl state 
            qh_zong_count = qh_list_qq[10]  #  Last total quantity 
            qh_stare_date = qh_list_qq[6]   #  Last crawl transaction date 
            qh_Log.qh_Log(" Stock flows are recorded , Record as ： Crawl state ：{}, Last total quantity ：{}, Last crawl transaction date ：{}".format(qh_pq_yy, qh_zong_count, qh_stare_date))
        else:
            qh_pq_yy = "N"  #  Crawl state 
            qh_zong_count = 0  #  Total quantity 
            qh_stare_date = "1989-01-01"    #  Default start date 
            qh_Log.qh_Log(" No record of stock flow , The default record is ： Crawl state ：{}, Last total quantity ：{}, Last crawl transaction date ：{}".format(qh_pq_yy, qh_zong_count, qh_stare_date))
    except:
        qh_pq_yy = "N"  #  Crawl state 
        qh_zong_count = 0  #  Total quantity 
        qh_stare_date = "1989-01-01"  #  Default start date 
        qh_Log.qh_Log(" Stock flow table not created , The default record is ： Crawl state ：{}, Last total quantity ：{}, Last crawl transaction date ：{}".format(qh_pq_yy, qh_zong_count, qh_stare_date))

    #  Determine whether to request data 
    if qh_pq_yy == "Y" and str(qh_end_date) == str(qh_stare_date):
        qh_Log.qh_Log(" Stock code （{}） Crawled , Last trading date {}, Que Hui ！".format(qh_code, qh_end_date))
    else:
        #  Random from IP Get a proxy from the list IP
        if len(qh_ip_list) > 0:
            qh_ip_kk = random.choice(qh_ip_list)
            qh_htt = qh_ip_kk[1]
            qh_htt = "https"
            qh_ip = qh_ip_kk[0]
            qh_htt_ip = "{}\\:{}".format(qh_htt, qh_ip)
            qh_proxy = {
                qh_htt: qh_htt_ip
            }
            qh_Log.qh_Log(" Get proxy randomly IP（{}）, Que Hui ！".format(qh_proxy))
        else:
            qh_proxy = {}
            qh_Log.qh_Log(" Don't use agents IP, Que Hui ！")
        #  Request data 
        qh_Log.qh_Log(" To request （{}）, Que Hui ！".format(qh_code))
        qh_his_jiaoyi_list = Qh_DF_His_Flow_Interface01(qh_Trading_day, qh_code,
                                                                        qh_stare_date=qh_stare_date,
                                                                        qh_end_date=qh_end_date, qh_klt=qh_klt,
                                                                        qh_fqt=qh_fqt,
                                                                        qh_spider_id="QH_Spider_00008",
                                                                        qh_out="qh_list", qh_day_t=False,
                                                                        qh_proxy=qh_proxy)
        qh_his_jiaoyi_list_len = len(qh_his_jiaoyi_list)  #  Count the amount of data this time 
        qh_new_table = qh_table_00.format(qh_code)  #  Generate table name 
        qh_Log.qh_Log(" Combined storage table name （{}）, Que Hui ！".format(qh_new_table))
        #  Store the data   Return to storage status 
        if qh_his_jiaoyi_list_len > 0:
            qh_Log.qh_Log(" request （{}） Data are successfully , Total {} strip , Que Hui ！".format(qh_code,qh_his_jiaoyi_list_len))
            qh_cc_star = qh_chcan.Qh_InF_To_DB(qh_path, qh_his_jiaoyi_list, qh_spider_id="QH_Spider_00008_D",
                                               qh_canshu=False, qh_new_table=qh_new_table)
        else:
            qh_cc_star = False
        # Judge whether the data is updated successfully , If the update is successful 
        if qh_cc_star:
            qh_Log.qh_Log(" Storage （{}） Data are successfully , Total {} strip , Que Hui ！".format(qh_code, qh_his_jiaoyi_list_len))
            qh_flow_jilu.append("Y")  # #qh_flow_jilu 5  The success of   Crawl state 
            qh_flow_jilu.append(qh_his_jiaoyi_list_len)  # #qh_flow_jilu 6  Quantity this time 
            qh_flow_jilu.append(qh_his_jiaoyi_list_len + int(qh_zong_count))  # #qh_flow_jilu 7  Total quantity 
            qh_flow_jilu = qh_chcan.Qh_TC_List(qh_flow_jilu, 18)  #  Supplement dissatisfaction 18 The field of is empty   #qh_flow_jilu 8 Fields that will be empty in the future 
            if qh_chcan.Qh_InF_To_DB(qh_path, [qh_flow_jilu], qh_spider_id="QH_Spider_99999"):  #  Update data storage status 
                qh_Log.qh_Log(" Update stock record stream succeeded , The stock record flow is ：{}, Que Hui ！".format(qh_flow_jilu))