Crawler request module

request modular

1、 Basic operation

''' -  Appoint url -  Initiate request  -  Get the data of the response  -  Persistent storage  '''
import requests
if __name__ == "__main__":
    #  Appoint url
    url = "https://www.sogou.com/"
    #  Send a request 
    response = requests.get(url=url)
    #  Obtain corresponding data ,text The corresponding data of string type is returned 
    page_text = response.text
    print(page_text)
    #  Persistent storage 
    with open("./sougou.html", "w", encoding="UTF-8") as fp:
        fp.write(page_text)

2、UA camouflage

# UA:user-agent
# UA camouflage 
import requests
if __name__ == "__main__":
    url = "https://www.sogou.com/web"

    kw = input('enter a word:')
    # UA camouflage 
    headers = {
    
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:96.0) Gecko/20100101 Firefox/96.0'
    }
    #  Dictionaries , amount to url Parameters in the request of 
    param = {
    
        'query': kw
    }
    #  agent , Normal use https and sockets5
    proxies = {
    "HTTP":"http://123.169.122.201:9999"}
    response = requests.get(url=url, params=param, headers=headers, proxies=proxies)
    page_text = response.text
    with open("./sougou.html", "w", encoding="UTF-8") as fp:
        fp.write(page_text)

3、 Small cases

#  Partial data of the whole page 
#  Crack Baidu translation 
''' - poat request ( With parameters ) -  The corresponding data is a set json data  json.load('json data ') It can be changed into string form  '''

import requests
import json
if __name__ == "__main__":
    post_url = "https://fanyi.baidu.com/sug"
    # post Requested parameters 
    data = {
    
        'kw': 'dog'
    }
    #UA camouflage 
    headers = {
    
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:96.0) Gecko/20100101 Firefox/96.0'
    }
    response = requests.post(url=post_url, data=data, headers=headers)
    #  Back to a json object ( The corresponding data must be json data )
    dic_obj = post_text = response.json()
    # print(dic_obj)
    #  Storage 
    with open('./dog.json', 'w', encoding='UTF-8') as fp:
        json.dump(dic_obj, fp=fp, ensure_ascii=False, indent=2)