Some simple uses of crawler requests Library

requests Use Official documents

1、 and urllib It's the same Is to HTTP Send a request

1、GET request

python
import requests
url ="http://www.baidu.com"
r =requests.get(url=url)

Parameterized GET request , And save

import requests
from fake_useragent import UserAgent

headers ={
    
    "User-Agent":UserAgent().chrome
}
data ={
    
    "kw":" book ",
}
url ="https://tieba.baidu.com/f?"
r =requests.get(url=url,params=data)
#print(r.text)
with open("tieba.html","wb")as fp:
    fp.write(r.content)

2、 Custom request header

import requests
from fake_useragent import UserAgent
headers ={
    
    "User-Agent":UserAgent().random
}
url ="http://www.baidu.com"
r =requests.get(url=url,headers=headers)

3、 Response object
r.text View the response as a string
r.content Byte type view response
r.encoding View or set the coding type （utf8 perhaps gbk etc. ）
r.status_code Check the status code
r.headers View the response header
r.url View request url
r.json json Response in format
4、POST request
Must translate

import requests
from fake_useragent import UserAgent
headers ={
    
    "User-Agent":UserAgent().chrome
}
formdata ={
    
    "from":"en",
    "text":"dog",
    "to":"zh - Hans"
}
url = "https://cn.bing.com/tlookupv3?isVertical=1&&IG=E2B1985FC4C04621828E387B864AB30C&IID=translator.5027.2"
r =requests.post(url=url,headers=headers,data=formdata)
print(r.status_code)

5、
6、
7 agent
Cicci agent

import requests
from fake_useragent import UserAgent
headers ={
    
    "User-Agent":UserAgent().chrome
}
proxies ={
    
    "http":"http://60.2.44.182:30963",
}
url ="https://www.baidu.com/s?ie=UTF-8&wd=ip"
r =requests.get(url=url,headers=headers,proxies=proxies)
# print(r.text)
with open("daili.html","wb")as fp:
    fp.write(r.content)

8、cookie Session login

import requests
from fake_useragent import UserAgent

headers ={
    
    "User-Agent":UserAgent().chrome
}
# Encounter the problem of conversation , You need to create a session first ,, Then the created session object is saved cookie Information 
# All operations below pass s Visit ,s.post s.get
s =requests.Session()

post_url ="http://www.renren.com/ajaxLogin/login?1=1&uniqueTimestamp=202043200369 "
formdata={
    
    "icode":"",
    "key_id":"1",
    "email":" Own account ",
    "rkey":"1675e02911435a4867105858c862a18c",
    "password":" password ",
    "origURL":"http://www.renren.com/home",
    "f":"https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3DOyMkLmY-DmwEtjONGn3IO2CEVIEZHHjxeeHGy6z3olK%26wd%3D%26eqid%3Dceab788e00072b59000000025ebbe1a2",
    "domain":"renren.com",
    "captcha_type":"web_login"
}
#
r =s.post(url=post_url,headers=headers,data=formdata)
# print(r.text)

get_url ="http://www.renren.com/974397449/profile"
#s.get I'll take cookie Go to the page after login 
r =s.get(get_url,headers=headers)
print(r.text)