爬虫Requests库的一些简单用法

requests的使用官方文档

1、 和urllib是一样的 都是给HTTP发送请求

1、GET请求

python
import requests
url ="http://www.baidu.com"
r =requests.get(url=url)

带参数的GET请求，并保存

import requests
from fake_useragent import UserAgent

headers ={
    
    "User-Agent":UserAgent().chrome
}
data ={
    
    "kw":"书",
}
url ="https://tieba.baidu.com/f?"
r =requests.get(url=url,params=data)
#print(r.text)
with open("tieba.html","wb")as fp:
    fp.write(r.content)

2、定制请求头

import requests
from fake_useragent import UserAgent
headers ={
    
    "User-Agent":UserAgent().random
}
url ="http://www.baidu.com"
r =requests.get(url=url,headers=headers)

3、响应对像
r.text 字符串形式查看响应
r.content 字节类型查看响应
r.encoding 查看或者设置编码类型（utf8或者gbk 等）
r.status_code 查看状态码
r.headers 查看响应头部
r.url 查看请求url
r.json json格式的响应
4、POST请求
必应翻译

import requests
from fake_useragent import UserAgent
headers ={
    
    "User-Agent":UserAgent().chrome
}
formdata ={
    
    "from":"en",
    "text":"dog",
    "to":"zh - Hans"
}
url = "https://cn.bing.com/tlookupv3?isVertical=1&&IG=E2B1985FC4C04621828E387B864AB30C&IID=translator.5027.2"
r =requests.post(url=url,headers=headers,data=formdata)
print(r.status_code)

5、
6、
7代理
西刺代理

import requests
from fake_useragent import UserAgent
headers ={
    
    "User-Agent":UserAgent().chrome
}
proxies ={
    
    "http":"http://60.2.44.182:30963",
}
url ="https://www.baidu.com/s?ie=UTF-8&wd=ip"
r =requests.get(url=url,headers=headers,proxies=proxies)
# print(r.text)
with open("daili.html","wb")as fp:
    fp.write(r.content)

8、cookie会话登录

import requests
from fake_useragent import UserAgent

headers ={
    
    "User-Agent":UserAgent().chrome
}
#碰到会话的问题，就要先创建会话,,那么创建的那个会话对象就保存了cookie信息
#往下所有的操作都通过s进行访问,s.post s.get
s =requests.Session()

post_url ="http://www.renren.com/ajaxLogin/login?1=1&uniqueTimestamp=202043200369 "
formdata={
    
    "icode":"",
    "key_id":"1",
    "email":"自己的账号",
    "rkey":"1675e02911435a4867105858c862a18c",
    "password":"密码",
    "origURL":"http://www.renren.com/home",
    "f":"https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3DOyMkLmY-DmwEtjONGn3IO2CEVIEZHHjxeeHGy6z3olK%26wd%3D%26eqid%3Dceab788e00072b59000000025ebbe1a2",
    "domain":"renren.com",
    "captcha_type":"web_login"
}
#
r =s.post(url=post_url,headers=headers,data=formdata)
# print(r.text)

get_url ="http://www.renren.com/974397449/profile"
#s.get就会带着cookie去访问登陆后的页面
r =s.get(get_url,headers=headers)
print(r.text)