urllib urllib2

#-*-coding:utf-8-*-

import urllib import urllib2 import cookielib ##urllib url="http://www.qq.com" header={"User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36"}

def urllib_study(url):

urldata=urllib.urlopen(url) print(urldata.read().decode('gbk'))##网页内容 print(urldata.getcode())##http状态码 print(urldata.geturl())## request url print(urldata.info()) ##header info urldata.info().getparam('charset')获取header 编码 urllib.urlretrieve(url,'D:\\pic\\')##下载文件 def urlretrieve_study(url,savepath): def callback(a,b,c): down_progess=100.0*a*b/c if down_progess>100: down_progess=100 print '%.2f%%'%down_progess urllib.urlretrieve(url,savepath,callback)

#urlretrieve_study(url,'D:\\pics\\qq.html')

##urllib2 def urllib2_study(url): urldata=urllib2.urlopen(url)  

登录后复制

      
      

       
       #-*-coding:utf-8-*-
       
       
import urllib2
       
       
import random
       
       
url=""
       
       
useragent=["Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36",
       
       

       
       
]
       
       
def get_content(url,useragent):
       
       
    """
       
       
    @
       
       
    """
       
       
    random_header=random.choice(useragent)
       
       
    req=urllib2.Request(url)
       
       
    req.add_header('User-Agent',random_header)
       
       
    req.add_header('Host',)
       
       
    req.add_header("Refer",)
       
       
    req.add_header('GET',)
       
       
    ret=urllib2.urlopen(req).read()
       
       
    return ret
      
      
      
      

       
       
1.
       
       
2.
       
       
3.
       
       
4.
       
       
5.
       
       
6.
       
       
7.
       
       
8.
       
       
9.
       
       
10.
       
       
11.
       
       
12.
       
       
13.
       
       
14.
       
       
15.
       
       
16.
       
       
17.
       
       
18.
       
       
19.
      
      

　　

print(dir(urldata))##urldata.read.decode('gbk') urldata.getcode() urldata.info() urldata.geturl() cookie=cookielib.CookieJar() opener=urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie)) data=opener.open(url)##urllib2.urlopen其实也是调用opener.open print(data.read().decode('gbk')) def urllib2_post(url): values={'name':'howhy','age':32} data=urllib.urlencode(values) req=urllib2.Request(url,data,header) response=urllib2.urlopen(req) the_page=response.read()

登录后复制

      
      

       
       def handler():##一般handler
       
       
    
       
       
    htp=urllib2.HTTPHandler()
       
       
    opener=urllib2.build_opener(htp)
       
       
    req=urllib2.Request("http://www.baidu.com/")
       
       
    print(opener.open(req).read())
       
       

       
       
def proxy():##代理handler
       
       
    hand=urllib2.ProxyHandler({"http":"219.141.153.41:80"})
       
       
    opener=urllib2.build_opener(hand)
       
       
    req=urllib2.Request("http://www.baidu.com/")
       
       
    print(opener.open(req).read())
      
      
      
      

       
       
1.
       
       
2.
       
       
3.
       
       
4.
       
       
5.
       
       
6.
       
       
7.
       
       
8.
       
       
9.
       
       
10.
       
       
11.
       
       
12.
      
      

登录后复制

      
      

       
       def authhandler():##基本认证handler
       
       
    htppwd=urllib2.HTTPPasswordMgrWithDefaultRealm()
       
       
    htppwd.add_password(None,"auth web","username","password")
       
       
    htp=urllib2.HTTPBasicAuthHandler(htppwd)
       
       
    opener=urllib2.build_opener(htp) ##此时可以加多个handler
       
       
    opener.addheaders=[("User-Agent","dsffsdfdsfd"),()]
       
       
    req=urllib2.Request("http://auth web")
       
       
    print(opener.open(req).read())
      
      
      
      

       
       
1.
       
       
2.
       
       
3.
       
       
4.
       
       
5.
       
       
6.
       
       
7.
       
       
8.