三国演义小说

import requests
from bs4 import BeautifulSoup
from fake_useragent import  UserAgent
#爬取三国演义小说所有章节标题和章节内容 https://www.shicimingju.com/book/sanguoyanyi.html
if __name__ =='__main__':
    headers={
    
        "User-Agent":UserAgent().chrome
    }
    get_url='https://www.shicimingju.com/book/sanguoyanyi.html'
    #发起请求，获取响应
    page_text=requests.get(url=get_url,headers=headers).text.encode('ISO-8859-1')
    #在首页中解析出章节标题和章节内容
    #1. 实例化BeautifulSoup对象，将html数据加载到该对象中
    soup=BeautifulSoup(page_text,'lxml')
    # print(soup)
    #2.解析章节标题和详情页的url
    list_data=soup.select('.book-mulu > ul > li')
    fp=open('./sanguo.text','w',encoding='utf-8')
    for i in list_data:
        title=i.a.text
        detail_url='https://www.shicimingju.com/'+ i.a['href']
        #对详情页的url发送请求，
        detail_text=requests.get(url=detail_url,headers=headers).text.encode('ISO-8859-1')
        detail_soup=BeautifulSoup(detail_text,'lxml')
        #获取章节内容
        content=detail_soup.find('div',class_='chapter_content').text

        #持久化存储
        fp.write(title+":"+content+"\n")
        print(title,'下载完成')