The first scratch crawler

scrapy The directory structure is as follows

What we want to crawl is the title of the book in the book network , author , And the description of the book

First, we need to define the model of crawling data , stay items.py In file

import scrapy 
 class MoveItem(scrapy.Item): #  Define the model of crawled data  title = scrapy.Field() auth = scrapy.Field() desc = scrapy.Field() 

The main thing is spiders In the catalog move.py file

import scrapy
from douban.items import MoveItem

class MovieSpider(scrapy.Spider):
    #  Indicates the name of the spider , The name of each spider must be unique 
    name = 'movie'
    #  Indicates the domain name filtered and crawled 
    allwed_domians = ['dushu.com']
    #  Indicates the first thing to crawl url
    start_urls = ['https://www.dushu.com/book/1188.html']

    def parse(self,response):
         li_list = response.xpath('/html/body/div[6]/div/div[2]/div[2]/ul/li')
        for li in li_list:
            item = MoveItem()
            item['title'] = li.xpath('div/h3/a/text()').extract_first()
            item['auth'] = li.xpath('div/p[1]/a/text()').extract_first()
            item['desc'] = li.xpath('div/p[2]/text()').extract_first()
            #  generator 
            yield item

        href_list = response.xpath('/html/body/div[6]/div/div[2]/div[3]/div/a/@href').extract()
        for href in href_list:
            #  Crawl on the page url completion 
            url = response.urljoin(href)
            #  A generator ,response Link inside , Then proceed to request, Keep executing parse, It's a recursion .
            yield scrapy.Request(url=url,callback=self.parse)

The only way to persist data is to save it ： stay settings.py Set in file

stay pipelines.py In the document ：

import pymongo


class DoubanPipeline(object):
    def __init__(self):
        self.mongo_client = pymongo.MongoClient('mongodb://39.108.188.19:27017')

    def process_item(self, item, spider):

        db = self.mongo_client.data
        message = db.messages
        message.insert(dict(item))
        return item