Taiyuan bus route crawling

8864 Bus route query

 Life is too short , I learned Python

# -*- codeing = utf-8 -*-

import requests
from lxml import etree
from fake_useragent import UserAgent
#** The list is used to save all information **
items = []

headers ={
    
    "User-Agent":UserAgent().chrome
}


def parse_navigation():
    start_url = "https://taiyuan.8684.cn/"
    r =requests.get(url=start_url,headers=headers)
    **# Parse the content to get all navigation links **
    tree =etree.HTML(r.text)
    **# For a link 
    # Begin with a number **
    number_href_list =tree.xpath('//div[@class="bus-layer depth w120"]/div[1]/div/a/@href')
    **# It starts with a letter **
    char_href_list =tree.xpath('//div[@class="bus-layer depth w120"]/div[2]/div/a/@href')
    **# All links will be returned **
    return number_href_list + char_href_list


def parse_erji_route(content):
    tree = etree.HTML(content)
    # Get every line link 
    route_list = tree.xpath('//div[@class="list clearfix"]/a/@href')
    route_name = tree.xpath('//div[@class="list clearfix"]/a/text()')
    # Traverse ,
    i = 0
    for route in route_list:
        print(" Start crawling %s Information about " % route_name[i])
        # Facilitate every route 
        route ="https://taiyuan.8684.cn" + route
        # Send a request , # Get a response 
        r =requests.get(url=route,headers=headers)
        parse_sanji_route(r.text)
        print(" End of crawling %s Information about " % route_name[i])
        i += 1


def parse_sanji_route(content):
    tree =etree.HTML(content)
    # Grab what you want 
    # Bus information 
    bus_num =tree.xpath('//div[@class="info"]/h1/text()')[0]
    # Get run time ：
    run_time = tree.xpath('//ul[@class="bus-desc"]/li[1]/text()')[0]
    # Fare information 
    ticket_price_info = tree.xpath('//ul[@class="bus-desc"]/li[2]/text()')[0]
    # Update time 
    updata_time =tree.xpath('//ul[@class="bus-desc"]/li[4]/text()')[0]
    # Get the total number of uplink stations 
    bus_station_list = tree.xpath('//div[@class="bus-excerpt mb15"]/div/div[@class="total"]/text()')
    up_total =bus_station_list[0]
    # Get the uplink station name 
    up_station_name =tree.xpath('//div[@class="bus-lzlist mb15"][1]')
    try:
        # Get the total number of downlink stations 
        down_total = bus_station_list[1]
        #  Get the downstream station name 
        down_station_name_list = tree.xpath('//div[@class="bus-lzlist mb15"][2]')
    except Exception as error:
        down_total = ""
        down_station_name_list =[]
    # Save every bus information , Store in dictionary 
    item ={
    
        " The name of the bus ":bus_num,
        " The elapsed time ":run_time,
        " Fare information ":ticket_price_info,
        " Update time ":updata_time,
        " Number of uplink stations ":up_total,
        " Uplink station name ":up_station_name,
        " Number of downlink stations ":down_total,
        " Downlink station name ":down_station_name_list,
    }
    items.append(item)


def parse_erji(navi_list):
    # Traverse the list and send the request once , Get all bus routes responding to each page 
    for first_url in navi_list:
        print(" Start crawling %s All bus information "%first_url)
        # Splice complete links 
        first_url = "https://taiyuan.8684.cn" +first_url
        # Send a request , Get a response 
        print(first_url)
        r =requests.get(first_url,headers=headers)
        content =r.text
        # Analyze the integrity of each bus url
        parse_erji_route(content)
        print(" End of crawling %s All bus information " % first_url)