Selenium basic knowledge paging processing

Sometimes you need to turn the page of the list ,Selenium The corresponding API.
Paging processing logic can be roughly divided into the following three steps ：
（1） Get total pages .
（2） Get all pages and cycle through .
（3） Carry out subsequent logical processing for each paging .

Baidu Post Bar Python For example ：

1. First navigate to paging div
2. Navigate to the last page button

# -*- coding: utf-8 -*-
from selenium import webdriver
import time
from selenium.webdriver.common.by import By

driver = webdriver.Edge()
data = {
    
    'index_url':'http://tieba.baidu.com/f?ie=utf-8&kw=python',
    'pagination_id':'frs_list_pager',
    '':''
}

#  visit Python Home page 

# frs_list_pager
driver.get(data['index_url'])
#  Locate until paging div
pagination_div = driver.find_element(by=By.ID,value=data['pagination_id'])
print(pagination_div)

#  Calculate the page number of the last page 
#  First click the last page button 
driver.find_element(by=By.CSS_SELECTOR,value='#frs_list_pager > a.last.pagination-item').click()
time.sleep(3)

#  Get the number of pages of the last page 
last_page_no = driver.find_element(by=By.CSS_SELECTOR,value='#frs_list_pager > span').text
time.sleep(2)
print(last_page_no)

#  Jump back to home page 
driver.get(data['index_url'])

#  loop last_page_no Get the data of each page once 
for index in last_page_no:
    #  Some code to collect data , Omit 
    time.sleep(2)
    driver.find_element(by=By.CSS_SELECTOR,value='#frs_list_pager > a.next.pagination-item').click()
# driver.quit()

After several trial runs, it was found that it stopped every time from page one to page five Then add an output statement to the loop Find every time index Namely 1 3 9 3

It's just the same as the total number of pages Just add another one type Statement output last_page_no Variable Found to be string type So there are only four cycles print(type(last_page_no))

The final code

# -*- coding: utf-8 -*-
from selenium import webdriver
import time
from selenium.webdriver.common.by import By

driver = webdriver.Edge()
data = {
    
    'index_url':'http://tieba.baidu.com/f?ie=utf-8&kw=python',
    'pagination_id':'frs_list_pager'
}

#  visit Python Home page 

# frs_list_pager
driver.get(data['index_url'])
#  Locate until paging div
pagination_div = driver.find_element(by=By.ID,value=data['pagination_id'])
print(pagination_div)

#  Calculate the page number of the last page 
#  First click the last page button 
driver.find_element(by=By.CSS_SELECTOR,value='#frs_list_pager > a.last.pagination-item').click()
time.sleep(2)

#  Get the number of pages of the last page 
last_page_no = driver.find_element(by=By.CSS_SELECTOR,value='#frs_list_pager > span').text
#  Now get last_page_no yes string type   Need to transform   Otherwise, there will only be four cycles  1 3 9 3  because lastpageno Is string ‘1393’
time.sleep(2)
print(type(last_page_no))
last_page_no = int(last_page_no)
print(type(last_page_no))
print(last_page_no)
#  Jump back to home page 
driver.get(data['index_url'])

#  loop last_page_no Get the data of each page once 
for index in range(last_page_no):
    #  Some code to collect data , Omit 
    print(index)
    time.sleep(2)
    driver.find_element(by=By.CSS_SELECTOR,value='#frs_list_pager > a.next.pagination-item').click()

# driver.quit()

Just like this ！