Selenium+bs4 parsing +mysql capturing BiliBili Tarot data

git Source code ：https://github.com/xuyanhuiwelcome/python-spider

It will be updated continuously , Crawling data spider

Data can be analyzed by Tarot , Don't talk much , Code up :

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup
from selenium.webdriver.support.wait import WebDriverWait
import lxml
import pymysql

browser = webdriver.Chrome()
def run(page):
    url = 'https://search.XXX.com/upuser?keyword=%E5%A1%94%E7%BD%97&page={page}'.format(
        page=page)
    browser.get(url)
    WebDriverWait(browser,30)
    html = browser.page_source
    soup = BeautifulSoup(html, 'lxml')
    save_data(soup)

def save_data(soup):
    lsts = soup.find(class_="body-contain").find_all(class_="user-item")
    for item in lsts:
        username = item.find(class_="title").get('title')
        level = item.find(class_="title").find("i")
        desc = item.find(class_="desc").text
        spans = item.find(class_="up-info clearfix").find_all("span")
        funs = spans[0].text.strip(" Manuscript ：")
        work = spans[1].text.strip(" fans ：")
        if work.count(' ten thousand ') > 0:
            work = float(work.strip(' ten thousand ')) * 10000
        centerUrl = item.find("a").get('href').strip("//")
        save_mysql(username,level,desc,funs,work,centerUrl)

def save_mysql(useranme,level,desc,funs,work,contenturl):
    #  Open database connection 
    db = pymysql.connect(host='localhost',
                         user='root',
                         password='root',
                         database='tesst')
    #  Use cursor() Method get operation cursor 
    cursor = db.cursor()

    # SQL  Insert statement 
    sql = "INSERT INTO `tarotuser`(`userName`,`funs`,`work`,`desc`,`centerUrl`) VALUES('"+useranme+"','"+funs+"','"+str(work)+"','"+desc+"','"+contenturl+"')"
    try:
        #  perform sql sentence 
        cursor.execute(sql)
        #  Commit to database execution 
        db.commit()
    except:
        #  Roll back if an error occurs 
        db.rollback()

    #  Close database connection 
    db.close()
def main():
    for page in range(1,50):
        run(page)

if __name__=='__main__':
    main()