[note] on May 28, 2022, data is obtained from the web page and written into the database

""" example11 -  Persistent crawler data with database  400 - Bad request. 401 - Unauthorized. 403 - Forbidden. 404 - Not Found. 405 - Method not allowed. 418 - I am a teapot. create table `tb_top_movie` ( `mov_id` bigint unsigned auto_increment comment ' Number ', `mov_title` varchar(200) not null comment ' title ', `mov_rating_num` decimal(3,1) not null comment ' score ', `mov_comments_count` bigint not null comment ' comments ', primary key (`mov_id`) ) engine=innodb auto_increment=1001 comment ' Movie data sheet '; Author: Hao Date: 2022/5/28 """
import bs4
import pymysql
import requests
from pymysql.cursors import Cursor


def fetch_page(session, url):
    """ Grab page  :param session: Session object  :param url:  Uniform resource locator  :return:  Page HTML Code  """
    resp = session.get(url=url)
    return resp.text if resp.status_code == 200 else ''


def parse_page(html_code):
    """ Parsing the page  :param html_code:  Page HTML Code  :return:  Data parsed from the page  """
    soup = bs4.BeautifulSoup(html_code, 'html.parser')
    movie_items_list = soup.select('#content > div > div.article > ol > li')
    data = []
    for movie_item in movie_items_list:
        title = movie_item.select_one('div > div.info > div.hd > a > span.title').text
        rating_num = movie_item.select_one('div > div.info > div.bd > div > span.rating_num').text
        comments_count = movie_item.select_one('div > div.info > div.bd > div > span:nth-child(4)').text[:-3]
        data.append((title, rating_num, comments_count))
    return data


def save_to_db(conn, data):
    """ Save the data to the database  :param conn:  Database connection  :param data:  data  """
    with conn.cursor() as cursor:  # type: Cursor
        cursor.executemany(
            'insert into tb_top_movie (mov_title, mov_rating_num, mov_comments_count) '
            'values (%s, %s, %s)',
            data
        )
    conn.commit()


def main():
    session = requests.Session()
    session.headers = {
    'User-Agent': 'Baiduspider'}
    conn = pymysql.connect(host='localhost', port=3306,
                           user='guest', password='Guest.618',
                           database='hrs', charset='utf8mb4')
    try:
        for page in range(10):
            url = f'https://movie.douban.com/top250?start={
      25 * page}'
            html_code = fetch_page(session, url)
            data = parse_page(html_code)
            save_to_db(conn, data)
    finally:
        conn.close()


if __name__ == '__main__':
    main()