Pymongo saves data in dataframe format (insert_one, insert_many, multi-threaded saving)

Use Pymongo The basic method of saving data （ Additions and deletions ） Please refer to ：Python Connect MongoDB, Use pymongo Add, delete, modify, etc

1. The basic method : Save line by line

This is the most basic way to save , You can fine tune the data itself , Then save

from pymongo import MongoClient
import pandas as pd
import numpy as np


def get_coll(database, collection, host="127.0.0.1"):
    """ Target database """
    mongo_conn = MongoClient(host=host, port=27017)
    mongo_db = mongo_conn.get_database(database)
    coll = mongo_db.get_collection(collection)
    return coll


def _save_or_update_mongodb(coll, dict_value):
    """ According to the inspection _id, If it exists, cover , If it doesn't exist, add """
    record = coll.find_one({
    "_id": dict_value['_id']})
    if not record:
        coll.insert_one(dict_value)
    else:
        coll.update_one(record, {
    
            "$set": dict_value,
        })


def save_dataframe_to_mongo(dataframe):
    coll = get_coll("test_db", "test_collection")
    for index, series in dataframe.iterrows():
        dict_value = series.to_dict()
        dict_value.update({
    
            "_id": index,
        })
        _save_or_update_mongodb(coll, dict_value)


if __name__ == '__main__':
    df = pd.DataFrame(np.random.randn(10, 4))
    df.columns = ['a', 'b', 'c', 'd']
    save_dataframe_to_mongo(df)

2. insert_many Save in bulk

You can save a batch of data at one time , Use insert_many Method can save data in batches

from pymongo import MongoClient
import pandas as pd
import numpy as np
import math


def get_coll(database, collection, host="127.0.0.1"):
    """ Target database """
    mongo_conn = MongoClient(host=host, port=27017)
    mongo_db = mongo_conn.get_database(database)
    coll = mongo_db.get_collection(collection)
    return coll


def save_dataframe_to_mongo(dataframe, step=20):
    coll = get_coll("test_db", "test_collection2")
    for i in range(math.ceil(dataframe.shape[0] / step)):
        dict_list = dataframe.iloc[step * i:step * (i + 1)].to_dict(orient="record")
        coll.insert_many(dict_list)


if __name__ == '__main__':
    df = pd.DataFrame(np.random.randn(900, 4))
    df.columns = ['a', 'b', 'c', 'd']
    save_dataframe_to_mongo(df)

3. Threading Multi thread save data

Pymongo Is multithread safe 、 Multi process unsafe , Therefore, you can use multithreading mode to save data recklessly , The sample code is as follows ：

from pymongo import MongoClient
import pandas as pd
import numpy as np
import math
import threading


def get_coll(database, collection, host="127.0.0.1"):
    """ Target database """
    mongo_conn = MongoClient(host=host, port=27017)
    mongo_db = mongo_conn.get_database(database)
    coll = mongo_db.get_collection(collection)
    return coll


def save_dataframe_to_mongo(dataframe, step=20):
    coll = get_coll("test_db", "test_collection2")
    thread_list = []
    for i in range(math.ceil(dataframe.shape[0] / step)):
        dict_list = dataframe.iloc[step * i:step * (i + 1)].to_dict(orient="record")  #  Data to be saved 
        #  Multithreading 
        thread = threading.Thread(target=coll.insert_many, args=(dict_list,))
        thread.start()
        thread_list.append(thread)
    #  Wait for all thread tasks to complete 
    for _thr in thread_list:
        _thr.join()


if __name__ == '__main__':
    df = pd.DataFrame(np.random.randn(900, 4))
    df.columns = ['a', 'b', 'c', 'd']
    save_dataframe_to_mongo(df)