当前位置:网站首页>Use go to add massive data to MySQL
Use go to add massive data to MySQL
2022-06-13 04:33:00 【gohxc】
use mysql Create a database Here to create test_big database
Create table
CREATE TABLE `student` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT ' Primary key ',
`user_name` varchar(255) DEFAULT NULL COMMENT ' full name ',
`add_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT ' Add the time ',
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8;
Class libraries that need to be used
github.com/go-sql-driver/mysql
github.com/satori/go.uuid
Use it directly go get xxx that will do
Probably 1000 10000 insert data needs 5 About minutes
The code is as follows
package main
import (
"database/sql"
"fmt"
_ "github.com/go-sql-driver/mysql"
"log"
"strconv"
"time"
)
func main() {
mysql()
}
// Structure
type Job struct {
db *sql.DB
ch chan int
total int
n int
}
func mysql(){
db, err := sql.Open("mysql", "root:[email protected]/test_big?charset=utf8")
if err != nil {
fmt.Println(" Error accessing database ", err)
return
}
defer db.Close()
db.SetConnMaxLifetime(time.Second * 500) // Set connection timeout 500 second
db.SetMaxOpenConns(100) // Set the maximum number of connections
total := 25000
gonum:=400
fmt.Println("====start=====")
start := time.Now()
// Test the function of inserting into the database , Maximum synchronization per time 20 Working process
jobChan := make(chan Job, 20)
go worker(jobChan)
// Count usage times
ch := make(chan int,gonum)
for n := 0; n < gonum; n++ {
job:=Job{
db:db,
ch:ch,
total:total,
n:n,
}
jobChan <- job
}
ii:=0
for{
<-ch
ii++
if(ii>=gonum){
break;
}
}
end := time.Now()
curr := end.Sub(start)
fmt.Println("run time:", curr)
}
func worker(jobChan <-chan Job) {
for job := range jobChan {
go sqlExec(job)
}
}
func sqlExec(job Job) {
buf:=make([]byte,0,job.total)
buf=append(buf," insert into student(user_name) values "...)
for i := 0; i < job.total; i++ {
myid, _ := uuid.NewV4()
userName := myid.String()
if i == job.total-1 {
buf=append(buf,"('" + userName + "');"...)
} else {
buf=append(buf,"('" + userName + "'),"...)
}
}
ss:=string(buf)
fmt.Println(" The first " + strconv.Itoa(job.n) + " Insert... Times 2.5 Ten thousand data !")
_, err := job.db.Exec(ss)
checkErr(err)
fmt.Println(" complete ---" + strconv.Itoa(job.n) + " Insert... Times 2.5 Ten thousand data !")
job.ch<- 1
}
func checkErr(err error) {
if err != nil {
panic(err)
}
}
边栏推荐
- dumi 搭建文檔型博客
- Catalan number
- EMC整改纲要
- Dumi construit un blog documentaire
- Message scrolling JS implementation
- Latex operation
- Express framework knowledge - Art template template, cookie, session
- Explanation of line segment tree
- Analysis of the implementation principle of an open source markdown to rich text editor
- A simple understanding of consistent hash
猜你喜欢

【Flutter 問題系列第 67 篇】在 Flutter 中使用 Get 插件在 Dialog 彈窗中不能二次跳轉路由問題的解决方案

前几年的互联网人vs现在的互联网人

Collection of wrong questions in soft test -- morning questions in the first half of 2011

R: Employee turnover forecast practice

Common encryption and decryption function encapsulation - AES encryption and decryption

Reread the classic: end to end object detection with transformers

The data obtained from mongodb query data command is null

Tree array explanation

2022 ICML | Pocket2Mol: Efficient Molecular Sampling Based on 3D Protein Pockets

Analysis of the implementation principle of an open source markdown to rich text editor
随机推荐
Common encryption and decryption function encapsulation - AES encryption and decryption
Differences and relations between three-tier architecture and MVC
ACM ICPC
Explanation of line segment tree
【Flutter 問題系列第 67 篇】在 Flutter 中使用 Get 插件在 Dialog 彈窗中不能二次跳轉路由問題的解决方案
Redis主从复制、哨兵模式、集群
2022 ICLR | CONTRASTIVE LEARNING OF IMAGE- AND STRUCTURE BASED REPRESENTATIONS IN DRUG DISCOVERY
php开发14 友情链接模块的编写
Notes on uni app
XOR prefix and +map maintenance
Small program input element moving up
记录一次排查问题的经过——视频通话无法接起
[flutter problem Series Chapter 67] the Solution to the problem of Routing cannot be jumped again in in dialog popup Using get plug - in in flutter
Redis persistence mode AOF and RDB
dumi 搭建文档型博客
Redis
This Sedata uses multiple methods to dynamically modify objects and values in arrays. Object calculation properties
EMC rectification outline
一致性哈希的简单认识
MySQL索引