当前位置：网站首页>Spark saving to external data source

Spark saving to external data source

2022-06-29 05:25:00 【wx5ba7ab4695f27】

List of articles

Save as sequenceFile
Save to HDFS
Save to mysql

Save as sequenceFile

       
        package write
        

        
import org.apache.hadoop.io.compress.GzipCodec
        
import org.apache.spark.{SparkConf, SparkContext}
        

        
object saveToSeq {
        
  def main(args: Array[String]): Unit = {
        
    val conf = new SparkConf().setMaster("local[*]")
        
      .setAppName("saveToSeq")
        
    val sc = new SparkContext(conf)
        

        
    val data = List(("name", "xiaoming"), ("age", "18"))
        
    val rddData = sc.parallelize(data, 1)
        
    rddData.saveAsSequenceFile("D:\\studyplace\\sparkBook\\chapter4\\result\\1",Some(classOf[GzipCodec]))
        
  }
        
}
       
       
        1.
        2.
        3.
        4.
        5.
        6.
        7.
        8.
        9.
        10.
        11.
        12.
        13.
        14.
        15.
        16.

among saveAsSequenceFile Of api The first parameter is the save file path , The second parameter is to set the compression mode

about ClassOf[xxxCodec] Objects must be encapsulated in Option In the collection SequenceFile In the method , stay scala in Option Two examples of are Some The collection and None aggregate , The latter means that there are no elements

In compression mode ,GzipCodec The compression ratio of is higher , If there are not enough disks, you can use this method , although Bzip Higher compression , But it's not suitable for frequent reading and writing scenarios

Save to HDFS

saveAsTextFile
Essentially called saveAsHadoopFile Method
saveAsHadoopFile
Yes URI Judge , With file:/// Save the data to the local file system , If schema yes hdfs:// Write data to hdfs In file

saveAsHadoopFile In the method , The default call is TextOutputFormat Implementation class as a formatting tool for output data

import org.apache.hadoop.io.{IntWritable, Text} import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat import org.apache.spark.{SparkConf, SparkContext} object saveTohadoop { def main(args: Array[String]): Unit = { val conf = new SparkConf().setAppName("saveTohadoop").setMaster("local[*]") val sc = new SparkContext(conf) val rddData = sc.parallelize(List(("cat",20),("dog",29),("pig",11)),1) rddData.saveAsNewAPIHadoopFile(" route ",classOf[Text],classOf[IntWritable],classOf[TextOutputFormat[Text,IntWritable]]) sc.stop() } }

Save to mysql

       
        package write
        

        
import java.sql.DriverManager
        

        
import org.apache.spark.{SparkConf, SparkContext}
        

        
object saveToMySQL {
        
  def main(args: Array[String]): Unit = {
        
    val conf = new SparkConf().setMaster("local[*]").setAppName("saveToMySQL")
        
    val sc = new SparkContext(conf)
        

        
    Class.forName("com.mysql.jdbc.Driver")
        
    val rddData = sc.parallelize(List(("tom",11),("jettty",19)))
        
    rddData.foreachPartition((iter:Iterator[(String,Int)]) => {
        
      val conn = DriverManager.getConnection("jdbc:mysql://localhost:3306/spark?useUnicode=true&characterEncoding=utf-8","root","123456")
        
      conn.setAutoCommit(false)
        
      val statement = conn.prepareStatement("insert into spark.person (name,age) VALUES (?,?);")
        
      iter.foreach( t => {
        
        statement.setString(1,t._1)
        
        statement.setInt(2,t._2)
        
        statement.addBatch()
        
      })
        
      statement.executeBatch()
        
      conn.commit()
        
      conn.close()
        
    })
        
    sc.stop()
        
  }
        
}
       
       
        1.
        2.
        3.
        4.
        5.
        6.
        7.
        8.
        9.
        10.
        11.
        12.
        13.
        14.
        15.
        16.
        17.
        18.
        19.
        20.
        21.
        22.
        23.
        24.
        25.
        26.
        27.
        28.
        29.