DataX JSON description

{
    "job": {
        "content": [
            {
                "reader": {
                    "name": "mysqlreader",
                    "parameter": {
                        "querySql": "", # Customize sql, Support multi table Association , When the user configures querySql when , Direct to ignore table、column、where Configuration of conditions .
                        "fetchSize": "", # Default 1024, This configuration item defines the number of batch data acquisition for plug-in and database server , This value determines DataX The number of network interactions with the server , It can greatly improve the performance of data extraction , Be careful , The value is too large (>2048) May cause DataX process OOM
                        "splitPk": "db_id", # Only plastic data segmentation is supported ; If specified splitPk, Indicates that the user wants to use splitPk Represents the field for data fragmentation , If the value is null , It means no segmentation , Using a single channel for extraction 
                        "column": [], #"*" Default all columns , Support column clipping , Column permutation 
                        "connection": [
                            {
                                "jdbcUrl": ["jdbc:mysql://IP:3306/database?useUnicode=true&characterEncoding=utf8"],
                                "table": [] # Multiple tables can be extracted at the same time 
                            }
                        ],
                        "password": "",
                        "username": "",
                        "where": "" # designated column、table、where Conditional mosaicing SQL, You can specify limit 10, You can also do incremental data synchronization , If the value is null , Represents to synchronize all information of the whole table 
                    }
                },
                "writer": {
                    "name": "hdfswriter",
                    "parameter": {
                        "column": [], # You must specify a field name , Field type ,{"name"："","tpye":""}
                        "compress": "", #hdfs File compression type , Not filling in by default means that there is no compression . among ：text Type files support compression. The types are gzip、bzip2;orc The compression types supported by type files are NONE、SNAPPY（ You need to install SnappyCodec）.
                        "defaultFS": "", #Hadoop hdfs file system namenode Node address .
                        "fieldDelimiter": "", # It needs to be guaranteed and created by users Hive The field separators of the table are consistent 
                        "fileName": "", #HdfsWriter The file name at the time of writing , You need to specify all field names and field types in the table , among ：name Specify the field name ,type Specify field type .
                        "fileType": "", # At present, only user configuration is supported ”text” or ”orc”
                        "path": "", # Store in Hadoop hdfs File system path information ,hive Table in hdfs Storage path on 
                        "hadoopConfig": {} #hadoopConfig It can be configured with Hadoop Some related advanced parameters , such as HA Configuration of .
                        "writeMode": "" #append, Do nothing before writing , File name does not conflict ;nonConflict, If there is fileName Prefix file , Direct error .
                    }
                }
            }
        ],
        "setting": {
            "speed": { # flow control 
                "byte": 1048576, # Control the transmission speed , Unit is byte/s,DataX Run as fast as you can, but don't exceed it 
                "channel": ""  # Control the number of concurrencies during synchronization 
                    }
            "errorLimit": { # Dirty data control 
                "record": 0 # Threshold of maximum number of records for dirty data （record value ） Or dirty data percentage threshold （percentage value , When quantity or percentage ,DataX Job Report an error and exit 
            }
        }
    }
}