当前位置：网站首页>Hudi record

Hudi record

2022-06-30 03:14:00 【Pangpang Pang Hu】

data structure

Insert picture description here

---->>> Flink on hudi

https://www.jianshu.com/p/f509429c2f20

hudi Sync hive

https://blog.csdn.net/hjl18309163914/article/details/107844269

Create table statement demo

CREATE TABLE sink_order_mysql_goods_order(
     `goods_order_id` bigint COMMENT ' Since the primary key id'
	, `goods_order_uid` string COMMENT ' Order uid'
	, `customer_uid` string COMMENT ' Customer uid'
	, `customer_name` string COMMENT ' Customer name'
	, `student_uid` string COMMENT ' Student uid'
	, `order_status` bigint COMMENT ' The order status  1: To be paid  2: Partial payment  3: Payment review  4: Paid  5: Cancelled '
	, `is_end` bigint COMMENT ' Whether the order is closed  1. Open  2. It's over '
	, `discount_deduction` bigint COMMENT ' Total amount of discount （ Company ： branch ）'
	, `contract_deduction` bigint COMMENT ' The amount deducted from the old contract （ Company ： branch ）'
	, `wallet_deduction` bigint COMMENT ' Wallet deduction amount （ Company ： branch ）'
	, `original_price` bigint COMMENT ' Original order price （ Company ： branch ）'
	, `real_price` bigint COMMENT ' Amount paid （ Company ： branch ）'
	, `pay_success_time` timestamp(3) COMMENT ' Full payment time '
	, `tags` string COMMENT ' Order label （1 New signature  2 renew  3 Extended family  4 To sign up - He Xin  5 Shift change - He Xin  6 renew - He Xin  7 audition - He Xin ）'
	, `status` bigint COMMENT ' Whether it works （1. take effect  2. invalid  3. Overdue payment ）'
	, `remark` string COMMENT ' Order notes '
	, `delete_flag` bigint COMMENT ' Whether or not to delete （1. no ,2. yes ）'
	, `test_flag` bigint COMMENT ' Test data （1. no ,2. yes ）'
	, `create_time` timestamp(3) COMMENT ' Creation time '
	, `update_time` timestamp(3) COMMENT ' Update time '
	, `create_by` string COMMENT ' founder uid（ Unique identification ）'
	, `update_by` string COMMENT ' Updated by uid（ Unique identification ）'
	, `belong_school` bigint COMMENT ' Belonging to the campus '
	, `share_sale_no` string COMMENT ' Share sales job number '
	
    ,PRIMARY KEY (goods_order_id) NOT ENFORCED
) COMMENT ' The order sheet '
WITH (
      'connector' = 'hudi'
    , 'path' = 'hdfs://hdfs-namenode-service:9000/hudi-warehouse/goods_order' --- The path will be created automatically 
    , 'hoodie.datasource.write.recordkey.field' = 'goods_order_id'  --  Primary key 
    , 'write.precombine.field' = 'update_time'             --  When the same key value , Take the maximum value of this field , Default ts Field 
    , 'read.streaming.skip_compaction' = 'true'            --  Avoid the problem of repeated consumption 
    , 'write.bucket_assign.tasks' = '2'              -- Written concurrently  bucekt  Count 
    , 'write.tasks' = '2'
    , 'compaction.tasks' = '1'
    , 'write.operation' = 'upsert'                          --UPSERT（ Insert update ）\INSERT（ Insert ）\BULK_INSERT（ Batch insert ）（upsert The performance will be lower , It is not suitable for burying point reporting ）
    , 'write.rate.limit' = '20000'                          --  Limit the number of messages per second 
    , 'table.type' = 'COPY_ON_WRITE'                      
    , 'compaction.async.enabled' = 'true'                  --  Online compression 
    , 'compaction.trigger.strategy' = 'num_or_time'        --  Compress by number of times 
    , 'compaction.delta_commits' = '20'               --  The default is 5
    , 'compaction.delta_seconds' = '60'                   --  The default is 1 Hours 
    , 'hive_sync.enable' = 'true'                          --  Enable hive Sync 
    , 'hive_sync.mode' = 'hms'                             --  Enable hive hms Sync , Default jdbc
    , 'hive_sync.metastore.uris' = 'thrift://hive-metastore-svc:9083'    -- required, metastore The port of 
    , 'hive_sync.jdbc_url' = 'jdbc:hive2://hive-service-svc:10000'     -- required, hiveServer Address 
    , 'hive_sync.table' = 'order_mysql_goods_order'                            -- required, hive  New table name   Will automatically synchronize hudi Table structure and data to hive
    , 'hive_sync.db' = 'cdc_ods'                              -- required, hive  New database name 
    , 'hive_sync.username' = 'root'                        -- required, HMS  user name 
    , 'hive_sync.password' = '123456'                            -- required, HMS  password 
    , 'hive_sync.skip_ro_suffix' = 'true'                  --  Remove ro suffix 
);

The problem record

hive Sync hudi Wrong task report

CREATE TABLE flink_cdc_sink_hudi_hive(
uuid varchar(20),
name varchar(10),
age int,
ts timestamp(3),
dt varchar(20)
)
PARTITIONED BY (dt)
with(
'connector'='hudi',
'path'= 'hdfs://hdfs-namenode-service:9000/flink_cdc_sink_hudi_hive', 
'table.type'= 'MERGE_ON_READ',
'hoodie.datasource.write.recordkey.field'= 'uuid', 
'write.precombine.field'= 'ts',
'write.tasks'= '1',
'write.rate.limit'= '2000', 
'compaction.tasks'= '1', 
'compaction.async.enabled'= 'true',
'compaction.trigger.strategy'= 'num_commits',
'compaction.delta_commits'= '1',
'changelog.enabled'= 'true',
'read.streaming.enabled'= 'true',
'read.streaming.check-interval'= '3',
'hive_sync.enable'= 'true',
'hive_sync.mode'= 'hms',
'hive_sync.metastore.uris'= 'thrift://hive-metastore-svc:9083',
'hive_sync.jdbc_url'= 'jdbc:hive2://hive-service-svc:10000',
'hive_sync.table'= 'flink_cdc_sink_hudi_hive',
'hive_sync.db'= 'default',
'hive_sync.username'= 'root',
'hive_sync.password'= '123456',
'hive_sync.support_timestamp'= 'true'
);