Kudu DDL

  1. CREATE TABLE customer
  2. (
  3.   c_custkey INT,
  4.   c_name STRING,
  5.   c_address STRING,
  6.   c_city STRING,
  7.   c_nation STRING,
  8.   c_region STRING,
  9.   c_phone STRING,
  10.   c_mktsegment STRING,
  11.   primary key(c_custkey)
  12. )
  13. PARTITION BY HASH PARTITIONS 16
  14. STORED AS KUDU;
  16. CREATE TABLE dates
  17. (
  18.   d_datekey INT,
  19.   d_date STRING,
  20.   d_dayofweek STRING,
  21.   d_month STRING,
  22.   d_year INT,
  23.   d_yearmonthnum INT,
  24.   d_yearmonth STRING,
  25.   d_daynuminweek INT,
  26.   d_daynuminmonth INT,
  27.   d_daynuminyear INT,
  28.   d_monthnuminyear INT,
  29.   d_weeknuminyear INT,
  30.   d_sellingseason STRING,
  31.   d_lastdayinweekfl INT,
  32.   d_lastdayinmonthfl INT,
  33.   d_holidayfl INT,
  34.   d_weekdayfl INT,
  35.   primary key(d_datekey)
  36. )
  37. PARTITION BY HASH PARTITIONS 16
  38. STORED AS KUDU;
  40. CREATE TABLE lineorder
  41. (
  42.   `lo_orderkey` bigint, 
  43.   `lo_linenumber` bigint, 
  44.   `lo_custkey` int, 
  45.   `lo_partkey` int, 
  46.   `lo_suppkey` int, 
  47.   `lo_orderdate` int, 
  48.   `lo_orderpriotity` string, 
  49.   `lo_shippriotity` int, 
  50.   `lo_quantity` bigint, 
  51.   `lo_extendedprice` bigint, 
  52.   `lo_ordtotalprice` bigint, 
  53.   `lo_discount` bigint, 
  54.   `lo_revenue` bigint, 
  55.   `lo_supplycost` bigint, 
  56.   `lo_tax` bigint, 
  57.   `lo_commitdate` int, 
  58.   `lo_shipmode` string,
  59.   PRIMARY KEY(lo_orderkey, lo_linenumber)
  60. )
  61. PARTITION BY HASH PARTITIONS 16
  62. STORED AS KUDU;
  64. CREATE TABLE part
  65. (
  66.   p_partkey INT,
  67.   p_name STRING,
  68.   p_mfgr STRING,
  69.   p_category STRING,
  70.   p_brand STRING,
  71.   p_color STRING,
  72.   p_type STRING,
  73.   p_size INT,
  74.   p_container STRING,
  75.   primary key(p_partkey)
  76. )
  77. PARTITION BY HASH PARTITIONS 16
  78. STORED AS KUDU;
  80. CREATE TABLE supplier
  81. (
  82.   s_suppkey INT,
  83.   s_name STRING,
  84.   s_address STRING,
  85.   s_city STRING,
  86.   s_nation STRING,
  87.   s_region STRING,
  88.   s_phone STRING,
  89.   primary key(s_suppkey)
  90. )
  91. PARTITION BY HASH PARTITIONS 16
  92. STORED AS KUDU;
  94. CREATE VIEW `p_lineorder` AS SELECT `lineorder`.`lo_orderkey`,         
  95. `lineorder`.`lo_linenumber`,    
  96. `lineorder`.`lo_custkey`,      
  97. `lineorder`.`lo_partkey`,         
  98. `lineorder`.`lo_suppkey`,    
  99. `lineorder`.`lo_orderdate`,    
  100. `lineorder`.`lo_orderpriotity`,    
  101. `lineorder`.`lo_shippriotity`,     
  102. `lineorder`.`lo_quantity`,         
  103. `lineorder`.`lo_extendedprice`,    
  104. `lineorder`.`lo_ordtotalprice`,    
  105. `lineorder`.`lo_discount`,         
  106. `lineorder`.`lo_revenue`,          
  107. `lineorder`.`lo_supplycost`,       
  108. `lineorder`.`lo_tax`,              
  109. `lineorder`.`lo_commitdate`,       
  110. `lineorder`.`lo_shipmode`,    
  111. `lineorder`.`lo_extendedprice`*`lineorder`.`lo_discount` AS `V_REVENUE`    
  112. FROM `LINEORDER`;

Flink DDL

  1. CREATE TABLE default_catalog.default_database.customer
  2. (
  3.   c_custkey INT,
  4.   c_name STRING,
  5.   c_address STRING,
  6.   c_city STRING,
  7.   c_nation STRING,
  8.   c_region STRING,
  9.   c_phone STRING,
  10.   c_mktsegment STRING
  11. ) with (
  12.     'connector.master' = 'hz-hadoop-test-199-141-27:7051',
  13.     'connector.table' = 'impala::yjuny_kudu.customer',
  14.   'connector.property-version' = '1',
  15.   'connector.type' = 'kudu-retract',
  16.   'connector.primary-key.0' = 'c_custkey'
  17. )
  19. CREATE TABLE default_catalog.default_database.dates
  20. (
  21.   d_datekey INT,
  22.   d_date STRING,
  23.   d_dayofweek STRING,
  24.   d_month STRING,
  25.   d_year INT,
  26.   d_yearmonthnum INT,
  27.   d_yearmonth STRING,
  28.   d_daynuminweek INT,
  29.   d_daynuminmonth INT,
  30.   d_daynuminyear INT,
  31.   d_monthnuminyear INT,
  32.   d_weeknuminyear INT,
  33.   d_sellingseason STRING,
  34.   d_lastdayinweekfl INT,
  35.   d_lastdayinmonthfl INT,
  36.   d_holidayfl INT,
  37.   d_weekdayfl INT
  38. ) with (
  39.     'connector.master' = 'hz-hadoop-test-199-141-27:7051',
  40.     'connector.table' = 'impala::yjuny_kudu.dates',
  41.   'connector.property-version' = '1',
  42.   'connector.type' = 'kudu-retract',
  43.   'connector.primary-key.0' = 'd_datekey'
  44. )
  46. CREATE TABLE default_catalog.default_database.lineorder
  47. (
  48.   `lo_orderkey` bigint, 
  49.   `lo_linenumber` bigint, 
  50.   `lo_custkey` int, 
  51.   `lo_partkey` int, 
  52.   `lo_suppkey` int, 
  53.   `lo_orderdate` int, 
  54.   `lo_orderpriotity` string, 
  55.   `lo_shippriotity` int, 
  56.   `lo_quantity` bigint, 
  57.   `lo_extendedprice` bigint, 
  58.   `lo_ordtotalprice` bigint, 
  59.   `lo_discount` bigint, 
  60.   `lo_revenue` bigint, 
  61.   `lo_supplycost` bigint, 
  62.   `lo_tax` bigint, 
  63.   `lo_commitdate` int, 
  64.   `lo_shipmode` string
  65. ) with (
  66.     'connector.master' = 'hz-hadoop-test-199-141-27:7051',
  67.     'connector.table' = 'impala::yjuny_kudu.lineorder',
  68.   'connector.property-version' = '1',
  69.   'connector.type' = 'kudu-retract',
  70.   'connector.primary-key.0' = 'lo_orderkey',
  71.   'connector.primary-key.1' = 'lo_linenumber'
  72. )
  74. CREATE TABLE default_catalog.default_database.part
  75. (
  76.   p_partkey INT,
  77.   p_name STRING,
  78.   p_mfgr STRING,
  79.   p_category STRING,
  80.   p_brand STRING,
  81.   p_color STRING,
  82.   p_type STRING,
  83.   p_size INT,
  84.   p_container STRING
  85. ) with (
  86.     'connector.master' = 'hz-hadoop-test-199-141-27:7051',
  87.     'connector.table' = 'impala::yjuny_kudu.part',
  88.   'connector.property-version' = '1',
  89.   'connector.type' = 'kudu-retract',
  90.   'connector.primary-key.0' = 'p_partkey'
  91. )
  93. CREATE TABLE default_catalog.default_database.supplier
  94. (
  95.   s_suppkey INT,
  96.   s_name STRING,
  97.   s_address STRING,
  98.   s_city STRING,
  99.   s_nation STRING,
  100.   s_region STRING,
  101.   s_phone STRING
  102. ) with (
  103.     'connector.master' = 'hz-hadoop-test-199-141-27:7051',
  104.     'connector.table' = 'impala::yjuny_kudu.supplier',
  105.   'connector.property-version' = '1',
  106.   'connector.type' = 'kudu-retract',
  107.   'connector.primary-key.0' = 's_suppkey'
  108. )

作业配置

  1. INSERT INTO job (job_id, name, description, tenant_id, project_id) VALUES (1, 'ssb_etl', null, 1, 1);
  2. INSERT INTO job_exec_env (job_id, mode, parallelism) VALUES (1, 'BATCH', 1);
  4. INSERT INTO job_script_sql (job_id, name, position, script, description, sql_type, dialect) VALUES (1, 'customer_insert', 1, 'insert into customer select * from flink_hive.ssb.customer', null, 'INSERT', 'FLINK');
  5. INSERT INTO job_script_sql (job_id, name, position, script, description, sql_type, dialect) VALUES (1, 'dates_insert', 1, 'insert into dates select * from flink_hive.ssb.dates', null, 'INSERT', 'FLINK');
  6. INSERT INTO job_script_sql (job_id, name, position, script, description, sql_type, dialect) VALUES (1, 'supplier_insert', 1, 'insert into lineorder select * from flink_hive.ssb.lineorder', null, 'INSERT', 'FLINK');
  7. INSERT INTO job_script_sql (job_id, name, position, script, description, sql_type, dialect) VALUES (1, 'part_insert', 1, 'insert into part select * from flink_hive.ssb.part', null, 'INSERT', 'FLINK');
  8. INSERT INTO job_script_sql (job_id, name, position, script, description, sql_type, dialect) VALUES (1, 'supplier_insert', 1, 'insert into supplier select * from flink_hive.ssb.supplier', null, 'INSERT', 'FLINK');
  11. INSERT INTO cluster (cluster_id, cluster_name, type, deploy_type, status, description, cluster_master_host, cluster_master_port) VALUES (1, 'flink1.10-yjuny', 'FLINK', 'YARN-SESSION', 'RUNNING', null, 'hz-hadoop-test-199-141-30', 43420);
  12. INSERT INTO cluster_yarn (cluster_id, application_id, user, queue) VALUES (1, 'application_1581335885532_12895', 'flink', 'root.flink');
  13. INSERT INTO cluster_config (cluster_id, key, value) VALUES (1, '__cmd.start', './bin/yarn-session.sh -tm 8192 -d -s 2 -jm 4096 -nm flink1.10-yjuny ');