1. public class Application {
    3.     private static String appName = "spark.demo";
    4.     private static String master = "local[*]";
    6.     public static void main(String[] args) {
    8.         SparkSession spark = SparkSession
    9.                 .builder()
    10.                 .appName("Java Spark Hive Example")
    11.                 .master("local[*]")
    12. //                .config("spark.sql.warehouse.dir","hdfs://172.29.108.184:8020/hive")
    13.                 .enableHiveSupport()
    14.                 .getOrCreate();
    16.         spark.sql("show databases").show();
    17.         spark.sql("show tables").show();
    18.         spark.sql("desc tmp.parse_log").show();
    20. //
    21. //
    22. //        JavaSparkContext sc = null;
    23. //        try {
    24. //            //初始化 JavaSparkContext
    25. //            SparkConf conf = new SparkConf()
    26. //                    .setAppName(appName)
    27. //                    .setMaster(master)
    28. //                    .set("hive.metastore.uris", "thrift://172.29.108.183:9083")
    29. //                    .set("spark.sql.sources.partitionOverwriteMode", "dynamic");
    30. //
    31. //            sc = new JavaSparkContext(conf);
    32. //
    33. //            List<SqlJob.LogStore> list = SqlJob.getList();
    34. //
    35. //            SQLContext sqlContext = new SQLContext(sc);
    36. //            // 注册成表
    37. //            Dataset<org.apache.spark.sql.Row> dataFrame = sqlContext.createDataFrame(list, SqlJob.LogStore.class);
    38. //
    39. //            sqlContext.table("parse_log").show();
    40. //            dataFrame.createOrReplaceTempView("tmp_table");
    41. //            System.out.println( "注册表 OK" );
    42. //            // 写入数据库
    43. //            sqlContext.sql("INSERT OVERWRITE TABLE tmp.parse_log partition (day='20200331') select tableNames, sql from tmp_table");
    45. //            sql.toDF().write().saveAsTable("tmp.parse_log");
    46. //            System.out.println( "写入数据表 OK" );
    47. //            sqlContext.sql("show tables").show();
    48. //        } catch (Exception e) {
    49. //            e.printStackTrace();
    50. //        } finally {
    51. //            if (sc != null) {
    52. //                sc.close();
    53. //            }
    54. //        }
    55.     }
    56. }
    1. ...
    3. <spark.version>2.4.0</spark.version>
    5. <dependency>
    6.     <groupId>org.apache.spark</groupId>
    7.     <artifactId>spark-core_2.11</artifactId>
    8.     <version>${spark.version}</version>
    9. </dependency>
    10. <dependency>
    11.     <groupId>org.apache.spark</groupId>
    12.     <artifactId>spark-sql_2.11</artifactId>
    13.     <version>${spark.version}</version>
    14. </dependency>
    15. <dependency>
    16.     <groupId>org.apache.spark</groupId>
    17.     <artifactId>spark-hive_2.11</artifactId>
    18.     <version>${spark.version}</version>
    19. </dependency>
    21. ...

    远端hive配置hive-site.xml
    image.png