创建SparkSession和sparkSQL的详细过程

//set up the spark configuration and create contexts
 val sparkConf = new SparkConf().setAppName("SparkSessionZipsExample").setMaster("local")
 
 
// your handle to SparkContext to access other context like SQLContext
 val sc = new SparkContext(sparkConf).set("spark.some.config.option", "some-value")
 val sqlContext = new org.apache.spark.sql.SQLContext(sc)

import org.apache.spark.sql.SparkSession
val spark: SparkSession = SparkSession.builder
 .appName("My Spark Application")  //设置 application 的名字
 .master("local[*]")             
 .enableHiveSupport() //增加支持 hive Support      
 .config("spark.sql.warehouse.dir", "target/spark-warehouse") //设置各种配置
 .getOrCreate     //获取或者新建一个 sparkSession

//配置设置
 spark.conf.set("spark.sql.shuffle.partitions", 6)
 spark.conf.set("spark.executor.memory", "2g")
 
 //获取配置，可以使用Scala的迭代器来读取configMap中的数据。 
 val configMap:Map[String, String] = spark.conf.getAll()

spark.catalog.listTables.show(false)
 spark.catalog.listDatabases.show(false)

// create a Dataset using spark.range starting from 5 to 100,
// with increments of 5
val numDS = spark.range(5, 100, 5)//创建dataset
 
// reverse the order and display first 5 items
numDS.orderBy(desc("id")).show(5)
 
//compute descriptive stats and display them
numDs.describe().show()
 
// create a DataFrame using spark.createDataFrame from a List or Seq
val langPercentDF = spark.createDataFrame( List( ("Scala", 35), 
    ("Python", 30), ("R", 15), ("Java", 20)) )//创建dataframe
 
//rename the columns
val lpDF = langPercentDF.withColumnRenamed("_1", "language").
    withColumnRenamed("_2", "percent")
 
//order the DataFrame in descending order of percentage
lpDF.orderBy(desc("percent")).show(false)

import spark.implicits //使RDD转化为DataFrame以及后续SQL操作
//读取JSON文件，生成DataFrame
val jsonFile = args(0)
val zipsDF = spark.read.json(jsonFile)

zipsDF.createOrReplaceTempView("zips_table")//对上面的dataframe创建一个表
zipsDF.cache()//缓存表
 
val resultsDF = spark.sql("SELECT city, pop, state, zip FROM zips_table")
 
//对表调用SQL语句
resultsDF.show(10)//展示结果

//drop the table if exists to get around existing table error
 spark.sql("DROP TABLE IF EXISTS zips_hive_table")
 
 //save as a hive table
 spark.table("zips_table").write.saveAsTable("zips_hive_table")
 
 //make a similar query against the hive table
 val resultsHiveDF = spark.sql("SELECT city, pop, state, 
        zip FROM zips_hive_table WHERE pop > 40000")
 
 resultsHiveDF.show(10)

from pyspark import SparkContext, SparkConf
from pyspark.sql import SparkSession, SQLContext, HiveContext
 
conf = SparkConf().setAppName('test').setMaster('yarn')
sc = SparkContext(conf=conf)
 
sqlc = SQLContext(sc)
print(dir(sqlc))
# 'cacheTable', 'clearCache', 'createDataFrame', 'createExternalTable', 'dropTempTable', 'getConf', 'getOrCreate', 'newSession', 'range', 'read', 'readStream',
# 'registerDataFrameAsTable', 'registerFunction', 'registerJavaFunction', 'setConf', 'sparkSession', 'sql', 'streams', 'table', 'tableNames', 'tables', 'udf', 'uncacheTable'
 
### sqlcontext 读取数据也自动生成 df
data = sqlc.read.text('/usr/yanshw/test.txt')
print(type(data))

from pyspark import SparkContext, SparkConf
from pyspark.sql import SparkSession, SQLContext, HiveContext
 
conf = SparkConf().setAppName('test').setMaster('yarn')
sc = SparkContext(conf=conf)
## 需要把 hive/conf/hive-site.xml 复制到 spark/conf 下
hivec = HiveContext(sc)
print(dir(hivec))
# 'cacheTable', 'clearCache', 'createDataFrame', 'createExternalTable', 'dropTempTable', 'getConf', 'getOrCreate', 'newSession', 'range', 'read', 'readStream','refreshTable',
# 'registerDataFrameAsTable', 'registerFunction', 'registerJavaFunction', 'setConf', 'sparkSession', 'sql', 'streams', 'table', 'tableNames', 'tables', 'udf', 'uncacheTable'
 
data = hivec.sql('''select * from hive1101.person limit 2''')
print(type(data))

from pyspark.sql import SparkSession
 
 
### method 1
sess = SparkSession.builder \
    .appName("aaa") \
    .config("spark.driver.extraClassPath", sparkClassPath) \
    .master("local") \
    .enableHiveSupport() \  # sparkSQL 连接 hive 时需要这句
    .getOrCreate()      # builder 方式必须有这句
 
### method 2
conf = SparkConf().setAppName('myapp1').setMaster('local[4]')   # 设定 appname 和 master
sess = SparkSession.builder.config(conf=conf).getOrCreate() # builder 方式必须有这句
 
### method 3
from pyspark import SparkContext, SparkConf
conf = SparkConf().setAppName('myapp1').setMaster('local[4]')   # 设定 appname 和 master
sc = SparkContext(conf=conf)
sess = SparkSession(sc)

from pyspark import SparkContext, SparkConf
from pyspark.sql import SparkSession, SQLContext, HiveContext
 
conf = SparkConf().setAppName('test').setMaster('yarn')
sc = SparkContext(conf=conf)
 
#### 替代了 SQLContext 和 HiveContext，其实只是简单的封装，提供了统一的接口
spark = SparkSession(sc)
print(dir(spark))
# 很多属性，我把私有属性删了
# 'Builder','builder', 'catalog', 'conf', 'createDataFrame', 'newSession', 'range', 'read', 'readStream','sparkContext', 'sql', 'stop', 'streams', 'table', 'udf', 'version'
 
### sess 读取数据自动生成 df
data = spark.read.text('/usr/yanshw/test.txt')      #read 可读类型 [ 'csv', 'format', 'jdbc', 'json', 'load', 'option', 'options', 'orc', 'parquet', 'schema', 'table', 'text']
print(type(data))       # <class 'pyspark.sql.dataframe.DataFrame'>

## 也需要把 hive/conf/hive-site.xml 复制到 spark/conf 下
spark = SparkSession.builder.appName('test').master('yarn').enableHiveSupport().getOrCreate()
 
hive_data = spark.sql('select * from hive1101.person limit 2')
print(hive_data)        # DataFrame[name: string, idcard: string]