说明
通过 DSL 实现分组查询
代码：
import org.apache.spark.SparkConf
import org.apache.spark.sql.catalyst.ScalaReflection
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.{DataFrame, SparkSession}
import java.sql.Timestamp
object Std4DSL {
case class Cdz(id: String, eleNum: Integer, time:Integer, startTime: Timestamp, endTime: Timestamp)
def main(args: Array[String]): Unit = {
val sc = new SparkConf().setMaster("local[*]").setAppName("std")
//创建SparkSession
val ss: SparkSession = SparkSession.builder().config(sc).getOrCreate()
ss.conf.set("spark.sql.shuffle.partitions", 1)
val df: DataFrame = ss.read
.option("header", "false") //默认为false，表示第一行不是表头
.option("sep", ",") //列分隔符
// 根据 样例类的主构造形参的顺序 匹配 文件的 列顺序
.schema(ScalaReflection.schemaFor[Cdz].dataType.asInstanceOf[StructType]) // 指定类型
.csv("cdz.txt") //读取文件
val resDF = df
.where("eleNum > 100")
.groupBy("id")
.sum("eleNum")
resDF.printSchema()
resDF.show()
ss.stop()
}
}