说明

分析性别分布，保存到mysql数据库

要求

将 mysql 驱动放入到spark的 jars 目录

代码

# 分析性别，保存到mysql数据库

import findspark
import datetime

findspark.init()

from pyspark.sql import  SparkSession 


def exe():
    spark = SparkSession.builder.\
    appName("gender").\
    master("local[*]").\
    enableHiveSupport().\
    getOrCreate()

    df = spark.sql("select gender,count(1) as `count`,'weibo' as oragin,substr(current_timestamp(),1,19) as `create_time` from yuqing_dwd.t_weibo group by gender")
    # df = spark.sql("select 'f' as gender,1 as `count`,'weibo' as oragin,substr(current_timestamp(),1,19) as `create_time`")

    df.show()
    df.printSchema()

    prop = {"user": "root","password": "root","driver": "com.mysql.jdbc.Driver"}
    df.write.jdbc("jdbc:mysql://hadoop1:3306/yuqing_dws?useSSL=false&characterEncoding=utf8", "t_gender_statistics","append", prop)

    spark.stop()

if __name__ == "__main__":
    exe()