基于python+大数据爬虫技术+数据可视化+Spark的电力能耗数据分析与可视化平台设计与实现

from pyspark.sql import SparkSession
from pyspark.sql.functions import *
import pyspark.sql.functions as F

# 初始化Spark会话
spark = SparkSession.builder.appName("EnergyConsumptionAnalysis").getOrCreate()

# 假设电力能耗数据已经通过爬虫技术爬取并保存到了CSV文件中
energyDataCSVPath = "path/to/energy_consumption_data.csv"

# 读取CSV文件到DataFrame
energyDataDF = spark.read.csv(energyDataCSVPath, header=True, inferSchema=True)

# 重命名列，以符合你的模型或分析需要
energyDataDF = energyDataDF.withColumnRenamed("date", "date") \
                           .withColumnRenamed("value", "energy_consumed")

# 转换日期格式，如果需要
energyDataDF = energyDataDF.withColumn("date", to_date(col("date"), "yyyy-MM-dd"))

# 按日期分组，并计算每日能耗总和
dailyEnergyConsumption = energyDataDF.groupBy("date").agg(sum("energy_consumed").alias("total_consumed"))

# 将结果显示为DataFrame
dailyEnergyConsumption.show()

# 关闭Spark会话
spark.stop()