pandas与pyspark计算效率对比分析

# 数据集构造
import pandas as pd
import numpy as np 
import pyarrow
import sys
import time
from pyspark.sql import SparkSession
df = pd.DataFrame(columns=['id','sales'])
df['id']= np.random.randint(1,10,800000000)    
df['sales']= np.random.randint(30,1000,800000000)   # 生成8亿数据
df = df.append(df)   # 数据量膨胀一倍
df.to_parquet('parquet_test')    # 写入本地文件
print(sys.getsizeof(df) / 1024 / 1024 / 1024)  # 总数据占用内存：23个g

def pandas_duration():
    start = time.time()
    # df.to_csv('data.txt',index=False,sep=',')
    df = pd.read_parquet('parquet_test')
    mid_time = time.time()
    print('pandas读取数据用时：{:.2f}'.format(mid_time-start))
    print(df.groupby('id',as_index=False).max())   # 分组求最大值
    end = time.time()
    print(end-start)

# 防止driver内存溢出，可以把资源调大点，笔者电脑64个g就随意填了个32g，分区数结合实际数据大小资源调整
spark = SparkSession.Builder()\
    .master("local[*]")\
    .config("spark.sql.shuffle.partitions",24)\
    .config("spark.driver.memory","32g")\
    .config("spark.driver.maxResultSize","32g")\
    .appName('pyspark')\
    .getOrCreate()
def pyspark_duration():
    start = time.time()
    # df.to_csv('data.txt',index=False,sep=',')
    spark_df = spark.read.parquet('parquet_test')
    mid_time = time.time()
    print('spark读取数据用时：{:.2f}'.format(mid_time-start))
    spark_df.groupBy('id').agg({"sales":"max"}).show()  # 分组求最大值
    end = time.time()
    print(end-start)