pandasでCSV, Pickle, Parquetの読み書き速度とファイルサイズを比較検証

import pandas as pd
import numpy as np
import time

# DataFrameを自動生成
def generate_large_dataframe(rows=1_000_000, cols=10):
    """ランダムデータを含むDataFrameを生成する"""
    data = np.random.rand(rows, cols)
    columns = [f'col_{i}' for i in range(cols)]
    df = pd.DataFrame(data, columns=columns)
    return df

# ベンチマーク関数
def benchmark(df, file_prefix='benchmark'):
    formats = ['parquet', 'pkl', 'csv']
    results = {}

    # 各形式での書き出しと読み込みを計測
    for fmt in formats:
        file_name = f'{file_prefix}.{fmt}'

        # 書き出し時間の計測
        start_time = time.time()
        if fmt == 'parquet':
            df.to_parquet(file_name, engine='pyarrow')
        elif fmt == 'pkl':
            df.to_pickle(file_name)
        elif fmt == 'csv':
            df.to_csv(file_name, index=False)
        write_time = time.time() - start_time

        # 読み込み時間の計測
        start_time = time.time()
        if fmt == 'parquet':
            df_loaded = pd.read_parquet(file_name, engine='pyarrow')
        elif fmt == 'pkl':
            df_loaded = pd.read_pickle(file_name)
        elif fmt == 'csv':
            df_loaded = pd.read_csv(file_name)
        read_time = time.time() - start_time

        # 結果を保存
        results[fmt] = {'write_time': write_time, 'read_time': read_time}

    return results

# 大きなDataFrameを生成
df = generate_large_dataframe()

# ベンチマークの実行
results = benchmark(df)

# 結果の表示
for fmt, times in results.items():
    print(f"{fmt.upper()} - Write time: {times['write_time']:.4f} seconds, Read time: {times['read_time']:.4f} seconds")

import pandas as pd
import time
import requests
from pathlib import Path

# データのダウンロード
def download_data(url, file_name):
    response = requests.get(url)
    with open(file_name, 'wb') as f:
        f.write(response.content)

# ベンチマーク関数
def benchmark(file_format, df, file_name):
    # 書き出し
    start_time = time.time()
    if file_format == 'csv':
        df.to_csv(file_name, index=False)
    elif file_format == 'parquet':
        df.to_parquet(file_name, index=False)
    elif file_format == 'pkl':
        df.to_pickle(file_name)
    write_time = time.time() - start_time

    # 読み込み
    start_time = time.time()
    if file_format == 'csv':
        df_read = pd.read_csv(file_name)
    elif file_format == 'parquet':
        df_read = pd.read_parquet(file_name)
    elif file_format == 'pkl':
        df_read = pd.read_pickle(file_name)
    read_time = time.time() - start_time

    return write_time, read_time

# データの準備
def load_data(file_name):
    return pd.read_parquet(file_name, engine='pyarrow')

# ダウンロードURLとファイル名の設定
data_url = 'https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2022-01.parquet'
csv_file_name = 'yellow_tripdata_2022.parquet'

# データのダウンロード
print("Downloading data...")
download_data(data_url, csv_file_name)

# データフレームの読み込み
df = load_data(csv_file_name)

# ベンチマーク実行
file_names = {
    'csv': 'nyc_taxi_data.csv',
    'parquet': 'nyc_taxi_data.parquet',
    'pkl': 'nyc_taxi_data.pkl'
}

results = {}
for file_format in file_names:
    print(f"Testing {file_format} format...")
    write_time, read_time = benchmark(file_format, df, file_names[file_format])
    results[file_format] = {'write_time': write_time, 'read_time': read_time}

# 結果の表示
print("\nBenchmark Results:")
for file_format, times in results.items():
    print(f"{file_format.upper()} - Write Time: {times['write_time']:.4f} seconds, Read Time: {times['read_time']:.4f} seconds")