Pandasで大規模データを扱う時にメモリ消費を抑える手法を紹介

import pandas as pd

df = pd.read_csv('large_data.csv', usecols=['column1', 'column2', 'column3'])

import pandas as pd

chunk_size = 10000 

for chunk in pd.read_csv('large_data.csv', chunksize=chunk_size):
    # 各チャンクに対して処理を行う
    # 例: 必要なデータをフィルタリング・統計計算、データクリーニング、ファイルへの保存など

df = pd.read_csv('large_data.csv', dtype={'column1': 'int32', 'column2': 'float32'})

import pandas as pd
import random

cols = 100
rows = 100_000
datas = {"column"+str(idx+1): [random.randint(0, 10000) for _ in range(rows)] for idx in range(cols)}


df = pd.DataFrame(datas)
df.to_csv("table.csv", index=False)

memory_usage = df.memory_usage(deep=True).sum()
print(f'DataFrame memory usage: {memory_usage/1024/1024 : .2f} Mbytes')
# DataFrame memory usage:  76.29 Mbytes

df = pd.read_csv("table.csv")
memory_usage = df.memory_usage(deep=True).sum()
print(f'DataFrame memory usage: {memory_usage/1024/1024 : .2f} Mbytes')
# DataFrame memory usage:  76.29 Mbytes

sel = ['column'+str(idx+1) for idx in range(0, cols, 10)]
df = pd.read_csv("table.csv", usecols=sel)
memory_usage = df.memory_usage(deep=True).sum()
print(f'DataFrame memory usage: {memory_usage/1024/1024 : .2f} Mbytes')
# DataFrame memory usage:  7.63 Mbytes

types = {'column'+str(idx+1): 'int16' for idx in range(cols)}
df = pd.read_csv("table.csv", dtype=types)
memory_usage = df.memory_usage(deep=True).sum()
print(f'DataFrame memory usage: {memory_usage/1024/1024 : .2f} Mbytes')
# DataFrame memory usage:  19.07 Mbytes

sel = ['column'+str(idx+1) for idx in range(0, cols, 10)]
types = {'column'+str(idx+1): 'int16' for idx in range(0, cols, 10)}
df = pd.read_csv("table.csv", usecols=sel, dtype=types)
memory_usage = df.memory_usage(deep=True).sum()
print(f'DataFrame memory usage: {memory_usage/1024/1024 : .2f} Mbytes')
# DataFrame memory usage:  1.91 Mbytes

chunk_size = 10000
for i, chunk in enumerate(pd.read_csv('table.csv', chunksize=chunk_size)):
    print(i, len(chunk))
    print(chunk)