テーブルデータ向けDNN”TabNet”で回帰問題に挑戦【Python】

pip install pytorch-tabnet

!pip install pytorch-tabnet

import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
from sklearn.datasets import make_regression
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error

from pytorch_tabnet.tab_model import TabNetRegressor
from pytorch_tabnet.pretraining import TabNetPretrainer

import torch

X, Y, coef = make_regression(
                       random_state=42, 
                       n_samples=10000, 
                       n_features=10,
                       n_informative=5,
                       noise=1,
                       bias=1.0,
                       coef=True)

for i, c in enumerate(coef):
  print(f"feature {i} : {c}")

train_rate, val_rate, test_rate = 0.7, 0.15, 0.15

X_train, X_test, y_train, y_test = train_test_split(X, Y, train_size=train_rate, random_state=42)
X_valid, X_test, y_valid, y_test = train_test_split(X_test, y_test, test_size=test_rate/(test_rate+val_rate), random_state=42)

y_train = y_train.reshape(-1, 1)
y_valid = y_valid.reshape(-1, 1)
y_test = y_test.reshape(-1, 1)

unsupervised_model = TabNetPretrainer(
    optimizer_fn=torch.optim.Adam,
    optimizer_params=dict(lr=2e-2),
    mask_type='entmax'
)

unsupervised_model.fit(
    X_train=X_train,
    eval_set=[X_valid],
    pretraining_ratio=0.8,
    max_epochs = 100,
)

tabnet_params = {
                 "optimizer_fn":torch.optim.Adam,
                 "optimizer_params":dict(lr=2e-2),
                 "scheduler_fn":torch.optim.lr_scheduler.ReduceLROnPlateau, #torch.optim.lr_scheduler.StepLR,
                 "mask_type":'entmax',
                 }

model = TabNetRegressor(**tabnet_params)

max_epochs = 100

model.fit(
    X_train=X_train, y_train=y_train,
    eval_set=[(X_train, y_train), (X_valid, y_valid)],
    eval_name=['train', 'valid'],
    eval_metric=['rmse'],
    max_epochs=max_epochs , patience=20,
    batch_size=1024, virtual_batch_size=128,
    num_workers=0,
    drop_last=False,
    augmentations=None,
    from_unsupervised=unsupervised_model
)

import matplotlib.pyplot as plt

for param in ['loss', 'lr', 'valid_rmse']:
    plt.plot(model.history[param], label=param)
    plt.xlabel('epoch')
    plt.grid()
    plt.legend()
    plt.show()

preds = model.predict(X_test)
test_mse = mean_squared_error(preds, y_test)


preds_valid = model.predict(X_valid)
valid_mse = mean_squared_error(preds_valid, y_valid)

print("valid_mse = ", valid_mse)
print("test_mse = ", test_mse)

importance = sorted([(i, n) for i, n in enumerate(model.feature_importances_)], key = lambda x: x[1], reverse = True)
label, y = [], []
for e in importance:
  print(f"feature {e[0]} : {e[1]}")
  label.append(e[0])
  y.append(e[1])

plt.bar([i for i in range(len(y))], y, tick_label = label)
plt.show()

from lightgbm import LGBMRegressor

lgb_params = {
  'n_estimators': 10000,
  'learning_rate': 0.01,
  'random_state': 42,
  'early_stopping_round': 100,
  'metric': 'rmse'
}

lgb = LGBMRegressor(**lgb_params)
lgb.fit(X_train, y_train, eval_set=[(X_valid, y_valid)])

preds = lgb.predict(X_test)
test_mse = mean_squared_error(preds, y_test)


preds_valid = lgb.predict(X_valid)
valid_mse = mean_squared_error(preds_valid, y_valid)

print("valid_mse = ", valid_mse)
print("test_mse = ", test_mse)