表形式データ向けDNN「TabNet」でクラス分類問題を解く（使い方）|Python

pip install pytorch-tabnet

!pip install pytorch-tabnet

import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score

from pytorch_tabnet.tab_model import TabNetClassifier
from pytorch_tabnet.pretraining import TabNetPretrainer

import torch

X, y = make_classification(n_samples=10000, 
                           n_features=10, 
                           n_redundant = 3,
                           n_informative = 5,
                           n_classes=2, 
                           random_state=42)

df = pd.DataFrame(X)
df['target'] = y
df.head(20)

train_rate, val_rate, test_rate = 0.7, 0.15, 0.15

X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=train_rate, random_state=42)
X_valid, X_test, y_valid, y_test = train_test_split(X_test, y_test, test_size=test_rate/(test_rate+val_rate), random_state=42)

unsupervised_model = TabNetPretrainer(
    optimizer_fn=torch.optim.Adam,
    optimizer_params=dict(lr=2e-2),
    mask_type='entmax' 
)

unsupervised_model.fit(
    X_train=X_train,
    eval_set=[X_valid],
    pretraining_ratio=0.8,
    max_epochs = 100,
)

tabnet_params = {
                 "optimizer_fn":torch.optim.Adam,
                 "optimizer_params":dict(lr=2e-2),
                 "scheduler_params":{"step_size":50, # how to use learning rate scheduler
                                 "gamma":0.9},
                 "scheduler_fn":torch.optim.lr_scheduler.StepLR,
                 "mask_type":'entmax',
                 }
max_epochs = 20

clf = TabNetClassifier(**tabnet_params
                      )

clf.fit(
    X_train=X_train, y_train=y_train,
    eval_set=[(X_train, y_train), (X_valid, y_valid)],
    eval_name=['train', 'valid'],
    eval_metric=['auc'],
    max_epochs=max_epochs , patience=20,
    batch_size=1024, virtual_batch_size=128,
    num_workers=0,
    weights=1,
    drop_last=False,
    augmentations=None,
    # from_unsupervised=unsupervised_model
)

preds = clf.predict_proba(X_test)
test_auc = roc_auc_score(y_score=preds[:,1], y_true=y_test)


preds_valid = clf.predict_proba(X_valid)
valid_auc = roc_auc_score(y_score=preds_valid[:,1], y_true=y_valid)

print("valid_auc = ", valid_auc)
print("test_auc = ", test_auc)

importance = sorted([(i, n) for i, n in enumerate(clf.feature_importances_)], key = lambda x: x[1], reverse = True)
label, y = [], []
for e in importance:
  print(f"feature {e[0]} : {e[1]}")
  label.append(e[0])
  y.append(e[1])

plt.bar([i for i in range(len(y))], y, tick_label = label)
plt.show()

from lightgbm import LGBMClassifier

lgb_params = {
  'n_estimators': 10000,
  'learning_rate': 0.05,
  'random_state': 42,
  'early_stopping_round': 20,
  'metric': 'auc'
}

lgb = LGBMClassifier(**lgb_params)
lgb.fit(X_train, y_train, eval_set=[(X_valid, y_valid)])

preds = lgb.predict_proba(X_test)
test_auc = roc_auc_score(y_score=preds[:,1], y_true=y_test)


preds_valid = lgb.predict_proba(X_valid)
valid_auc = roc_auc_score(y_score=preds_valid[:,1], y_true=y_valid)

print("valid_auc = ", valid_auc)
print("test_auc = ", test_auc)