Use Quadratic Weighted Kappa

from sklearn.datasets import make_multilabel_classification
from sklearn.model_selection import train_test_split
from sklearn.metrics import cohen_kappa_score
from bokbokbok.eval_metrics.classification import QuadraticWeightedKappaMetric
from bokbokbok.utils import clip_sigmoid

X, y = make_multilabel_classification(n_samples=1000, 
                                      n_features=10,
                                      n_classes=2,
                                      n_labels=1,
                                      random_state=41114)
y = y.sum(axis=1)
X_train, X_valid, y_train, y_valid = train_test_split(X, 
                                                      y, 
                                                      test_size=0.25, 
                                                      random_state=41114)

Usage in LightGBM

import lightgbm as lgb

train = lgb.Dataset(X_train, y_train)
valid = lgb.Dataset(X_valid, y_valid, reference=train)
params = {
     'n_estimators': 300,
     'seed': 41114,
     'n_jobs': 8,
     'learning_rate': 0.1,
    'objective':'multiclass',
    'num_class': 3
   }

clf = lgb.train(params=params,
                train_set=train,
                valid_sets=[train, valid],
                valid_names=['train','valid'],
                feval=QuadraticWeightedKappaMetric(),
                early_stopping_rounds=100)

Usage in XGBoost

import xgboost as xgb

dtrain = xgb.DMatrix(X_train, y_train)
dvalid = xgb.DMatrix(X_valid, y_valid)

params = {
     'seed': 41114,
     'learning_rate': 0.1,
    'disable_default_eval_metric': 1,
    'objective': 'multi:softprob',
    'num_class': 3
   }

bst = xgb.train(params,
          dtrain=dtrain,
          num_boost_round=300,
          early_stopping_rounds=100,
          verbose_eval=10,
          feval=QuadraticWeightedKappaMetric(XGBoost=True),
          maximize=True,
          evals=[(dtrain, 'dtrain'), (dvalid, 'dvalid')])