Use F1 Score

When to use F-Score

The F-score or F-measure is a measure of a test's accuracy. It is calculated from the precision and recall of the test, where the precision is the number of true positive results divided by the number of all positive results, including those not identified correctly, and the recall is the number of true positive results divided by the number of all samples that should have been identified as positive. Precision is also known as positive predictive value, and recall is also known as sensitivity in diagnostic binary classification.

The highest possible value of an F-score is 1.0, indicating perfect precision and recall, and the lowest possible value is 0, if either the precision or the recall is zero.

from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
from bokbokbok.eval_metrics.classification import F1_Score_Binary
from bokbokbok.utils import clip_sigmoid

X, y = make_classification(n_samples=1000, 
                           n_features=10, 
                           random_state=41114)

X_train, X_valid, y_train, y_valid = train_test_split(X, 
                                                      y, 
                                                      test_size=0.25, 
                                                      random_state=41114)

Usage in LightGBM

import lightgbm as lgb

train = lgb.Dataset(X_train, y_train)
valid = lgb.Dataset(X_valid, y_valid, reference=train)
params = {
     'n_estimators': 300,
     'objective': 'binary',
     'seed': 41114,
     'n_jobs': 8,
     'learning_rate': 0.1,
   }

clf = lgb.train(params=params,
                train_set=train,
                valid_sets=[train, valid],
                valid_names=['train','valid'],
                feval=F1_Score_Binary(average='micro'),
                early_stopping_rounds=100)

roc_auc_score(y_valid, clf.predict(X_valid))

Usage in XGBoost

import xgboost as xgb

dtrain = xgb.DMatrix(X_train, y_train)
dvalid = xgb.DMatrix(X_valid, y_valid)

params = {
     'seed': 41114,
     'objective':'binary:logistic',
     'learning_rate': 0.1,
    'disable_default_eval_metric': 1
   }

bst = xgb.train(params,
          dtrain=dtrain,
          num_boost_round=300,
          early_stopping_rounds=10,
          verbose_eval=10,
          maximize=True,
          feval=F1_Score_Binary(average='micro', XGBoost=True),
          evals=[(dtrain, 'dtrain'), (dvalid, 'dvalid')])

roc_auc_score(y_valid, clip_sigmoid(bst.predict(dvalid)))