HistGradientBoostingClassifier drops down into 0 loss and 0 accuracy after a certain iteration

[Loci-Tran]

Hi,

Does anyone know why the model vanishes loss and accuracy?
I implemented 5-folds cross validation and trained the HistGradientBoostingClassifier models for 300 iterations.

from load_data import load_data
folds, test_samples, input_shape = load_data(flag=False) # in-house data

Build model:

from sklearn.ensemble import HistGradientBoostingClassifier

clfs = []
for i in range(5):
    train, val, test = folds[i].values()
    clf_i = HistGradientBoostingClassifier(max_iter=300).fit(train['x'], train['y'])
    clfs.append(clf_i)

# print out the parameters of the model
params = clf_i.get_params()
params

{'categorical_features': None,
 'class_weight': None,
 'early_stopping': 'auto',
 'interaction_cst': None,
 'l2_regularization': 0.0,
 'learning_rate': 0.1,
 'loss': 'log_loss',
 'max_bins': 255,
 'max_depth': None,
 'max_iter': 300,
 'max_leaf_nodes': 31,
 'min_samples_leaf': 20,
 'monotonic_cst': None,
 'n_iter_no_change': 10,
 'random_state': None,
 'scoring': 'loss',
 'tol': 1e-07,
 'validation_fraction': 0.1,
 'verbose': 0,
 'warm_start': False}

Visualization the loss and accuracy of 300 iterations

import numpy as np
from sklearn import ensemble
from sklearn.metrics import accuracy_score

def acc_score(clf, x, y):
    acc_ = np.zeros((clf.get_params()['max_iter'],), dtype=np.float64)
    for i, y_pred in enumerate(clf.staged_predict(x)):
        acc_[i] = accuracy_score(y, y_pred)
    return acc_

def loss_score(clf, x, y):
    loss_ = np.zeros((clf.get_params()['max_iter'],), dtype=np.float64)
    for i, y_pred in enumerate(clf.staged_decision_function(x)):
        loss_[i] = clf._loss(y, y_pred)
    return loss_

entities = ['train', 'val', 'test']
colors = ['blue', 'red', 'green']
linestyles = ['-', '-', '-.']
linewidths = [1, 2, 2]

# Plot learning curve
fig, axes = plt.subplots(2, 5, figsize=(25, 10))
fig.suptitle('Loss and Accuracy', fontsize=35)
for fold, ax, clf in zip(range(5), axes[0], clfs):
    train_loss = loss_score(clf, train['x'], train['y'])
    val_loss = loss_score(clf, val['x'], val['y'])
    test_loss = loss_score(clf, test['x'], test['y'])
    for entity_loss, entity, color, linestyle, linewidth in zip([train_loss, val_loss, test_loss], entities, colors, linestyles, linewidths):
        ax.plot(entity_loss, label=entity, color=color, linestyle=linestyle, linewidth=linewidth)
    ax.set_title(f'Split {fold+1}', fontsize=30)
    ax.legend(fontsize=15)
    ax.grid(axis='y')
    ax.tick_params(labelsize=20)
    ax.xticks = np.arange(0, params["max_iter"], 10)
    ax.yticks = np.arange(0, 1, 0.2)
    ax.set_xlim(0, params["max_iter"])
    ax.set_ylim(0, 1)
    
for fold, ax, clf in zip(range(5), axes[1], clfs):
    train_acc = acc_score(clf, train['x'], train['y'])
    val_acc = acc_score(clf, val['x'], val['y'])
    test_acc = acc_score(clf, test['x'], test['y'])
    for entity_acc, entity, color, linestyle, linewidth in zip([train_acc, val_acc, test_acc], entities, colors, linestyles, linewidths):
        ax.plot(entity_acc, label=entity, color=color, linestyle=linestyle, linewidth=linewidth)
    ax.set_xlabel('max_iter', fontsize=30)
    ax.xticks = np.arange(0, params["max_iter"], 10)
    ax.legend(fontsize=15)
    ax.grid(axis='y')
    ax.tick_params(labelsize=20)
    ax.set_xlim(0, params["max_iter"])
    ax.set_ylim(0, 1)
    

axes[0, 0].set_ylabel('Loss', fontsize=30)
axes[1, 0].set_ylabel('Accuracy', fontsize=30)

plt.tight_layout()
plt.show()

This is the result

You can notice that the curves of both train, test, and validation loss/accuracy immediately dropping down after a certain iteration.
Very appreciate if you share your thoughts. Thank you.

[glemaitre]

Could you investigate that you really get 0.0 or instead NaN?

Somehow a losses of zero would mean that you get perfect classification which apparently is not the case. So I would wonder if there not invalid gradients or values at the iteration where you observe the drop.