Optuna: Modern Hyperparameter Optimization That Beats Grid Search

import optuna
import lightgbm as lgb
from sklearn.model_selection import cross_val_score
from sklearn.datasets import load_breast_cancer

X, y = load_breast_cancer(return_X_y=True)

def objective(trial: optuna.Trial) -> float:
    params = {
        "n_estimators": trial.suggest_int("n_estimators", 100, 2000),
        "num_leaves": trial.suggest_int("num_leaves", 20, 300),
        "learning_rate": trial.suggest_float("learning_rate", 1e-4, 0.3, log=True),
        "min_child_samples": trial.suggest_int("min_child_samples", 5, 100),
        "subsample": trial.suggest_float("subsample", 0.5, 1.0),
        "colsample_bytree": trial.suggest_float("colsample_bytree", 0.5, 1.0),
    }
    model = lgb.LGBMClassifier(**params, random_state=42)
    scores = cross_val_score(model, X, y, cv=5, scoring="roc_auc", n_jobs=-1)
    return scores.mean()

study = optuna.create_study(direction="maximize")
study.optimize(objective, n_trials=100, n_jobs=1)

print(f"Best AUC: {study.best_value:.4f}")
print(f"Best params: {study.best_params}")

import optuna
import pytorch_lightning as pl
from optuna.integration import PyTorchLightningPruningCallback

def objective(trial: optuna.Trial) -> float:
    lr = trial.suggest_float("lr", 1e-5, 1e-2, log=True)
    hidden_size = trial.suggest_categorical("hidden_size", [64, 128, 256, 512])

    model = MyModel(lr=lr, hidden_size=hidden_size)
    trainer = pl.Trainer(
        max_epochs=50,
        callbacks=[PyTorchLightningPruningCallback(trial, monitor="val_loss")],
    )
    trainer.fit(model, train_loader, val_loader)
    return trainer.callback_metrics["val_loss"].item()

study = optuna.create_study(
    direction="minimize",
    pruner=optuna.pruners.MedianPruner(n_startup_trials=5, n_warmup_steps=10),
)
study.optimize(objective, n_trials=100)

import optuna

# All workers share this study via PostgreSQL
study = optuna.create_study(
    study_name="distributed_lgbm",
    storage="postgresql://user:pass@db-host/optuna",
    direction="maximize",
    load_if_exists=True,
)
study.optimize(objective, n_trials=50)  # run on each machine

from optuna.visualization import (
    plot_optimization_history,
    plot_param_importances,
    plot_parallel_coordinate,
)

# Which trials converged to best values
fig = plot_optimization_history(study)
fig.show()

# Which hyperparameters matter most (Fanova-based)
fig = plot_param_importances(study)
fig.show()

# Parallel coordinates  -  see correlations between params and objective
fig = plot_parallel_coordinate(study)
fig.show()

pip install optuna-dashboard
optuna-dashboard postgresql://user:pass@db-host/optuna