表格数据 ML 完全指南:精通 XGBoost、LightGBM、CatBoost 和 TabNet

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from scipy import stats

# 查看基本信息
def eda_overview(df):
    print("=== 数据概览 ===")
    print(f"Shape: {df.shape}")
    print(f"\n数据类型:\n{df.dtypes}")
    print(f"\n缺失值情况:\n{df.isnull().sum()}")
    print(f"\n缺失值比例:\n{df.isnull().mean() * 100:.2f}%")
    print(f"\n数值型统计:\n{df.describe()}")
    print(f"\n类别特征唯一值数量:")
    for col in df.select_dtypes(include='object').columns:
        print(f"  {col}: {df[col].nunique()} unique values")

# 查看目标分布
def plot_target_distribution(df, target_col):
    fig, axes = plt.subplots(1, 2, figsize=(14, 5))

    # 分布直方图
    axes[0].hist(df[target_col], bins=50, color='steelblue', edgecolor='white')
    axes[0].set_title(f'{target_col} 分布')
    axes[0].set_xlabel(target_col)

    # QQ plot(正态性检验)
    stats.probplot(df[target_col].dropna(), dist="norm", plot=axes[1])
    axes[1].set_title('Q-Q Plot (正态性检验)')

    plt.tight_layout()
    plt.show()

# 数值特征间的相关性
def plot_correlation_matrix(df, target_col, top_n=20):
    numeric_df = df.select_dtypes(include=[np.number])

    # 按与目标的相关性选出前 N 个
    corr_with_target = abs(numeric_df.corr()[target_col]).sort_values(ascending=False)
    top_features = corr_with_target.head(top_n + 1).index.tolist()

    corr_matrix = numeric_df[top_features].corr()

    plt.figure(figsize=(12, 10))
    mask = np.triu(np.ones_like(corr_matrix, dtype=bool))
    sns.heatmap(
        corr_matrix, mask=mask, annot=True, fmt='.2f',
        cmap='RdYlBu_r', center=0, square=True, linewidths=0.5
    )
    plt.title(f'排名前 {top_n} 的特征相关性')
    plt.tight_layout()
    plt.show()

# 异常值检测(IQR 方法)
def detect_outliers_iqr(df, columns):
    outlier_info = {}
    for col in columns:
        Q1 = df[col].quantile(0.25)
        Q3 = df[col].quantile(0.75)
        IQR = Q3 - Q1
        lower = Q1 - 1.5 * IQR
        upper = Q3 + 1.5 * IQR
        outliers = df[(df[col] < lower) | (df[col] > upper)]
        outlier_info[col] = {
            'count': len(outliers),
            'ratio': len(outliers) / len(df),
            'lower': lower,
            'upper': upper
        }
    return pd.DataFrame(outlier_info).T

from sklearn.impute import SimpleImputer, KNNImputer
from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer

# 1. 简单填补
def simple_imputation(df):
    # 数值型:用中位数填补
    numeric_cols = df.select_dtypes(include=[np.number]).columns
    num_imputer = SimpleImputer(strategy='median')
    df[numeric_cols] = num_imputer.fit_transform(df[numeric_cols])

    # 类别型:用众数填补
    cat_cols = df.select_dtypes(include='object').columns
    cat_imputer = SimpleImputer(strategy='most_frequent')
    df[cat_cols] = cat_imputer.fit_transform(df[cat_cols])

    return df

# 2. KNN 填补(对小规模数据集有效)
def knn_imputation(df, n_neighbors=5):
    numeric_cols = df.select_dtypes(include=[np.number]).columns
    imputer = KNNImputer(n_neighbors=n_neighbors)
    df[numeric_cols] = imputer.fit_transform(df[numeric_cols])
    return df

# 3. MICE(Multiple Imputation by Chained Equations)
def mice_imputation(df, max_iter=10):
    numeric_cols = df.select_dtypes(include=[np.number]).columns
    imputer = IterativeImputer(max_iter=max_iter, random_state=42)
    df[numeric_cols] = imputer.fit_transform(df[numeric_cols])
    return df

# 4. 添加缺失指示变量(当缺失模式本身就是信息时)
def add_missing_indicators(df):
    cols_with_missing = df.columns[df.isnull().any()].tolist()
    for col in cols_with_missing:
        df[f'{col}_missing'] = df[col].isnull().astype(int)
    return df

H(S) = -sum(p_i * log2(p_i))

IG(S, A) = H(S) - sum(|S_v|/|S| * H(S_v))

Gini(S) = 1 - sum(p_i^2)

import numpy as np
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
from sklearn.tree import export_text, plot_tree
import matplotlib.pyplot as plt

# 训练决策树
def train_decision_tree(X_train, y_train, task='classification'):
    if task == 'classification':
        model = DecisionTreeClassifier(
            criterion='gini',      # 'gini' 或 'entropy'
            max_depth=5,           # 树的最大深度
            min_samples_split=10,  # 划分所需的最小样本数
            min_samples_leaf=5,    # 叶节点最小样本数
            max_features=None,     # 考虑的最大特征数
            random_state=42
        )
    else:
        model = DecisionTreeRegressor(
            criterion='squared_error',
            max_depth=5,
            min_samples_split=10,
            min_samples_leaf=5,
            random_state=42
        )

    model.fit(X_train, y_train)
    return model

# 可视化决策树
def visualize_tree(model, feature_names, class_names=None, max_depth=3):
    plt.figure(figsize=(20, 10))
    plot_tree(
        model,
        feature_names=feature_names,
        class_names=class_names,
        filled=True,
        rounded=True,
        max_depth=max_depth,
        fontsize=10
    )
    plt.title('Decision Tree Visualization')
    plt.tight_layout()
    plt.show()

    # 以文本形式输出
    print(export_text(model, feature_names=feature_names, max_depth=3))

from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
from sklearn.model_selection import cross_val_score
import joblib

def train_random_forest(X_train, y_train, task='classification'):
    if task == 'classification':
        model = RandomForestClassifier(
            n_estimators=300,         # 树的数量
            max_depth=None,           # 不限制(注意过拟合)
            min_samples_split=2,
            min_samples_leaf=1,
            max_features='sqrt',      # 每次划分考虑的特征数
            bootstrap=True,           # 使用自助采样
            oob_score=True,          # 计算 OOB 分数
            n_jobs=-1,               # 并行处理
            random_state=42
        )
    else:
        model = RandomForestRegressor(
            n_estimators=300,
            max_features='sqrt',
            oob_score=True,
            n_jobs=-1,
            random_state=42
        )

    model.fit(X_train, y_train)

    print(f"OOB Score: {model.oob_score_:.4f}")
    return model

# 特征重要度分析
def plot_feature_importance(model, feature_names, top_n=20):
    importances = pd.Series(
        model.feature_importances_,
        index=feature_names
    ).sort_values(ascending=False)

    plt.figure(figsize=(10, 8))
    importances.head(top_n).plot(kind='barh', color='steelblue')
    plt.title(f'随机森林特征重要度(前 {top_n} 名)')
    plt.xlabel('Importance')
    plt.gca().invert_yaxis()
    plt.tight_layout()
    plt.show()

    return importances

# 排列重要度(更可靠的重要度指标)
from sklearn.inspection import permutation_importance

def compute_permutation_importance(model, X_val, y_val, feature_names):
    result = permutation_importance(
        model, X_val, y_val,
        n_repeats=10,
        random_state=42,
        n_jobs=-1
    )

    perm_imp = pd.DataFrame({
        'feature': feature_names,
        'importance_mean': result.importances_mean,
        'importance_std': result.importances_std
    }).sort_values('importance_mean', ascending=False)

    return perm_imp

r_i = -[dL(y_i, F(x_i)) / dF(x_i)]

from sklearn.ensemble import GradientBoostingClassifier, GradientBoostingRegressor

def train_sklearn_gbm(X_train, y_train, task='classification'):
    if task == 'classification':
        model = GradientBoostingClassifier(
            n_estimators=200,
            learning_rate=0.1,
            max_depth=3,
            subsample=0.8,        # 每棵树使用的样本比例
            max_features='sqrt',
            random_state=42
        )
    else:
        model = GradientBoostingRegressor(
            n_estimators=200,
            learning_rate=0.1,
            max_depth=3,
            subsample=0.8,
            random_state=42
        )

    model.fit(X_train, y_train)
    return model

import xgboost as xgb
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, roc_auc_score

# XGBoost 完整参数指南
xgb_params = {
    # === 学习参数 ===
    'objective': 'binary:logistic',   # 目标函数
    # 分类:'binary:logistic', 'multi:softprob'
    # 回归:'reg:squarederror', 'reg:absoluteerror', 'reg:tweedie'
    # 排序:'rank:pairwise'

    'eval_metric': 'auc',             # 评估指标
    # 'logloss', 'rmse', 'mae', 'auc', 'aucpr', 'merror', 'mlogloss'

    # === 树参数 ===
    'n_estimators': 1000,             # 树的数量(配合早停使用)
    'max_depth': 6,                   # 树的最大深度(默认:6)
    'min_child_weight': 1,            # 叶节点最小权重和(防止过拟合)
    'gamma': 0,                       # 分裂所需的最小损失减少量(为 0 则总会分裂)
    'max_delta_step': 0,              # 每棵树权重的最大变化量(对不平衡数据有用)

    # === 采样参数 ===
    'subsample': 0.8,                 # 每棵树训练时的样本比例(0.5~0.9)
    'colsample_bytree': 0.8,          # 每棵树的特征采样比例
    'colsample_bylevel': 1.0,         # 每层的特征采样
    'colsample_bynode': 1.0,          # 每个分裂节点的特征采样

    # === 正则化参数 ===
    'reg_alpha': 0,                   # L1 正则化(特征选择效果)
    'reg_lambda': 1,                  # L2 正则化(默认:1)

    # === 学习率 ===
    'learning_rate': 0.01,            # 学习率(eta)。越低越能防止过拟合

    # === 其他 ===
    'scale_pos_weight': 1,            # 不平衡数据:负/正样本比例
    'tree_method': 'hist',            # 'hist'(快速)、'exact'、'approx'、'gpu_hist'
    'seed': 42,
    'n_jobs': -1,
}

def train_xgboost_full(X, y, task='binary'):
    X_train, X_val, y_train, y_val = train_test_split(
        X, y, test_size=0.2, random_state=42, stratify=y if task=='binary' else None
    )

    if task == 'binary':
        objective = 'binary:logistic'
        eval_metric = 'auc'
    elif task == 'multiclass':
        objective = 'multi:softprob'
        eval_metric = 'mlogloss'
    else:  # regression
        objective = 'reg:squarederror'
        eval_metric = 'rmse'

    model = xgb.XGBClassifier(
        n_estimators=1000,
        max_depth=6,
        min_child_weight=1,
        gamma=0,
        subsample=0.8,
        colsample_bytree=0.8,
        reg_alpha=0.1,
        reg_lambda=1.0,
        learning_rate=0.01,
        objective=objective,
        eval_metric=eval_metric,
        tree_method='hist',
        random_state=42,
        n_jobs=-1,
    )

    # 早停(early stopping)
    model.fit(
        X_train, y_train,
        eval_set=[(X_train, y_train), (X_val, y_val)],
        early_stopping_rounds=50,
        verbose=100,
    )

    print(f"Best iteration: {model.best_iteration}")
    print(f"Best score: {model.best_score:.4f}")

    return model, X_val, y_val

# 用 SHAP 值解释模型
def explain_with_shap(model, X_val, feature_names):
    import shap

    explainer = shap.TreeExplainer(model)
    shap_values = explainer.shap_values(X_val)

    # 汇总图
    plt.figure(figsize=(10, 8))
    shap.summary_plot(shap_values, X_val, feature_names=feature_names, show=False)
    plt.tight_layout()
    plt.show()

    # 特征重要度(mean absolute SHAP)
    mean_abs_shap = pd.DataFrame({
        'feature': feature_names,
        'importance': np.abs(shap_values).mean(axis=0)
    }).sort_values('importance', ascending=False)

    return mean_abs_shap

# 使用 GPU 的 XGBoost(需要 NVIDIA GPU)
model_gpu = xgb.XGBClassifier(
    n_estimators=1000,
    tree_method='gpu_hist',    # GPU 直方图方法
    predictor='gpu_predictor', # GPU 预测
    device='cuda',             # XGBoost 2.0+ 中使用 device='cuda'
    random_state=42
)

import lightgbm as lgb
from sklearn.model_selection import StratifiedKFold
import numpy as np

# LightGBM 完整参数指南
lgb_params = {
    # === 核心参数 ===
    'objective': 'binary',           # 目标函数
    # 'binary', 'multiclass', 'regression', 'regression_l1', 'huber'
    # 'cross_entropy', 'mape', 'gamma', 'tweedie'

    'metric': 'auc',                 # 评估指标
    # 'auc', 'binary_logloss', 'rmse', 'mae', 'mse', 'multi_logloss'

    # === 树参数 ===
    'n_estimators': 1000,
    'num_leaves': 31,                # 叶节点数量(设置为小于 2^max_depth)
    'max_depth': -1,                 # -1 表示不限制(由 num_leaves 控制)
    'min_child_samples': 20,         # 叶节点最小样本数(防止过拟合)
    'min_child_weight': 0.001,       # 叶节点最小权重和
    'max_bin': 255,                  # 直方图最大区间数

    # === 采样 ===
    'subsample': 0.8,                # 行采样比例
    'subsample_freq': 1,             # 采样周期
    'colsample_bytree': 0.8,         # 列(特征)采样比例

    # === 正则化 ===
    'reg_alpha': 0.1,                # L1 正则化
    'reg_lambda': 0.1,               # L2 正则化
    'min_split_gain': 0.0,           # 最小分裂增益

    # === 学习率 ===
    'learning_rate': 0.01,

    # === 类别特征 ===
    'cat_smooth': 10,                # 类别特征平滑

    # === 进阶 ===
    'boosting_type': 'gbdt',         # 'gbdt', 'rf', 'dart', 'goss'
    'is_unbalance': False,           # 不平衡数据
    'scale_pos_weight': 1,

    # === 系统 ===
    'device': 'cpu',                 # 'cpu', 'gpu', 'cuda'
    'n_jobs': -1,
    'random_state': 42,
    'verbose': -1,
}

def train_lightgbm_cv(X, y, n_folds=5):
    """用 Stratified K-Fold CV 训练 LightGBM"""
    skf = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=42)

    oof_preds = np.zeros(len(X))
    models = []

    for fold, (train_idx, val_idx) in enumerate(skf.split(X, y)):
        print(f"\n--- Fold {fold + 1}/{n_folds} ---")

        X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]
        y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]

        model = lgb.LGBMClassifier(
            n_estimators=2000,
            num_leaves=31,
            learning_rate=0.05,
            subsample=0.8,
            colsample_bytree=0.8,
            reg_alpha=0.1,
            reg_lambda=0.1,
            min_child_samples=20,
            random_state=42,
            n_jobs=-1,
            verbose=-1,
        )

        model.fit(
            X_train, y_train,
            eval_set=[(X_val, y_val)],
            callbacks=[
                lgb.early_stopping(stopping_rounds=100, verbose=True),
                lgb.log_evaluation(period=200),
            ],
        )

        oof_preds[val_idx] = model.predict_proba(X_val)[:, 1]
        models.append(model)

        fold_auc = roc_auc_score(y_val, oof_preds[val_idx])
        print(f"Fold {fold+1} AUC: {fold_auc:.4f}")

    overall_auc = roc_auc_score(y, oof_preds)
    print(f"\nOverall OOF AUC: {overall_auc:.4f}")

    return models, oof_preds

# 类别特征处理
def lgbm_with_categorical(X_train, y_train, X_val, y_val, cat_features):
    """使用 LightGBM 内置的类别特征处理"""

    # 将类别特征编码为 integer
    for col in cat_features:
        X_train[col] = X_train[col].astype('category').cat.codes
        X_val[col] = X_val[col].astype('category').cat.codes

    train_data = lgb.Dataset(
        X_train, y_train,
        categorical_feature=cat_features,
        free_raw_data=False
    )
    val_data = lgb.Dataset(
        X_val, y_val,
        reference=train_data,
        categorical_feature=cat_features
    )

    params = {
        'objective': 'binary',
        'metric': 'auc',
        'num_leaves': 31,
        'learning_rate': 0.05,
        'verbose': -1,
    }

    model = lgb.train(
        params, train_data,
        num_boost_round=1000,
        valid_sets=[val_data],
        callbacks=[lgb.early_stopping(100), lgb.log_evaluation(100)],
    )

    return model

from catboost import CatBoostClassifier, CatBoostRegressor, Pool
import pandas as pd
import numpy as np

def train_catboost(X_train, y_train, X_val, y_val, cat_features=None):
    """训练 CatBoost - 类别特征可以直接以字符串形式传入"""

    # 创建 CatBoost Pool(高效的数据处理)
    train_pool = Pool(
        data=X_train,
        label=y_train,
        cat_features=cat_features  # 类别特征的索引或名称列表
    )
    val_pool = Pool(
        data=X_val,
        label=y_val,
        cat_features=cat_features
    )

    model = CatBoostClassifier(
        # === 基本参数 ===
        iterations=1000,              # 与 n_estimators 相同
        learning_rate=0.05,
        depth=6,                      # 树的深度(默认:6,最大:16)

        # === 正则化 ===
        l2_leaf_reg=3.0,             # L2 正则化
        min_data_in_leaf=1,           # 叶节点最小样本数

        # === 采样 ===
        subsample=0.8,               # 行采样(Bernoulli/MVS 提升)
        colsample_bylevel=0.8,       # 每层的列采样

        # === 类别特征处理 ===
        cat_features=cat_features,
        one_hot_max_size=2,          # 应用 OHE 的最大唯一值数量

        # === 提升类型 ===
        boosting_type='Ordered',      # 'Ordered' 或 'Plain'
        bootstrap_type='Bayesian',    # 'Bayesian', 'Bernoulli', 'MVS', 'No'
        bagging_temperature=1.0,      # Bayesian bootstrap 温度

        # === 损失函数 ===
        loss_function='Logloss',      # 分类:'Logloss', 'CrossEntropy'
        eval_metric='AUC',           # 'AUC', 'Accuracy', 'F1', 'RMSE'

        # === 系统 ===
        task_type='CPU',             # 'CPU' 或 'GPU'
        devices='0',                 # GPU 设备 ID
        random_seed=42,
        verbose=100,
        use_best_model=True,         # 使用最优迭代处的模型
        early_stopping_rounds=50,
    )

    model.fit(
        train_pool,
        eval_set=val_pool,
        plot=False,                   # 学习曲线可视化
    )

    print(f"Best iteration: {model.get_best_iteration()}")
    print(f"Best score: {model.get_best_score()}")

    return model

# XGBoost vs LightGBM vs CatBoost 比较
def compare_gbm_models(X, y, cat_features=None):
    """比较三种模型的性能与速度"""
    import time
    from sklearn.model_selection import cross_val_score

    results = {}

    # XGBoost(类别特征需要标签编码)
    start = time.time()
    xgb_model = xgb.XGBClassifier(
        n_estimators=300, max_depth=6, learning_rate=0.1,
        subsample=0.8, colsample_bytree=0.8, random_state=42, n_jobs=-1
    )
    xgb_scores = cross_val_score(xgb_model, X, y, cv=5, scoring='roc_auc')
    results['XGBoost'] = {
        'mean_auc': xgb_scores.mean(),
        'std_auc': xgb_scores.std(),
        'time': time.time() - start
    }

    # LightGBM
    start = time.time()
    lgb_model = lgb.LGBMClassifier(
        n_estimators=300, num_leaves=31, learning_rate=0.1,
        subsample=0.8, colsample_bytree=0.8, random_state=42, n_jobs=-1, verbose=-1
    )
    lgb_scores = cross_val_score(lgb_model, X, y, cv=5, scoring='roc_auc')
    results['LightGBM'] = {
        'mean_auc': lgb_scores.mean(),
        'std_auc': lgb_scores.std(),
        'time': time.time() - start
    }

    # CatBoost
    start = time.time()
    cb_model = CatBoostClassifier(
        iterations=300, depth=6, learning_rate=0.1,
        random_seed=42, verbose=0
    )
    cb_scores = cross_val_score(cb_model, X, y, cv=5, scoring='roc_auc')
    results['CatBoost'] = {
        'mean_auc': cb_scores.mean(),
        'std_auc': cb_scores.std(),
        'time': time.time() - start
    }

    # 输出结果
    result_df = pd.DataFrame(results).T
    print("=== GBM 模型比较 ===")
    print(result_df.to_string())

    return result_df

from sklearn.preprocessing import (
    StandardScaler, MinMaxScaler, RobustScaler,
    PowerTransformer, QuantileTransformer
)
from scipy.stats import boxcox
import numpy as np

def transform_numeric_features(df, columns):
    """数值特征变换策略"""
    transformed = df.copy()

    for col in columns:
        # 1. 对数变换(正偏度、右尾)
        if (df[col] > 0).all():
            transformed[f'{col}_log'] = np.log1p(df[col])

        # 2. 平方根变换
        if (df[col] >= 0).all():
            transformed[f'{col}_sqrt'] = np.sqrt(df[col])

        # 3. Box-Cox 变换(仅限正数)
        if (df[col] > 0).all():
            transformed[f'{col}_boxcox'], _ = boxcox(df[col] + 1)

        # 4. Yeo-Johnson 变换(可包含负数)
        pt = PowerTransformer(method='yeo-johnson')
        transformed[f'{col}_yeojohnson'] = pt.fit_transform(df[[col]])

        # 5. 分位数变换(转换为正态分布)
        qt = QuantileTransformer(output_distribution='normal', n_quantiles=1000)
        transformed[f'{col}_quantile'] = qt.fit_transform(df[[col]])

    return transformed

# 分箱(Binning)
def create_bins(df, col, n_bins=10, strategy='quantile'):
    """将连续变量转换为区间"""
    from sklearn.preprocessing import KBinsDiscretizer

    kbd = KBinsDiscretizer(n_bins=n_bins, encode='ordinal', strategy=strategy)
    # strategy: 'uniform', 'quantile', 'kmeans'
    df[f'{col}_bin'] = kbd.fit_transform(df[[col]]).astype(int)

    return df

from category_encoders import (
    TargetEncoder, LeaveOneOutEncoder,
    CatBoostEncoder, BinaryEncoder, HashingEncoder
)

def encode_categorical_features(X_train, X_val, y_train, cat_features):
    """多种类别特征编码方法"""
    results = {}

    # 1. One-Hot Encoding(低基数)
    from sklearn.preprocessing import OneHotEncoder
    ohe = OneHotEncoder(sparse_output=False, handle_unknown='ignore')
    # 建议只用于基数在 10 以下的特征

    # 2. Label Encoding(有序类别)
    from sklearn.preprocessing import LabelEncoder
    le = LabelEncoder()

    # 3. Target Encoding(中~高基数)
    # 注意:在训练数据上可能发生目标泄漏 -> 只能在 CV 内部使用
    te = TargetEncoder(cols=cat_features, smoothing=1.0)
    X_train_te = te.fit_transform(X_train, y_train)
    X_val_te = te.transform(X_val)
    results['target_enc'] = (X_train_te, X_val_te)

    # 4. Leave-One-Out Encoding(防止目标泄漏)
    loo = LeaveOneOutEncoder(cols=cat_features, sigma=0.05)
    X_train_loo = loo.fit_transform(X_train, y_train)
    X_val_loo = loo.transform(X_val)
    results['loo_enc'] = (X_train_loo, X_val_loo)

    # 5. CatBoost Encoding(Ordered TS)
    cbe = CatBoostEncoder(cols=cat_features)
    X_train_cbe = cbe.fit_transform(X_train, y_train)
    X_val_cbe = cbe.transform(X_val)
    results['catboost_enc'] = (X_train_cbe, X_val_cbe)

    return results

# 日期/时间特征工程
def extract_datetime_features(df, date_col):
    """提取日期/时间特征"""
    df[date_col] = pd.to_datetime(df[date_col])

    df[f'{date_col}_year'] = df[date_col].dt.year
    df[f'{date_col}_month'] = df[date_col].dt.month
    df[f'{date_col}_day'] = df[date_col].dt.day
    df[f'{date_col}_dayofweek'] = df[date_col].dt.dayofweek
    df[f'{date_col}_dayofyear'] = df[date_col].dt.dayofyear
    df[f'{date_col}_weekofyear'] = df[date_col].dt.isocalendar().week.astype(int)
    df[f'{date_col}_quarter'] = df[date_col].dt.quarter
    df[f'{date_col}_hour'] = df[date_col].dt.hour
    df[f'{date_col}_is_weekend'] = (df[date_col].dt.dayofweek >= 5).astype(int)
    df[f'{date_col}_is_month_start'] = df[date_col].dt.is_month_start.astype(int)
    df[f'{date_col}_is_month_end'] = df[date_col].dt.is_month_end.astype(int)

    # 周期性编码(sin/cos 变换)
    df[f'{date_col}_month_sin'] = np.sin(2 * np.pi * df[f'{date_col}_month'] / 12)
    df[f'{date_col}_month_cos'] = np.cos(2 * np.pi * df[f'{date_col}_month'] / 12)
    df[f'{date_col}_hour_sin'] = np.sin(2 * np.pi * df[f'{date_col}_hour'] / 24)
    df[f'{date_col}_hour_cos'] = np.cos(2 * np.pi * df[f'{date_col}_hour'] / 24)

    return df

# 聚合特征(GroupBy 特征)
def create_aggregate_features(df, group_cols, agg_cols):
    """按分组生成聚合统计特征"""
    for group_col in group_cols:
        for agg_col in agg_cols:
            prefix = f'{agg_col}_by_{group_col}'
            agg = df.groupby(group_col)[agg_col].agg([
                'mean', 'std', 'min', 'max', 'median',
                lambda x: x.quantile(0.25),
                lambda x: x.quantile(0.75)
            ])
            agg.columns = [
                f'{prefix}_mean', f'{prefix}_std',
                f'{prefix}_min', f'{prefix}_max',
                f'{prefix}_median', f'{prefix}_q25', f'{prefix}_q75'
            ]
            df = df.join(agg, on=group_col)

    return df

from sklearn.model_selection import cross_val_predict, StratifiedKFold
from sklearn.linear_model import LogisticRegression, Ridge
from sklearn.metrics import roc_auc_score
import numpy as np
import pandas as pd

class StackingEnsemble:
    """Stacking 集成实现"""

    def __init__(self, base_models, meta_model, n_folds=5):
        self.base_models = base_models
        self.meta_model = meta_model
        self.n_folds = n_folds
        self.trained_base_models = []

    def fit(self, X, y):
        """训练基础模型并生成元特征"""
        meta_features_train = np.zeros((len(X), len(self.base_models)))
        skf = StratifiedKFold(n_splits=self.n_folds, shuffle=True, random_state=42)

        for i, (name, model) in enumerate(self.base_models):
            print(f"Training base model: {name}")
            fold_models = []

            for fold, (train_idx, val_idx) in enumerate(skf.split(X, y)):
                if hasattr(X, 'iloc'):
                    X_tr, X_val = X.iloc[train_idx], X.iloc[val_idx]
                    y_tr, y_val = y.iloc[train_idx], y.iloc[val_idx]
                else:
                    X_tr, X_val = X[train_idx], X[val_idx]
                    y_tr, y_val = y[train_idx], y[val_idx]

                model_clone = type(model)(**model.get_params())
                model_clone.fit(X_tr, y_tr)
                fold_models.append(model_clone)

                if hasattr(model_clone, 'predict_proba'):
                    meta_features_train[val_idx, i] = model_clone.predict_proba(X_val)[:, 1]
                else:
                    meta_features_train[val_idx, i] = model_clone.predict(X_val)

            self.trained_base_models.append((name, fold_models))

        # 训练元模型
        self.meta_model.fit(meta_features_train, y)

        meta_auc = roc_auc_score(y, meta_features_train.mean(axis=1))
        print(f"Meta features mean AUC: {meta_auc:.4f}")

        return self

    def predict_proba(self, X):
        """对测试数据进行预测"""
        meta_features_test = np.zeros((len(X), len(self.base_models)))

        for i, (name, fold_models) in enumerate(self.trained_base_models):
            fold_preds = []
            for model in fold_models:
                if hasattr(model, 'predict_proba'):
                    fold_preds.append(model.predict_proba(X)[:, 1])
                else:
                    fold_preds.append(model.predict(X))
            meta_features_test[:, i] = np.mean(fold_preds, axis=0)

        return self.meta_model.predict_proba(meta_features_test)

# 实战集成实现
def build_ensemble(X_train, y_train, X_test):
    base_models = [
        ('xgb', xgb.XGBClassifier(
            n_estimators=500, max_depth=6, learning_rate=0.05,
            subsample=0.8, colsample_bytree=0.8, random_state=42
        )),
        ('lgb', lgb.LGBMClassifier(
            n_estimators=500, num_leaves=31, learning_rate=0.05,
            subsample=0.8, colsample_bytree=0.8, random_state=42, verbose=-1
        )),
        ('cb', CatBoostClassifier(
            iterations=500, depth=6, learning_rate=0.05,
            random_seed=42, verbose=0
        )),
    ]

    meta_model = LogisticRegression(C=1.0, max_iter=1000)

    stacker = StackingEnsemble(base_models, meta_model, n_folds=5)
    stacker.fit(X_train, y_train)

    return stacker.predict_proba(X_test)[:, 1]

# 加权集成(Weighted Blending)
def weighted_blend(predictions, weights):
    """
    predictions: list of arrays(每个模型的预测概率)
    weights: list of floats(每个模型的权重,合计 = 1.0)
    """
    weights = np.array(weights) / sum(weights)
    blended = sum(w * p for w, p in zip(weights, predictions))
    return blended

from pytorch_tabnet.tab_model import TabNetClassifier, TabNetRegressor
import numpy as np
from sklearn.preprocessing import LabelEncoder

def train_tabnet(X_train, y_train, X_val, y_val, cat_features=None, cat_dims=None):
    """训练 TabNet"""

    # 类别特征处理
    if cat_features is None:
        cat_features = []
        cat_dims = []

    model = TabNetClassifier(
        # === 架构 ===
        n_d=64,                      # 预测层维度(与 n_a 相同)
        n_a=64,                      # Attention 嵌入维度
        n_steps=5,                   # Sequential attention 步骤数
        gamma=1.3,                   # 特征复用系数(1.0~2.0)
        n_independent=2,             # 独立 GLU 层数
        n_shared=2,                  # 共享 GLU 层数

        # === 类别嵌入 ===
        cat_idxs=list(range(len(cat_features))),
        cat_dims=cat_dims,
        cat_emb_dim=1,               # 类别嵌入维度

        # === 正则化 ===
        lambda_sparse=1e-3,          # Sparsity 正则化系数
        momentum=0.02,               # BatchNorm momentum
        epsilon=1e-15,               # 数值稳定性

        # === 训练 ===
        optimizer_fn=torch.optim.Adam,
        optimizer_params=dict(lr=2e-2),
        scheduler_params=dict(
            mode='min', patience=5, min_lr=1e-5, factor=0.9
        ),
        scheduler_fn=torch.optim.lr_scheduler.ReduceLROnPlateau,
        mask_type='entmax',          # 'sparsemax' 或 'entmax'

        # === 其他 ===
        verbose=10,
        seed=42,
        device_name='auto',          # 'cpu', 'cuda', 'auto'
    )

    # 训练
    model.fit(
        X_train=X_train.values if hasattr(X_train, 'values') else X_train,
        y_train=y_train.values if hasattr(y_train, 'values') else y_train,
        eval_set=[(
            X_val.values if hasattr(X_val, 'values') else X_val,
            y_val.values if hasattr(y_val, 'values') else y_val
        )],
        eval_name=['val'],
        eval_metric=['auc'],
        max_epochs=200,
        patience=20,                 # Early stopping patience
        batch_size=1024,
        virtual_batch_size=128,      # Ghost batch normalization
        num_workers=0,
        drop_last=False,
        pretraining_ratio=0.8,       # 预训练掩码比例
    )

    return model

# TabNet 特征重要度
def tabnet_feature_importance(model, feature_names):
    importances = model.feature_importances_
    imp_df = pd.DataFrame({
        'feature': feature_names,
        'importance': importances
    }).sort_values('importance', ascending=False)

    plt.figure(figsize=(10, 8))
    imp_df.head(20).plot(x='feature', y='importance', kind='barh', color='coral')
    plt.title('TabNet 特征重要度')
    plt.gca().invert_yaxis()
    plt.tight_layout()
    plt.show()

    return imp_df

from sklearn.model_selection import (
    StratifiedKFold, KFold, GroupKFold,
    StratifiedGroupKFold, TimeSeriesSplit
)
import numpy as np

def kaggle_cv_pipeline(X, y, model, groups=None, time_col=None, n_folds=5):
    """Kaggle 级别的 CV 流水线"""

    # 选择 CV 策略
    if time_col is not None:
        # 时间序列数据
        cv = TimeSeriesSplit(n_splits=n_folds)
        print("使用 TimeSeriesSplit")
    elif groups is not None:
        # 分组数据(同一组进入同一折)
        cv = StratifiedGroupKFold(n_splits=n_folds, shuffle=True, random_state=42)
        print("使用 StratifiedGroupKFold")
    else:
        # 标准分类
        cv = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=42)
        print("使用 StratifiedKFold")

    oof_predictions = np.zeros(len(X))
    feature_importances = []

    for fold, (train_idx, val_idx) in enumerate(
        cv.split(X, y, groups) if groups is not None else cv.split(X, y)
    ):
        print(f"\n{'='*50}")
        print(f"Fold {fold + 1}/{n_folds}")
        print(f"Train size: {len(train_idx)}, Val size: {len(val_idx)}")

        if hasattr(X, 'iloc'):
            X_tr, X_val = X.iloc[train_idx], X.iloc[val_idx]
            y_tr, y_val = y.iloc[train_idx], y.iloc[val_idx]
        else:
            X_tr, X_val = X[train_idx], X[val_idx]
            y_tr, y_val = y[train_idx], y[val_idx]

        model.fit(X_tr, y_tr)

        if hasattr(model, 'predict_proba'):
            oof_predictions[val_idx] = model.predict_proba(X_val)[:, 1]
        else:
            oof_predictions[val_idx] = model.predict(X_val)

        # 收集特征重要度
        if hasattr(model, 'feature_importances_'):
            feature_importances.append(model.feature_importances_)

        fold_score = roc_auc_score(y_val, oof_predictions[val_idx])
        print(f"Fold {fold+1} AUC: {fold_score:.5f}")

    overall_score = roc_auc_score(y, oof_predictions)
    print(f"\n{'='*50}")
    print(f"Overall OOF AUC: {overall_score:.5f}")

    # 平均特征重要度
    if feature_importances:
        mean_importance = np.mean(feature_importances, axis=0)
    else:
        mean_importance = None

    return oof_predictions, mean_importance, overall_score

# 防止泄漏(Leakage)
def prevent_leakage_pipeline(X_train, X_val, y_train):
    """
    防止数据泄漏的预处理流水线
    核心:fit 只在训练数据上进行,transform 在训练/验证数据上都要应用
    """
    from sklearn.pipeline import Pipeline
    from sklearn.preprocessing import StandardScaler
    from sklearn.impute import SimpleImputer

    # 错误的做法(会发生泄漏):
    # scaler = StandardScaler()
    # X_train_scaled = scaler.fit_transform(X_train)  # 用全部数据 fit
    # X_val_scaled = scaler.transform(X_val)

    # 正确的做法(使用流水线):
    pipeline = Pipeline([
        ('imputer', SimpleImputer(strategy='median')),
        ('scaler', StandardScaler()),
    ])

    # fit 只在训练数据上进行
    X_train_processed = pipeline.fit_transform(X_train)
    # transform 应用在验证/测试数据上
    X_val_processed = pipeline.transform(X_val)

    return X_train_processed, X_val_processed, pipeline

# pip install boruta
from boruta import BorutaPy
from sklearn.ensemble import RandomForestClassifier

def boruta_feature_selection(X, y, max_iter=100):
    """用 Boruta 算法进行特征选择"""
    rf = RandomForestClassifier(
        n_estimators=200,
        max_depth=7,
        random_state=42,
        n_jobs=-1
    )

    boruta = BorutaPy(
        estimator=rf,
        n_estimators='auto',
        perc=100,           # 百分位数(100 = 与最大值比较)
        alpha=0.05,         # 显著性水平
        max_iter=max_iter,
        random_state=42,
        verbose=1
    )

    boruta.fit(X.values, y.values)

    # 结果分析
    feature_ranking = pd.DataFrame({
        'feature': X.columns,
        'ranking': boruta.ranking_,
        'selected': boruta.support_,
        'tentative': boruta.support_weak_
    }).sort_values('ranking')

    selected_features = X.columns[boruta.support_].tolist()
    tentative_features = X.columns[boruta.support_weak_].tolist()

    print(f"已选特征: {len(selected_features)} 个")
    print(f"待定特征: {len(tentative_features)} 个")
    print(f"已剔除特征: {len(X.columns) - len(selected_features) - len(tentative_features)} 个")

    return selected_features, tentative_features, feature_ranking

# 基于 SHAP 的特征选择
def shap_feature_selection(model, X_val, threshold=0.01):
    """基于 SHAP 值的特征选择"""
    import shap

    explainer = shap.TreeExplainer(model)
    shap_values = explainer.shap_values(X_val)

    if isinstance(shap_values, list):
        shap_abs = np.abs(shap_values[1])
    else:
        shap_abs = np.abs(shap_values)

    mean_shap = shap_abs.mean(axis=0)
    total_shap = mean_shap.sum()

    feature_importance = pd.DataFrame({
        'feature': X_val.columns,
        'mean_abs_shap': mean_shap,
        'shap_ratio': mean_shap / total_shap
    }).sort_values('mean_abs_shap', ascending=False)

    # 按累计 SHAP 比例选择
    feature_importance['cumulative_ratio'] = feature_importance['shap_ratio'].cumsum()
    selected = feature_importance[feature_importance['mean_abs_shap'] > threshold * total_shap]

    print(f"基于 SHAP 的选择: {len(selected)} 个 / 共 {len(X_val.columns)} 个")

    return selected['feature'].tolist(), feature_importance

def complete_kaggle_pipeline(train_df, test_df, target_col, cat_features=None):
    """
    完整的 Kaggle ML 流水线
    - EDA -> 预处理 -> 特征工程 -> 模型训练 -> 集成
    """
    from sklearn.metrics import roc_auc_score

    # 1. 分离
    y = train_df[target_col]
    X = train_df.drop(columns=[target_col])

    # 2. 缺失值处理
    for col in X.select_dtypes(include=[np.number]).columns:
        X[col] = X[col].fillna(X[col].median())
        test_df[col] = test_df[col].fillna(X[col].median())

    # 3. 类别特征编码
    if cat_features:
        for col in cat_features:
            X[col] = X[col].astype('category').cat.codes
            test_df[col] = test_df[col].astype('category').cat.codes

    # 4. 训练集成模型
    skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

    xgb_oof = np.zeros(len(X))
    lgb_oof = np.zeros(len(X))
    cb_oof = np.zeros(len(X))

    xgb_test_preds = np.zeros(len(test_df))
    lgb_test_preds = np.zeros(len(test_df))
    cb_test_preds = np.zeros(len(test_df))

    for fold, (train_idx, val_idx) in enumerate(skf.split(X, y)):
        X_tr, X_val = X.iloc[train_idx], X.iloc[val_idx]
        y_tr, y_val = y.iloc[train_idx], y.iloc[val_idx]

        # XGBoost
        xgb_model = xgb.XGBClassifier(
            n_estimators=1000, max_depth=6, learning_rate=0.05,
            subsample=0.8, colsample_bytree=0.8, reg_alpha=0.1,
            random_state=42, n_jobs=-1, eval_metric='auc'
        )
        xgb_model.fit(X_tr, y_tr, eval_set=[(X_val, y_val)],
                      early_stopping_rounds=50, verbose=False)
        xgb_oof[val_idx] = xgb_model.predict_proba(X_val)[:, 1]
        xgb_test_preds += xgb_model.predict_proba(test_df)[:, 1] / 5

        # LightGBM
        lgb_model = lgb.LGBMClassifier(
            n_estimators=1000, num_leaves=31, learning_rate=0.05,
            subsample=0.8, colsample_bytree=0.8, random_state=42,
            n_jobs=-1, verbose=-1
        )
        lgb_model.fit(X_tr, y_tr, eval_set=[(X_val, y_val)],
                      callbacks=[lgb.early_stopping(50, verbose=False),
                                lgb.log_evaluation(-1)])
        lgb_oof[val_idx] = lgb_model.predict_proba(X_val)[:, 1]
        lgb_test_preds += lgb_model.predict_proba(test_df)[:, 1] / 5

        # CatBoost
        cb_model = CatBoostClassifier(
            iterations=1000, depth=6, learning_rate=0.05,
            random_seed=42, verbose=0, early_stopping_rounds=50
        )
        cb_model.fit(X_tr, y_tr, eval_set=(X_val, y_val))
        cb_oof[val_idx] = cb_model.predict_proba(X_val)[:, 1]
        cb_test_preds += cb_model.predict_proba(test_df)[:, 1] / 5

    print(f"XGBoost OOF AUC: {roc_auc_score(y, xgb_oof):.5f}")
    print(f"LightGBM OOF AUC: {roc_auc_score(y, lgb_oof):.5f}")
    print(f"CatBoost OOF AUC: {roc_auc_score(y, cb_oof):.5f}")

    # 集成
    ensemble_oof = (xgb_oof + lgb_oof + cb_oof) / 3
    ensemble_test = (xgb_test_preds + lgb_test_preds + cb_test_preds) / 3

    print(f"Ensemble OOF AUC: {roc_auc_score(y, ensemble_oof):.5f}")

    return ensemble_test, ensemble_oof

# 运行示例
if __name__ == "__main__":
    # 生成示例数据
    from sklearn.datasets import make_classification
    from sklearn.model_selection import train_test_split

    X, y = make_classification(
        n_samples=10000, n_features=30, n_informative=20,
        n_redundant=5, random_state=42
    )
    X = pd.DataFrame(X, columns=[f'feature_{i}' for i in range(30)])
    y = pd.Series(y)

    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.2, random_state=42
    )

    # LightGBM CV 训练
    models, oof_preds = train_lightgbm_cv(X_train, y_train)
    print(f"Final OOF AUC: {roc_auc_score(y_train, oof_preds):.4f}")