# 检查 Python 版本
python --version
python3 --version

# 用 pyenv 安装指定版本
pyenv install 3.12.0
pyenv global 3.12.0

# 创建虚拟环境
python -m venv ml_env

# 激活 (Linux/Mac)
source ml_env/bin/activate

# 激活 (Windows)
ml_env\Scripts\activate

# 停用
deactivate

# 创建环境
conda create -n ml_env python=3.12

# 激活
conda activate ml_env

# 安装包
conda install numpy pandas scikit-learn matplotlib

# 环境列表
conda env list

# 导出环境
conda env export > environment.yml

# 恢复环境
conda env create -f environment.yml

# 安装 Poetry
curl -sSL https://install.python-poetry.org | python3 -

# 初始化项目
poetry new ml_project
cd ml_project

# 添加包
poetry add numpy pandas scikit-learn torch

# 添加开发依赖
poetry add --dev pytest black flake8

# 运行环境
poetry run python train.py

# 安装 JupyterLab
pip install jupyterlab

# 注册内核
python -m ipykernel install --user --name=ml_env --display-name "ML Environment"

# 运行 JupyterLab
jupyter lab

# 安装常用扩展
pip install jupyterlab-git
pip install nbformat

c.ServerApp.open_browser = True
c.ServerApp.port = 8888
c.ServerApp.ip = '0.0.0.0'

# 检查 CUDA 版本
nvidia-smi
nvcc --version

# 安装带 CUDA 的 PyTorch (CUDA 12.1)
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121

# 带 GPU 支持的 TensorFlow
pip install tensorflow[and-cuda]

# 检查 GPU 是否可用 (PyTorch)
python -c "import torch; print(torch.cuda.is_available())"

# 检查 cuDNN
python -c "import torch; print(torch.backends.cudnn.version())"

# requirements.txt
numpy>=1.24.0
pandas>=2.0.0
matplotlib>=3.7.0
seaborn>=0.12.0
scikit-learn>=1.3.0
scipy>=1.11.0
torch>=2.0.0
torchvision>=0.15.0
tensorflow>=2.13.0
xgboost>=1.7.0
lightgbm>=4.0.0
optuna>=3.3.0
wandb>=0.15.0
tqdm>=4.65.0
jupyterlab>=4.0.0
black>=23.0.0
flake8>=6.0.0
pytest>=7.4.0

# 一次性安装
pip install -r requirements.txt

import numpy as np

# 基本数组创建
arr1 = np.array([1, 2, 3, 4, 5])
arr2 = np.array([[1, 2, 3], [4, 5, 6]])

print(arr1.shape)   # (5,)
print(arr2.shape)   # (2, 3)
print(arr2.dtype)   # int64
print(arr2.ndim)    # 2
print(arr2.size)    # 6

# 特殊数组
zeros = np.zeros((3, 4))          # 所有元素为 0
ones = np.ones((2, 3, 4))         # 所有元素为 1
full = np.full((3, 3), 7)         # 所有元素为 7
eye = np.eye(4)                    # 单位矩阵
empty = np.empty((2, 3))          # 未初始化的数组

# 区间数组
arange = np.arange(0, 10, 2)      # [0, 2, 4, 6, 8]
linspace = np.linspace(0, 1, 5)   # [0, 0.25, 0.5, 0.75, 1.0]
logspace = np.logspace(0, 3, 4)   # [1, 10, 100, 1000]

# 随机数数组
np.random.seed(42)
rand_uniform = np.random.rand(3, 4)       # [0, 1) 均匀分布
rand_normal = np.random.randn(3, 4)       # 标准正态分布
rand_int = np.random.randint(0, 10, (3, 4))  # 整数随机数
rand_choice = np.random.choice([1, 2, 3, 4, 5], size=10, replace=True)

# 高级随机数(推荐方式)
rng = np.random.default_rng(42)
samples = rng.normal(loc=0, scale=1, size=(100, 3))

import numpy as np

a = np.array([[1, 2, 3], [4, 5, 6]])
b = np.array([[7, 8, 9], [10, 11, 12]])

# 基本算术运算(逐元素)
print(a + b)   # 逐元素加法
print(a - b)   # 逐元素减法
print(a * b)   # 逐元素乘法
print(a / b)   # 逐元素除法
print(a ** 2)  # 逐元素平方
print(a % 2)   # 逐元素取余

# Broadcasting - 对不同形状的数组做运算
# 规则:维度不匹配时,将大小为 1 的维度扩展
x = np.array([[1], [2], [3]])  # shape: (3, 1)
y = np.array([10, 20, 30])     # shape: (3,) → (1, 3)

# Broadcasting 结果:(3, 3)
result = x + y
print(result)
# [[11, 21, 31],
#  [12, 22, 32],
#  [13, 23, 33]]

# Broadcasting 实用示例
# 批量数据归一化
data = np.random.randn(100, 10)  # 100 个样本,10 个特征
mean = data.mean(axis=0)         # 各特征的均值 (shape: 10,)
std = data.std(axis=0)           # 各特征的标准差 (shape: 10,)

normalized = (data - mean) / std  # 用 Broadcasting 归一化
print(normalized.mean(axis=0).round(10))  # ≈ 0
print(normalized.std(axis=0).round(10))   # ≈ 1

import numpy as np

arr = np.arange(24).reshape(4, 6)
print(arr)
# [[ 0  1  2  3  4  5]
#  [ 6  7  8  9 10 11]
#  [12 13 14 15 16 17]
#  [18 19 20 21 22 23]]

# 基本索引
print(arr[0, 0])    # 0
print(arr[3, 5])    # 23
print(arr[-1, -1])  # 23

# 切片
print(arr[1:3, 2:5])  # 第 2~3 行,第 3~5 列
print(arr[:, 0])      # 所有行的第 0 列
print(arr[::2, ::2])  # 按 2 间隔采样

# Fancy 索引
rows = np.array([0, 2])
cols = np.array([1, 4])
print(arr[rows, cols])  # [arr[0,1], arr[2,4]] = [1, 16]

# Boolean 索引(掩码)
mask = arr > 12
print(arr[mask])  # 大于 12 的元素

# 用条件过滤
data = np.array([1, -2, 3, -4, 5, -6])
positive = data[data > 0]  # [1, 3, 5]
print(positive)

# np.where - 按条件选择
result = np.where(data > 0, data, 0)  # 正数保留,负数置 0
print(result)  # [1, 0, 3, 0, 5, 0]

# 用 np.where 查找索引
indices = np.where(data > 0)
print(indices)  # (array([0, 2, 4]),)

import numpy as np

arr = np.arange(12)

# reshape
a = arr.reshape(3, 4)
b = arr.reshape(2, 2, 3)
c = arr.reshape(-1, 4)   # -1 表示自动计算:(3, 4)

# flatten vs ravel
flat1 = a.flatten()  # 返回副本
flat2 = a.ravel()    # 尽可能返回视图(更省内存)

# transpose
mat = np.random.randn(3, 4)
transposed = mat.T          # (4, 3)
transposed2 = mat.transpose()  # 结果相同
transposed3 = np.transpose(mat, (1, 0))  # 指定轴顺序

# 3D 数组 transpose
tensor = np.random.randn(2, 3, 4)
# 批次、通道、空间 → 批次、空间、通道
reordered = tensor.transpose(0, 2, 1)  # (2, 4, 3)

# squeeze 与 expand_dims
x = np.array([[[1, 2, 3]]])  # shape: (1, 1, 3)
squeezed = np.squeeze(x)     # (3,)
expanded = np.expand_dims(squeezed, axis=0)  # (1, 3)

# 数组拼接
a = np.array([[1, 2], [3, 4]])
b = np.array([[5, 6], [7, 8]])

hstack = np.hstack([a, b])  # 水平堆叠 (2, 4)
vstack = np.vstack([a, b])  # 垂直堆叠 (4, 2)
concat0 = np.concatenate([a, b], axis=0)  # 与 vstack 相同
concat1 = np.concatenate([a, b], axis=1)  # 与 hstack 相同

import numpy as np

x = np.array([0, np.pi/6, np.pi/4, np.pi/3, np.pi/2])

# 三角函数
sin_x = np.sin(x)
cos_x = np.cos(x)
tan_x = np.tan(x)

# 指数/对数
exp_x = np.exp(x)       # e^x
log_x = np.log(x + 1)   # 自然对数 (ln)
log2_x = np.log2(x + 1) # 以 2 为底的对数
log10_x = np.log10(x + 1)  # 常用对数

# 平方/平方根
sqrt_x = np.sqrt(x)
square_x = np.square(x)  # x^2
power_x = np.power(x, 3)  # x^3

# 绝对值、四舍五入
abs_x = np.abs(x)
ceil_x = np.ceil(x)    # 向上取整
floor_x = np.floor(x)  # 向下取整
round_x = np.round(x, 2)  # 四舍五入

# Sigmoid(手动实现)
def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def softmax(x):
    e_x = np.exp(x - x.max())  # 为数值稳定性减去 max
    return e_x / e_x.sum()

z = np.array([1.0, 2.0, 3.0])
print(sigmoid(z))   # [0.731, 0.880, 0.952]
print(softmax(z))   # [0.090, 0.245, 0.665]

import numpy as np

A = np.array([[1, 2], [3, 4]])
B = np.array([[5, 6], [7, 8]])

# 矩阵乘法
C = np.dot(A, B)         # 传统方式
C = A @ B                # Python 3.5+ 推荐方式
C = np.matmul(A, B)      # 与 np.dot 相同 (2D)

# 批量矩阵乘法(3D 以上)
batch_A = np.random.randn(32, 3, 4)
batch_B = np.random.randn(32, 4, 5)
batch_C = batch_A @ batch_B  # (32, 3, 5)

# 线性代数函数
det = np.linalg.det(A)               # 行列式
inv = np.linalg.inv(A)               # 逆矩阵
rank = np.linalg.matrix_rank(A)      # 秩
trace = np.trace(A)                   # 迹(对角线之和)

# 特征值分解
eigenvalues, eigenvectors = np.linalg.eig(A)

# 奇异值分解 (SVD)
U, S, Vt = np.linalg.svd(A)

# 求解线性方程组 Ax = b
b = np.array([5, 6])
x = np.linalg.solve(A, b)

# 范数 (Norm)
v = np.array([3, 4])
l1_norm = np.linalg.norm(v, ord=1)   # L1 范数:7
l2_norm = np.linalg.norm(v, ord=2)   # L2 范数:5
inf_norm = np.linalg.norm(v, ord=np.inf)  # 最大值范数:4

import numpy as np

data = np.random.randn(100, 5)

# 基本统计
print(data.mean())         # 整体均值
print(data.mean(axis=0))   # 按列均值 (shape: 5,)
print(data.mean(axis=1))   # 按行均值 (shape: 100,)

print(data.std())          # 标准差
print(data.var())          # 方差
print(data.sum())          # 求和
print(data.min())          # 最小值
print(data.max())          # 最大值

# 累积运算
cumsum = data.cumsum(axis=0)   # 累积和
cumprod = data.cumprod(axis=0) # 累积积

# 排序
sorted_arr = np.sort(data, axis=0)
sort_indices = np.argsort(data, axis=0)  # 排序索引

# 分位数
q25 = np.percentile(data, 25)
q50 = np.percentile(data, 50)  # 中位数
q75 = np.percentile(data, 75)
median = np.median(data)

# 相关系数
corr = np.corrcoef(data.T)  # 5x5 相关系数矩阵

# 直方图
counts, bin_edges = np.histogram(data[:, 0], bins=20)

import numpy as np
import time

n = 1_000_000
a = np.random.randn(n)
b = np.random.randn(n)

# for 循环方式
start = time.time()
result_loop = []
for i in range(n):
    result_loop.append(a[i] * b[i])
loop_time = time.time() - start
print(f"For loop: {loop_time:.4f}秒")

# 向量化方式
start = time.time()
result_vec = a * b
vec_time = time.time() - start
print(f"Vectorized: {vec_time:.4f}秒")

print(f"速度提升: {loop_time / vec_time:.1f}倍")
# 通常快 100~1000 倍

import numpy as np

class SimpleNeuralNetwork:
    """仅用 NumPy 实现的两层神经网络"""

    def __init__(self, input_size, hidden_size, output_size, seed=42):
        np.random.seed(seed)
        # He 初始化
        self.W1 = np.random.randn(input_size, hidden_size) * np.sqrt(2.0 / input_size)
        self.b1 = np.zeros((1, hidden_size))
        self.W2 = np.random.randn(hidden_size, output_size) * np.sqrt(2.0 / hidden_size)
        self.b2 = np.zeros((1, output_size))

    def relu(self, z):
        return np.maximum(0, z)

    def relu_derivative(self, z):
        return (z > 0).astype(float)

    def softmax(self, z):
        exp_z = np.exp(z - z.max(axis=1, keepdims=True))
        return exp_z / exp_z.sum(axis=1, keepdims=True)

    def forward(self, X):
        # 第 1 层
        self.Z1 = X @ self.W1 + self.b1
        self.A1 = self.relu(self.Z1)
        # 第 2 层
        self.Z2 = self.A1 @ self.W2 + self.b2
        self.A2 = self.softmax(self.Z2)
        return self.A2

    def cross_entropy_loss(self, y_pred, y_true):
        m = y_true.shape[0]
        log_probs = -np.log(y_pred[range(m), y_true] + 1e-8)
        return log_probs.mean()

    def backward(self, X, y_true, learning_rate=0.01):
        m = X.shape[0]

        # 输出层梯度
        dZ2 = self.A2.copy()
        dZ2[range(m), y_true] -= 1
        dZ2 /= m

        dW2 = self.A1.T @ dZ2
        db2 = dZ2.sum(axis=0, keepdims=True)

        # 隐藏层梯度
        dA1 = dZ2 @ self.W2.T
        dZ1 = dA1 * self.relu_derivative(self.Z1)

        dW1 = X.T @ dZ1
        db1 = dZ1.sum(axis=0, keepdims=True)

        # 更新权重
        self.W1 -= learning_rate * dW1
        self.b1 -= learning_rate * db1
        self.W2 -= learning_rate * dW2
        self.b2 -= learning_rate * db2

    def train(self, X, y, epochs=100, learning_rate=0.01):
        losses = []
        for epoch in range(epochs):
            y_pred = self.forward(X)
            loss = self.cross_entropy_loss(y_pred, y)
            losses.append(loss)
            self.backward(X, y, learning_rate)

            if epoch % 10 == 0:
                acc = (y_pred.argmax(axis=1) == y).mean()
                print(f"Epoch {epoch:3d}: Loss={loss:.4f}, Acc={acc:.4f}")
        return losses


# 测试
from sklearn.datasets import make_classification

X, y = make_classification(n_samples=1000, n_features=20,
                             n_classes=3, n_informative=15,
                             random_state=42)

nn = SimpleNeuralNetwork(input_size=20, hidden_size=64, output_size=3)
losses = nn.train(X, y, epochs=50, learning_rate=0.1)

import pandas as pd
import numpy as np

# 创建 Series
s1 = pd.Series([1, 2, 3, 4, 5])
s2 = pd.Series([10, 20, 30], index=['a', 'b', 'c'])
s3 = pd.Series({'x': 100, 'y': 200, 'z': 300})

print(s2['a'])      # 10
print(s2[['a', 'c']])  # a=10, c=30

# 创建 DataFrame
data = {
    'name': ['Alice', 'Bob', 'Charlie', 'David', 'Eve'],
    'age': [25, 30, 35, 28, 22],
    'score': [88.5, 92.3, 78.1, 95.7, 83.2],
    'passed': [True, True, False, True, True]
}
df = pd.DataFrame(data)

print(df.head())
print(df.tail(3))
print(df.info())
print(df.describe())
print(df.dtypes)
print(df.shape)  # (5, 4)

import pandas as pd

# CSV
df_csv = pd.read_csv('data.csv',
                      sep=',',
                      header=0,
                      index_col=0,
                      parse_dates=['date'],
                      encoding='utf-8',
                      na_values=['N/A', 'null', ''])

df_csv.to_csv('output.csv', index=False, encoding='utf-8-sig')

# Excel
df_excel = pd.read_excel('data.xlsx',
                          sheet_name='Sheet1',
                          header=0)
df_excel.to_excel('output.xlsx', sheet_name='Result', index=False)

# JSON
df_json = pd.read_json('data.json', orient='records')
df_json.to_json('output.json', orient='records', force_ascii=False, indent=2)

# Parquet(高性能列式格式)
df.to_parquet('data.parquet', engine='pyarrow', compression='snappy')
df_parquet = pd.read_parquet('data.parquet')

# SQL(以 SQLite 为例)
import sqlite3
conn = sqlite3.connect('database.db')
df_sql = pd.read_sql_query("SELECT * FROM users WHERE age > 25", conn)
df.to_sql('new_table', conn, if_exists='replace', index=False)

import pandas as pd
import numpy as np

df = pd.DataFrame({
    'A': range(10),
    'B': range(10, 20),
    'C': range(20, 30)
}, index=[f'row{i}' for i in range(10)])

# loc:基于标签的索引
print(df.loc['row0', 'A'])          # 单个值
print(df.loc['row0':'row3', 'A':'B'])  # 范围(含末端)
print(df.loc[['row1', 'row5'], 'C'])   # 列表

# iloc:基于位置的索引
print(df.iloc[0, 0])           # 第 0 行第 0 列
print(df.iloc[0:4, 0:2])      # 范围(不含末端)
print(df.iloc[[1, 5], 2])     # 列表

# 基于条件的选择
mask = df['A'] > 5
filtered = df[mask]
filtered2 = df[df['B'].between(12, 17)]
filtered3 = df.query('A > 5 and B < 18')

# 复合条件
condition = (df['A'] > 3) & (df['B'] < 17) | (df['C'] >= 28)
result = df[condition]

# at/iat(单值访问 - 更快)
val = df.at['row3', 'A']
val2 = df.iat[3, 0]

import pandas as pd
import numpy as np

# 生成含缺失值的数据
df = pd.DataFrame({
    'age': [25, np.nan, 35, np.nan, 22],
    'income': [50000, 60000, np.nan, 80000, np.nan],
    'city': ['Seoul', 'Busan', None, 'Incheon', 'Seoul'],
    'score': [88.5, 92.3, 78.1, np.nan, 83.2]
})

# 检查缺失值
print(df.isnull())
print(df.isnull().sum())         # 按列统计缺失值数
print(df.isnull().sum() / len(df) * 100)  # 缺失比例(%)

# 删除缺失值
df_dropped_rows = df.dropna()              # 删除含缺失值的行
df_dropped_cols = df.dropna(axis=1)       # 删除含缺失值的列
df_thresh = df.dropna(thresh=3)           # 至少需要 3 个非缺失值

# 填充缺失值
df_filled_0 = df.fillna(0)               # 用 0 填充
df_filled_mean = df.fillna(df.mean())    # 用均值填充
df_filled_dict = df.fillna({
    'age': df['age'].mean(),
    'income': df['income'].median(),
    'city': 'Unknown',
    'score': df['score'].mean()
})

# 用前/后值填充
df_ffill = df.fillna(method='ffill')  # 用前一个值填充
df_bfill = df.fillna(method='bfill')  # 用后一个值填充

# 插值
df_interpolated = df.interpolate(method='linear')

# 检查缺失值后处理的模式
for col in df.columns:
    missing_pct = df[col].isnull().mean()
    if missing_pct > 0.5:
        df.drop(columns=[col], inplace=True)
    elif df[col].dtype == 'object':
        df[col].fillna(df[col].mode()[0], inplace=True)
    else:
        df[col].fillna(df[col].median(), inplace=True)

import pandas as pd
import numpy as np

df = pd.DataFrame({
    'text': ['hello world', 'PYTHON IS GREAT', 'data science'],
    'value': [1, 2, 3],
    'category': ['A', 'B', 'A']
})

# apply:应用函数
df['text_upper'] = df['text'].apply(str.upper)
df['text_length'] = df['text'].apply(len)

# 复杂函数
def process_text(text):
    return ' '.join(word.capitalize() for word in text.lower().split())

df['text_processed'] = df['text'].apply(process_text)

# 同时应用于多列
def feature_engineer(row):
    return pd.Series({
        'value_squared': row['value'] ** 2,
        'category_is_A': int(row['category'] == 'A')
    })

new_features = df.apply(feature_engineer, axis=1)
df = pd.concat([df, new_features], axis=1)

# map:应用映射表
category_map = {'A': 'Alpha', 'B': 'Beta', 'C': 'Gamma'}
df['category_name'] = df['category'].map(category_map)

# transform:组内变换(保持组大小)
df['numeric'] = [10, 20, 30, 40, 50, 60]
df['category'] = ['A', 'B', 'A', 'B', 'A', 'B']
df['group_mean'] = df.groupby('category')['numeric'].transform('mean')

# 字符串运算(向量化)
texts = pd.Series(['Hello World', 'Python 3.12', 'Machine Learning'])
print(texts.str.lower())
print(texts.str.split())
print(texts.str.contains('Python'))
print(texts.str.extract(r'(\w+)\s+(\w+)'))

import pandas as pd
import numpy as np

np.random.seed(42)
df = pd.DataFrame({
    'team': np.random.choice(['A', 'B', 'C'], 100),
    'role': np.random.choice(['dev', 'ds', 'pm'], 100),
    'score': np.random.randint(60, 100, 100),
    'salary': np.random.randint(3000, 8000, 100)
})

# 基本 groupby
grouped = df.groupby('team')
print(grouped['score'].mean())
print(grouped['salary'].describe())

# 多重键
multi_grouped = df.groupby(['team', 'role'])
print(multi_grouped['score'].mean().unstack())

# 聚合函数
agg_result = df.groupby('team').agg(
    avg_score=('score', 'mean'),
    total_salary=('salary', 'sum'),
    count=('score', 'count'),
    max_score=('score', 'max'),
    min_salary=('salary', 'min')
)
print(agg_result)

# 自定义聚合
def iqr(x):
    return x.quantile(0.75) - x.quantile(0.25)

custom_agg = df.groupby('team')['score'].agg([
    'mean', 'median', 'std', iqr
])

# filter:只选出满足条件的组
large_teams = df.groupby('team').filter(lambda x: len(x) > 30)

import pandas as pd

users = pd.DataFrame({
    'user_id': [1, 2, 3, 4, 5],
    'name': ['Alice', 'Bob', 'Charlie', 'David', 'Eve'],
    'age': [25, 30, 35, 28, 22]
})

orders = pd.DataFrame({
    'order_id': [101, 102, 103, 104, 105, 106],
    'user_id': [1, 2, 1, 3, 5, 6],
    'amount': [150, 250, 80, 320, 190, 440]
})

# Inner Join(交集)
inner = pd.merge(users, orders, on='user_id', how='inner')

# Left Join
left = pd.merge(users, orders, on='user_id', how='left')

# Right Join
right = pd.merge(users, orders, on='user_id', how='right')

# Outer Join(并集)
outer = pd.merge(users, orders, on='user_id', how='outer')

# 用不同键合并
merged = pd.merge(users, orders,
                   left_on='user_id', right_on='user_id',
                   suffixes=('_user', '_order'))

# concat
df1 = pd.DataFrame({'A': [1, 2], 'B': [3, 4]})
df2 = pd.DataFrame({'A': [5, 6], 'B': [7, 8]})
df3 = pd.DataFrame({'C': [9, 10], 'D': [11, 12]})

vertical = pd.concat([df1, df2], axis=0, ignore_index=True)
horizontal = pd.concat([df1, df3], axis=1)

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler, LabelEncoder

def preprocess_ml_data(filepath):
    """面向 AI 训练的数据预处理流水线"""

    # 1. 加载数据
    df = pd.read_csv(filepath)
    print(f"原始数据: {df.shape}")

    # 2. 去重
    df = df.drop_duplicates()
    print(f"去重后: {df.shape}")

    # 3. 处理缺失值
    numeric_cols = df.select_dtypes(include=[np.number]).columns
    cat_cols = df.select_dtypes(include=['object']).columns

    for col in numeric_cols:
        df[col].fillna(df[col].median(), inplace=True)

    for col in cat_cols:
        df[col].fillna(df[col].mode()[0], inplace=True)

    # 4. 处理离群值(IQR 方法)
    for col in numeric_cols:
        Q1 = df[col].quantile(0.25)
        Q3 = df[col].quantile(0.75)
        IQR = Q3 - Q1
        lower = Q1 - 1.5 * IQR
        upper = Q3 + 1.5 * IQR
        df[col] = df[col].clip(lower=lower, upper=upper)

    # 5. 类别变量编码
    le = LabelEncoder()
    for col in cat_cols:
        if df[col].nunique() <= 10:
            df[col] = le.fit_transform(df[col].astype(str))
        else:
            # 高基数:频率编码
            freq_map = df[col].value_counts().to_dict()
            df[col] = df[col].map(freq_map)

    # 6. 特征工程
    if 'date' in df.columns:
        df['date'] = pd.to_datetime(df['date'])
        df['year'] = df['date'].dt.year
        df['month'] = df['date'].dt.month
        df['dayofweek'] = df['date'].dt.dayofweek
        df.drop('date', axis=1, inplace=True)

    return df


# 泰坦尼克数据预处理示例
def preprocess_titanic(df):
    df = df.copy()

    # 特征工程
    df['Title'] = df['Name'].str.extract(r' ([A-Za-z]+)\.')
    title_map = {'Mr': 0, 'Miss': 1, 'Mrs': 2, 'Master': 3}
    df['Title'] = df['Title'].map(title_map).fillna(4)

    df['FamilySize'] = df['SibSp'] + df['Parch'] + 1
    df['IsAlone'] = (df['FamilySize'] == 1).astype(int)

    # 处理缺失值
    df['Age'].fillna(df.groupby('Title')['Age'].transform('median'), inplace=True)
    df['Fare'].fillna(df['Fare'].median(), inplace=True)
    df['Embarked'].fillna(df['Embarked'].mode()[0], inplace=True)

    # 编码
    df['Sex'] = (df['Sex'] == 'male').astype(int)
    df['Embarked'] = df['Embarked'].map({'S': 0, 'C': 1, 'Q': 2})

    features = ['Pclass', 'Sex', 'Age', 'Fare', 'Embarked',
                'FamilySize', 'IsAlone', 'Title']

    return df[features]

import matplotlib.pyplot as plt
import numpy as np

# 韩文字体设置 (Mac)
import matplotlib.font_manager as fm
plt.rcParams['font.family'] = 'AppleGothic'
plt.rcParams['axes.unicode_minus'] = False

# 基本设置
fig, axes = plt.subplots(2, 3, figsize=(15, 10))

# 折线图
x = np.linspace(0, 2 * np.pi, 100)
axes[0, 0].plot(x, np.sin(x), 'b-', linewidth=2, label='sin(x)')
axes[0, 0].plot(x, np.cos(x), 'r--', linewidth=2, label='cos(x)')
axes[0, 0].set_title('三角函数')
axes[0, 0].legend()
axes[0, 0].grid(True, alpha=0.3)

# 柱状图
categories = ['分类', '回归', '聚类', '降维']
values = [85, 72, 68, 91]
bars = axes[0, 1].bar(categories, values, color=['#3498db', '#e74c3c', '#2ecc71', '#f39c12'])
axes[0, 1].set_title('各算法准确率')
axes[0, 1].set_ylabel('准确率 (%)')
for bar, val in zip(bars, values):
    axes[0, 1].text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.5,
                    f'{val}%', ha='center', va='bottom')

# 散点图
np.random.seed(42)
x_scatter = np.random.randn(100)
y_scatter = 2 * x_scatter + np.random.randn(100) * 0.5
axes[0, 2].scatter(x_scatter, y_scatter, alpha=0.6, c=y_scatter, cmap='viridis')
axes[0, 2].set_title('散点图')

# 直方图
data = np.concatenate([
    np.random.normal(0, 1, 500),
    np.random.normal(4, 1.5, 300)
])
axes[1, 0].hist(data, bins=50, density=True, alpha=0.7, color='steelblue')
axes[1, 0].set_title('数据分布')

# 箱线图
box_data = [np.random.normal(i, 1, 100) for i in range(5)]
axes[1, 1].boxplot(box_data, labels=[f'模型{i+1}' for i in range(5)])
axes[1, 1].set_title('各模型性能分布')

# 饼图
sizes = [35, 25, 20, 12, 8]
labels = ['Python', 'R', 'Scala', 'Java', '其他']
explode = (0.05, 0, 0, 0, 0)
axes[1, 2].pie(sizes, labels=labels, explode=explode, autopct='%1.1f%%',
               shadow=True, startangle=90)
axes[1, 2].set_title('语言使用比例')

plt.tight_layout()
plt.savefig('basic_plots.png', dpi=150, bbox_inches='tight')
plt.show()

import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np

# 样式设置
sns.set_theme(style='whitegrid', palette='husl', font_scale=1.2)

# 示例数据
df = pd.DataFrame({
    'model': np.repeat(['ResNet', 'VGG', 'EfficientNet', 'ViT'], 50),
    'accuracy': np.concatenate([
        np.random.normal(92, 2, 50),
        np.random.normal(88, 3, 50),
        np.random.normal(94, 1.5, 50),
        np.random.normal(95, 2.5, 50)
    ]),
    'params_M': np.concatenate([
        np.random.normal(25, 2, 50),
        np.random.normal(138, 5, 50),
        np.random.normal(5.3, 0.3, 50),
        np.random.normal(86, 3, 50)
    ]),
    'training_time': np.concatenate([
        np.random.exponential(10, 50),
        np.random.exponential(20, 50),
        np.random.exponential(8, 50),
        np.random.exponential(15, 50)
    ])
})

fig, axes = plt.subplots(2, 3, figsize=(18, 12))

# 小提琴图
sns.violinplot(data=df, x='model', y='accuracy', ax=axes[0, 0])
axes[0, 0].set_title('各模型准确率分布')

# 箱线图 + swarm
sns.boxplot(data=df, x='model', y='accuracy', ax=axes[0, 1])
sns.swarmplot(data=df, x='model', y='accuracy', color='black',
              size=2, ax=axes[0, 1])
axes[0, 1].set_title('准确率详细分布')

# 热力图(相关系数)
corr_data = df[['accuracy', 'params_M', 'training_time']].corr()
sns.heatmap(corr_data, annot=True, fmt='.2f', cmap='RdYlGn',
            center=0, ax=axes[0, 2])
axes[0, 2].set_title('变量间相关关系')

# 散点图 + 回归线
sns.regplot(data=df, x='params_M', y='accuracy',
            scatter_kws={'alpha': 0.4}, ax=axes[1, 0])
axes[1, 0].set_title('参数量 vs 准确率')

# 分布图(KDE + 直方图)
for model in df['model'].unique():
    subset = df[df['model'] == model]
    sns.kdeplot(data=subset, x='accuracy', label=model, ax=axes[1, 1])
axes[1, 1].set_title('各模型准确率分布 (KDE)')
axes[1, 1].legend()

# Facet Grid(高级)
# axes[1, 2] 单独处理
axes[1, 2].remove()

plt.tight_layout()
plt.savefig('seaborn_plots.png', dpi=150, bbox_inches='tight')
plt.show()

import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
from sklearn.metrics import confusion_matrix

def plot_learning_curve(train_losses, val_losses, train_accs, val_accs):
    """学习曲线可视化"""
    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5))

    epochs = range(1, len(train_losses) + 1)

    ax1.plot(epochs, train_losses, 'b-', label='训练损失', linewidth=2)
    ax1.plot(epochs, val_losses, 'r--', label='验证损失', linewidth=2)
    ax1.fill_between(epochs, train_losses, val_losses, alpha=0.1, color='gray')
    ax1.set_xlabel('Epoch')
    ax1.set_ylabel('损失')
    ax1.set_title('损失曲线')
    ax1.legend()
    ax1.grid(True, alpha=0.3)

    ax2.plot(epochs, train_accs, 'b-', label='训练准确率', linewidth=2)
    ax2.plot(epochs, val_accs, 'r--', label='验证准确率', linewidth=2)
    best_epoch = np.argmax(val_accs)
    ax2.axvline(x=best_epoch + 1, color='g', linestyle=':', label=f'最优 Epoch ({best_epoch+1})')
    ax2.set_xlabel('Epoch')
    ax2.set_ylabel('准确率')
    ax2.set_title('准确率曲线')
    ax2.legend()
    ax2.grid(True, alpha=0.3)

    plt.tight_layout()
    return fig


def plot_confusion_matrix(y_true, y_pred, class_names):
    """混淆矩阵可视化"""
    cm = confusion_matrix(y_true, y_pred)
    cm_pct = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]

    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5))

    # 绝对值
    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues',
                xticklabels=class_names, yticklabels=class_names, ax=ax1)
    ax1.set_title('混淆矩阵 (绝对值)')
    ax1.set_ylabel('真实标签')
    ax1.set_xlabel('预测标签')

    # 比例
    sns.heatmap(cm_pct, annot=True, fmt='.2%', cmap='Greens',
                xticklabels=class_names, yticklabels=class_names, ax=ax2)
    ax2.set_title('混淆矩阵 (比例)')
    ax2.set_ylabel('真实标签')
    ax2.set_xlabel('预测标签')

    plt.tight_layout()
    return fig

from sklearn.preprocessing import (
    StandardScaler, MinMaxScaler, RobustScaler,
    LabelEncoder, OneHotEncoder, OrdinalEncoder
)
import numpy as np

X = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]], dtype=float)

# StandardScaler:均值 0,标准差 1
scaler = StandardScaler()
X_standard = scaler.fit_transform(X)

# MinMaxScaler:[0, 1] 区间
min_max = MinMaxScaler(feature_range=(0, 1))
X_minmax = min_max.fit_transform(X)

# RobustScaler:对离群值稳健(使用中位数、IQR)
robust = RobustScaler()
X_robust = robust.fit_transform(X)

# LabelEncoder:类别 → 数字
le = LabelEncoder()
labels = ['cat', 'dog', 'bird', 'cat', 'dog']
encoded = le.fit_transform(labels)  # [0, 2, 1, 0, 2]
decoded = le.inverse_transform(encoded)

# OneHotEncoder
ohe = OneHotEncoder(sparse_output=False, handle_unknown='ignore')
categories = np.array([['red'], ['green'], ['blue'], ['red']])
encoded_ohe = ohe.fit_transform(categories)

from sklearn.decomposition import PCA
from sklearn.feature_selection import (
    SelectKBest, f_classif, mutual_info_classif,
    RFE, SelectFromModel
)
from sklearn.ensemble import RandomForestClassifier
import numpy as np

X = np.random.randn(200, 20)
y = (X[:, 0] + X[:, 1] + np.random.randn(200) * 0.1 > 0).astype(int)

# PCA(主成分分析)
pca = PCA(n_components=10)
X_pca = pca.fit_transform(X)
print(f"解释方差比例: {pca.explained_variance_ratio_.sum():.2%}")

# 累积解释方差图
import matplotlib.pyplot as plt
cumsum = np.cumsum(pca.explained_variance_ratio_)
plt.figure(figsize=(8, 4))
plt.plot(range(1, len(cumsum)+1), cumsum * 100)
plt.xlabel('主成分数')
plt.ylabel('累积解释方差 (%)')
plt.axhline(y=95, color='r', linestyle='--', label='95%')
plt.legend()
plt.grid(True)

# SelectKBest
selector = SelectKBest(f_classif, k=5)
X_kbest = selector.fit_transform(X, y)
selected_features = selector.get_support(indices=True)
print(f"选中的特征索引: {selected_features}")

# RFE(递归特征消除)
rf = RandomForestClassifier(n_estimators=100, random_state=42)
rfe = RFE(estimator=rf, n_features_to_select=5)
X_rfe = rfe.fit_transform(X, y)

# 基于特征重要性的选择
rf.fit(X, y)
sfm = SelectFromModel(rf, threshold='mean')
X_sfm = sfm.fit_transform(X, y)

from sklearn.linear_model import (
    LinearRegression, LogisticRegression,
    Ridge, Lasso, ElasticNet
)
from sklearn.datasets import make_classification, make_regression
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, r2_score, accuracy_score
import numpy as np

# 回归
X_reg, y_reg = make_regression(n_samples=500, n_features=20,
                                 noise=0.1, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X_reg, y_reg, test_size=0.2)

# Linear Regression
lr = LinearRegression()
lr.fit(X_train, y_train)
y_pred = lr.predict(X_test)
print(f"Linear R2: {r2_score(y_test, y_pred):.4f}")

# Ridge (L2 正则化)
ridge = Ridge(alpha=1.0)
ridge.fit(X_train, y_train)
print(f"Ridge R2: {r2_score(y_test, ridge.predict(X_test)):.4f}")

# Lasso (L1 正则化,特征选择效果)
lasso = Lasso(alpha=0.01)
lasso.fit(X_train, y_train)
print(f"Lasso R2: {r2_score(y_test, lasso.predict(X_test)):.4f}")
print(f"Non-zero coefficients: {np.sum(lasso.coef_ != 0)}")

# 分类
X_cls, y_cls = make_classification(n_samples=500, n_features=20,
                                    n_classes=3, n_informative=10,
                                    random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X_cls, y_cls, test_size=0.2)

logistic = LogisticRegression(C=1.0, max_iter=1000, solver='lbfgs',
                               multi_class='multinomial')
logistic.fit(X_train, y_train)
print(f"Logistic Accuracy: {accuracy_score(y_test, logistic.predict(X_test)):.4f}")

from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import (
    RandomForestClassifier, GradientBoostingClassifier,
    AdaBoostClassifier, ExtraTreesClassifier
)
import xgboost as xgb
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
import matplotlib.pyplot as plt
import numpy as np

X, y = make_classification(n_samples=1000, n_features=20,
                             n_classes=2, n_informative=10,
                             random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

models = {
    'Decision Tree': DecisionTreeClassifier(max_depth=5, random_state=42),
    'Random Forest': RandomForestClassifier(n_estimators=100, random_state=42, n_jobs=-1),
    'Gradient Boosting': GradientBoostingClassifier(n_estimators=100, random_state=42),
    'XGBoost': xgb.XGBClassifier(n_estimators=100, random_state=42, eval_metric='logloss'),
}

results = {}
for name, model in models.items():
    model.fit(X_train, y_train)
    y_pred = model.predict(X_test)
    acc = (y_pred == y_test).mean()
    results[name] = acc
    print(f"{name}: {acc:.4f}")

# 特征重要性可视化
rf_model = models['Random Forest']
importances = rf_model.feature_importances_
indices = np.argsort(importances)[::-1][:10]

plt.figure(figsize=(10, 6))
plt.bar(range(10), importances[indices])
plt.xticks(range(10), [f'F{i}' for i in indices])
plt.title('前 10 个重要特征')
plt.xlabel('特征')
plt.ylabel('重要度')
plt.tight_layout()
plt.show()

from sklearn.model_selection import (
    cross_val_score, StratifiedKFold,
    GridSearchCV, RandomizedSearchCV
)
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classification_report, roc_auc_score
from sklearn.datasets import make_classification
import numpy as np

X, y = make_classification(n_samples=1000, n_features=20, random_state=42)

# K-Fold 交叉验证
rf = RandomForestClassifier(n_estimators=100, random_state=42)
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
scores = cross_val_score(rf, X, y, cv=cv, scoring='accuracy', n_jobs=-1)
print(f"CV Accuracy: {scores.mean():.4f} (+/- {scores.std() * 2:.4f})")

# GridSearchCV
param_grid = {
    'n_estimators': [50, 100, 200],
    'max_depth': [None, 5, 10],
    'min_samples_split': [2, 5, 10]
}

grid_search = GridSearchCV(
    RandomForestClassifier(random_state=42),
    param_grid,
    cv=5,
    scoring='accuracy',
    n_jobs=-1,
    verbose=1
)
grid_search.fit(X, y)

print(f"最优参数: {grid_search.best_params_}")
print(f"最高 CV 得分: {grid_search.best_score_:.4f}")

# 用最终模型在测试集上评估
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
best_model = grid_search.best_estimator_
best_model.fit(X_train, y_train)
y_pred = best_model.predict(X_test)
print(classification_report(y_test, y_pred))

from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer
from sklearn.ensemble import RandomForestClassifier
import pandas as pd
import numpy as np

# 示例数据
np.random.seed(42)
n = 500
df = pd.DataFrame({
    'age': np.random.randint(18, 70, n).astype(float),
    'income': np.random.randint(20000, 100000, n).astype(float),
    'education': np.random.choice(['High School', 'Bachelor', 'Master', 'PhD'], n),
    'city': np.random.choice(['Seoul', 'Busan', 'Incheon', 'Daegu'], n),
    'target': np.random.randint(0, 2, n)
})

# 加入部分缺失值
df.loc[np.random.choice(n, 50), 'age'] = np.nan
df.loc[np.random.choice(n, 30), 'income'] = np.nan
df.loc[np.random.choice(n, 20), 'education'] = None

X = df.drop('target', axis=1)
y = df['target']

# 拆分数值型、类别型列
numeric_features = ['age', 'income']
categorical_features = ['education', 'city']

# 数值型预处理流水线
numeric_transformer = Pipeline(steps=[
    ('imputer', SimpleImputer(strategy='median')),
    ('scaler', StandardScaler())
])

# 类别型预处理流水线
categorical_transformer = Pipeline(steps=[
    ('imputer', SimpleImputer(strategy='most_frequent')),
    ('encoder', OneHotEncoder(handle_unknown='ignore', sparse_output=False))
])

# 用 ColumnTransformer 合并
preprocessor = ColumnTransformer(transformers=[
    ('num', numeric_transformer, numeric_features),
    ('cat', categorical_transformer, categorical_features)
])

# 完整流水线
full_pipeline = Pipeline(steps=[
    ('preprocessor', preprocessor),
    ('classifier', RandomForestClassifier(n_estimators=100, random_state=42))
])

# 训练与评估
from sklearn.model_selection import cross_val_score
scores = cross_val_score(full_pipeline, X, y, cv=5, scoring='accuracy')
print(f"流水线 CV 准确率: {scores.mean():.4f} ± {scores.std():.4f}")

import pandas as pd
import numpy as np
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import cross_val_score, GridSearchCV
from sklearn.metrics import classification_report, roc_auc_score
import matplotlib.pyplot as plt

# 加载数据(实际中使用 pd.read_csv('titanic.csv'))
# 此处生成示例数据
np.random.seed(42)
n = 891
df = pd.DataFrame({
    'Pclass': np.random.choice([1, 2, 3], n, p=[0.24, 0.21, 0.55]),
    'Sex': np.random.choice(['male', 'female'], n, p=[0.65, 0.35]),
    'Age': np.random.uniform(1, 80, n),
    'SibSp': np.random.randint(0, 5, n),
    'Parch': np.random.randint(0, 5, n),
    'Fare': np.random.exponential(50, n),
    'Embarked': np.random.choice(['S', 'C', 'Q'], n, p=[0.72, 0.19, 0.09]),
    'Survived': np.random.randint(0, 2, n)
})

# 加入缺失值
df.loc[np.random.choice(n, 177), 'Age'] = np.nan

# 特征工程
df['FamilySize'] = df['SibSp'] + df['Parch'] + 1
df['IsAlone'] = (df['FamilySize'] == 1).astype(int)
df['Sex_binary'] = (df['Sex'] == 'male').astype(int)
df['Fare_log'] = np.log1p(df['Fare'])

feature_cols = ['Pclass', 'Sex_binary', 'Age', 'FamilySize', 'IsAlone',
                'Fare_log', 'Embarked']
X = df[feature_cols]
y = df['Survived']

# 预处理流水线
numeric_features = ['Age', 'FamilySize', 'Fare_log', 'Pclass']
categorical_features = ['Embarked']
binary_features = ['Sex_binary', 'IsAlone']

preprocessor = ColumnTransformer(transformers=[
    ('num', Pipeline([
        ('imputer', SimpleImputer(strategy='median')),
        ('scaler', StandardScaler())
    ]), numeric_features),
    ('cat', Pipeline([
        ('imputer', SimpleImputer(strategy='most_frequent')),
        ('encoder', OneHotEncoder(handle_unknown='ignore', sparse_output=False))
    ]), categorical_features),
    ('bin', 'passthrough', binary_features)
])

# 模型流水线
pipeline = Pipeline([
    ('preprocessor', preprocessor),
    ('classifier', GradientBoostingClassifier(random_state=42))
])

# 超参数搜索
param_grid = {
    'classifier__n_estimators': [100, 200],
    'classifier__max_depth': [3, 5],
    'classifier__learning_rate': [0.05, 0.1]
}

grid_search = GridSearchCV(pipeline, param_grid, cv=5, scoring='roc_auc', n_jobs=-1)
grid_search.fit(X, y)

print(f"最优 AUC: {grid_search.best_score_:.4f}")
print(f"最优参数: {grid_search.best_params_}")