torchaudio 完全指南 — 从音频处理到语音识别、TTS、音乐分析

pip install torch torchaudio

import torch
import torchaudio

# 音频加载
waveform, sample_rate = torchaudio.load("speech.wav")
print(f"Shape: {waveform.shape}")    # [channels, samples]
print(f"Sample Rate: {sample_rate}")  # 16000
print(f"Duration: {waveform.shape[1] / sample_rate:.2f}s")

# 声道: 单声道(1) vs 立体声(2)
if waveform.shape[0] == 2:
    mono = waveform.mean(dim=0, keepdim=True)  # 立体声 → 单声道

# 重采样 (44100Hz → 16000Hz)
resampler = torchaudio.transforms.Resample(
    orig_freq=44100, new_freq=16000
)
waveform_16k = resampler(waveform)

# 保存
torchaudio.save("output.wav", waveform_16k, 16000)

# 支持格式: wav, flac, mp3, ogg, opus, sphere
# 后端: sox, soundfile, ffmpeg
print(torchaudio.list_audio_backends())

import matplotlib.pyplot as plt

# 波形 (Waveform)
fig, axes = plt.subplots(3, 1, figsize=(12, 8))

# 1. 时间域 (波形)
time_axis = torch.arange(0, waveform.shape[1]) / sample_rate
axes[0].plot(time_axis, waveform[0])
axes[0].set_title("Waveform")
axes[0].set_xlabel("Time (s)")
axes[0].set_ylabel("Amplitude")

# 2. 频谱图
spectrogram = torchaudio.transforms.Spectrogram(n_fft=1024)(waveform)
axes[1].imshow(
    spectrogram[0].log2().numpy(),
    aspect='auto', origin='lower', cmap='magma'
)
axes[1].set_title("Spectrogram")

# 3. Mel 频谱图
mel_spec = torchaudio.transforms.MelSpectrogram(
    sample_rate=sample_rate, n_fft=1024, n_mels=80
)(waveform)
axes[2].imshow(
    mel_spec[0].log2().numpy(),
    aspect='auto', origin='lower', cmap='magma'
)
axes[2].set_title("Mel Spectrogram")

plt.tight_layout()
plt.savefig("audio_analysis.png", dpi=150)

# STFT (Short-Time Fourier Transform)
# 从时间域转换到时间+频率域
spectrogram_transform = torchaudio.transforms.Spectrogram(
    n_fft=1024,       # FFT 窗口大小 (频率分辨率)
    hop_length=256,    # 窗口移动间隔 (时间分辨率)
    win_length=1024,   # 窗口长度
    power=2.0,         # 2.0=功率, 1.0=振幅
)

spec = spectrogram_transform(waveform)
# shape: [channels, n_freq_bins, time_frames]
# n_freq_bins = n_fft // 2 + 1 = 513

n_fft 与 hop_length 的权衡:

n_fft ↑ → 频率分辨率 ↑，时间分辨率 ↓
n_fft ↓ → 频率分辨率 ↓，时间分辨率 ↑

常见设置:
├── 语音: n_fft=400~512, hop=160 (以 16kHz 为准)
├── 音乐: n_fft=2048, hop=512 (以 44.1kHz 为准)
└── 通用: n_fft=1024, hop=256

# 人耳对低频敏感，对高频迟钝
# Mel 尺度 = 反映人类听觉特性的频率尺度

mel_transform = torchaudio.transforms.MelSpectrogram(
    sample_rate=16000,
    n_fft=1024,
    hop_length=256,
    n_mels=80,          # Mel 滤波器数量 (通常 40~128)
    f_min=0,            # 最低频率
    f_max=8000,         # 最高频率 (Nyquist)
)

mel_spec = mel_transform(waveform)
# shape: [1, 80, time_frames]

# dB 尺度转换 (对数压缩)
amplitude_to_db = torchaudio.transforms.AmplitudeToDB(stype='power', top_db=80)
mel_spec_db = amplitude_to_db(mel_spec)

Mel 频率转换公式:
  mel = 2595 × log10(1 + freq / 700)

频率 → Mel:
  100 Hz  →  150 mel   (低频: 密集)
  1000 Hz →  1000 mel
  4000 Hz →  2146 mel
  8000 Hz →  2840 mel  (高频: 稀疏)

→ 低频分析得精细，高频归并得粗略
→ 一种接近人耳听感的表示方式!

# MFCC = Mel 频谱图 + DCT (离散余弦变换)
# 表示语音「形状」的核心特征

mfcc_transform = torchaudio.transforms.MFCC(
    sample_rate=16000,
    n_mfcc=13,          # MFCC 系数个数 (通常 13~40)
    melkwargs={
        'n_fft': 1024,
        'n_mels': 80,
        'hop_length': 256,
    }
)

mfcc = mfcc_transform(waveform)
# shape: [1, 13, time_frames]

# Delta (一阶导数) + Delta-Delta (二阶导数)
# → 加入语音的变化率信息
delta = torchaudio.functional.compute_deltas(mfcc)
delta_delta = torchaudio.functional.compute_deltas(delta)

# 最终特征: 拼接 [MFCC, Delta, Delta-Delta]
features = torch.cat([mfcc, delta, delta_delta], dim=1)
# shape: [1, 39, time_frames]

应用场景:
├── MFCC: 传统语音识别 (HMM-GMM)、说话人识别
├── Mel Spectrogram: 深度学习语音识别 (Wav2Vec2, Whisper)
├── Spectrogram: 音乐分析、环境声分类
└── Raw Waveform: 端到端模型 (最新趋势)

# 时间掩码 (SpecAugment)
time_masking = torchaudio.transforms.TimeMasking(
    time_mask_param=30   # 最多掩盖 30 帧
)

# 频率掩码 (SpecAugment)
freq_masking = torchaudio.transforms.FrequencyMasking(
    freq_mask_param=15   # 最多掩盖 15 个通道
)

# SpecAugment (显著提升语音识别准确率!)
augmented_spec = time_masking(freq_masking(mel_spec))

# 时间伸缩
time_stretch = torchaudio.transforms.TimeStretch()
stretched = time_stretch(complex_spec, overriding_rate=1.2)  # 加快 20%

# 音高变换
pitch_shift = torchaudio.transforms.PitchShift(
    sample_rate=16000, n_steps=4  # 升高 4 个半音
)
shifted = pitch_shift(waveform)

# 添加噪声
def add_noise(waveform, snr_db=10):
    """以指定 SNR dB 添加白噪声"""
    noise = torch.randn_like(waveform)
    signal_power = waveform.norm(p=2)
    noise_power = noise.norm(p=2)
    snr = 10 ** (snr_db / 20)
    scale = signal_power / (snr * noise_power)
    return waveform + scale * noise

import torchaudio
from torchaudio.pipelines import WAV2VEC2_ASR_BASE_960H

# 加载 pipeline
bundle = WAV2VEC2_ASR_BASE_960H
model = bundle.get_model()
labels = bundle.get_labels()  # token 列表

# 推理
waveform, sr = torchaudio.load("speech.wav")
if sr != bundle.sample_rate:
    waveform = torchaudio.transforms.Resample(sr, bundle.sample_rate)(waveform)

with torch.no_grad():
    emissions, _ = model(waveform)

# CTC 解码 (Greedy)
def greedy_decode(emissions, labels):
    indices = torch.argmax(emissions, dim=-1)[0]
    tokens = []
    prev = -1
    for idx in indices:
        if idx != prev and idx != 0:  # 0 = blank
            tokens.append(labels[idx])
        prev = idx
    return "".join(tokens).replace("|", " ").strip()

text = greedy_decode(emissions, labels)
print(f"识别结果: {text}")

from torchaudio.pipelines import HUBERT_BASE

bundle = HUBERT_BASE
model = bundle.get_model()

with torch.no_grad():
    features, _ = model(waveform)
# features: [1, time_frames, 768]
# → 语音的语义表示向量
# → 用于说话人识别、情感分析、语音分类

# 语音与文本的时间对齐!
# → 字幕生成、歌词同步所必需

from torchaudio.pipelines import MMS_FA  # Multilingual!

bundle = MMS_FA
model = bundle.get_model()
tokenizer = bundle.get_tokenizer()
aligner = bundle.get_aligner()

transcript = "你好 很高兴认识你"
tokens = tokenizer(transcript)

with torch.no_grad():
    emissions, _ = model(waveform)

token_spans = aligner(emissions[0], tokens)
# 以帧为单位返回每个 token 的起止时间!

for span, token in zip(token_spans, transcript):
    start_time = span.start * model.hop_length / sample_rate
    end_time = span.end * model.hop_length / sample_rate
    print(f"  '{token}': {start_time:.3f}s ~ {end_time:.3f}s")

# torchaudio.functional — GPU 加速音频处理

import torchaudio.functional as F

# 音量调节
loud = F.gain(waveform, gain_db=6.0)    # +6dB
quiet = F.gain(waveform, gain_db=-6.0)  # -6dB

# 高通/低通滤波器
highpass = F.highpass_biquad(waveform, sample_rate, cutoff_freq=300)
lowpass = F.lowpass_biquad(waveform, sample_rate, cutoff_freq=3000)

# 均衡器
eq = F.equalizer_biquad(
    waveform, sample_rate,
    center_freq=1000,  # 1kHz 附近
    gain=5.0,          # +5dB 提升
    Q=0.707
)

# 混响
rir, _ = torchaudio.load("room_impulse_response.wav")  # RIR 文件
reverb = F.fftconvolve(waveform, rir)

# 淡入/淡出
fade = torchaudio.transforms.Fade(
    fade_in_len=sample_rate,      # 1 秒淡入
    fade_out_len=sample_rate * 3  # 3 秒淡出
)
faded = fade(waveform)

# VAD (Voice Activity Detection)
vad = torchaudio.transforms.Vad(sample_rate=16000)
speech_only = vad(waveform)  # 去除静音区间

import torch.nn as nn

class AudioClassifier(nn.Module):
    def __init__(self, n_classes=10):
        super().__init__()
        self.mel = torchaudio.transforms.MelSpectrogram(
            sample_rate=16000, n_fft=1024, n_mels=64
        )
        self.db = torchaudio.transforms.AmplitudeToDB()

        # 把 Mel 频谱图当作「图像」输入 CNN!
        self.cnn = nn.Sequential(
            nn.Conv2d(1, 32, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2),
            nn.Conv2d(32, 64, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2),
            nn.Conv2d(64, 128, 3, padding=1), nn.ReLU(),
            nn.AdaptiveAvgPool2d((1, 1)),
        )
        self.fc = nn.Linear(128, n_classes)

    def forward(self, waveform):
        # [B, 1, samples] → [B, 1, n_mels, time]
        x = self.mel(waveform)
        x = self.db(x)
        x = self.cnn(x)
        x = x.view(x.size(0), -1)
        return self.fc(x)

# Mel 频谱图 = 音频的「图像」
# → 可以用 CNN(ResNet, EfficientNet)分类!

from torchaudio.io import StreamReader

# 麦克风输入实时处理
reader = StreamReader(src=":0", format="avfoundation")  # macOS
reader.add_basic_audio_stream(
    frames_per_chunk=16000,  # 以 1 秒为单位
    sample_rate=16000,
)

for (chunk,) in reader.stream():
    # chunk: [1, 16000]
    mel = mel_transform(chunk)
    with torch.no_grad():
        prediction = model(mel)
    print(f"检测结果: {labels[prediction.argmax()]}")